EPFL ALiBi and training provenance 701-750 of 1184

epfl_final/train_logs/bench_moba.log

@@ -0,0 +1,183 @@
+New run: Checkpoints will be saved to /tmp/bench_m/run_20260713_051713
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/tmp/bench_m/run_20260713_051713",
+  "resume_from_checkpoint": null,
+  "force_new_run": true,
+  "num_epochs": 1,
+  "max_steps": 50,
+  "per_device_batch_size": 2,
+  "gradient_accumulation_steps": 1,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 5,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 100000,
+  "eval_steps": 100000,
+  "eval_batches": 50,
+  "logging_steps": 5,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": false,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "365m resampled 8k nrb31",
+  "wandb_tags": [
+    "qwen3",
+    "365m",
+    "8k",
+    "scaleladder",
+    "resampled"
+  ],
+  "hidden_size": 768,
+  "intermediate_size": 3072,
+  "num_attention_heads": 6,
+  "num_key_value_heads": 2,
+  "num_hidden_layers": 28,
+  "head_dim": null,
+  "attention_type": "moba",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 1,
+  "sparse_mask_pool_size": 1000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "sparse_mask_seed": 0,
+  "dense_mode": false,
+  "dense_step_intervals": null,
+  "moba_block_size": 128,
+  "moba_top_k": 32,
+  "asentmax_alpha": 1.5,
+  "asentmax_learn_temp": true,
+  "asentmax_init_temp": 1.0
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Initializing model from scratch (random weights)...
+Checkpoint activations: False
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 768
+  num_hidden_layers: 28
+  num_attention_heads: 6
+  num_key_value_heads: 2
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: moba
+
+Parameter count:
+  Total parameters: 358,958,848
+  Trainable parameters: 358,958,848
+  Non-trainable parameters: 0
+  Model size (BF16): 0.72 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+===========================================================================================================================================================
+Layer (type:depth-idx)                                  Input Shape               Output Shape              Param #                   Trainable
+===========================================================================================================================================================
+Qwen3ForCausalLM                                        [2, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                       --                        [2, 8192, 768]            --                        True
+│    └─Embedding: 2-1                                   [2, 8192]                 [2, 8192, 768]            116,686,848               True
+│    └─ModuleList: 2-2                                  --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                      [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-2                      [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-3                      [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-4                      [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-5                      [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-6                      [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-7                      [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-8                      [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-9                      [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-10                     [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-11                     [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-12                     [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-13                     [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-14                     [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-15                     [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-16                     [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-17                     [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-18                     [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-19                     [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-20                     [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-21                     [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-22                     [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-23                     [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-24                     [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-25                     [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-26                     [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-27                     [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-28                     [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    └─Qwen3RMSNorm: 2-3                                [2, 8192, 768]            [2, 8192, 768]            768                       True
+├─Linear: 1-2                                           [2, 8192, 768]            [2, 8192, 151936]         116,686,848               True
+===========================================================================================================================================================
+Total params: 475,645,696
+Trainable params: 475,645,696
+Non-trainable params: 0
+Total mult-adds (Units.MEGABYTES): 951.29
+===========================================================================================================================================================
+Input size (MB): 0.13
+Forward/backward pass size (MB): 31197.23
+Params size (MB): 951.29
+Estimated Total Size (MB): 32148.66
+===========================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    2
+World size (GPUs):                        1
+Gradient accumulation steps:              1
+Effective batch size (seqs/step):         2
+Sequence length:                          8192
+Tokens per optimizer step:                16,384
+Micro-batches per epoch (per rank):       1513882
+Optimizer steps per epoch:                1513882
+Total optimizer steps:                    50  (capped by max_steps; 0.00 epochs)
+Starting training...
+Switching attention mode for step 1: sparse_stochastic (configured sparse attention)
+Step 5/50 | Loss: 11.5636 | LR: 5.00e-04 | 23723.5ms/step | 0.1 samples/s
+Step 10/50 | Loss: 9.9292 | LR: 4.85e-04 | 369.1ms/step | 5.4 samples/s
+Step 15/50 | Loss: 8.3805 | LR: 4.42e-04 | 368.1ms/step | 5.4 samples/s
+Step 20/50 | Loss: 7.9042 | LR: 3.75e-04 | 368.3ms/step | 5.4 samples/s
+Step 25/50 | Loss: 7.8506 | LR: 2.93e-04 | 369.3ms/step | 5.4 samples/s
+Step 30/50 | Loss: 7.7661 | LR: 2.07e-04 | 370.3ms/step | 5.4 samples/s
+Step 35/50 | Loss: 7.8439 | LR: 1.25e-04 | 370.2ms/step | 5.4 samples/s
+Step 40/50 | Loss: 7.8447 | LR: 5.85e-05 | 370.8ms/step | 5.4 samples/s
+Step 45/50 | Loss: 7.7862 | LR: 1.51e-05 | 370.5ms/step | 5.4 samples/s
+Step 50/50 | Loss: 7.8643 | LR: 0.00e+00 | 370.3ms/step | 5.4 samples/s
+Saved checkpoint to /tmp/bench_m/run_20260713_051713/checkpoint-50
+Training complete!
+[rank0]:[W713 05:19:46.550723983 ProcessGroupNCCL.cpp:1575] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())

epfl_final/train_logs/bench_resamp.log

@@ -0,0 +1,187 @@
+New run: Checkpoints will be saved to /tmp/bench_r/run_20260713_052924
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/tmp/bench_r/run_20260713_052924",
+  "resume_from_checkpoint": null,
+  "force_new_run": true,
+  "num_epochs": 1,
+  "max_steps": 50,
+  "per_device_batch_size": 2,
+  "gradient_accumulation_steps": 1,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 5,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 100000,
+  "eval_steps": 100000,
+  "eval_batches": 50,
+  "logging_steps": 5,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": false,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "365m resampled 8k nrb31",
+  "wandb_tags": [
+    "qwen3",
+    "365m",
+    "8k",
+    "scaleladder",
+    "resampled"
+  ],
+  "hidden_size": 768,
+  "intermediate_size": 3072,
+  "num_attention_heads": 6,
+  "num_key_value_heads": 2,
+  "num_hidden_layers": 28,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 8,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/tmp/bench_maskcache",
+  "sparse_mask_seed": 0,
+  "dense_mode": false,
+  "dense_step_intervals": null,
+  "moba_block_size": 128,
+  "moba_top_k": 32,
+  "asentmax_alpha": 1.5,
+  "asentmax_learn_temp": true,
+  "asentmax_init_temp": 1.0
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Initializing model from scratch (random weights)...
+Checkpoint activations: False
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 768
+  num_hidden_layers: 28
+  num_attention_heads: 6
+  num_key_value_heads: 2
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 8
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 358,958,848
+  Trainable parameters: 358,958,848
+  Non-trainable parameters: 0
+  Model size (BF16): 0.72 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [2, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [2, 8192, 768]            --                        True
+│    └─Embedding: 2-1                                        [2, 8192]                 [2, 8192, 768]            116,686,848               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-2                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-3                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-4                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-5                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-6                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-7                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-8                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-9                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-10                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-11                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-12                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-13                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-14                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-15                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-16                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-17                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-18                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-19                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-20                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-21                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-22                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-23                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-24                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-25                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-26                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-27                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-28                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    └─Qwen3RMSNorm: 2-3                                     [2, 8192, 768]            [2, 8192, 768]            768                       True
+├─Linear: 1-2                                                [2, 8192, 768]            [2, 8192, 151936]         116,686,848               True
+================================================================================================================================================================
+Total params: 475,645,696
+Trainable params: 475,645,696
+Non-trainable params: 0
+Total mult-adds (Units.MEGABYTES): 951.29
+================================================================================================================================================================
+Input size (MB): 0.13
+Forward/backward pass size (MB): 31197.23
+Params size (MB): 951.29
+Estimated Total Size (MB): 32148.66
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    2
+World size (GPUs):                        1
+Gradient accumulation steps:              1
+Effective batch size (seqs/step):         2
+Sequence length:                          8192
+Tokens per optimizer step:                16,384
+Micro-batches per epoch (per rank):       1513882
+Optimizer steps per epoch:                1513882
+Total optimizer steps:                    50  (capped by max_steps; 0.00 epochs)
+Starting training...
+Switching attention mode for step 1: sparse_stochastic (configured sparse attention)
+Step 5/50 | Loss: 11.5599 | LR: 5.00e-04 | 11537.8ms/step | 0.2 samples/s
+Step 10/50 | Loss: 9.9272 | LR: 4.85e-04 | 315.0ms/step | 6.4 samples/s
+Step 15/50 | Loss: 8.3778 | LR: 4.42e-04 | 314.6ms/step | 6.4 samples/s
+Step 20/50 | Loss: 7.9042 | LR: 3.75e-04 | 314.1ms/step | 6.4 samples/s
+Step 25/50 | Loss: 7.8506 | LR: 2.93e-04 | 314.3ms/step | 6.4 samples/s
+Step 30/50 | Loss: 7.7669 | LR: 2.07e-04 | 314.8ms/step | 6.4 samples/s
+Step 35/50 | Loss: 7.8439 | LR: 1.25e-04 | 314.8ms/step | 6.4 samples/s
+Step 40/50 | Loss: 7.8443 | LR: 5.85e-05 | 314.9ms/step | 6.4 samples/s
+Step 45/50 | Loss: 7.7864 | LR: 1.51e-05 | 315.2ms/step | 6.3 samples/s
+Step 50/50 | Loss: 7.8643 | LR: 0.00e+00 | 315.2ms/step | 6.3 samples/s
+Saved checkpoint to /tmp/bench_r/run_20260713_052924/checkpoint-50
+Training complete!
+[rank0]:[W713 05:31:16.338746416 ProcessGroupNCCL.cpp:1575] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())

epfl_final/train_logs/build_30k_then_train.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+set -u
+
+BASE=/mloscratch/homes/navasard/sparse
+REPO="$BASE/SparseAtt-baselines"
+CONFIG=configs/365m_mixed_30k_2gpu_seed42.json
+CACHE="$BASE/mask_cache_30k_365m_mixed_seed42"
+PY=/mloscratch/homes/navasard/conda/bin/python3.13
+BUILDER="$BASE/train_logs/build_layer_mixed_seeded.py"
+LOG="$BASE/train_logs/mixed30k365m_s42.cache.log"
+WATCHDOG="$BASE/train_logs/watchdog_mixed_training.sh"
+
+cd "$REPO" || exit 2
+mkdir -p "$CACHE"
+
+CUDA_VISIBLE_DEVICES=2 PYTHONPATH="$REPO" HF_HOME=/mloscratch/homes/navasard/hf_cache \
+  "$PY" "$BUILDER" --config "$CONFIG" --cache-dir "$CACHE" \
+  --layers 0,2,4,6,8,10,12,14,16,18,20,22,24,26 >> "$LOG" 2>&1 &
+pid_even=$!
+
+CUDA_VISIBLE_DEVICES=3 PYTHONPATH="$REPO" HF_HOME=/mloscratch/homes/navasard/hf_cache \
+  "$PY" "$BUILDER" --config "$CONFIG" --cache-dir "$CACHE" \
+  --layers 1,3,5,7,9,11,13,15,17,19,21,23,25,27 >> "$LOG" 2>&1 &
+pid_odd=$!
+
+status=0
+wait "$pid_even" || status=1
+wait "$pid_odd" || status=1
+
+files=$(find "$CACHE" -maxdepth 1 -type f -name 'masks_layer_*.pt' | wc -l)
+echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) cache status=$status files=$files" >> "$LOG"
+if [ "$status" -ne 0 ] || [ "$files" -ne 28 ]; then
+  exit 1
+fi
+
+nohup "$WATCHDOG" mixed30k365m_s42 "$CONFIG" \
+  "$BASE/checkpoints/l30k365m_mixed_seed42" 11700 2,3 2 \
+  > "$BASE/train_logs/mixed30k365m_s42.supervisor.log" 2>&1 < /dev/null &

epfl_final/train_logs/build_layer_mixed_seeded.py

@@ -0,0 +1,58 @@
+import argparse
+import json
+import os
+import time
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--config", required=True)
+    parser.add_argument("--cache-dir", required=True)
+    parser.add_argument("--layers", required=True)
+    args = parser.parse_args()
+
+    with open(args.config) as config_file:
+        raw = json.load(config_file)
+
+    model = raw["model"]
+    data = raw["data"]
+    attention = raw["attention"]
+
+    from sparseatt.config import Qwen3Config
+    from sparseatt.sparse_attention import Qwen3SparseStochasticAttention
+
+    config = Qwen3Config(
+        hidden_size=model["hidden_size"],
+        intermediate_size=model["intermediate_size"],
+        num_attention_heads=model["num_attention_heads"],
+        num_key_value_heads=model["num_key_value_heads"],
+        num_hidden_layers=model["num_hidden_layers"],
+        attention_type="sparse_stochastic",
+        sparse_block_size=attention["sparse_block_size"],
+        sparse_max_seq_length=data["max_seq_length"],
+        sparse_num_random_blocks=attention["sparse_num_random_blocks"],
+        sparse_mask_pool_size=attention["sparse_mask_pool_size"],
+        sparse_mixed_nrb_set=attention["sparse_mixed_nrb_set"],
+        sparse_mask_seed=attention.get("sparse_mask_seed", 0),
+    )
+
+    os.makedirs(args.cache_dir, exist_ok=True)
+    for layer_index in map(int, args.layers.split(",")):
+        output = os.path.join(args.cache_dir, f"masks_layer_{layer_index}.pt")
+        if os.path.isfile(output):
+            print(f"layer {layer_index}: exists", flush=True)
+            continue
+        started = time.time()
+        Qwen3SparseStochasticAttention(
+            config,
+            layer_idx=layer_index,
+            mask_cache_dir=args.cache_dir,
+        )
+        print(
+            f"layer {layer_index}: built in {time.time() - started:.0f}s",
+            flush=True,
+        )
+
+
+if __name__ == "__main__":
+    main()

epfl_final/train_logs/dense8k3b_60b.watch.log

@@ -0,0 +1,35 @@
+2026-07-20T07:42:01Z watchdog start pid=151
+2026-07-20T07:42:01Z launch GPUs=0,1,2,3 nproc=4 latest_step=
+2026-07-20T17:25:14Z watchdog start pid=19659
+2026-07-20T17:25:14Z launch GPUs=0,1,2,3 nproc=4 latest_step=5000
+2026-07-20T20:12:53Z watchdog start pid=146
+2026-07-20T20:12:53Z launch GPUs=0,1,2,3 nproc=4 latest_step=6000
+2026-07-21T07:53:01Z watchdog start pid=129
+2026-07-21T07:53:43Z watchdog start pid=187
+2026-07-21T07:53:43Z launch GPUs=0,1,2,3,4,5 nproc=6 latest_step=12000
+2026-07-21T20:14:57Z watchdog start pid=143
+2026-07-21T20:14:57Z launch GPUs=0,1,2,3,4,5 nproc=6 latest_step=21500
+2026-07-22T08:14:30Z watchdog start pid=145
+2026-07-22T08:14:30Z launch GPUs=0,1,2,3,4,5 nproc=6 latest_step=30750
+2026-07-22T18:04:53Z watchdog start pid=182
+2026-07-22T18:05:20Z watchdog start pid=216
+2026-07-22T18:05:20Z launch GPUs=0,1,2,3 nproc=4 latest_step=31500
+2026-07-22T23:31:22Z watchdog start pid=160
+2026-07-22T23:31:22Z launch GPUs=0,1,2,3,4,5 nproc=6 latest_step=34250
+2026-07-23T07:08:12Z watchdog start pid=154
+2026-07-23T07:08:12Z launch GPUs=0,1,2,3,4,5 nproc=6 latest_step=40000
+2026-07-23T18:47:42Z watchdog start pid=153
+2026-07-23T18:47:42Z launch GPUs=0,1,2,3,4,5 nproc=6 latest_step=48750
+2026-07-24T06:02:30Z watchdog start pid=154
+2026-07-24T06:02:30Z launch GPUs=0,1,2,3,4,5 nproc=6 latest_step=57500
+2026-07-25T17:36:18Z watchdog start pid=154
+2026-07-25T17:36:19Z launch GPUs=0,1 nproc=2 latest_step=62250
+2026-07-25T19:02:01Z watchdog start pid=5777
+2026-07-25T19:02:01Z launch GPUs=0,1,2,3 nproc=4 latest_step=62250
+2026-07-26T09:05:22Z watchdog start pid=144 latest_step=62250
+2026-07-26T09:06:16Z watchdog start pid=211 latest_step=62250
+2026-07-26T09:06:16Z launch checkpoint=/mloscratch/homes/navasard/sparse/checkpoints/l8k3b_dense_60b/run_20260720_074219/checkpoint-62250 GPUs=0,1,2,3,4,5 nproc=6
+2026-07-26T17:33:02Z watchdog start pid=144 latest_step=63250
+2026-07-26T17:33:02Z launch checkpoint=/mloscratch/homes/navasard/sparse/checkpoints/l8k3b_dense_60b/run_20260720_074219/checkpoint-63250 GPUs=0,1,2,3,4,5 nproc=6
+2026-07-28T03:20:00Z watchdog start pid=298 latest_step=78000
+2026-07-28T03:20:01Z launch checkpoint=/mloscratch/homes/navasard/sparse/checkpoints/l8k3b_dense_60b/run_20260720_074219/checkpoint-78000 GPUs=0,1,2,3,4,5 nproc=6

epfl_final/train_logs/eval30k_fixed.log

@@ -0,0 +1,39 @@
+windows gutenberg (16, 30720)
+windows arxiv (16, 30720)
+windows pubmed (16, 30720)
+Checkpoint activations: False
+built topk=240 attn=moba layer0.top_k=240 (compiling flex...)
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/compile_fx.py:322: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance.
+  warnings.warn(
+topk 240 gutenberg L30720 ppl    47.221 nll 3.8548 win 16  58s
+topk 240 arxiv     L30720 ppl   176.422 nll 5.1729 win 16  10s
+topk 240 pubmed    L30720 ppl    44.365 nll 3.7924 win 16  10s
+Checkpoint activations: False
+built topk=120 attn=moba layer0.top_k=120 (compiling flex...)
+topk 120 gutenberg L30720 ppl    44.040 nll 3.7851 win 16  57s
+topk 120 arxiv     L30720 ppl   155.744 nll 5.0482 win 16  9s
+topk 120 pubmed    L30720 ppl    39.222 nll 3.6692 win 16  9s
+Checkpoint activations: False
+built topk=60 attn=moba layer0.top_k=60 (compiling flex...)
+topk  60 gutenberg L30720 ppl    34.065 nll 3.5283 win 16  25s
+topk  60 arxiv     L30720 ppl   108.078 nll 4.6829 win 16  7s
+topk  60 pubmed    L30720 ppl    25.856 nll 3.2526 win 16  7s
+Checkpoint activations: False
+built topk=30 attn=moba layer0.top_k=30 (compiling flex...)
+topk  30 gutenberg L30720 ppl    26.893 nll 3.2918 win 16  54s
+topk  30 arxiv     L30720 ppl    78.182 nll 4.3590 win 16  7s
+topk  30 pubmed    L30720 ppl    17.987 nll 2.8897 win 16  7s
+Checkpoint activations: False
+built topk=15 attn=moba layer0.top_k=15 (compiling flex...)
+topk  15 gutenberg L30720 ppl    24.577 nll 3.2018 win 16  56s
+topk  15 arxiv     L30720 ppl    69.587 nll 4.2426 win 16  6s
+topk  15 pubmed    L30720 ppl    16.606 nll 2.8097 win 16  6s
+Checkpoint activations: False
+built topk=8 attn=moba layer0.top_k=8 (compiling flex...)
+topk   8 gutenberg L30720 ppl    25.154 nll 3.2250 win 16  54s
+topk   8 arxiv     L30720 ppl    73.669 nll 4.2996 win 16  6s
+topk   8 pubmed    L30720 ppl    18.223 nll 2.9027 win 16  6s
+wrote /mloscratch/homes/navasard/sparse/SparseAtt/eval/results/L365M_8K/ppl_fixed365_L30720_gutenberg.tsv
+wrote /mloscratch/homes/navasard/sparse/SparseAtt/eval/results/L365M_8K/ppl_fixed365_L30720_arxiv.tsv
+wrote /mloscratch/homes/navasard/sparse/SparseAtt/eval/results/L365M_8K/ppl_fixed365_L30720_pubmed.tsv
+DONE fixed365

epfl_final/train_logs/eval30k_moba.log

@@ -0,0 +1,39 @@
+windows gutenberg (16, 30720)
+windows arxiv (16, 30720)
+windows pubmed (16, 30720)
+Checkpoint activations: False
+built topk=240 attn=moba layer0.top_k=240 (compiling flex...)
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/compile_fx.py:322: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance.
+  warnings.warn(
+topk 240 gutenberg L30720 ppl    41.571 nll 3.7274 win 16  59s
+topk 240 arxiv     L30720 ppl   142.291 nll 4.9579 win 16  10s
+topk 240 pubmed    L30720 ppl    34.256 nll 3.5339 win 16  10s
+Checkpoint activations: False
+built topk=120 attn=moba layer0.top_k=120 (compiling flex...)
+topk 120 gutenberg L30720 ppl    37.896 nll 3.6349 win 16  58s
+topk 120 arxiv     L30720 ppl   123.961 nll 4.8200 win 16  9s
+topk 120 pubmed    L30720 ppl    29.609 nll 3.3881 win 16  9s
+Checkpoint activations: False
+built topk=60 attn=moba layer0.top_k=60 (compiling flex...)
+topk  60 gutenberg L30720 ppl    29.481 nll 3.3838 win 16  25s
+topk  60 arxiv     L30720 ppl    90.316 nll 4.5033 win 16  7s
+topk  60 pubmed    L30720 ppl    20.388 nll 3.0149 win 16  7s
+Checkpoint activations: False
+built topk=30 attn=moba layer0.top_k=30 (compiling flex...)
+topk  30 gutenberg L30720 ppl    25.224 nll 3.2278 win 16  53s
+topk  30 arxiv     L30720 ppl    73.057 nll 4.2912 win 16  7s
+topk  30 pubmed    L30720 ppl    16.815 nll 2.8223 win 16  7s
+Checkpoint activations: False
+built topk=15 attn=moba layer0.top_k=15 (compiling flex...)
+topk  15 gutenberg L30720 ppl    24.842 nll 3.2125 win 16  55s
+topk  15 arxiv     L30720 ppl    72.465 nll 4.2831 win 16  6s
+topk  15 pubmed    L30720 ppl    17.629 nll 2.8696 win 16  6s
+Checkpoint activations: False
+built topk=8 attn=moba layer0.top_k=8 (compiling flex...)
+topk   8 gutenberg L30720 ppl    26.242 nll 3.2674 win 16  55s
+topk   8 arxiv     L30720 ppl    82.900 nll 4.4176 win 16  6s
+topk   8 pubmed    L30720 ppl    20.312 nll 3.0112 win 16  6s
+wrote /mloscratch/homes/navasard/sparse/SparseAtt/eval/results/L365M_8K/ppl_moba365_L30720_gutenberg.tsv
+wrote /mloscratch/homes/navasard/sparse/SparseAtt/eval/results/L365M_8K/ppl_moba365_L30720_arxiv.tsv
+wrote /mloscratch/homes/navasard/sparse/SparseAtt/eval/results/L365M_8K/ppl_moba365_L30720_pubmed.tsv
+DONE moba365

epfl_final/train_logs/launch_1p2b_30k_mixed_after_moba.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+set -u
+
+BASE=/mloscratch/homes/navasard/sparse
+MOBA_OUT="$BASE/checkpoints/l30k1_2b_moba"
+MOBA_LOG="$BASE/train_logs/resume_moba30k_h100_2-0-6.log"
+MIXED_OUT="$BASE/checkpoints/l30k1_2b_mixed"
+MIXED_CONFIG=configs/1p2b_mixed_30k_4gpu_seed42.json
+WATCHDOG="$BASE/train_logs/watchdog_mixed_training.sh"
+LOG="$BASE/train_logs/wait_1p2b_30k_mixed_after_moba.log"
+LOCK="$BASE/train_logs/wait_1p2b_30k_mixed_after_moba.lock"
+
+log() {
+  echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*" >> "$LOG"
+}
+
+latest_moba_step() {
+  ls -1d "$MOBA_OUT"/run_*/checkpoint-* 2>/dev/null |
+    sed -E 's#.*/checkpoint-##' |
+    grep -E '^[0-9]+$' |
+    sort -n |
+    tail -1
+}
+
+moba_complete() {
+  local step
+  step=$(latest_moba_step)
+  if [ -n "${step:-}" ] && [ "$step" -ge 37465 ] 2>/dev/null; then
+    return 0
+  fi
+  grep -qs "Training complete!" "$MOBA_LOG" 2>/dev/null
+}
+
+if [ -f "$LOCK" ]; then
+  old_pid=$(cat "$LOCK" 2>/dev/null || true)
+  if [ -n "${old_pid:-}" ] && kill -0 "$old_pid" 2>/dev/null; then
+    exit 0
+  fi
+fi
+
+echo $$ > "$LOCK"
+log "waiter start pid=$$ latest_moba_step=$(latest_moba_step)"
+
+while ! moba_complete; do
+  sleep 30
+done
+
+while pgrep -f 'train.py --config configs/train_config_30k_moba_b128.json' >/dev/null 2>&1; do
+  sleep 10
+done
+
+log "MoBA complete; launching native-30k 1.2B Mixed"
+nohup "$WATCHDOG" mixed30k1p2b_s42 "$MIXED_CONFIG" "$MIXED_OUT" \
+  37465 0,1,2,3 4 \
+  > "$BASE/train_logs/mixed30k1p2b_s42.supervisor.log" 2>&1 < /dev/null &
+log "Mixed watchdog pid=$!"
+rm -f "$LOCK"

epfl_final/train_logs/launch_1p2b_mixed_8k_seed43.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -u
+
+BASE=/mloscratch/homes/navasard/sparse
+WATCHDOG="$BASE/train_logs/watchdog_mixed_training.sh"
+
+nohup "$WATCHDOG" mixed8k1p2b_s43 \
+  configs/1p2b_mixed_8k_seed43_4gpu.json \
+  "$BASE/checkpoints/l8k1_2b_mixed_seed43" \
+  50462 0,1,2,3 4 \
+  > "$BASE/train_logs/mixed8k1p2b_s43.supervisor.log" 2>&1 < /dev/null &
+
+echo "mixed8k1p2b_s43 watchdog pid=$!"

epfl_final/train_logs/launch_3b_dense_4gpu.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -u
+
+BASE=/mloscratch/homes/navasard/sparse
+WATCHDOG="$BASE/train_logs/watchdog_mixed_training.sh"
+
+nohup "$WATCHDOG" dense8k3b_60b \
+  configs/ladder8k/dense_3b_60b.json \
+  "$BASE/checkpoints/l8k3b_dense_60b" \
+  122070 0,1,2,3 4 \
+  > "$BASE/train_logs/dense8k3b_60b.supervisor.log" 2>&1 < /dev/null &
+
+echo "dense8k3b_60b watchdog pid=$!"

epfl_final/train_logs/log_3b_dense_gpu_usage.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -u
+
+LOG=/mloscratch/homes/navasard/sparse/train_logs/dense8k3b_60b.gpu.csv
+
+if [ ! -s "$LOG" ]; then
+  echo "timestamp,index,name,memory_used_mib,memory_total_mib,utilization_gpu_pct,power_draw_w" > "$LOG"
+fi
+
+while pgrep -f 'train.py --config configs/ladder8k/dense_3b_60b.json' >/dev/null 2>&1; do
+  timestamp=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+  nvidia-smi \
+    --query-gpu=index,name,memory.used,memory.total,utilization.gpu,power.draw \
+    --format=csv,noheader,nounits |
+    sed "s/^/$timestamp,/" >> "$LOG"
+  sleep 60
+done

epfl_final/train_logs/mixed30k365m_s42.cache.log

@@ -0,0 +1,29 @@
+layer 1: built in 1179s
+layer 0: built in 1203s
+layer 3: built in 1187s
+layer 2: built in 1195s
+layer 5: built in 1174s
+layer 4: built in 1175s
+layer 7: built in 1193s
+layer 6: built in 1193s
+layer 9: built in 1152s
+layer 8: built in 1212s
+layer 11: built in 1206s
+layer 10: built in 1201s
+layer 13: built in 1179s
+layer 12: built in 1174s
+layer 15: built in 1198s
+layer 14: built in 1168s
+layer 17: built in 1161s
+layer 16: built in 1184s
+layer 19: built in 1130s
+layer 18: built in 1163s
+layer 21: built in 1174s
+layer 20: built in 1184s
+layer 23: built in 1171s
+layer 22: built in 1174s
+layer 25: built in 1167s
+layer 24: built in 1195s
+layer 27: built in 1176s
+layer 26: built in 1187s
+2026-07-20T04:18:32Z cache status=0 files=28

epfl_final/train_logs/mixed30k365m_s42.watch.log

@@ -0,0 +1,5 @@
+2026-07-19T23:35:00Z watchdog start pid=330
+2026-07-19T23:35:00Z launch GPUs=2,3 nproc=2 latest_step=
+2026-07-20T04:18:32Z watchdog start pid=16098
+2026-07-20T04:18:32Z launch GPUs=2,3 nproc=2 latest_step=
+2026-07-21T21:03:35Z complete latest_step=11700

epfl_final/train_logs/mixed8k1p2b_s43.watch.log

@@ -0,0 +1,8 @@
+2026-07-20T18:55:59Z watchdog start pid=171
+2026-07-20T18:55:59Z launch GPUs=0,1,2,3 nproc=4 latest_step=
+2026-07-22T18:11:22Z watchdog start pid=181
+2026-07-22T18:11:22Z launch GPUs=0,1,2,3 nproc=4 latest_step=20000
+2026-07-22T21:37:57Z watchdog start pid=42334
+2026-07-22T21:37:57Z launch GPUs=0,1,2,3 nproc=4 latest_step=20000
+2026-07-23T06:33:52Z watchdog start pid=155
+2026-07-23T06:33:52Z launch GPUs=0,1,2,3 nproc=4 latest_step=24000

epfl_final/train_logs/mixed8k365m_s42.watch.log

@@ -0,0 +1,5 @@
+2026-07-19T23:35:00Z watchdog start pid=328
+2026-07-19T23:35:00Z launch GPUs=0 nproc=1 latest_step=
+2026-07-19T23:41:40Z watchdog start pid=1439
+2026-07-19T23:41:40Z launch GPUs=0 nproc=1 latest_step=
+2026-07-21T19:33:03Z complete latest_step=44000

epfl_final/train_logs/mixed8k365m_s43.watch.log

@@ -0,0 +1,5 @@
+2026-07-19T23:35:00Z watchdog start pid=329
+2026-07-19T23:35:00Z launch GPUs=1 nproc=1 latest_step=
+2026-07-19T23:41:40Z watchdog start pid=1440
+2026-07-19T23:41:40Z launch GPUs=1 nproc=1 latest_step=
+2026-07-21T19:49:04Z complete latest_step=44000

epfl_final/train_logs/mobafix_arxiv.log

@@ -0,0 +1,28 @@
+[sparse-eval] loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base
+[sparse-eval] building packed windows (eval_len=30720, max_windows=16)
+[sparse-eval] 16 windows x 30720 tokens
+[sparse-eval] /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_moba/run_20260709_041217/checkpoint-44000 @ deploy_sparsity=31 pattern=random mask_mode=fixed
+Checkpoint activations: False
+[sparse-eval]   ppl=132.1137 nll=4.8837 density=0.1370 tokens=491504
+{
+  "generated_at": "2026-07-13T05:45:15.873088+00:00",
+  "eval_len": 30720,
+  "block_size": 128,
+  "results": [
+    {
+      "checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_moba/run_20260709_041217/checkpoint-44000",
+      "deploy_sparsity": "31",
+      "deploy_pattern": "random",
+      "mask_mode": "fixed",
+      "force_sparse_layers": "all",
+      "eval_len": 30720,
+      "dataset": "/mloscratch/homes/navasard/data/arxiv30k_ood_windows",
+      "avg_nll": 4.883662664749422,
+      "perplexity": 132.11366687744194,
+      "tokens": 491504,
+      "windows": 16,
+      "realized_block_density": 0.1369763880655997
+    }
+  ]
+}
+[sparse-eval] TSV written to /mloscratch/homes/navasard/sparse/SparseAtt/eval/results/L365M_8K/moba_fixedmask30k_arxiv.tsv

epfl_final/train_logs/mobafix_gutenberg.log

@@ -0,0 +1,28 @@
+[sparse-eval] loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base
+[sparse-eval] building packed windows (eval_len=30720, max_windows=16)
+[sparse-eval] 16 windows x 30720 tokens
+[sparse-eval] /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_moba/run_20260709_041217/checkpoint-44000 @ deploy_sparsity=31 pattern=random mask_mode=fixed
+Checkpoint activations: False
+[sparse-eval]   ppl=32.5508 nll=3.4828 density=0.1370 tokens=491504
+{
+  "generated_at": "2026-07-13T05:45:15.827144+00:00",
+  "eval_len": 30720,
+  "block_size": 128,
+  "results": [
+    {
+      "checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_moba/run_20260709_041217/checkpoint-44000",
+      "deploy_sparsity": "31",
+      "deploy_pattern": "random",
+      "mask_mode": "fixed",
+      "force_sparse_layers": "all",
+      "eval_len": 30720,
+      "dataset": "/mloscratch/homes/navasard/data/gutenberg_ood_windows",
+      "avg_nll": 3.4828026100754013,
+      "perplexity": 32.55082161861716,
+      "tokens": 491504,
+      "windows": 16,
+      "realized_block_density": 0.1369763880655997
+    }
+  ]
+}
+[sparse-eval] TSV written to /mloscratch/homes/navasard/sparse/SparseAtt/eval/results/L365M_8K/moba_fixedmask30k_gutenberg.tsv

epfl_final/train_logs/mobafix_pubmed.log

@@ -0,0 +1,28 @@
+[sparse-eval] loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base
+[sparse-eval] building packed windows (eval_len=30720, max_windows=16)
+[sparse-eval] 16 windows x 30720 tokens
+[sparse-eval] /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_moba/run_20260709_041217/checkpoint-44000 @ deploy_sparsity=31 pattern=random mask_mode=fixed
+Checkpoint activations: False
+[sparse-eval]   ppl=27.3365 nll=3.3082 density=0.1370 tokens=491504
+{
+  "generated_at": "2026-07-13T05:45:14.517909+00:00",
+  "eval_len": 30720,
+  "block_size": 128,
+  "results": [
+    {
+      "checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_moba/run_20260709_041217/checkpoint-44000",
+      "deploy_sparsity": "31",
+      "deploy_pattern": "random",
+      "mask_mode": "fixed",
+      "force_sparse_layers": "all",
+      "eval_len": 30720,
+      "dataset": "/mloscratch/homes/navasard/data/pubmed_ood_windows",
+      "avg_nll": 3.3082236646344687,
+      "perplexity": 27.336523484532226,
+      "tokens": 491504,
+      "windows": 16,
+      "realized_block_density": 0.1369763880655997
+    }
+  ]
+}
+[sparse-eval] TSV written to /mloscratch/homes/navasard/sparse/SparseAtt/eval/results/L365M_8K/moba_fixedmask30k_pubmed.tsv

epfl_final/train_logs/native_fixed.log

@@ -0,0 +1,7 @@
+}
+[sparse-eval] TSV written to /mloscratch/homes/navasard/sparse/SparseAtt/eval/results/L365M_8K/native30k_fixed_gutenberg.tsv
+}
+[sparse-eval] TSV written to /mloscratch/homes/navasard/sparse/SparseAtt/eval/results/L365M_8K/native30k_fixed_arxiv.tsv
+}
+[sparse-eval] TSV written to /mloscratch/homes/navasard/sparse/SparseAtt/eval/results/L365M_8K/native30k_fixed_pubmed.tsv
+DONE_NATIVE fixed

epfl_final/train_logs/native_moba.log

@@ -0,0 +1,14 @@
+windows gutenberg (16, 30720)
+windows arxiv (16, 30720)
+windows pubmed (16, 30720)
+Checkpoint activations: False
+built topk=32 attn=moba layer0.top_k=32 (compiling flex...)
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/compile_fx.py:322: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance.
+  warnings.warn(
+topk  32 gutenberg L30720 ppl    25.410 nll 3.2352 win 16  58s
+topk  32 arxiv     L30720 ppl    73.912 nll 4.3029 win 16  7s
+topk  32 pubmed    L30720 ppl    16.908 nll 2.8278 win 16  7s
+wrote /mloscratch/homes/navasard/sparse/SparseAtt/eval/results/L365M_8K/ppl_moba_native30k_L30720_gutenberg.tsv
+wrote /mloscratch/homes/navasard/sparse/SparseAtt/eval/results/L365M_8K/ppl_moba_native30k_L30720_arxiv.tsv
+wrote /mloscratch/homes/navasard/sparse/SparseAtt/eval/results/L365M_8K/ppl_moba_native30k_L30720_pubmed.tsv
+DONE moba_native30k

epfl_final/train_logs/native_resamp.log

@@ -0,0 +1,7 @@
+}
+[sparse-eval] TSV written to /mloscratch/homes/navasard/sparse/SparseAtt/eval/results/L365M_8K/native30k_resamp_gutenberg.tsv
+}
+[sparse-eval] TSV written to /mloscratch/homes/navasard/sparse/SparseAtt/eval/results/L365M_8K/native30k_resamp_arxiv.tsv
+}
+[sparse-eval] TSV written to /mloscratch/homes/navasard/sparse/SparseAtt/eval/results/L365M_8K/native30k_resamp_pubmed.tsv
+DONE_NATIVE resamp

epfl_final/train_logs/resume_4b_h100_3-0-0.log

@@ -0,0 +1,591 @@
+W0717 23:36:36.781000 162 site-packages/torch/distributed/run.py:851] 
+W0717 23:36:36.781000 162 site-packages/torch/distributed/run.py:851] *****************************************
+W0717 23:36:36.781000 162 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0717 23:36:36.781000 162 site-packages/torch/distributed/run.py:851] *****************************************
+Auto-resuming from latest checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k4b_resampled/run_20260715_232017/checkpoint-11000
+Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k4b_resampled/run_20260715_232017
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k4b_resampled/run_20260715_232017",
+  "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k4b_resampled/run_20260715_232017/checkpoint-11000",
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": null,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 15,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": true,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "4b resampled 8k",
+  "wandb_tags": [
+    "qwen3",
+    "4b",
+    "8k",
+    "resampled",
+    "scaleladder"
+  ],
+  "hidden_size": 2560,
+  "intermediate_size": 9728,
+  "num_attention_heads": 32,
+  "num_key_value_heads": 8,
+  "num_hidden_layers": 36,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 10000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_36L",
+  "sparse_mask_seed": 0,
+  "sparse_mixed_nrb_set": null,
+  "dense_mode": false,
+  "dense_step_intervals": null,
+  "moba_block_size": 128,
+  "moba_top_k": 32,
+  "asentmax_alpha": 1.5,
+  "asentmax_learn_temp": true,
+  "asentmax_init_temp": 1.0
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k4b_resampled/run_20260715_232017/checkpoint-11000
+Checkpoint activations: TrueCheckpoint activations: True
+
+Checkpoint activations: True
+Checkpoint activations: True
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 2560
+  num_hidden_layers: 36
+  num_attention_heads: 32
+  num_key_value_heads: 8
+  head_dim: 128
+  intermediate_size: 9728
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 10000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 4,022,468,096
+  Trainable parameters: 4,022,468,096
+  Non-trainable parameters: 0
+  Model size (BF16): 8.04 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 8192, 2560]           --                        True
+│    └─Embedding: 2-1                                        [1, 8192]                 [1, 8192, 2560]           388,956,160               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-29                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-30                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-31                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-32                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-33                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-34                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-35                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    │    └─Qwen3DecoderLayer: 3-36                          [1, 8192, 2560]           [1, 8192, 2560]           100,930,816               True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 8192, 2560]           [1, 8192, 2560]           2,560                     True
+├─Linear: 1-2                                                [1, 8192, 2560]           [1, 8192, 151936]         388,956,160               True
+================================================================================================================================================================
+Total params: 4,411,424,256
+Trainable params: 4,411,424,256
+Non-trainable params: 0
+Total mult-adds (Units.GIGABYTES): 4.41
+================================================================================================================================================================
+Input size (MB): 0.07
+Forward/backward pass size (MB): 53464.79
+Params size (MB): 8822.85
+Estimated Total Size (MB): 62287.71
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (388,956,160 params). True unique parameter count: 4,022,468,096 (4022.5M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    1
+World size (GPUs):                        4
+Gradient accumulation steps:              15
+Effective batch size (seqs/step):         60
+Sequence length:                          8192
+Tokens per optimizer step:                491,520
+Micro-batches per epoch (per rank):       756941
+Optimizer steps per epoch:                50462
+Total optimizer steps:                    50462
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k4b_resampled/run_20260715_232017/checkpoint-11000...
+Overriding checkpoint LR with config LR: 0.0005
+Resumed from checkpoint at step 11000 (epoch 0, micro_step 165000)
+Resuming training from step 11000
+Resuming wandb run w2xx8aku
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt-baselines/wandb/run-20260717_234254-w2xx8aku
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Resuming run 4b resampled 8k
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/w2xx8aku
+Starting training...
+Validation at step 11000: Loss = 2.6263, Perplexity = 13.82
+wandb: WARNING Tried to log to step 11000 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 11001: sparse_stochastic (configured sparse attention)
+Step 11010/50462 | Loss: 2.5954 | LR: 4.49e-04 | 49735.6ms/step | 1.2 samples/s
+wandb: WARNING Tried to log to step 11010 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11020/50462 | Loss: 2.5819 | LR: 4.49e-04 | 14254.3ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11020 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11030/50462 | Loss: 2.6228 | LR: 4.49e-04 | 14200.1ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11030 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11040/50462 | Loss: 2.6200 | LR: 4.49e-04 | 14292.7ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11040 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11050/50462 | Loss: 2.5650 | LR: 4.49e-04 | 14347.1ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11050 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11060/50462 | Loss: 2.5858 | LR: 4.49e-04 | 14300.3ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11060 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11070/50462 | Loss: 2.6413 | LR: 4.49e-04 | 14251.3ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11070 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11080/50462 | Loss: 2.6296 | LR: 4.49e-04 | 14250.2ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11080 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11090/50462 | Loss: 2.6252 | LR: 4.48e-04 | 14206.4ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11090 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11100/50462 | Loss: 2.6364 | LR: 4.48e-04 | 14353.1ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11100 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11110/50462 | Loss: 2.6242 | LR: 4.48e-04 | 14310.0ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11110 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11120/50462 | Loss: 2.6168 | LR: 4.48e-04 | 14330.6ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11120 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11130/50462 | Loss: 2.6110 | LR: 4.48e-04 | 14231.4ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11130 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11140/50462 | Loss: 2.5972 | LR: 4.48e-04 | 14244.5ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11140 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11150/50462 | Loss: 2.6062 | LR: 4.48e-04 | 14215.0ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11150 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11160/50462 | Loss: 2.5759 | LR: 4.48e-04 | 14332.6ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11160 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11170/50462 | Loss: 2.6306 | LR: 4.48e-04 | 14374.6ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11170 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11180/50462 | Loss: 2.6173 | LR: 4.48e-04 | 14357.1ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11180 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11190/50462 | Loss: 2.6270 | LR: 4.48e-04 | 14308.5ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11190 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11200/50462 | Loss: 2.6146 | LR: 4.47e-04 | 14350.0ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11200 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11210/50462 | Loss: 2.6149 | LR: 4.47e-04 | 14303.0ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11210 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11220/50462 | Loss: 2.6336 | LR: 4.47e-04 | 14372.6ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11220 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11230/50462 | Loss: 2.5965 | LR: 4.47e-04 | 14237.3ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11230 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11240/50462 | Loss: 2.5996 | LR: 4.47e-04 | 14337.2ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11240 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11250/50462 | Loss: 2.6064 | LR: 4.47e-04 | 14367.5ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11250 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Validation at step 11250: Loss = 2.6185, Perplexity = 13.71
+Step 11260/50462 | Loss: 2.5925 | LR: 4.47e-04 | 17298.9ms/step | 3.5 samples/s
+wandb: WARNING Tried to log to step 11260 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11270/50462 | Loss: 2.5896 | LR: 4.47e-04 | 14336.8ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11270 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11280/50462 | Loss: 2.6218 | LR: 4.47e-04 | 14433.7ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11280 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11290/50462 | Loss: 2.6088 | LR: 4.47e-04 | 14268.5ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11290 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11300/50462 | Loss: 2.6138 | LR: 4.46e-04 | 14340.6ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11300 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11310/50462 | Loss: 2.6041 | LR: 4.46e-04 | 14355.7ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11310 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11320/50462 | Loss: 2.5947 | LR: 4.46e-04 | 14295.5ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11320 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11330/50462 | Loss: 2.6053 | LR: 4.46e-04 | 14387.1ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11330 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11340/50462 | Loss: 2.6326 | LR: 4.46e-04 | 14339.9ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11340 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11350/50462 | Loss: 2.5940 | LR: 4.46e-04 | 14281.1ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11350 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11360/50462 | Loss: 2.6508 | LR: 4.46e-04 | 14246.1ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11360 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11370/50462 | Loss: 2.6168 | LR: 4.46e-04 | 14027.9ms/step | 4.3 samples/s
+wandb: WARNING Tried to log to step 11370 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11380/50462 | Loss: 2.6236 | LR: 4.46e-04 | 14135.1ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11380 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11390/50462 | Loss: 2.5941 | LR: 4.46e-04 | 14151.9ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11390 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11400/50462 | Loss: 2.6092 | LR: 4.45e-04 | 14199.4ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11400 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11410/50462 | Loss: 2.5794 | LR: 4.45e-04 | 14376.3ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11410 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11420/50462 | Loss: 2.6125 | LR: 4.45e-04 | 14093.2ms/step | 4.3 samples/s
+wandb: WARNING Tried to log to step 11420 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11430/50462 | Loss: 2.6393 | LR: 4.45e-04 | 13994.3ms/step | 4.3 samples/s
+wandb: WARNING Tried to log to step 11430 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11440/50462 | Loss: 2.6282 | LR: 4.45e-04 | 14082.8ms/step | 4.3 samples/s
+wandb: WARNING Tried to log to step 11440 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11450/50462 | Loss: 2.5904 | LR: 4.45e-04 | 14022.1ms/step | 4.3 samples/s
+wandb: WARNING Tried to log to step 11450 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11460/50462 | Loss: 2.6100 | LR: 4.45e-04 | 14042.2ms/step | 4.3 samples/s
+wandb: WARNING Tried to log to step 11460 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11470/50462 | Loss: 2.6161 | LR: 4.45e-04 | 14285.3ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11470 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11480/50462 | Loss: 2.5954 | LR: 4.45e-04 | 14216.2ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11480 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11490/50462 | Loss: 2.6367 | LR: 4.45e-04 | 14230.6ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11490 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11500/50462 | Loss: 2.5893 | LR: 4.44e-04 | 14069.9ms/step | 4.3 samples/s
+wandb: WARNING Tried to log to step 11500 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Validation at step 11500: Loss = 2.6169, Perplexity = 13.69
+Step 11510/50462 | Loss: 2.5693 | LR: 4.44e-04 | 17599.7ms/step | 3.4 samples/s
+wandb: WARNING Tried to log to step 11510 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11520/50462 | Loss: 2.5712 | LR: 4.44e-04 | 14230.1ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11520 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11530/50462 | Loss: 2.6118 | LR: 4.44e-04 | 14130.3ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11530 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11540/50462 | Loss: 2.6292 | LR: 4.44e-04 | 14280.1ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11540 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11550/50462 | Loss: 2.6081 | LR: 4.44e-04 | 14306.4ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11550 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11560/50462 | Loss: 2.6182 | LR: 4.44e-04 | 14238.0ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11560 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11570/50462 | Loss: 2.6115 | LR: 4.44e-04 | 14244.7ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11570 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11580/50462 | Loss: 2.6060 | LR: 4.44e-04 | 14269.8ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11580 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11590/50462 | Loss: 2.5900 | LR: 4.44e-04 | 14210.0ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11590 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11600/50462 | Loss: 2.6133 | LR: 4.43e-04 | 14280.2ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11600 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11610/50462 | Loss: 2.6216 | LR: 4.43e-04 | 14358.9ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11610 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11620/50462 | Loss: 2.5978 | LR: 4.43e-04 | 14892.6ms/step | 4.0 samples/s
+wandb: WARNING Tried to log to step 11620 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11630/50462 | Loss: 2.5902 | LR: 4.43e-04 | 14014.3ms/step | 4.3 samples/s
+wandb: WARNING Tried to log to step 11630 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11640/50462 | Loss: 2.6036 | LR: 4.43e-04 | 16400.1ms/step | 3.7 samples/s
+wandb: WARNING Tried to log to step 11640 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11650/50462 | Loss: 2.6032 | LR: 4.43e-04 | 14211.7ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11650 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11660/50462 | Loss: 2.5804 | LR: 4.43e-04 | 14211.6ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11660 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11670/50462 | Loss: 2.5974 | LR: 4.43e-04 | 14213.7ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11670 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11680/50462 | Loss: 2.5904 | LR: 4.43e-04 | 14380.5ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11680 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11690/50462 | Loss: 2.5974 | LR: 4.43e-04 | 14103.0ms/step | 4.3 samples/s
+wandb: WARNING Tried to log to step 11690 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11700/50462 | Loss: 2.5817 | LR: 4.42e-04 | 14414.2ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11700 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11710/50462 | Loss: 2.5938 | LR: 4.42e-04 | 14318.4ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 11710 that is less than the current step 11711. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 11720/50462 | Loss: 2.5779 | LR: 4.42e-04 | 14277.0ms/step | 4.2 samples/s
+Step 11730/50462 | Loss: 2.6133 | LR: 4.42e-04 | 14266.9ms/step | 4.2 samples/s
+Step 11740/50462 | Loss: 2.6008 | LR: 4.42e-04 | 14275.7ms/step | 4.2 samples/s
+Step 11750/50462 | Loss: 2.5863 | LR: 4.42e-04 | 14269.5ms/step | 4.2 samples/s
+Validation at step 11750: Loss = 2.6126, Perplexity = 13.63
+Step 11760/50462 | Loss: 2.5870 | LR: 4.42e-04 | 17395.7ms/step | 3.4 samples/s
+Step 11770/50462 | Loss: 2.5906 | LR: 4.42e-04 | 14293.4ms/step | 4.2 samples/s
+Step 11780/50462 | Loss: 2.6323 | LR: 4.42e-04 | 14189.0ms/step | 4.2 samples/s
+Step 11790/50462 | Loss: 2.6043 | LR: 4.42e-04 | 14035.0ms/step | 4.3 samples/s
+Step 11800/50462 | Loss: 2.5894 | LR: 4.41e-04 | 14101.5ms/step | 4.3 samples/s
+Step 11810/50462 | Loss: 2.6194 | LR: 4.41e-04 | 14166.1ms/step | 4.2 samples/s
+Step 11820/50462 | Loss: 2.6241 | LR: 4.41e-04 | 14177.1ms/step | 4.2 samples/s
+Step 11830/50462 | Loss: 2.5896 | LR: 4.41e-04 | 14154.3ms/step | 4.2 samples/s
+Step 11840/50462 | Loss: 2.5729 | LR: 4.41e-04 | 14135.7ms/step | 4.2 samples/s
+Step 11850/50462 | Loss: 2.5995 | LR: 4.41e-04 | 14350.7ms/step | 4.2 samples/s
+Step 11860/50462 | Loss: 2.6041 | LR: 4.41e-04 | 14299.5ms/step | 4.2 samples/s
+Step 11870/50462 | Loss: 2.6049 | LR: 4.41e-04 | 14180.7ms/step | 4.2 samples/s
+Step 11880/50462 | Loss: 2.5891 | LR: 4.41e-04 | 14375.9ms/step | 4.2 samples/s
+Step 11890/50462 | Loss: 2.5981 | LR: 4.41e-04 | 14321.9ms/step | 4.2 samples/s
+Step 11900/50462 | Loss: 2.5953 | LR: 4.40e-04 | 14276.3ms/step | 4.2 samples/s
+Step 11910/50462 | Loss: 2.5908 | LR: 4.40e-04 | 14368.1ms/step | 4.2 samples/s
+Step 11920/50462 | Loss: 2.5633 | LR: 4.40e-04 | 14338.2ms/step | 4.2 samples/s
+Step 11930/50462 | Loss: 2.5696 | LR: 4.40e-04 | 14285.2ms/step | 4.2 samples/s
+Step 11940/50462 | Loss: 2.5739 | LR: 4.40e-04 | 14364.8ms/step | 4.2 samples/s
+Step 11950/50462 | Loss: 2.6007 | LR: 4.40e-04 | 14310.8ms/step | 4.2 samples/s
+Step 11960/50462 | Loss: 2.5726 | LR: 4.40e-04 | 14233.1ms/step | 4.2 samples/s
+Step 11970/50462 | Loss: 2.6061 | LR: 4.40e-04 | 14338.6ms/step | 4.2 samples/s
+Step 11980/50462 | Loss: 2.6188 | LR: 4.40e-04 | 14301.2ms/step | 4.2 samples/s
+Step 11990/50462 | Loss: 2.5990 | LR: 4.40e-04 | 14320.8ms/step | 4.2 samples/s
+Step 12000/50462 | Loss: 2.6520 | LR: 4.39e-04 | 14248.6ms/step | 4.2 samples/s
+Validation at step 12000: Loss = 2.6067, Perplexity = 13.55
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k4b_resampled/run_20260715_232017/checkpoint-12000
+Step 12010/50462 | Loss: 2.6089 | LR: 4.39e-04 | 20247.7ms/step | 3.0 samples/s
+Step 12020/50462 | Loss: 2.5815 | LR: 4.39e-04 | 14257.8ms/step | 4.2 samples/s
+Step 12030/50462 | Loss: 2.5929 | LR: 4.39e-04 | 14251.6ms/step | 4.2 samples/s
+Step 12040/50462 | Loss: 2.5986 | LR: 4.39e-04 | 14340.9ms/step | 4.2 samples/s
+Step 12050/50462 | Loss: 2.5946 | LR: 4.39e-04 | 14351.1ms/step | 4.2 samples/s
+Step 12060/50462 | Loss: 2.5676 | LR: 4.39e-04 | 14227.7ms/step | 4.2 samples/s
+Step 12070/50462 | Loss: 2.6060 | LR: 4.39e-04 | 14314.8ms/step | 4.2 samples/s
+Step 12080/50462 | Loss: 2.6400 | LR: 4.39e-04 | 14163.1ms/step | 4.2 samples/s
+Step 12090/50462 | Loss: 2.5591 | LR: 4.39e-04 | 14166.2ms/step | 4.2 samples/s
+Step 12100/50462 | Loss: 2.5721 | LR: 4.38e-04 | 14345.9ms/step | 4.2 samples/s
+Step 12110/50462 | Loss: 2.6135 | LR: 4.38e-04 | 14245.5ms/step | 4.2 samples/s
+Step 12120/50462 | Loss: 2.5827 | LR: 4.38e-04 | 14226.2ms/step | 4.2 samples/s
+Step 12130/50462 | Loss: 2.5970 | LR: 4.38e-04 | 14239.7ms/step | 4.2 samples/s
+Step 12140/50462 | Loss: 2.5805 | LR: 4.38e-04 | 14373.1ms/step | 4.2 samples/s
+Step 12150/50462 | Loss: 2.5914 | LR: 4.38e-04 | 14354.5ms/step | 4.2 samples/s
+Step 12160/50462 | Loss: 2.6137 | LR: 4.38e-04 | 14147.1ms/step | 4.2 samples/s
+Step 12170/50462 | Loss: 2.5860 | LR: 4.38e-04 | 14348.5ms/step | 4.2 samples/s
+Step 12180/50462 | Loss: 2.5858 | LR: 4.38e-04 | 14404.9ms/step | 4.2 samples/s
+Step 12190/50462 | Loss: 2.5820 | LR: 4.37e-04 | 14248.6ms/step | 4.2 samples/s
+Step 12200/50462 | Loss: 2.5967 | LR: 4.37e-04 | 14220.6ms/step | 4.2 samples/s
+Step 12210/50462 | Loss: 2.5801 | LR: 4.37e-04 | 14205.8ms/step | 4.2 samples/s
+Step 12220/50462 | Loss: 2.6018 | LR: 4.37e-04 | 14177.0ms/step | 4.2 samples/s
+Step 12230/50462 | Loss: 2.6005 | LR: 4.37e-04 | 14202.3ms/step | 4.2 samples/s
+Step 12240/50462 | Loss: 2.6186 | LR: 4.37e-04 | 14322.1ms/step | 4.2 samples/s
+Step 12250/50462 | Loss: 2.5861 | LR: 4.37e-04 | 14271.8ms/step | 4.2 samples/s
+Validation at step 12250: Loss = 2.6029, Perplexity = 13.50
+Step 12260/50462 | Loss: 2.5980 | LR: 4.37e-04 | 17410.3ms/step | 3.4 samples/s
+Step 12270/50462 | Loss: 2.5938 | LR: 4.37e-04 | 14266.5ms/step | 4.2 samples/s
+Step 12280/50462 | Loss: 2.5920 | LR: 4.37e-04 | 14315.1ms/step | 4.2 samples/s
+Step 12290/50462 | Loss: 2.5909 | LR: 4.36e-04 | 14056.3ms/step | 4.3 samples/s
+Step 12300/50462 | Loss: 2.5949 | LR: 4.36e-04 | 14381.4ms/step | 4.2 samples/s
+Step 12310/50462 | Loss: 2.6029 | LR: 4.36e-04 | 14315.5ms/step | 4.2 samples/s
+Step 12320/50462 | Loss: 2.5889 | LR: 4.36e-04 | 14227.6ms/step | 4.2 samples/s
+Step 12330/50462 | Loss: 2.5696 | LR: 4.36e-04 | 14284.4ms/step | 4.2 samples/s
+Step 12340/50462 | Loss: 2.5925 | LR: 4.36e-04 | 14285.8ms/step | 4.2 samples/s
+Step 12350/50462 | Loss: 2.5621 | LR: 4.36e-04 | 14338.5ms/step | 4.2 samples/s
+Step 12360/50462 | Loss: 2.5983 | LR: 4.36e-04 | 14315.3ms/step | 4.2 samples/s
+Step 12370/50462 | Loss: 2.5870 | LR: 4.36e-04 | 14278.2ms/step | 4.2 samples/s
+Step 12380/50462 | Loss: 2.5919 | LR: 4.35e-04 | 14394.0ms/step | 4.2 samples/s
+Step 12390/50462 | Loss: 2.6121 | LR: 4.35e-04 | 14269.6ms/step | 4.2 samples/s
+Step 12400/50462 | Loss: 2.5843 | LR: 4.35e-04 | 14270.8ms/step | 4.2 samples/s
+Step 12410/50462 | Loss: 2.5770 | LR: 4.35e-04 | 14370.1ms/step | 4.2 samples/s
+Step 12420/50462 | Loss: 2.6123 | LR: 4.35e-04 | 14231.6ms/step | 4.2 samples/s
+Step 12430/50462 | Loss: 2.5725 | LR: 4.35e-04 | 14204.4ms/step | 4.2 samples/s
+Step 12440/50462 | Loss: 2.5615 | LR: 4.35e-04 | 14066.2ms/step | 4.3 samples/s
+Step 12450/50462 | Loss: 2.5933 | LR: 4.35e-04 | 14169.3ms/step | 4.2 samples/s
+Step 12460/50462 | Loss: 2.5783 | LR: 4.35e-04 | 14124.6ms/step | 4.2 samples/s
+Step 12470/50462 | Loss: 2.5761 | LR: 4.35e-04 | 14167.7ms/step | 4.2 samples/s
+Step 12480/50462 | Loss: 2.5854 | LR: 4.34e-04 | 14072.6ms/step | 4.3 samples/s
+Step 12490/50462 | Loss: 2.6493 | LR: 4.34e-04 | 14003.8ms/step | 4.3 samples/s
+Step 12500/50462 | Loss: 2.5995 | LR: 4.34e-04 | 14153.9ms/step | 4.2 samples/s
+Validation at step 12500: Loss = 2.5970, Perplexity = 13.42
+Step 12510/50462 | Loss: 2.5956 | LR: 4.34e-04 | 17224.9ms/step | 3.5 samples/s
+Step 12520/50462 | Loss: 2.5764 | LR: 4.34e-04 | 14025.1ms/step | 4.3 samples/s
+Step 12530/50462 | Loss: 2.5870 | LR: 4.34e-04 | 14117.7ms/step | 4.2 samples/s
+Step 12540/50462 | Loss: 2.5740 | LR: 4.34e-04 | 14103.9ms/step | 4.3 samples/s
+Step 12550/50462 | Loss: 2.5515 | LR: 4.34e-04 | 14097.2ms/step | 4.3 samples/s
+Step 12560/50462 | Loss: 2.5628 | LR: 4.34e-04 | 13938.9ms/step | 4.3 samples/s
+Step 12570/50462 | Loss: 2.5677 | LR: 4.33e-04 | 14144.9ms/step | 4.2 samples/s
+Step 12580/50462 | Loss: 2.6235 | LR: 4.33e-04 | 13899.3ms/step | 4.3 samples/s
+Step 12590/50462 | Loss: 2.6015 | LR: 4.33e-04 | 13939.6ms/step | 4.3 samples/s
+Step 12600/50462 | Loss: 2.5614 | LR: 4.33e-04 | 14108.2ms/step | 4.3 samples/s
+Step 12610/50462 | Loss: 2.5814 | LR: 4.33e-04 | 14321.5ms/step | 4.2 samples/s
+Step 12620/50462 | Loss: 2.5841 | LR: 4.33e-04 | 14099.6ms/step | 4.3 samples/s
+Step 12630/50462 | Loss: 2.5772 | LR: 4.33e-04 | 13951.8ms/step | 4.3 samples/s
+Step 12640/50462 | Loss: 2.5967 | LR: 4.33e-04 | 14017.9ms/step | 4.3 samples/s
+Step 12650/50462 | Loss: 2.5508 | LR: 4.33e-04 | 14170.5ms/step | 4.2 samples/s
+Step 12660/50462 | Loss: 2.5745 | LR: 4.32e-04 | 14047.8ms/step | 4.3 samples/s
+Step 12670/50462 | Loss: 2.5854 | LR: 4.32e-04 | 14050.3ms/step | 4.3 samples/s
+Step 12680/50462 | Loss: 2.5807 | LR: 4.32e-04 | 14056.7ms/step | 4.3 samples/s
+Step 12690/50462 | Loss: 2.5458 | LR: 4.32e-04 | 14041.8ms/step | 4.3 samples/s
+Step 12700/50462 | Loss: 2.5568 | LR: 4.32e-04 | 14037.9ms/step | 4.3 samples/s
+Step 12710/50462 | Loss: 2.5923 | LR: 4.32e-04 | 13975.7ms/step | 4.3 samples/s
+Step 12720/50462 | Loss: 2.5682 | LR: 4.32e-04 | 13956.0ms/step | 4.3 samples/s
+Step 12730/50462 | Loss: 2.5702 | LR: 4.32e-04 | 13990.8ms/step | 4.3 samples/s
+Step 12740/50462 | Loss: 2.5531 | LR: 4.32e-04 | 13929.8ms/step | 4.3 samples/s
+Step 12750/50462 | Loss: 2.6041 | LR: 4.31e-04 | 13959.4ms/step | 4.3 samples/s
+Validation at step 12750: Loss = 2.5929, Perplexity = 13.37
+Step 12760/50462 | Loss: 2.5955 | LR: 4.31e-04 | 17196.7ms/step | 3.5 samples/s
+Step 12770/50462 | Loss: 2.5653 | LR: 4.31e-04 | 14066.1ms/step | 4.3 samples/s
+Step 12780/50462 | Loss: 2.6091 | LR: 4.31e-04 | 14061.7ms/step | 4.3 samples/s
+Step 12790/50462 | Loss: 2.5577 | LR: 4.31e-04 | 13938.1ms/step | 4.3 samples/s
+Step 12800/50462 | Loss: 2.5876 | LR: 4.31e-04 | 14095.7ms/step | 4.3 samples/s
+Step 12810/50462 | Loss: 2.5831 | LR: 4.31e-04 | 14173.5ms/step | 4.2 samples/s
+Step 12820/50462 | Loss: 2.6138 | LR: 4.31e-04 | 14091.9ms/step | 4.3 samples/s
+Step 12830/50462 | Loss: 2.5509 | LR: 4.31e-04 | 14115.7ms/step | 4.3 samples/s
+Step 12840/50462 | Loss: 2.5742 | LR: 4.31e-04 | 14069.7ms/step | 4.3 samples/s
+Step 12850/50462 | Loss: 2.5667 | LR: 4.30e-04 | 14160.0ms/step | 4.2 samples/s
+Step 12860/50462 | Loss: 2.5649 | LR: 4.30e-04 | 14046.3ms/step | 4.3 samples/s
+Step 12870/50462 | Loss: 2.5470 | LR: 4.30e-04 | 14039.9ms/step | 4.3 samples/s
+Step 12880/50462 | Loss: 2.5606 | LR: 4.30e-04 | 14065.5ms/step | 4.3 samples/s
+Step 12890/50462 | Loss: 2.5957 | LR: 4.30e-04 | 13956.1ms/step | 4.3 samples/s
+Step 12900/50462 | Loss: 2.5894 | LR: 4.30e-04 | 13964.5ms/step | 4.3 samples/s
+Step 12910/50462 | Loss: 2.5958 | LR: 4.30e-04 | 14046.8ms/step | 4.3 samples/s
+Step 12920/50462 | Loss: 2.6042 | LR: 4.30e-04 | 14028.5ms/step | 4.3 samples/s
+Step 12930/50462 | Loss: 2.5619 | LR: 4.30e-04 | 14022.1ms/step | 4.3 samples/s
+Step 12940/50462 | Loss: 2.5747 | LR: 4.29e-04 | 13952.0ms/step | 4.3 samples/s
+Step 12950/50462 | Loss: 2.5708 | LR: 4.29e-04 | 14105.4ms/step | 4.3 samples/s
+Step 12960/50462 | Loss: 2.5634 | LR: 4.29e-04 | 13972.2ms/step | 4.3 samples/s
+Step 12970/50462 | Loss: 2.5613 | LR: 4.29e-04 | 13970.1ms/step | 4.3 samples/s
+Step 12980/50462 | Loss: 2.5507 | LR: 4.29e-04 | 13972.6ms/step | 4.3 samples/s
+Step 12990/50462 | Loss: 2.5502 | LR: 4.29e-04 | 14001.6ms/step | 4.3 samples/s
+Step 13000/50462 | Loss: 2.5795 | LR: 4.29e-04 | 13937.4ms/step | 4.3 samples/s
+Validation at step 13000: Loss = 2.5911, Perplexity = 13.34
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k4b_resampled/run_20260715_232017/checkpoint-13000
+Step 13010/50462 | Loss: 2.5199 | LR: 4.29e-04 | 20033.0ms/step | 3.0 samples/s
+Step 13020/50462 | Loss: 2.6041 | LR: 4.29e-04 | 13928.0ms/step | 4.3 samples/s
+Step 13030/50462 | Loss: 2.5694 | LR: 4.28e-04 | 14014.3ms/step | 4.3 samples/s
+Step 13040/50462 | Loss: 2.5797 | LR: 4.28e-04 | 13923.4ms/step | 4.3 samples/s
+Step 13050/50462 | Loss: 2.5530 | LR: 4.28e-04 | 13985.7ms/step | 4.3 samples/s
+Step 13060/50462 | Loss: 2.6086 | LR: 4.28e-04 | 13933.9ms/step | 4.3 samples/s
+Step 13070/50462 | Loss: 2.5911 | LR: 4.28e-04 | 14016.1ms/step | 4.3 samples/s
+Step 13080/50462 | Loss: 2.5745 | LR: 4.28e-04 | 14108.5ms/step | 4.3 samples/s
+Step 13090/50462 | Loss: 2.5692 | LR: 4.28e-04 | 13913.8ms/step | 4.3 samples/s
+Step 13100/50462 | Loss: 2.5116 | LR: 4.28e-04 | 13971.7ms/step | 4.3 samples/s
+Step 13110/50462 | Loss: 2.5768 | LR: 4.28e-04 | 13983.7ms/step | 4.3 samples/s
+Step 13120/50462 | Loss: 2.6022 | LR: 4.27e-04 | 13994.8ms/step | 4.3 samples/s
+Step 13130/50462 | Loss: 2.5785 | LR: 4.27e-04 | 13947.4ms/step | 4.3 samples/s
+Step 13140/50462 | Loss: 2.5832 | LR: 4.27e-04 | 14066.6ms/step | 4.3 samples/s
+Step 13150/50462 | Loss: 2.6107 | LR: 4.27e-04 | 14127.5ms/step | 4.2 samples/s
+Step 13160/50462 | Loss: 2.5719 | LR: 4.27e-04 | 14083.5ms/step | 4.3 samples/s
+Step 13170/50462 | Loss: 2.6064 | LR: 4.27e-04 | 13991.5ms/step | 4.3 samples/s
+Step 13180/50462 | Loss: 2.5646 | LR: 4.27e-04 | 13941.1ms/step | 4.3 samples/s
+Step 13190/50462 | Loss: 2.5801 | LR: 4.27e-04 | 13888.1ms/step | 4.3 samples/s
+Step 13200/50462 | Loss: 2.5550 | LR: 4.27e-04 | 14076.2ms/step | 4.3 samples/s
+Step 13210/50462 | Loss: 2.5759 | LR: 4.26e-04 | 13965.5ms/step | 4.3 samples/s
+Step 13220/50462 | Loss: 2.5861 | LR: 4.26e-04 | 13986.2ms/step | 4.3 samples/s
+Step 13230/50462 | Loss: 2.5531 | LR: 4.26e-04 | 14063.4ms/step | 4.3 samples/s
+Step 13240/50462 | Loss: 2.5918 | LR: 4.26e-04 | 14162.7ms/step | 4.2 samples/s
+Step 13250/50462 | Loss: 2.5624 | LR: 4.26e-04 | 14084.3ms/step | 4.3 samples/s
+Validation at step 13250: Loss = 2.5871, Perplexity = 13.29
+Step 13260/50462 | Loss: 2.5755 | LR: 4.26e-04 | 17267.5ms/step | 3.5 samples/s
+Step 13270/50462 | Loss: 2.6160 | LR: 4.26e-04 | 14068.4ms/step | 4.3 samples/s
+Step 13280/50462 | Loss: 2.5750 | LR: 4.26e-04 | 14074.5ms/step | 4.3 samples/s
+Step 13290/50462 | Loss: 2.5732 | LR: 4.26e-04 | 14035.6ms/step | 4.3 samples/s
+Step 13300/50462 | Loss: 2.6098 | LR: 4.25e-04 | 14077.6ms/step | 4.3 samples/s
+Step 13310/50462 | Loss: 2.5955 | LR: 4.25e-04 | 14018.9ms/step | 4.3 samples/s
+Step 13320/50462 | Loss: 2.5718 | LR: 4.25e-04 | 13928.2ms/step | 4.3 samples/s
+Step 13330/50462 | Loss: 2.5319 | LR: 4.25e-04 | 13971.3ms/step | 4.3 samples/s
+Step 13340/50462 | Loss: 2.5819 | LR: 4.25e-04 | 14135.9ms/step | 4.2 samples/s
+Step 13350/50462 | Loss: 2.5790 | LR: 4.25e-04 | 14090.0ms/step | 4.3 samples/s
+Step 13360/50462 | Loss: 2.5234 | LR: 4.25e-04 | 14141.1ms/step | 4.2 samples/s
+Step 13370/50462 | Loss: 2.5831 | LR: 4.25e-04 | 14059.4ms/step | 4.3 samples/s
+Step 13380/50462 | Loss: 2.5655 | LR: 4.25e-04 | 14040.5ms/step | 4.3 samples/s
+Step 13390/50462 | Loss: 2.5655 | LR: 4.24e-04 | 14200.6ms/step | 4.2 samples/s
+Step 13400/50462 | Loss: 2.5607 | LR: 4.24e-04 | 14143.4ms/step | 4.2 samples/s
+Step 13410/50462 | Loss: 2.5844 | LR: 4.24e-04 | 13989.7ms/step | 4.3 samples/s
+Step 13420/50462 | Loss: 2.5770 | LR: 4.24e-04 | 14109.9ms/step | 4.3 samples/s
+Step 13430/50462 | Loss: 2.5838 | LR: 4.24e-04 | 13967.5ms/step | 4.3 samples/s
+Step 13440/50462 | Loss: 2.5621 | LR: 4.24e-04 | 14050.6ms/step | 4.3 samples/s
+Step 13450/50462 | Loss: 2.5668 | LR: 4.24e-04 | 13960.1ms/step | 4.3 samples/s
+Step 13460/50462 | Loss: 2.5588 | LR: 4.24e-04 | 13962.0ms/step | 4.3 samples/s
+Step 13470/50462 | Loss: 2.5556 | LR: 4.23e-04 | 13952.5ms/step | 4.3 samples/s
+Step 13480/50462 | Loss: 2.5612 | LR: 4.23e-04 | 14033.8ms/step | 4.3 samples/s
+Step 13490/50462 | Loss: 2.5435 | LR: 4.23e-04 | 14200.3ms/step | 4.2 samples/s
+Step 13500/50462 | Loss: 2.5732 | LR: 4.23e-04 | 13919.7ms/step | 4.3 samples/s
+Validation at step 13500: Loss = 2.5830, Perplexity = 13.24
+Step 13510/50462 | Loss: 2.5691 | LR: 4.23e-04 | 17420.8ms/step | 3.4 samples/s
+Step 13520/50462 | Loss: 2.5399 | LR: 4.23e-04 | 14372.9ms/step | 4.2 samples/s
+Step 13530/50462 | Loss: 2.5754 | LR: 4.23e-04 | 14206.6ms/step | 4.2 samples/s
+Step 13540/50462 | Loss: 2.6025 | LR: 4.23e-04 | 14312.2ms/step | 4.2 samples/s
+Step 13550/50462 | Loss: 2.5548 | LR: 4.23e-04 | 14309.2ms/step | 4.2 samples/s
+Step 13560/50462 | Loss: 2.5764 | LR: 4.22e-04 | 14257.4ms/step | 4.2 samples/s
+Step 13570/50462 | Loss: 2.5783 | LR: 4.22e-04 | 14367.4ms/step | 4.2 samples/s
+Step 13580/50462 | Loss: 2.5845 | LR: 4.22e-04 | 14306.6ms/step | 4.2 samples/s
+Step 13590/50462 | Loss: 2.5601 | LR: 4.22e-04 | 14372.0ms/step | 4.2 samples/s
+Step 13600/50462 | Loss: 2.5534 | LR: 4.22e-04 | 14273.0ms/step | 4.2 samples/s
+Step 13610/50462 | Loss: 2.5714 | LR: 4.22e-04 | 14221.4ms/step | 4.2 samples/s
+Step 13620/50462 | Loss: 2.5432 | LR: 4.22e-04 | 14311.3ms/step | 4.2 samples/s
+Step 13630/50462 | Loss: 2.5734 | LR: 4.22e-04 | 14185.8ms/step | 4.2 samples/s
+Step 13640/50462 | Loss: 2.6152 | LR: 4.22e-04 | 14269.0ms/step | 4.2 samples/s
+Step 13650/50462 | Loss: 2.5486 | LR: 4.21e-04 | 14278.9ms/step | 4.2 samples/s
+Step 13660/50462 | Loss: 2.5540 | LR: 4.21e-04 | 14221.4ms/step | 4.2 samples/s
+Step 13670/50462 | Loss: 2.5798 | LR: 4.21e-04 | 14353.2ms/step | 4.2 samples/s
+Step 13680/50462 | Loss: 2.5896 | LR: 4.21e-04 | 14339.7ms/step | 4.2 samples/s
+Step 13690/50462 | Loss: 2.5659 | LR: 4.21e-04 | 14338.1ms/step | 4.2 samples/s
+Step 13700/50462 | Loss: 2.6174 | LR: 4.21e-04 | 14304.7ms/step | 4.2 samples/s
+Step 13710/50462 | Loss: 2.5524 | LR: 4.21e-04 | 14331.6ms/step | 4.2 samples/s
+Step 13720/50462 | Loss: 2.5554 | LR: 4.21e-04 | 14414.8ms/step | 4.2 samples/s
+Step 13730/50462 | Loss: 2.5403 | LR: 4.21e-04 | 14249.5ms/step | 4.2 samples/s
+Step 13740/50462 | Loss: 2.5709 | LR: 4.20e-04 | 14344.5ms/step | 4.2 samples/s
+Step 13750/50462 | Loss: 2.5583 | LR: 4.20e-04 | 14310.8ms/step | 4.2 samples/s
+Validation at step 13750: Loss = 2.5774, Perplexity = 13.16
+Step 13760/50462 | Loss: 2.5573 | LR: 4.20e-04 | 17501.0ms/step | 3.4 samples/s
+Step 13770/50462 | Loss: 2.5858 | LR: 4.20e-04 | 14267.8ms/step | 4.2 samples/s
+Step 13780/50462 | Loss: 2.5847 | LR: 4.20e-04 | 14237.3ms/step | 4.2 samples/s
+Step 13790/50462 | Loss: 2.5463 | LR: 4.20e-04 | 14372.2ms/step | 4.2 samples/s
+Step 13800/50462 | Loss: 2.5644 | LR: 4.20e-04 | 14346.7ms/step | 4.2 samples/s
+Step 13810/50462 | Loss: 2.5539 | LR: 4.20e-04 | 14273.2ms/step | 4.2 samples/s
+Step 13820/50462 | Loss: 2.5298 | LR: 4.19e-04 | 14352.4ms/step | 4.2 samples/s
+Step 13830/50462 | Loss: 2.5386 | LR: 4.19e-04 | 14333.0ms/step | 4.2 samples/s
+Step 13840/50462 | Loss: 2.5582 | LR: 4.19e-04 | 14216.5ms/step | 4.2 samples/s
+Step 13850/50462 | Loss: 2.5536 | LR: 4.19e-04 | 14234.9ms/step | 4.2 samples/s
+Step 13860/50462 | Loss: 2.5623 | LR: 4.19e-04 | 14233.2ms/step | 4.2 samples/s
+Step 13870/50462 | Loss: 2.5580 | LR: 4.19e-04 | 14359.6ms/step | 4.2 samples/s
+Step 13880/50462 | Loss: 2.5751 | LR: 4.19e-04 | 14412.0ms/step | 4.2 samples/s
+Step 13890/50462 | Loss: 2.5549 | LR: 4.19e-04 | 14314.3ms/step | 4.2 samples/s
+Step 13900/50462 | Loss: 2.5681 | LR: 4.19e-04 | 14389.9ms/step | 4.2 samples/s
+Step 13910/50462 | Loss: 2.5807 | LR: 4.18e-04 | 14233.7ms/step | 4.2 samples/s
+Step 13920/50462 | Loss: 2.5907 | LR: 4.18e-04 | 14226.5ms/step | 4.2 samples/s
+Step 13930/50462 | Loss: 2.5716 | LR: 4.18e-04 | 14151.3ms/step | 4.2 samples/s
+Step 13940/50462 | Loss: 2.5529 | LR: 4.18e-04 | 14308.7ms/step | 4.2 samples/s
+Step 13950/50462 | Loss: 2.5273 | LR: 4.18e-04 | 14337.3ms/step | 4.2 samples/s

epfl_final/train_logs/wait_1p2b_30k_mixed_after_moba.log

@@ -0,0 +1,5 @@
+2026-07-20T07:15:35Z waiter start pid=101422 latest_moba_step=28200
+2026-07-20T07:17:47Z waiter start pid=101557 latest_moba_step=28200
+2026-07-21T01:59:37Z waiter start pid=145 latest_moba_step=35400
+2026-07-21T05:57:01Z MoBA complete; launching native-30k 1.2B Mixed
+2026-07-21T05:57:01Z Mixed watchdog pid=15034

epfl_final/train_logs/watchdog_generic_fork.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+# Persistent in-pod training watchdog.
+# Relaunches torchrun (auto-resumes from latest checkpoint) ONLY when the run is
+# truly dead: GPU compute apps == 0 AND no matching train procs, sustained.
+# Exits (stops watching) once the run has completed.
+# Args: JOB CONFIG OUTPUT_DIR PROC_PAT TOTAL [NPROC]
+set -u
+JOB="$1"; CFG="$2"; OUTDIR="$3"; PROC_PAT="$4"; TOTAL="$5"; NPROC="${6:-4}"
+BASE=/mloscratch/homes/navasard/sparse
+SPA="$BASE/SparseAtt-baselines"
+TR=/mloscratch/homes/navasard/conda/bin/torchrun
+export HF_HOME=/mloscratch/homes/navasard/hf_cache
+WLOG="$BASE/train_logs/wd_${JOB}.watch.log"
+RUNLOG="$BASE/train_logs/wd_${JOB}.run.log"
+LOCK="$BASE/train_logs/wd_${JOB}.lock"
+DOWN_THRESHOLD=10   # x30s = 5 min sustained-dead before relaunch
+
+log(){ echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*" >> "$WLOG"; }
+
+# ---- singleton guard ----
+if [ -f "$LOCK" ]; then
+  oldpid=$(cat "$LOCK" 2>/dev/null || true)
+  if [ -n "${oldpid:-}" ] && kill -0 "$oldpid" 2>/dev/null; then
+    log "duplicate start: watchdog already running pid=$oldpid; exiting"; exit 0
+  fi
+fi
+echo $$ > "$LOCK"
+log "=== watchdog start pid=$$ job=$JOB cfg=$CFG total=$TOTAL nproc=$NPROC ==="
+
+latest_ckpt_step(){
+  ls -1d "$OUTDIR"/run_*/checkpoint-* 2>/dev/null \
+    | sed -E 's#.*/checkpoint-##' | grep -E '^[0-9]+$' | sort -n | tail -1
+}
+
+is_complete(){
+  # 1) explicit completion marker in any of this job's run logs
+  if grep -qs "Training complete!" "$BASE"/train_logs/*"${JOB}"*.log 2>/dev/null; then return 0; fi
+  # 2) a checkpoint at/after the final step exists
+  local s; s=$(latest_ckpt_step)
+  if [ -n "${s:-}" ] && [ "$s" -ge "$TOTAL" ] 2>/dev/null; then return 0; fi
+  return 1
+}
+
+n_procs(){ pgrep -f "$PROC_PAT" 2>/dev/null | grep -c '[0-9]'; }
+n_gpu_apps(){ nvidia-smi --query-compute-apps=pid --format=csv,noheader 2>/dev/null | grep -c '[0-9]'; }
+
+relaunch(){
+  cd "$SPA" || { log "ERR cd $SPA failed"; return 1; }
+  log "RELAUNCH torchrun nproc=$NPROC cfg=$CFG (latest ckpt step=$(latest_ckpt_step))"
+  setsid nohup "$TR" --standalone --nproc_per_node="$NPROC" train.py --config "$CFG" \
+      >> "$RUNLOG" 2>&1 < /dev/null &
+  sleep 240
+  if [ "$(n_procs)" -ge 1 ] && { [ "$(n_gpu_apps)" -ge 1 ] \
+       || grep -qE "Step [0-9]+/|Resuming from checkpoint|Starting training|New run:" "$RUNLOG" 2>/dev/null; }; then
+    log "RELAUNCH OK (procs=$(n_procs) gpu_apps=$(n_gpu_apps))"; return 0
+  fi
+  log "RELAUNCH FAILED (procs=$(n_procs) gpu_apps=$(n_gpu_apps)); runlog tail:"
+  tail -n 8 "$RUNLOG" >> "$WLOG" 2>/dev/null
+  return 1
+}
+
+zero=0
+while true; do
+  if is_complete; then log "COMPLETE detected (ckpt step=$(latest_ckpt_step)); watchdog exiting"; rm -f "$LOCK"; exit 0; fi
+  procs=$(n_procs); apps=$(n_gpu_apps)
+  if [ "$procs" -eq 0 ] && [ "$apps" -eq 0 ]; then zero=$((zero+1)); else zero=0; fi
+  if [ "$zero" -ge "$DOWN_THRESHOLD" ]; then
+    log "SUSTAINED DEAD (${DOWN_THRESHOLD}x30s: procs=0 gpu_apps=0)"
+    if is_complete; then log "actually complete; exit"; rm -f "$LOCK"; exit 0; fi
+    ok=0
+    for a in 1 2 3; do
+      if relaunch; then ok=1; break; fi
+      log "relaunch attempt $a failed; retry in 60s"; sleep 60
+    done
+    [ "$ok" -eq 1 ] && log "recovered" || log "!!! RELAUNCH EXHAUSTED - manual action needed !!!"
+    zero=0
+  fi
+  sleep 30
+done

epfl_final/train_logs/watchdog_mixed_training.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+set -u
+
+JOB="$1"
+CFG="$2"
+OUTDIR="$3"
+TOTAL="$4"
+GPUSET="$5"
+NPROC="$6"
+
+BASE=/mloscratch/homes/navasard/sparse
+REPO="$BASE/SparseAtt-baselines"
+TORCHRUN=/mloscratch/homes/navasard/conda/bin/torchrun
+WLOG="$BASE/train_logs/${JOB}.watch.log"
+RUNLOG="$BASE/train_logs/${JOB}.run.log"
+LOCK="$BASE/train_logs/${JOB}.lock"
+PROC_PAT="train.py --config $CFG"
+
+export HF_HOME=/mloscratch/homes/navasard/hf_cache
+
+log() {
+  echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*" >> "$WLOG"
+}
+
+latest_step() {
+  ls -1d "$OUTDIR"/run_*/checkpoint-* 2>/dev/null |
+    sed -E 's#.*/checkpoint-##' |
+    grep -E '^[0-9]+$' |
+    sort -n |
+    tail -1
+}
+
+is_complete() {
+  if grep -qs "Training complete!" "$RUNLOG" 2>/dev/null; then
+    return 0
+  fi
+  local step
+  step=$(latest_step)
+  [ -n "${step:-}" ] && [ "$step" -ge "$TOTAL" ] 2>/dev/null
+}
+
+process_count() {
+  pgrep -f "$PROC_PAT" 2>/dev/null | grep -c '[0-9]'
+}
+
+launch() {
+  cd "$REPO" || return 1
+  log "launch GPUs=$GPUSET nproc=$NPROC latest_step=$(latest_step)"
+  setsid nohup env CUDA_VISIBLE_DEVICES="$GPUSET" \
+    "$TORCHRUN" --standalone --nproc_per_node="$NPROC" \
+    train.py --config "$CFG" >> "$RUNLOG" 2>&1 < /dev/null &
+}
+
+if [ -f "$LOCK" ]; then
+  old_pid=$(cat "$LOCK" 2>/dev/null || true)
+  if [ -n "${old_pid:-}" ] && kill -0 "$old_pid" 2>/dev/null; then
+    exit 0
+  fi
+fi
+
+echo $$ > "$LOCK"
+log "watchdog start pid=$$"
+
+if [ "$(process_count)" -eq 0 ] && ! is_complete; then
+  launch
+fi
+
+dead_checks=0
+while true; do
+  if is_complete; then
+    log "complete latest_step=$(latest_step)"
+    rm -f "$LOCK"
+    exit 0
+  fi
+
+  if [ "$(process_count)" -eq 0 ]; then
+    dead_checks=$((dead_checks + 1))
+  else
+    dead_checks=0
+  fi
+
+  if [ "$dead_checks" -ge 10 ]; then
+    log "process absent for 5 minutes; relaunching"
+    launch
+    dead_checks=0
+  fi
+  sleep 30
+done

epfl_final/train_logs/wd_moba30k.watch.log

@@ -0,0 +1,5 @@
+2026-07-17T23:42:59Z === watchdog start pid=125853 job=moba30k cfg=configs/train_config_30k_moba_b128.json total=37465 nproc=4 ===
+2026-07-18T01:55:15Z === watchdog start pid=129870 job=moba30k cfg=configs/train_config_30k_moba_b128.json total=37465 nproc=4 ===
+2026-07-18T03:17:12Z === watchdog start pid=716 job=moba30k cfg=configs/train_config_30k_moba_b128.json total=37465 nproc=4 ===
+2026-07-21T01:59:37Z === watchdog start pid=144 job=moba30k cfg=configs/train_config_30k_moba_b128.json total=37465 nproc=4 ===
+2026-07-21T05:56:47Z COMPLETE detected (ckpt step=37465); watchdog exiting

epfl_final/train_logs/wd_moba_matched.watch.log

@@ -0,0 +1,2 @@
+2026-07-13T19:57:29Z === watchdog start pid=97404 job=moba_matched cfg=configs/ladder8k/moba_1p2b_matched.json total=50462 nproc=3 ===
+2026-07-15T22:15:55Z COMPLETE detected (ckpt step=50462); watchdog exiting

epfl_final/train_logs/wd_resampled4b.lock

@@ -0,0 +1 @@
+163

epfl_final/train_logs/wd_resampled4b.watch.log

@@ -0,0 +1 @@
+2026-07-17T23:36:29Z === watchdog start pid=163 job=resampled4b cfg=configs/ladder8k/resampled_4b.json total=50462 nproc=4 ===