EPFL ALiBi and training provenance 751-800 of 1184

epfl_final/train_logs/dense30k365m_s43.run.log.INTERRUPTED_STEP40_20260722T225915Z

@@ -0,0 +1,181 @@
+New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense_seed43/run_20260722_221751
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/longctx30k/tokenized",
+  "val_path": null,
+  "max_seq_length": 30720,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense_seed43/run_20260722_221751",
+  "resume_from_checkpoint": null,
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 11700,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 20,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 43,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "365m dense 30k seed43",
+  "wandb_tags": [
+    "qwen3",
+    "365m",
+    "30k",
+    "dense",
+    "native",
+    "seed43",
+    "multiseed"
+  ],
+  "hidden_size": 768,
+  "intermediate_size": 3072,
+  "num_attention_heads": 6,
+  "num_key_value_heads": 2,
+  "num_hidden_layers": 28,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 120,
+  "sparse_mask_pool_size": 1,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "sparse_mask_seed": 0,
+  "sparse_mixed_nrb_set": null,
+  "dense_mode": true,
+  "dense_step_intervals": null,
+  "moba_block_size": 128,
+  "moba_top_k": 32,
+  "asentmax_alpha": 1.5,
+  "asentmax_learn_temp": true,
+  "asentmax_init_temp": 1.0
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Initializing model from scratch (random weights)...
+Checkpoint activations: False
+dense_mode=True: sparse attention layers will use standard causal SDPA.
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 768
+  num_hidden_layers: 28
+  num_attention_heads: 6
+  num_key_value_heads: 2
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 120
+  sparse_mask_pool_size: 1
+  sparse_max_seq_length: 30720
+
+Parameter count:
+  Total parameters: 358,958,848
+  Trainable parameters: 358,958,848
+  Non-trainable parameters: 0
+  Model size (BF16): 0.72 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 30720]                --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 30720, 768]           --                        True
+│    └─Embedding: 2-1                                        [1, 30720]                [1, 30720, 768]           116,686,848               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 30720, 768]           [1, 30720, 768]           768                       True
+├─Linear: 1-2                                                [1, 30720, 768]           [1, 30720, 151936]        116,686,848               True
+================================================================================================================================================================
+Total params: 475,645,696
+Trainable params: 475,645,696
+Non-trainable params: 0
+Total mult-adds (Units.MEGABYTES): 475.65
+================================================================================================================================================================
+Input size (MB): 0.25
+Forward/backward pass size (MB): 58494.81
+Params size (MB): 951.29
+Estimated Total Size (MB): 59446.35
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)
+Dataset split: 749299 train, 39437 validation
+Training examples (packed, train split):  749299
+Validation examples (packed, val split):  39437
+Per-device batch size:                    1
+World size (GPUs):                        1
+Gradient accumulation steps:              20
+Effective batch size (seqs/step):         20
+Sequence length:                          30720
+Tokens per optimizer step:                614,400
+Micro-batches per epoch (per rank):       749299
+Optimizer steps per epoch:                37464
+Total optimizer steps:                    11700  (capped by max_steps; 0.31 epochs)
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt-baselines/wandb/run-20260722_221844-g0u38cyr
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run 365m dense 30k seed43
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/g0u38cyr
+Starting training...
+Switching attention mode for step 1: dense (dense step interval)
+Step 10/11700 | Loss: 12.0652 | LR: 6.67e-06 | 32362.0ms/step | 0.6 samples/s
+Step 20/11700 | Loss: 11.9952 | LR: 1.33e-05 | 24539.7ms/step | 0.8 samples/s
+Step 30/11700 | Loss: 11.7724 | LR: 2.00e-05 | 24563.8ms/step | 0.8 samples/s
+Step 40/11700 | Loss: 11.2788 | LR: 2.67e-05 | 24533.2ms/step | 0.8 samples/s

epfl_final/train_logs/dense30k365m_s43.watch.log

@@ -0,0 +1,14 @@
+2026-07-22T22:17:37Z watchdog start pid=308
+2026-07-22T22:17:37Z launch GPUs=0 nproc=1 latest_step=
+2026-07-22T22:59:15Z watchdog start pid=184
+2026-07-22T22:59:15Z launch GPUs=0 nproc=1 latest_step=
+2026-07-24T07:03:34Z watchdog start pid=351
+2026-07-24T07:03:34Z launch GPUs=0 nproc=1 latest_step=
+2026-07-24T17:38:52Z watchdog start pid=198
+2026-07-24T17:38:52Z launch GPUs=0 nproc=1 latest_step=
+2026-07-25T06:17:28Z watchdog start pid=188
+2026-07-25T06:17:28Z launch GPUs=0 nproc=1 latest_step=
+2026-07-25T07:46:31Z process absent for 5 minutes; relaunching
+2026-07-25T07:46:31Z launch GPUs=0 nproc=1 latest_step=
+2026-07-25T07:51:31Z process absent for 5 minutes; relaunching
+2026-07-25T07:51:31Z launch GPUs=0 nproc=1 latest_step=

epfl_final/train_logs/dense30k365m_s44.run.log.INTERRUPTED_STEP40_20260722T225915Z

@@ -0,0 +1,181 @@
+New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense_seed44/run_20260722_221751
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/longctx30k/tokenized",
+  "val_path": null,
+  "max_seq_length": 30720,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense_seed44/run_20260722_221751",
+  "resume_from_checkpoint": null,
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 11700,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 20,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 44,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "365m dense 30k seed44",
+  "wandb_tags": [
+    "qwen3",
+    "365m",
+    "30k",
+    "dense",
+    "native",
+    "seed44",
+    "multiseed"
+  ],
+  "hidden_size": 768,
+  "intermediate_size": 3072,
+  "num_attention_heads": 6,
+  "num_key_value_heads": 2,
+  "num_hidden_layers": 28,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 120,
+  "sparse_mask_pool_size": 1,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "sparse_mask_seed": 0,
+  "sparse_mixed_nrb_set": null,
+  "dense_mode": true,
+  "dense_step_intervals": null,
+  "moba_block_size": 128,
+  "moba_top_k": 32,
+  "asentmax_alpha": 1.5,
+  "asentmax_learn_temp": true,
+  "asentmax_init_temp": 1.0
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Initializing model from scratch (random weights)...
+Checkpoint activations: False
+dense_mode=True: sparse attention layers will use standard causal SDPA.
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 768
+  num_hidden_layers: 28
+  num_attention_heads: 6
+  num_key_value_heads: 2
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 120
+  sparse_mask_pool_size: 1
+  sparse_max_seq_length: 30720
+
+Parameter count:
+  Total parameters: 358,958,848
+  Trainable parameters: 358,958,848
+  Non-trainable parameters: 0
+  Model size (BF16): 0.72 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 30720]                --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 30720, 768]           --                        True
+│    └─Embedding: 2-1                                        [1, 30720]                [1, 30720, 768]           116,686,848               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 30720, 768]           [1, 30720, 768]           768                       True
+├─Linear: 1-2                                                [1, 30720, 768]           [1, 30720, 151936]        116,686,848               True
+================================================================================================================================================================
+Total params: 475,645,696
+Trainable params: 475,645,696
+Non-trainable params: 0
+Total mult-adds (Units.MEGABYTES): 475.65
+================================================================================================================================================================
+Input size (MB): 0.25
+Forward/backward pass size (MB): 58494.81
+Params size (MB): 951.29
+Estimated Total Size (MB): 59446.35
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)
+Dataset split: 749299 train, 39437 validation
+Training examples (packed, train split):  749299
+Validation examples (packed, val split):  39437
+Per-device batch size:                    1
+World size (GPUs):                        1
+Gradient accumulation steps:              20
+Effective batch size (seqs/step):         20
+Sequence length:                          30720
+Tokens per optimizer step:                614,400
+Micro-batches per epoch (per rank):       749299
+Optimizer steps per epoch:                37464
+Total optimizer steps:                    11700  (capped by max_steps; 0.31 epochs)
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt-baselines/wandb/run-20260722_221844-8dm0uevk
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run 365m dense 30k seed44
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/8dm0uevk
+Starting training...
+Switching attention mode for step 1: dense (dense step interval)
+Step 10/11700 | Loss: 12.0542 | LR: 6.67e-06 | 32412.1ms/step | 0.6 samples/s
+Step 20/11700 | Loss: 11.9953 | LR: 1.33e-05 | 24631.0ms/step | 0.8 samples/s
+Step 30/11700 | Loss: 11.7596 | LR: 2.00e-05 | 24615.4ms/step | 0.8 samples/s
+Step 40/11700 | Loss: 11.2855 | LR: 2.67e-05 | 24588.6ms/step | 0.8 samples/s

epfl_final/train_logs/dense30k365m_s44.watch.log

@@ -0,0 +1,14 @@
+2026-07-22T22:17:37Z watchdog start pid=310
+2026-07-22T22:17:37Z launch GPUs=1 nproc=1 latest_step=
+2026-07-22T22:59:15Z watchdog start pid=185
+2026-07-22T22:59:15Z launch GPUs=1 nproc=1 latest_step=
+2026-07-24T07:03:34Z watchdog start pid=352
+2026-07-24T07:03:34Z launch GPUs=1 nproc=1 latest_step=
+2026-07-24T17:38:52Z watchdog start pid=199
+2026-07-24T17:38:52Z launch GPUs=1 nproc=1 latest_step=
+2026-07-25T06:17:28Z watchdog start pid=190
+2026-07-25T06:17:28Z launch GPUs=1 nproc=1 latest_step=
+2026-07-25T07:47:01Z process absent for 5 minutes; relaunching
+2026-07-25T07:47:01Z launch GPUs=1 nproc=1 latest_step=
+2026-07-25T07:52:01Z process absent for 5 minutes; relaunching
+2026-07-25T07:52:01Z launch GPUs=1 nproc=1 latest_step=

epfl_final/train_logs/dense8k365m_s43.watch.log

@@ -0,0 +1,3 @@
+2026-07-22T22:17:37Z watchdog start pid=160
+2026-07-22T22:17:37Z launch GPUs=0 nproc=1 latest_step=
+2026-07-24T22:04:50Z complete latest_step=44000

epfl_final/train_logs/dense8k365m_s44.run.log.NAN_20260722T222304Z

@@ -0,0 +1,216 @@
+New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed44/run_20260722_221755
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed44/run_20260722_221755",
+  "resume_from_checkpoint": null,
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 44000,
+  "per_device_batch_size": 2,
+  "gradient_accumulation_steps": 10,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 44,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "365m dense 8k seed44",
+  "wandb_tags": [
+    "qwen3",
+    "365m",
+    "8k",
+    "dense",
+    "seed44",
+    "multiseed"
+  ],
+  "hidden_size": 768,
+  "intermediate_size": 3072,
+  "num_attention_heads": 6,
+  "num_key_value_heads": 2,
+  "num_hidden_layers": 28,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 1,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "sparse_mask_seed": 0,
+  "sparse_mixed_nrb_set": null,
+  "dense_mode": true,
+  "dense_step_intervals": null,
+  "moba_block_size": 128,
+  "moba_top_k": 32,
+  "asentmax_alpha": 1.5,
+  "asentmax_learn_temp": true,
+  "asentmax_init_temp": 1.0
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Initializing model from scratch (random weights)...
+Checkpoint activations: False
+dense_mode=True: sparse attention layers will use standard causal SDPA.
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 768
+  num_hidden_layers: 28
+  num_attention_heads: 6
+  num_key_value_heads: 2
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 1
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 358,958,848
+  Trainable parameters: 358,958,848
+  Non-trainable parameters: 0
+  Model size (BF16): 0.72 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [2, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [2, 8192, 768]            --                        True
+│    └─Embedding: 2-1                                        [2, 8192]                 [2, 8192, 768]            116,686,848               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-2                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-3                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-4                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-5                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-6                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-7                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-8                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-9                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-10                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-11                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-12                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-13                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-14                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-15                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-16                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-17                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-18                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-19                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-20                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-21                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-22                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-23                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-24                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-25                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-26                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-27                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-28                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    └─Qwen3RMSNorm: 2-3                                     [2, 8192, 768]            [2, 8192, 768]            768                       True
+├─Linear: 1-2                                                [2, 8192, 768]            [2, 8192, 151936]         116,686,848               True
+================================================================================================================================================================
+Total params: 475,645,696
+Trainable params: 475,645,696
+Non-trainable params: 0
+Total mult-adds (Units.MEGABYTES): 951.29
+================================================================================================================================================================
+Input size (MB): 0.13
+Forward/backward pass size (MB): 31197.23
+Params size (MB): 951.29
+Estimated Total Size (MB): 32148.66
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    2
+World size (GPUs):                        1
+Gradient accumulation steps:              10
+Effective batch size (seqs/step):         20
+Sequence length:                          8192
+Tokens per optimizer step:                163,840
+Micro-batches per epoch (per rank):       1513882
+Optimizer steps per epoch:                151388
+Total optimizer steps:                    44000  (capped by max_steps; 0.29 epochs)
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run 62sdl4ys
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt-baselines/wandb/run-20260722_221828-62sdl4ys
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run 365m dense 8k seed44
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/62sdl4ys
+Starting training...
+Switching attention mode for step 1: dense (dense step interval)
+Step 10/44000 | Loss: nan | LR: 6.67e-06 | 11545.3ms/step | 1.7 samples/s
+Step 20/44000 | Loss: nan | LR: 1.33e-05 | 3436.3ms/step | 5.8 samples/s
+Step 30/44000 | Loss: nan | LR: 2.00e-05 | 3431.8ms/step | 5.8 samples/s
+Step 40/44000 | Loss: nan | LR: 2.67e-05 | 3424.7ms/step | 5.8 samples/s
+Step 50/44000 | Loss: nan | LR: 3.33e-05 | 3426.9ms/step | 5.8 samples/s
+W0722 22:22:59.300000 237 site-packages/torch/distributed/elastic/agent/server/api.py:739] Received 15 death signal, shutting down workers
+W0722 22:22:59.302000 237 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 535 closing signal SIGTERM
+Traceback (most recent call last):
+  File "/mloscratch/homes/navasard/conda/bin/torchrun", line 6, in <module>
+    sys.exit(main())
+             ~~~~^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper
+    return f(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main
+    run(args)
+    ~~~^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run
+    elastic_launch(
+    ~~~~~~~~~~~~~~~
+        config=config,
+        ~~~~~~~~~~~~~~
+        entrypoint=cmd,
+        ~~~~~~~~~~~~~~~
+    )(*cmd_args)
+    ~^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 308, in launch_agent
+    result = agent.run()
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper
+    result = f(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 731, in run
+    result = self._invoke_run(role)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 908, in _invoke_run
+    time.sleep(monitor_interval)
+    ~~~~~~~~~~^^^^^^^^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 237 got signal: 15

epfl_final/train_logs/dense8k365m_s44.run.log.REPRO_NAN_20260722T222840Z

@@ -0,0 +1,214 @@
+New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed44/run_20260722_222547
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed44/run_20260722_222547",
+  "resume_from_checkpoint": null,
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 44000,
+  "per_device_batch_size": 2,
+  "gradient_accumulation_steps": 10,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 44,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "365m dense 8k seed44",
+  "wandb_tags": [
+    "qwen3",
+    "365m",
+    "8k",
+    "dense",
+    "seed44",
+    "multiseed"
+  ],
+  "hidden_size": 768,
+  "intermediate_size": 3072,
+  "num_attention_heads": 6,
+  "num_key_value_heads": 2,
+  "num_hidden_layers": 28,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 1,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "sparse_mask_seed": 0,
+  "sparse_mixed_nrb_set": null,
+  "dense_mode": true,
+  "dense_step_intervals": null,
+  "moba_block_size": 128,
+  "moba_top_k": 32,
+  "asentmax_alpha": 1.5,
+  "asentmax_learn_temp": true,
+  "asentmax_init_temp": 1.0
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Initializing model from scratch (random weights)...
+Checkpoint activations: False
+dense_mode=True: sparse attention layers will use standard causal SDPA.
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 768
+  num_hidden_layers: 28
+  num_attention_heads: 6
+  num_key_value_heads: 2
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 1
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 358,958,848
+  Trainable parameters: 358,958,848
+  Non-trainable parameters: 0
+  Model size (BF16): 0.72 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [2, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [2, 8192, 768]            --                        True
+│    └─Embedding: 2-1                                        [2, 8192]                 [2, 8192, 768]            116,686,848               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-2                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-3                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-4                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-5                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-6                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-7                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-8                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-9                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-10                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-11                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-12                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-13                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-14                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-15                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-16                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-17                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-18                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-19                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-20                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-21                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-22                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-23                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-24                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-25                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-26                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-27                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-28                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    └─Qwen3RMSNorm: 2-3                                     [2, 8192, 768]            [2, 8192, 768]            768                       True
+├─Linear: 1-2                                                [2, 8192, 768]            [2, 8192, 151936]         116,686,848               True
+================================================================================================================================================================
+Total params: 475,645,696
+Trainable params: 475,645,696
+Non-trainable params: 0
+Total mult-adds (Units.MEGABYTES): 951.29
+================================================================================================================================================================
+Input size (MB): 0.13
+Forward/backward pass size (MB): 31197.23
+Params size (MB): 951.29
+Estimated Total Size (MB): 32148.66
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    2
+World size (GPUs):                        1
+Gradient accumulation steps:              10
+Effective batch size (seqs/step):         20
+Sequence length:                          8192
+Tokens per optimizer step:                163,840
+Micro-batches per epoch (per rank):       1513882
+Optimizer steps per epoch:                151388
+Total optimizer steps:                    44000  (capped by max_steps; 0.29 epochs)
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run yus9oo3t
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt-baselines/wandb/run-20260722_222615-yus9oo3t
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run 365m dense 8k seed44
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/yus9oo3t
+Starting training...
+Switching attention mode for step 1: dense (dense step interval)
+Step 10/44000 | Loss: nan | LR: 6.67e-06 | 4618.6ms/step | 4.3 samples/s
+Step 20/44000 | Loss: nan | LR: 1.33e-05 | 3682.9ms/step | 5.4 samples/s
+Step 30/44000 | Loss: nan | LR: 2.00e-05 | 3687.9ms/step | 5.4 samples/s
+W0722 22:28:35.485000 3997 site-packages/torch/distributed/elastic/agent/server/api.py:739] Received 15 death signal, shutting down workers
+W0722 22:28:35.502000 3997 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 4071 closing signal SIGTERM
+Traceback (most recent call last):
+  File "/mloscratch/homes/navasard/conda/bin/torchrun", line 6, in <module>
+    sys.exit(main())
+             ~~~~^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper
+    return f(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main
+    run(args)
+    ~~~^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run
+    elastic_launch(
+    ~~~~~~~~~~~~~~~
+        config=config,
+        ~~~~~~~~~~~~~~
+        entrypoint=cmd,
+        ~~~~~~~~~~~~~~~
+    )(*cmd_args)
+    ~^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 308, in launch_agent
+    result = agent.run()
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper
+    result = f(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 731, in run
+    result = self._invoke_run(role)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 908, in _invoke_run
+    time.sleep(monitor_interval)
+    ~~~~~~~~~~^^^^^^^^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 3997 got signal: 15

epfl_final/train_logs/dense8k365m_s44.watch.log

@@ -0,0 +1,7 @@
+2026-07-22T22:17:37Z watchdog start pid=162
+2026-07-22T22:17:37Z launch GPUs=1 nproc=1 latest_step=
+2026-07-22T22:25:38Z watchdog start pid=3978
+2026-07-22T22:25:38Z launch GPUs=1 nproc=1 latest_step=
+2026-07-22T22:39:25Z watchdog start pid=143
+2026-07-22T22:39:25Z launch GPUs=0 nproc=1 latest_step=
+2026-07-24T22:15:31Z complete latest_step=44000

epfl_final/train_logs/dense8k365m_s45.watch.log

@@ -0,0 +1,3 @@
+2026-07-22T22:17:37Z watchdog start pid=164
+2026-07-22T22:17:37Z launch GPUs=2 nproc=1 latest_step=
+2026-07-24T22:38:52Z complete latest_step=44000

epfl_final/train_logs/dense8k365m_s46.run.log.GPU1_NAN_20260722T223303Z

@@ -0,0 +1,214 @@
+New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed46/run_20260722_222917
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed46/run_20260722_222917",
+  "resume_from_checkpoint": null,
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 44000,
+  "per_device_batch_size": 2,
+  "gradient_accumulation_steps": 10,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 46,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "365m dense 8k seed46",
+  "wandb_tags": [
+    "qwen3",
+    "365m",
+    "8k",
+    "dense",
+    "seed46",
+    "multiseed",
+    "replacement"
+  ],
+  "hidden_size": 768,
+  "intermediate_size": 3072,
+  "num_attention_heads": 6,
+  "num_key_value_heads": 2,
+  "num_hidden_layers": 28,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 1,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "sparse_mask_seed": 0,
+  "sparse_mixed_nrb_set": null,
+  "dense_mode": true,
+  "dense_step_intervals": null,
+  "moba_block_size": 128,
+  "moba_top_k": 32,
+  "asentmax_alpha": 1.5,
+  "asentmax_learn_temp": true,
+  "asentmax_init_temp": 1.0
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Initializing model from scratch (random weights)...
+Checkpoint activations: False
+dense_mode=True: sparse attention layers will use standard causal SDPA.
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 768
+  num_hidden_layers: 28
+  num_attention_heads: 6
+  num_key_value_heads: 2
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 1
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 358,958,848
+  Trainable parameters: 358,958,848
+  Non-trainable parameters: 0
+  Model size (BF16): 0.72 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [2, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [2, 8192, 768]            --                        True
+│    └─Embedding: 2-1                                        [2, 8192]                 [2, 8192, 768]            116,686,848               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-2                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-3                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-4                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-5                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-6                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-7                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-8                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-9                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-10                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-11                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-12                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-13                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-14                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-15                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-16                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-17                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-18                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-19                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-20                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-21                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-22                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-23                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-24                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-25                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-26                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-27                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-28                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    └─Qwen3RMSNorm: 2-3                                     [2, 8192, 768]            [2, 8192, 768]            768                       True
+├─Linear: 1-2                                                [2, 8192, 768]            [2, 8192, 151936]         116,686,848               True
+================================================================================================================================================================
+Total params: 475,645,696
+Trainable params: 475,645,696
+Non-trainable params: 0
+Total mult-adds (Units.MEGABYTES): 951.29
+================================================================================================================================================================
+Input size (MB): 0.13
+Forward/backward pass size (MB): 31197.23
+Params size (MB): 951.29
+Estimated Total Size (MB): 32148.66
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    2
+World size (GPUs):                        1
+Gradient accumulation steps:              10
+Effective batch size (seqs/step):         20
+Sequence length:                          8192
+Tokens per optimizer step:                163,840
+Micro-batches per epoch (per rank):       1513882
+Optimizer steps per epoch:                151388
+Total optimizer steps:                    44000  (capped by max_steps; 0.29 epochs)
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt-baselines/wandb/run-20260722_222942-xnsczozs
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run 365m dense 8k seed46
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/xnsczozs
+Starting training...
+Switching attention mode for step 1: dense (dense step interval)
+Step 10/44000 | Loss: nan | LR: 6.67e-06 | 4519.0ms/step | 4.4 samples/s
+Step 20/44000 | Loss: nan | LR: 1.33e-05 | 3693.3ms/step | 5.4 samples/s
+Step 30/44000 | Loss: nan | LR: 2.00e-05 | 3699.0ms/step | 5.4 samples/s
+W0722 22:31:55.321000 4881 site-packages/torch/distributed/elastic/agent/server/api.py:739] Received 15 death signal, shutting down workers
+W0722 22:31:55.324000 4881 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 4988 closing signal SIGTERM
+Traceback (most recent call last):
+  File "/mloscratch/homes/navasard/conda/bin/torchrun", line 6, in <module>
+    sys.exit(main())
+             ~~~~^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper
+    return f(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main
+    run(args)
+    ~~~^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run
+    elastic_launch(
+    ~~~~~~~~~~~~~~~
+        config=config,
+        ~~~~~~~~~~~~~~
+        entrypoint=cmd,
+        ~~~~~~~~~~~~~~~
+    )(*cmd_args)
+    ~^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 308, in launch_agent
+    result = agent.run()
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper
+    result = f(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 731, in run
+    result = self._invoke_run(role)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 908, in _invoke_run
+    time.sleep(monitor_interval)
+    ~~~~~~~~~~^^^^^^^^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 4881 got signal: 15

epfl_final/train_logs/dense8k365m_s46.watch.log

@@ -0,0 +1,6 @@
+2026-07-22T22:29:06Z watchdog start pid=4862
+2026-07-22T22:29:06Z launch GPUs=1 nproc=1 latest_step=
+2026-07-22T22:29:29Z watchdog start pid=5132
+2026-07-22T22:39:25Z watchdog start pid=144
+2026-07-22T22:39:25Z launch GPUs=1 nproc=1 latest_step=
+2026-07-24T22:43:33Z complete latest_step=44000

epfl_final/train_logs/launch_3b_dense_resume.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+set -eu
+
+GPU_COUNT="${1:?usage: launch_3b_dense_resume.sh <4|5|6>}"
+BASE=/mloscratch/homes/navasard/sparse
+WATCHDOG="$BASE/train_logs/watchdog_mixed_training.sh"
+
+case "$GPU_COUNT" in
+  4)
+    CFG=configs/ladder8k/dense_3b_60b.json
+    GPUSET=0,1,2,3
+    ;;
+  5)
+    CFG=configs/ladder8k/dense_3b_60b_5gpu.json
+    GPUSET=0,1,2,3,4
+    ;;
+  6)
+    CFG=configs/ladder8k/dense_3b_60b_6gpu.json
+    GPUSET=0,1,2,3,4,5
+    ;;
+  *)
+    echo "GPU count must be 4, 5, or 6" >&2
+    exit 2
+    ;;
+esac
+
+rm -f "$BASE/train_logs/dense8k3b_60b.lock"
+nohup "$WATCHDOG" dense8k3b_60b \
+  "$CFG" \
+  "$BASE/checkpoints/l8k3b_dense_60b" \
+  122070 "$GPUSET" "$GPU_COUNT" \
+  > "$BASE/train_logs/dense8k3b_60b.supervisor.log" 2>&1 < /dev/null &
+
+echo "dense8k3b_60b resume watchdog pid=$! GPUs=$GPU_COUNT config=$CFG"

epfl_final/train_logs/launch_3b_mixed.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+set -eu
+
+GPU_COUNT="${1:?usage: launch_3b_mixed.sh <4|5|6>}"
+BASE=/mloscratch/homes/navasard/sparse
+WATCHDOG="$BASE/train_logs/watchdog_mixed_training.sh"
+
+case "$GPU_COUNT" in
+  4)
+    CFG=configs/ladder8k/mixed_3b_60b.json
+    GPUSET=0,1,2,3
+    ;;
+  5)
+    CFG=configs/ladder8k/mixed_3b_60b_5gpu.json
+    GPUSET=0,1,2,3,4
+    ;;
+  6)
+    CFG=configs/ladder8k/mixed_3b_60b_6gpu.json
+    GPUSET=0,1,2,3,4,5
+    ;;
+  *)
+    echo "GPU count must be 4, 5, or 6" >&2
+    exit 2
+    ;;
+esac
+
+rm -f "$BASE/train_logs/mixed8k3b_60b.lock"
+nohup "$WATCHDOG" mixed8k3b_60b \
+  "$CFG" \
+  "$BASE/checkpoints/l8k3b_mixed_60b" \
+  122070 "$GPUSET" "$GPU_COUNT" \
+  > "$BASE/train_logs/mixed8k3b_60b.supervisor.log" 2>&1 < /dev/null &
+
+echo "mixed8k3b_60b watchdog pid=$! GPUs=$GPU_COUNT config=$CFG"

epfl_final/train_logs/mixed30k1p2b_s42.watch.log

@@ -0,0 +1,19 @@
+2026-07-21T05:57:01Z watchdog start pid=15034
+2026-07-21T05:57:01Z launch GPUs=0,1,2,3 nproc=4 latest_step=
+2026-07-21T18:02:27Z watchdog start pid=128
+2026-07-21T18:02:27Z launch GPUs=0,1,2,3 nproc=4 latest_step=2700
+2026-07-22T01:28:12Z watchdog start pid=144
+2026-07-22T01:28:12Z launch GPUs=0,1,2,3 nproc=4 latest_step=5100
+2026-07-24T17:55:47Z watchdog start pid=167
+2026-07-24T17:55:47Z launch GPUs=0,1,2,3 nproc=4 latest_step=29400
+2026-07-25T05:52:00Z watchdog start pid=153
+2026-07-25T05:52:01Z launch GPUs=0,1,2,3 nproc=4 latest_step=33600
+2026-07-25T06:02:32Z watchdog start pid=3407
+2026-07-25T06:02:32Z launch GPUs=0,1,2,3 nproc=4 latest_step=33600
+2026-07-25T06:24:56Z watchdog start pid=142
+2026-07-25T06:24:56Z launch GPUs=0,1,2,3 nproc=4 latest_step=33600
+2026-07-25T07:48:02Z process absent for 5 minutes; relaunching
+2026-07-25T07:48:02Z launch GPUs=0,1,2,3 nproc=4 latest_step=33600
+2026-07-25T17:20:50Z watchdog start pid=145
+2026-07-25T17:20:51Z launch GPUs=0,1,2,3 nproc=4 latest_step=36900
+2026-07-25T19:01:58Z complete latest_step=37465

epfl_final/train_logs/mixed30k365m_s43.run.log.INTERRUPTED_STEP40_20260722T225915Z

@@ -0,0 +1,196 @@
+New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_mixed_seed43/run_20260722_221751
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/longctx30k/tokenized",
+  "val_path": null,
+  "max_seq_length": 30720,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l30k365m_mixed_seed43/run_20260722_221751",
+  "resume_from_checkpoint": null,
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 11700,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 20,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 43,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "365m mixed 30k seed43",
+  "wandb_tags": [
+    "qwen3",
+    "365m",
+    "30k",
+    "mixed",
+    "native",
+    "seed43",
+    "multiseed"
+  ],
+  "hidden_size": 768,
+  "intermediate_size": 3072,
+  "num_attention_heads": 6,
+  "num_key_value_heads": 2,
+  "num_hidden_layers": 28,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 120,
+  "sparse_mask_pool_size": 1000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_30k_365m_mixed_seed42",
+  "sparse_mask_seed": 42,
+  "sparse_mixed_nrb_set": [
+    239,
+    179,
+    120,
+    79,
+    39
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null,
+  "moba_block_size": 128,
+  "moba_top_k": 32,
+  "asentmax_alpha": 1.5,
+  "asentmax_learn_temp": true,
+  "asentmax_init_temp": 1.0
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Initializing model from scratch (random weights)...
+Checkpoint activations: False
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 768
+  num_hidden_layers: 28
+  num_attention_heads: 6
+  num_key_value_heads: 2
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 120
+  sparse_mask_pool_size: 1000
+  sparse_max_seq_length: 30720
+
+Parameter count:
+  Total parameters: 358,958,848
+  Trainable parameters: 358,958,848
+  Non-trainable parameters: 0
+  Model size (BF16): 0.72 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 30720]                --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 30720, 768]           --                        True
+│    └─Embedding: 2-1                                        [1, 30720]                [1, 30720, 768]           116,686,848               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 30720, 768]           [1, 30720, 768]           768                       True
+├─Linear: 1-2                                                [1, 30720, 768]           [1, 30720, 151936]        116,686,848               True
+================================================================================================================================================================
+Total params: 475,645,696
+Trainable params: 475,645,696
+Non-trainable params: 0
+Total mult-adds (Units.MEGABYTES): 475.65
+================================================================================================================================================================
+Input size (MB): 0.25
+Forward/backward pass size (MB): 58494.81
+Params size (MB): 951.29
+Estimated Total Size (MB): 59446.35
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)
+Dataset split: 749299 train, 39437 validation
+Training examples (packed, train split):  749299
+Validation examples (packed, val split):  39437
+Per-device batch size:                    1
+World size (GPUs):                        1
+Gradient accumulation steps:              20
+Effective batch size (seqs/step):         20
+Sequence length:                          30720
+Tokens per optimizer step:                614,400
+Micro-batches per epoch (per rank):       749299
+Optimizer steps per epoch:                37464
+Total optimizer steps:                    11700  (capped by max_steps; 0.31 epochs)
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run x6bzdeb3
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt-baselines/wandb/run-20260722_221947-x6bzdeb3
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run 365m mixed 30k seed43
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/x6bzdeb3
+Starting training...
+Switching attention mode for step 1: sparse_stochastic (configured sparse attention)
+Step 10/11700 | Loss: 12.0641 | LR: 6.67e-06 | 29622.4ms/step | 0.7 samples/s
+Step 20/11700 | Loss: 11.9864 | LR: 1.33e-05 | 23511.1ms/step | 0.9 samples/s
+Step 30/11700 | Loss: 11.7461 | LR: 2.00e-05 | 23295.2ms/step | 0.9 samples/s
+Step 40/11700 | Loss: 11.2480 | LR: 2.67e-05 | 23382.1ms/step | 0.9 samples/s

epfl_final/train_logs/mixed30k365m_s43.watch.log

@@ -0,0 +1,14 @@
+2026-07-22T22:17:37Z watchdog start pid=312
+2026-07-22T22:17:37Z launch GPUs=2 nproc=1 latest_step=
+2026-07-22T22:59:15Z watchdog start pid=186
+2026-07-22T22:59:15Z launch GPUs=2 nproc=1 latest_step=
+2026-07-24T07:03:34Z watchdog start pid=353
+2026-07-24T07:03:34Z launch GPUs=2 nproc=1 latest_step=
+2026-07-24T17:38:52Z watchdog start pid=200
+2026-07-24T17:38:52Z launch GPUs=2 nproc=1 latest_step=
+2026-07-25T06:17:28Z watchdog start pid=192
+2026-07-25T06:17:28Z launch GPUs=2 nproc=1 latest_step=
+2026-07-25T07:50:01Z process absent for 5 minutes; relaunching
+2026-07-25T07:50:01Z launch GPUs=2 nproc=1 latest_step=
+2026-07-25T07:55:02Z process absent for 5 minutes; relaunching
+2026-07-25T07:55:02Z launch GPUs=2 nproc=1 latest_step=

epfl_final/train_logs/mixed30k365m_s44.run.log.INTERRUPTED_STEP40_20260722T225915Z

@@ -0,0 +1,196 @@
+New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_mixed_seed44/run_20260722_221751
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/longctx30k/tokenized",
+  "val_path": null,
+  "max_seq_length": 30720,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l30k365m_mixed_seed44/run_20260722_221751",
+  "resume_from_checkpoint": null,
+  "force_new_run": false,
+  "num_epochs": 1,
+  "max_steps": 11700,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 20,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 250,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 44,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": true,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "365m mixed 30k seed44",
+  "wandb_tags": [
+    "qwen3",
+    "365m",
+    "30k",
+    "mixed",
+    "native",
+    "seed44",
+    "multiseed"
+  ],
+  "hidden_size": 768,
+  "intermediate_size": 3072,
+  "num_attention_heads": 6,
+  "num_key_value_heads": 2,
+  "num_hidden_layers": 28,
+  "head_dim": null,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 120,
+  "sparse_mask_pool_size": 1000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_30k_365m_mixed_seed42",
+  "sparse_mask_seed": 42,
+  "sparse_mixed_nrb_set": [
+    239,
+    179,
+    120,
+    79,
+    39
+  ],
+  "dense_mode": false,
+  "dense_step_intervals": null,
+  "moba_block_size": 128,
+  "moba_top_k": 32,
+  "asentmax_alpha": 1.5,
+  "asentmax_learn_temp": true,
+  "asentmax_init_temp": 1.0
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Initializing model from scratch (random weights)...
+Checkpoint activations: False
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 768
+  num_hidden_layers: 28
+  num_attention_heads: 6
+  num_key_value_heads: 2
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 120
+  sparse_mask_pool_size: 1000
+  sparse_max_seq_length: 30720
+
+Parameter count:
+  Total parameters: 358,958,848
+  Trainable parameters: 358,958,848
+  Non-trainable parameters: 0
+  Model size (BF16): 0.72 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [1, 30720]                --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [1, 30720, 768]           --                        True
+│    └─Embedding: 2-1                                        [1, 30720]                [1, 30720, 768]           116,686,848               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-2                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-3                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-4                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-5                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-6                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-7                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-8                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-9                           [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-10                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-11                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-12                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-13                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-14                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-15                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-16                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-17                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-18                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-19                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-20                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-21                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-22                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-23                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-24                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-25                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-26                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-27                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-28                          [1, 30720, 768]           [1, 30720, 768]           8,652,544                 True
+│    └─Qwen3RMSNorm: 2-3                                     [1, 30720, 768]           [1, 30720, 768]           768                       True
+├─Linear: 1-2                                                [1, 30720, 768]           [1, 30720, 151936]        116,686,848               True
+================================================================================================================================================================
+Total params: 475,645,696
+Trainable params: 475,645,696
+Non-trainable params: 0
+Total mult-adds (Units.MEGABYTES): 475.65
+================================================================================================================================================================
+Input size (MB): 0.25
+Forward/backward pass size (MB): 58494.81
+Params size (MB): 951.29
+Estimated Total Size (MB): 59446.35
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/longctx30k/tokenized...
+Loaded 788736 pre-tokenized examples (stored_len=30720, pack_factor=1, packed_examples=788736)
+Dataset split: 749299 train, 39437 validation
+Training examples (packed, train split):  749299
+Validation examples (packed, val split):  39437
+Per-device batch size:                    1
+World size (GPUs):                        1
+Gradient accumulation steps:              20
+Effective batch size (seqs/step):         20
+Sequence length:                          30720
+Tokens per optimizer step:                614,400
+Micro-batches per epoch (per rank):       749299
+Optimizer steps per epoch:                37464
+Total optimizer steps:                    11700  (capped by max_steps; 0.31 epochs)
+wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
+wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
+wandb: setting up run 3id6yvxn
+wandb: Tracking run with wandb version 0.26.1
+wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt-baselines/wandb/run-20260722_221947-3id6yvxn
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run 365m mixed 30k seed44
+wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
+wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/3id6yvxn
+Starting training...
+Switching attention mode for step 1: sparse_stochastic (configured sparse attention)
+Step 10/11700 | Loss: 12.0550 | LR: 6.67e-06 | 29814.3ms/step | 0.7 samples/s
+Step 20/11700 | Loss: 11.9902 | LR: 1.33e-05 | 23570.5ms/step | 0.8 samples/s
+Step 30/11700 | Loss: 11.7343 | LR: 2.00e-05 | 23561.8ms/step | 0.8 samples/s
+Step 40/11700 | Loss: 11.2565 | LR: 2.67e-05 | 23593.3ms/step | 0.8 samples/s

epfl_final/train_logs/mixed30k365m_s44.watch.log

@@ -0,0 +1,14 @@
+2026-07-22T22:17:37Z watchdog start pid=315
+2026-07-22T22:17:37Z launch GPUs=3 nproc=1 latest_step=
+2026-07-22T22:59:15Z watchdog start pid=187
+2026-07-22T22:59:15Z launch GPUs=3 nproc=1 latest_step=
+2026-07-24T07:03:34Z watchdog start pid=354
+2026-07-24T07:03:34Z launch GPUs=3 nproc=1 latest_step=
+2026-07-24T17:38:52Z watchdog start pid=201
+2026-07-24T17:38:52Z launch GPUs=3 nproc=1 latest_step=
+2026-07-25T06:17:28Z watchdog start pid=195
+2026-07-25T06:17:28Z launch GPUs=3 nproc=1 latest_step=
+2026-07-25T07:50:31Z process absent for 5 minutes; relaunching
+2026-07-25T07:50:31Z launch GPUs=3 nproc=1 latest_step=
+2026-07-25T07:55:32Z process absent for 5 minutes; relaunching
+2026-07-25T07:55:32Z launch GPUs=3 nproc=1 latest_step=

epfl_final/train_logs/mixed8k365m_s44.watch.log

@@ -0,0 +1,3 @@
+2026-07-22T22:17:37Z watchdog start pid=168
+2026-07-22T22:17:37Z launch GPUs=3 nproc=1 latest_step=
+2026-07-24T20:54:14Z complete latest_step=44000

epfl_final/train_logs/mixed8k365m_s45.watch.log

@@ -0,0 +1,3 @@
+2026-07-22T22:39:25Z watchdog start pid=145
+2026-07-22T22:39:25Z launch GPUs=2 nproc=1 latest_step=
+2026-07-24T21:16:26Z complete latest_step=44000

epfl_final/train_logs/mixed8k365m_s46.watch.log

@@ -0,0 +1,3 @@
+2026-07-22T22:39:25Z watchdog start pid=146
+2026-07-22T22:39:25Z launch GPUs=3 nproc=1 latest_step=
+2026-07-24T21:33:58Z complete latest_step=44000

epfl_final/train_logs/mixed8k3b_60b.watch.log

@@ -0,0 +1,16 @@
+2026-07-22T07:59:55Z watchdog start pid=223898
+2026-07-22T07:59:55Z launch GPUs=0,1,2,3 nproc=4 latest_step=
+2026-07-22T08:42:15Z watchdog start pid=225027
+2026-07-22T08:42:15Z launch GPUs=0,1,2,3 nproc=4 latest_step=
+2026-07-23T07:31:40Z watchdog start pid=256637
+2026-07-23T07:31:40Z launch GPUs=0,1,2,3 nproc=4 latest_step=4250
+2026-07-23T07:45:57Z watchdog start pid=258463
+2026-07-23T07:45:57Z launch GPUs=0,1,2,3 nproc=4 latest_step=4250
+2026-07-24T17:11:14Z process absent for 5 minutes; relaunching
+2026-07-24T17:11:14Z launch GPUs=0,1,2,3 nproc=4 latest_step=11000
+2026-07-24T18:19:57Z watchdog start pid=307075
+2026-07-24T18:19:57Z launch GPUs=0,1,2,3 nproc=4 latest_step=11000
+2026-07-25T07:50:44Z process absent for 5 minutes; relaunching
+2026-07-25T07:50:44Z launch GPUs=0,1,2,3 nproc=4 latest_step=13500
+2026-07-25T16:57:06Z watchdog start pid=340455
+2026-07-25T16:57:06Z launch GPUs=0,1,2,3 nproc=4 latest_step=13500

epfl_final/train_logs/mixed8k3b_prepare.log

@@ -0,0 +1 @@
+mixed8k3b_60b watchdog pid=225027 GPUs=4 config=configs/ladder8k/mixed_3b_60b.json

epfl_final/train_logs/prebuild_mixed_mask_layers.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+"""Build missing mixed-attention mask-cache files for selected layers."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+import torch
+
+REPO = Path("/mloscratch/homes/navasard/sparse/SparseAtt-baselines")
+if str(REPO) not in sys.path:
+    sys.path.insert(0, str(REPO))
+
+from sparseatt import Qwen3Config
+from sparseatt.sparse_attention import Qwen3SparseStochasticAttention
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--config", required=True)
+    parser.add_argument("--layers", nargs="+", type=int, required=True)
+    args = parser.parse_args()
+
+    raw = json.loads(Path(args.config).read_text())
+    model = raw["model"]
+    data = raw["data"]
+    attention = raw["attention"]
+    cache_dir = Path(attention["sparse_mask_cache_dir"])
+    cache_dir.mkdir(parents=True, exist_ok=True)
+
+    config_values = {
+        "hidden_size": model["hidden_size"],
+        "intermediate_size": model["intermediate_size"],
+        "num_hidden_layers": model["num_hidden_layers"],
+        "num_attention_heads": model["num_attention_heads"],
+        "num_key_value_heads": model["num_key_value_heads"],
+        "attention_type": attention["type"],
+        "sparse_block_size": attention["sparse_block_size"],
+        "sparse_num_random_blocks": attention["sparse_num_random_blocks"],
+        "sparse_mask_pool_size": attention["sparse_mask_pool_size"],
+        "sparse_max_seq_length": data["max_seq_length"],
+        "sparse_mask_cache_dir": str(cache_dir),
+        "sparse_mask_seed": attention["sparse_mask_seed"],
+        "sparse_mixed_nrb_set": attention["sparse_mixed_nrb_set"],
+    }
+    if model.get("head_dim") is not None:
+        config_values["head_dim"] = model["head_dim"]
+    config = Qwen3Config(**config_values)
+
+    for layer_index in args.layers:
+        cache_path = cache_dir / f"masks_layer_{layer_index}.pt"
+        if cache_path.is_file() and cache_path.stat().st_size > 0:
+            print(f"layer={layer_index} already complete bytes={cache_path.stat().st_size}")
+            continue
+        print(f"layer={layer_index} building", flush=True)
+        module = Qwen3SparseStochasticAttention(
+            config,
+            layer_index,
+            mask_cache_dir=str(cache_dir),
+        )
+        if not cache_path.is_file() or cache_path.stat().st_size == 0:
+            raise RuntimeError(f"Mask cache was not created: {cache_path}")
+        print(f"layer={layer_index} complete bytes={cache_path.stat().st_size}", flush=True)
+        del module
+        torch.cuda.empty_cache()
+
+
+if __name__ == "__main__":
+    main()

epfl_final/train_logs/prepare_365m_seed_matrix.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+import argparse
+import copy
+import json
+from pathlib import Path
+
+
+REPO = Path("/mloscratch/homes/navasard/sparse/SparseAtt-baselines")
+CONFIG_DIR = REPO / "configs/multiseed"
+
+
+def write_config(base_path, output_name, output_dir, seed, wandb_name, tags, updates=None):
+    config = json.loads((REPO / base_path).read_text())
+    config = copy.deepcopy(config)
+    config["training"]["output_dir"] = output_dir
+    config["training"]["resume_from_checkpoint"] = None
+    config["training"]["force_new_run"] = False
+    config["training"]["seed"] = seed
+    config["wandb"]["name"] = wandb_name
+    config["wandb"]["tags"] = tags
+
+    for section, values in (updates or {}).items():
+        config[section].update(values)
+
+    CONFIG_DIR.mkdir(parents=True, exist_ok=True)
+    destination = CONFIG_DIR / output_name
+    temporary = destination.with_suffix(".json.tmp")
+    temporary.write_text(json.dumps(config, indent=2) + "\n")
+    temporary.replace(destination)
+    return destination
+
+
+def prepare_30k():
+    jobs = []
+    for seed, gpu in ((43, 0), (44, 1)):
+        output_dir = f"/mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense_seed{seed}"
+        config = write_config(
+            "configs/ladder30k/dense.json",
+            f"dense_365m_30k_seed{seed}.json",
+            output_dir,
+            seed,
+            f"365m dense 30k seed{seed}",
+            ["qwen3", "365m", "30k", "dense", "native", f"seed{seed}", "multiseed"],
+        )
+        jobs.append((f"dense30k365m_s{seed}", config, output_dir, gpu))
+
+    for seed, gpu in ((43, 2), (44, 3)):
+        output_dir = f"/mloscratch/homes/navasard/sparse/checkpoints/l30k365m_mixed_seed{seed}"
+        config = write_config(
+            "configs/365m_mixed_30k_2gpu_seed42.json",
+            f"mixed_365m_30k_seed{seed}.json",
+            output_dir,
+            seed,
+            f"365m mixed 30k seed{seed}",
+            ["qwen3", "365m", "30k", "mixed", "native", f"seed{seed}", "multiseed"],
+            updates={"training": {"gradient_accumulation_steps": 20}},
+        )
+        jobs.append((f"mixed30k365m_s{seed}", config, output_dir, gpu))
+    return jobs
+
+
+def prepare_8k():
+    jobs = []
+    for seed, gpu in ((43, 0), (44, 1), (45, 2)):
+        output_dir = f"/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed{seed}"
+        config = write_config(
+            "configs/ladder8k/dense.json",
+            f"dense_365m_8k_seed{seed}.json",
+            output_dir,
+            seed,
+            f"365m dense 8k seed{seed}",
+            ["qwen3", "365m", "8k", "dense", f"seed{seed}", "multiseed"],
+        )
+        jobs.append((f"dense8k365m_s{seed}", config, output_dir, gpu))
+
+    seed = 44
+    output_dir = f"/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed{seed}"
+    config = write_config(
+        "configs/365m_mixed_8k_seed42.json",
+        f"mixed_365m_8k_seed{seed}.json",
+        output_dir,
+        seed,
+        f"365m mixed 8k seed{seed}",
+        ["qwen3", "365m", "8k", "mixed", f"seed{seed}", "multiseed"],
+    )
+    jobs.append((f"mixed8k365m_s{seed}", config, output_dir, 3))
+    return jobs
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--matrix", choices=("8k", "30k"), required=True)
+    args = parser.parse_args()
+    jobs = prepare_8k() if args.matrix == "8k" else prepare_30k()
+    for job, config, output_dir, gpu in jobs:
+        print(
+            json.dumps(
+                {
+                    "job": job,
+                    "config": str(config.relative_to(REPO)),
+                    "output_dir": output_dir,
+                    "gpu": gpu,
+                    "max_steps": 44000 if args.matrix == "8k" else 11700,
+                }
+            )
+        )
+
+
+if __name__ == "__main__":
+    main()

epfl_final/train_logs/prepare_then_launch_3b_mixed.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+set -euo pipefail
+
+BASE=/mloscratch/homes/navasard/sparse
+REPO="$BASE/SparseAtt-baselines"
+PYTHON=/mloscratch/homes/navasard/conda/bin/python3.13
+BUILDER="$BASE/train_logs/prebuild_mixed_mask_layers.py"
+CONFIG=configs/ladder8k/mixed_3b_60b.json
+LOGDIR="$BASE/train_logs/mixed8k3b_mask_build"
+
+mkdir -p "$LOGDIR"
+cd "$REPO"
+
+pids=()
+for spec in "0:28:32" "1:29:33" "2:30:34" "3:31:35"; do
+  IFS=: read -r gpu first second <<< "$spec"
+  CUDA_VISIBLE_DEVICES="$gpu" "$PYTHON" "$BUILDER" \
+    --config "$CONFIG" --layers "$first" "$second" \
+    > "$LOGDIR/gpu${gpu}.log" 2>&1 &
+  pids+=("$!")
+done
+
+for pid in "${pids[@]}"; do
+  wait "$pid"
+done
+
+for layer in $(seq 0 35); do
+  test -s "$BASE/mask_cache_8k_mixed/masks_layer_${layer}.pt"
+done
+touch "$BASE/mask_cache_8k_mixed/.BUILD_DONE_36"
+
+"$BASE/train_logs/launch_3b_mixed.sh" 4