EPFL ALiBi and training provenance 851-900 of 1184

epfl_final/train_logs/chain_next_40a.log

@@ -0,0 +1,50 @@
+START gpu0 3b_dense__ppl_16k_sink_local_fineweb
+START gpu1 3b_dense__ppl_16k_sliding_fineweb
+START gpu2 3b_dense__ppl_16k_dense_fineweb
+START gpu3 3b_dense__passkey_16k_random
+DONE gpu1 3b_dense__ppl_16k_sliding_fineweb status=retrying
+START gpu1 3b_mixed__ppl_16k_quest_fineweb
+DONE gpu0 3b_dense__ppl_16k_sink_local_fineweb status=retrying
+START gpu0 3b_mixed__ppl_16k_moba_fineweb
+DONE gpu3 3b_dense__passkey_16k_random status=retrying
+START gpu3 3b_mixed__ppl_16k_h2o_fineweb
+DONE gpu2 3b_dense__ppl_16k_dense_fineweb status=complete
+START gpu2 3b_mixed__ruler_random_niah_multikey_16k
+DONE gpu0 3b_mixed__ppl_16k_moba_fineweb status=retrying
+START gpu0 3b_mixed__efficiency_16k
+DONE gpu1 3b_mixed__ppl_16k_quest_fineweb status=retrying
+START gpu1 3b_dense__harness_gsm8k
+DONE gpu1 3b_dense__harness_gsm8k status=retrying
+START gpu1 3b_mixed__harness_arc_challenge
+DONE gpu3 3b_mixed__ppl_16k_h2o_fineweb status=complete
+START gpu3 3b_dense__ppl_8k_random_gutenberg
+DONE gpu1 3b_mixed__harness_arc_challenge status=complete
+START gpu1 3b_dense__ppl_8k_random_fwtail
+DONE gpu3 3b_dense__ppl_8k_random_gutenberg status=complete
+START gpu3 3b_dense__ppl_8k_topk_pg19
+DONE gpu1 3b_dense__ppl_8k_random_fwtail status=complete
+START gpu1 3b_dense__efficiency_8k
+DONE gpu3 3b_dense__ppl_8k_topk_pg19 status=complete
+START gpu3 3b_mixed__ppl_8k_random_pubmed
+DONE gpu1 3b_dense__efficiency_8k status=complete
+START gpu1 3b_mixed__ppl_8k_topk_fineweb
+DONE gpu0 3b_mixed__efficiency_16k status=complete
+START gpu0 3b_mixed__ppl_8k_topk_arxiv
+DONE gpu2 3b_mixed__ruler_random_niah_multikey_16k status=complete
+START gpu2 3b_dense__ppl_16k_sliding_fineweb
+DONE gpu2 3b_dense__ppl_16k_sliding_fineweb status=failed
+START gpu2 3b_dense__ppl_16k_sink_local_fineweb
+DONE gpu2 3b_dense__ppl_16k_sink_local_fineweb status=failed
+START gpu2 3b_dense__passkey_16k_random
+DONE gpu2 3b_dense__passkey_16k_random status=failed
+START gpu2 3b_mixed__ppl_16k_moba_fineweb
+DONE gpu1 3b_mixed__ppl_8k_topk_fineweb status=complete
+START gpu1 3b_mixed__ppl_16k_quest_fineweb
+DONE gpu2 3b_mixed__ppl_16k_moba_fineweb status=failed
+START gpu2 3b_dense__harness_gsm8k
+DONE gpu2 3b_dense__harness_gsm8k status=failed
+DONE gpu1 3b_mixed__ppl_16k_quest_fineweb status=failed
+DONE gpu3 3b_mixed__ppl_8k_random_pubmed status=complete
+DONE gpu0 3b_mixed__ppl_8k_topk_arxiv status=complete
+finished complete=14/20 failed=6
+Failed jobs: 3b_dense__ppl_16k_sink_local_fineweb, 3b_dense__ppl_16k_sliding_fineweb, 3b_dense__passkey_16k_random, 3b_mixed__ppl_16k_quest_fineweb, 3b_mixed__ppl_16k_moba_fineweb, 3b_dense__harness_gsm8k

epfl_final/train_logs/chain_next_40b.log

@@ -0,0 +1,54 @@
+START gpu0 3b_dense__ppl_16k_quest_fineweb
+START gpu1 3b_dense__ppl_16k_moba_fineweb
+START gpu2 3b_dense__ppl_16k_h2o_fineweb
+START gpu3 3b_dense__ruler_random_niah_multikey_16k
+DONE gpu1 3b_dense__ppl_16k_moba_fineweb status=retrying
+START gpu1 3b_dense__efficiency_16k
+DONE gpu0 3b_dense__ppl_16k_quest_fineweb status=retrying
+START gpu0 3b_mixed__ppl_16k_sink_local_fineweb
+DONE gpu0 3b_mixed__ppl_16k_sink_local_fineweb status=retrying
+START gpu0 3b_mixed__ppl_16k_sliding_fineweb
+DONE gpu0 3b_mixed__ppl_16k_sliding_fineweb status=retrying
+START gpu0 3b_mixed__ppl_16k_dense_fineweb
+DONE gpu2 3b_dense__ppl_16k_h2o_fineweb status=complete
+START gpu2 3b_mixed__passkey_16k_random
+DONE gpu0 3b_mixed__ppl_16k_dense_fineweb status=complete
+START gpu0 3b_dense__harness_mmlu
+DONE gpu2 3b_mixed__passkey_16k_random status=retrying
+START gpu2 3b_dense__harness_arc_challenge
+DONE gpu2 3b_dense__harness_arc_challenge status=complete
+START gpu2 3b_mixed__harness_mmlu
+DONE gpu0 3b_dense__harness_mmlu status=complete
+START gpu0 3b_mixed__harness_gsm8k
+DONE gpu0 3b_mixed__harness_gsm8k status=retrying
+START gpu0 3b_dense__ppl_8k_random_pubmed
+DONE gpu2 3b_mixed__harness_mmlu status=complete
+START gpu2 3b_dense__ppl_8k_topk_fineweb
+DONE gpu2 3b_dense__ppl_8k_topk_fineweb status=complete
+START gpu2 3b_dense__ppl_8k_topk_arxiv
+DONE gpu0 3b_dense__ppl_8k_random_pubmed status=complete
+START gpu0 3b_mixed__ppl_8k_random_gutenberg
+DONE gpu2 3b_dense__ppl_8k_topk_arxiv status=complete
+START gpu2 3b_mixed__ppl_8k_random_fwtail
+DONE gpu1 3b_dense__efficiency_16k status=complete
+START gpu1 3b_mixed__ppl_8k_topk_pg19
+DONE gpu3 3b_dense__ruler_random_niah_multikey_16k status=complete
+START gpu3 3b_mixed__efficiency_8k
+DONE gpu1 3b_mixed__ppl_8k_topk_pg19 status=complete
+START gpu1 3b_dense__ppl_16k_moba_fineweb
+DONE gpu1 3b_dense__ppl_16k_moba_fineweb status=failed
+START gpu1 3b_dense__ppl_16k_quest_fineweb
+DONE gpu1 3b_dense__ppl_16k_quest_fineweb status=failed
+START gpu1 3b_mixed__ppl_16k_sink_local_fineweb
+DONE gpu1 3b_mixed__ppl_16k_sink_local_fineweb status=failed
+START gpu1 3b_mixed__ppl_16k_sliding_fineweb
+DONE gpu1 3b_mixed__ppl_16k_sliding_fineweb status=failed
+START gpu1 3b_mixed__passkey_16k_random
+DONE gpu1 3b_mixed__passkey_16k_random status=failed
+START gpu1 3b_mixed__harness_gsm8k
+DONE gpu1 3b_mixed__harness_gsm8k status=failed
+DONE gpu0 3b_mixed__ppl_8k_random_gutenberg status=complete
+DONE gpu2 3b_mixed__ppl_8k_random_fwtail status=complete
+DONE gpu3 3b_mixed__efficiency_8k status=complete
+finished complete=15/21 failed=6
+Failed jobs: 3b_dense__ppl_16k_quest_fineweb, 3b_dense__ppl_16k_moba_fineweb, 3b_mixed__ppl_16k_sink_local_fineweb, 3b_mixed__ppl_16k_sliding_fineweb, 3b_mixed__passkey_16k_random, 3b_mixed__harness_gsm8k

epfl_final/train_logs/chain_next_80g.log

@@ -0,0 +1,4 @@
+START gpu0 3b_dense__ppl_30k_random_gutenberg
+START gpu1 3b_dense__ruler_random_niah_multikey_30k
+START gpu2 3b_dense__passkey_30k_random
+START gpu3 3b_dense__efficiency_30k

epfl_final/train_logs/chain_next_h100.log

@@ -0,0 +1,7 @@
+START gpu0 3b_dense__ppl_30k_random_pubmed
+DONE gpu0 3b_dense__ppl_30k_random_pubmed status=complete
+START gpu0 3b_mixed__ppl_30k_random_gutenberg
+DONE gpu0 3b_mixed__ppl_30k_random_gutenberg status=complete
+START gpu0 3b_mixed__longbench_qasper
+DONE gpu0 3b_mixed__longbench_qasper status=complete
+finished complete=4/4 failed=0

epfl_final/train_logs/dense8k3b_60b.4gpu.lock

@@ -0,0 +1 @@
+137

epfl_final/train_logs/dense8k3b_60b.4gpu.watch.log

@@ -0,0 +1,6 @@
+2026-07-26T18:19:29Z watchdog start pid=176 latest_step=63500
+2026-07-26T18:19:29Z launch checkpoint=/mloscratch/homes/navasard/sparse/checkpoints/l8k3b_dense_60b/run_20260720_074219/checkpoint-63500 GPUs=0,1,2,3 nproc=4
+2026-07-27T06:24:00Z watchdog start pid=138 latest_step=69750
+2026-07-27T06:24:00Z launch checkpoint=/mloscratch/homes/navasard/sparse/checkpoints/l8k3b_dense_60b/run_20260720_074219/checkpoint-69750 GPUs=0,1,2,3 nproc=4
+2026-07-27T16:47:09Z watchdog start pid=137 latest_step=72500
+2026-07-27T16:47:10Z launch checkpoint=/mloscratch/homes/navasard/sparse/checkpoints/l8k3b_dense_60b/run_20260720_074219/checkpoint-72500 GPUs=0,1,2,3 nproc=4

epfl_final/train_logs/dense8k3b_60b.lock

@@ -0,0 +1 @@
+298

epfl_final/train_logs/dense_dropout_p28_8k365m_s42.run.log

@@ -0,0 +1,264 @@
+nohup: ignoring input
+New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_dropout_p28_seed42/run_20260819_050018
+Running config:
+{
+  "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 8,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_dropout_p28_seed42/run_20260819_050018",
+  "resume_from_checkpoint": null,
+  "force_new_run": true,
+  "num_epochs": 1,
+  "max_steps": 44000,
+  "per_device_batch_size": 2,
+  "gradient_accumulation_steps": 10,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 1000,
+  "eval_steps": 1000,
+  "eval_batches": 50,
+  "logging_steps": 10,
+  "seed": 42,
+  "verbose": true,
+  "checkpoint_activations": false,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": false,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "l8k365m_dense_dropout_p28_seed42",
+  "wandb_tags": [
+    "qwen3",
+    "365m",
+    "8k",
+    "scaleladder",
+    "dense"
+  ],
+  "hidden_size": 768,
+  "intermediate_size": 3072,
+  "num_attention_heads": 6,
+  "num_key_value_heads": 2,
+  "num_hidden_layers": 28,
+  "head_dim": null,
+  "attention_dropout": 0.28,
+  "attention_type": "sparse_stochastic",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 31,
+  "sparse_mask_pool_size": 1,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "sparse_mask_seed": 0,
+  "sparse_mixed_nrb_set": null,
+  "dense_mode": true,
+  "dense_step_intervals": null
+}
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Initializing model from scratch (random weights)...
+Checkpoint activations: False
+dense_mode=True: sparse attention layers will use standard causal SDPA.
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 768
+  num_hidden_layers: 28
+  num_attention_heads: 6
+  num_key_value_heads: 2
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 1
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 358,958,848
+  Trainable parameters: 358,958,848
+  Non-trainable parameters: 0
+  Model size (BF16): 0.72 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [2, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [2, 8192, 768]            --                        True
+│    └─Embedding: 2-1                                        [2, 8192]                 [2, 8192, 768]            116,686,848               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-2                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-3                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-4                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-5                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-6                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-7                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-8                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-9                           [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-10                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-11                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-12                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-13                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-14                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-15                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-16                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-17                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-18                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-19                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-20                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-21                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-22                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-23                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-24                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-25                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-26                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-27                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    │    └─Qwen3DecoderLayer: 3-28                          [2, 8192, 768]            [2, 8192, 768]            8,652,544                 True
+│    └─Qwen3RMSNorm: 2-3                                     [2, 8192, 768]            [2, 8192, 768]            768                       True
+├─Linear: 1-2                                                [2, 8192, 768]            [2, 8192, 151936]         116,686,848               True
+================================================================================================================================================================
+Total params: 475,645,696
+Trainable params: 475,645,696
+Non-trainable params: 0
+Total mult-adds (Units.MEGABYTES): 951.29
+================================================================================================================================================================
+Input size (MB): 0.13
+Forward/backward pass size (MB): 31197.23
+Params size (MB): 951.29
+Estimated Total Size (MB): 32148.66
+================================================================================================================================================================
+  NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M)
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    2
+World size (GPUs):                        1
+Gradient accumulation steps:              10
+Effective batch size (seqs/step):         20
+Sequence length:                          8192
+Tokens per optimizer step:                163,840
+Micro-batches per epoch (per rank):       1513882
+Optimizer steps per epoch:                151388
+Total optimizer steps:                    44000  (capped by max_steps; 0.29 epochs)
+Starting training...
+Switching attention mode for step 1: dense (dense step interval)
+Step 10/44000 | Loss: 12.0380 | LR: 6.67e-06 | 7516.2ms/step | 2.7 samples/s
+Step 20/44000 | Loss: 11.9235 | LR: 1.33e-05 | 1552.6ms/step | 12.9 samples/s
+Step 30/44000 | Loss: 11.5564 | LR: 2.00e-05 | 1550.7ms/step | 12.9 samples/s
+Step 40/44000 | Loss: 11.0157 | LR: 2.67e-05 | 1546.6ms/step | 12.9 samples/s
+Step 50/44000 | Loss: 10.7348 | LR: 3.33e-05 | 1546.9ms/step | 12.9 samples/s
+Step 60/44000 | Loss: 10.4908 | LR: 4.00e-05 | 1542.1ms/step | 13.0 samples/s
+Step 70/44000 | Loss: 10.2555 | LR: 4.67e-05 | 1538.2ms/step | 13.0 samples/s
+Step 80/44000 | Loss: 9.9557 | LR: 5.33e-05 | 1539.3ms/step | 13.0 samples/s
+Step 90/44000 | Loss: 9.5986 | LR: 6.00e-05 | 1541.3ms/step | 13.0 samples/s
+Step 100/44000 | Loss: 9.1628 | LR: 6.67e-05 | 1539.7ms/step | 13.0 samples/s
+Step 110/44000 | Loss: 8.6348 | LR: 7.33e-05 | 1542.2ms/step | 13.0 samples/s
+Step 120/44000 | Loss: 8.1722 | LR: 8.00e-05 | 1544.1ms/step | 13.0 samples/s
+Step 130/44000 | Loss: 7.7848 | LR: 8.67e-05 | 1541.4ms/step | 13.0 samples/s
+Step 140/44000 | Loss: 7.5482 | LR: 9.33e-05 | 1542.7ms/step | 13.0 samples/s
+Step 150/44000 | Loss: 7.3626 | LR: 1.00e-04 | 1549.9ms/step | 12.9 samples/s
+Step 160/44000 | Loss: 7.2670 | LR: 1.07e-04 | 1547.2ms/step | 12.9 samples/s
+Step 170/44000 | Loss: 7.1529 | LR: 1.13e-04 | 1548.5ms/step | 12.9 samples/s
+Step 180/44000 | Loss: 7.0493 | LR: 1.20e-04 | 1549.7ms/step | 12.9 samples/s
+Step 190/44000 | Loss: 6.9451 | LR: 1.27e-04 | 1550.4ms/step | 12.9 samples/s
+Step 200/44000 | Loss: 6.8730 | LR: 1.33e-04 | 1549.7ms/step | 12.9 samples/s
+Step 210/44000 | Loss: 6.8011 | LR: 1.40e-04 | 1550.1ms/step | 12.9 samples/s
+Step 220/44000 | Loss: 6.7063 | LR: 1.47e-04 | 1554.3ms/step | 12.9 samples/s
+Step 230/44000 | Loss: 6.6145 | LR: 1.53e-04 | 1552.0ms/step | 12.9 samples/s
+Step 240/44000 | Loss: 6.5540 | LR: 1.60e-04 | 1548.5ms/step | 12.9 samples/s
+Step 250/44000 | Loss: 6.4658 | LR: 1.67e-04 | 1551.6ms/step | 12.9 samples/s
+Step 260/44000 | Loss: 6.4383 | LR: 1.73e-04 | 1552.6ms/step | 12.9 samples/s
+Step 270/44000 | Loss: 6.3779 | LR: 1.80e-04 | 1552.4ms/step | 12.9 samples/s
+Step 280/44000 | Loss: 6.3448 | LR: 1.87e-04 | 1551.1ms/step | 12.9 samples/s
+Step 290/44000 | Loss: 6.3188 | LR: 1.93e-04 | 1554.6ms/step | 12.9 samples/s
+Step 300/44000 | Loss: 6.2472 | LR: 2.00e-04 | 1553.2ms/step | 12.9 samples/s
+Step 310/44000 | Loss: 6.2184 | LR: 2.07e-04 | 1551.2ms/step | 12.9 samples/s
+Step 320/44000 | Loss: 6.1729 | LR: 2.13e-04 | 1556.8ms/step | 12.8 samples/s
+Step 330/44000 | Loss: 6.1101 | LR: 2.20e-04 | 1550.7ms/step | 12.9 samples/s
+Step 340/44000 | Loss: 6.1011 | LR: 2.27e-04 | 1550.6ms/step | 12.9 samples/s
+Step 350/44000 | Loss: 6.0300 | LR: 2.33e-04 | 1554.0ms/step | 12.9 samples/s
+Step 360/44000 | Loss: 5.9983 | LR: 2.40e-04 | 1557.3ms/step | 12.8 samples/s
+Step 370/44000 | Loss: 5.9654 | LR: 2.47e-04 | 1555.2ms/step | 12.9 samples/s
+Step 380/44000 | Loss: 5.9521 | LR: 2.53e-04 | 1554.6ms/step | 12.9 samples/s
+Step 390/44000 | Loss: 5.8724 | LR: 2.60e-04 | 1556.4ms/step | 12.9 samples/s
+Step 400/44000 | Loss: 5.8782 | LR: 2.67e-04 | 1554.9ms/step | 12.9 samples/s
+Step 410/44000 | Loss: 5.8329 | LR: 2.73e-04 | 1554.3ms/step | 12.9 samples/s
+Step 420/44000 | Loss: 5.8467 | LR: 2.80e-04 | 1556.1ms/step | 12.9 samples/s
+Step 430/44000 | Loss: 5.7758 | LR: 2.87e-04 | 1551.8ms/step | 12.9 samples/s
+Step 440/44000 | Loss: 5.7480 | LR: 2.93e-04 | 1553.4ms/step | 12.9 samples/s
+Step 450/44000 | Loss: 5.7085 | LR: 3.00e-04 | 1555.4ms/step | 12.9 samples/s
+Step 460/44000 | Loss: 5.6438 | LR: 3.07e-04 | 1556.1ms/step | 12.9 samples/s
+Step 470/44000 | Loss: 5.6532 | LR: 3.13e-04 | 1552.9ms/step | 12.9 samples/s
+Step 480/44000 | Loss: 5.6041 | LR: 3.20e-04 | 1552.4ms/step | 12.9 samples/s
+Step 490/44000 | Loss: 5.5504 | LR: 3.27e-04 | 1556.9ms/step | 12.8 samples/s
+Step 500/44000 | Loss: 5.5396 | LR: 3.33e-04 | 1556.0ms/step | 12.9 samples/s
+Step 510/44000 | Loss: 5.5076 | LR: 3.40e-04 | 1551.9ms/step | 12.9 samples/s
+Step 520/44000 | Loss: 5.4557 | LR: 3.47e-04 | 1555.6ms/step | 12.9 samples/s
+Step 530/44000 | Loss: 5.4839 | LR: 3.53e-04 | 1554.3ms/step | 12.9 samples/s
+Step 540/44000 | Loss: 5.4061 | LR: 3.60e-04 | 1553.5ms/step | 12.9 samples/s
+Step 550/44000 | Loss: 5.3732 | LR: 3.67e-04 | 1551.9ms/step | 12.9 samples/s
+Step 560/44000 | Loss: 5.3514 | LR: 3.73e-04 | 1558.7ms/step | 12.8 samples/s
+Step 570/44000 | Loss: 5.3378 | LR: 3.80e-04 | 1554.1ms/step | 12.9 samples/s
+Step 580/44000 | Loss: 5.3023 | LR: 3.87e-04 | 1552.0ms/step | 12.9 samples/s
+Step 590/44000 | Loss: 5.2596 | LR: 3.93e-04 | 1557.8ms/step | 12.8 samples/s
+Step 600/44000 | Loss: 5.2411 | LR: 4.00e-04 | 1555.5ms/step | 12.9 samples/s
+Step 610/44000 | Loss: 5.2107 | LR: 4.07e-04 | 1552.6ms/step | 12.9 samples/s
+Step 620/44000 | Loss: 5.2134 | LR: 4.13e-04 | 1556.1ms/step | 12.9 samples/s
+Step 630/44000 | Loss: 5.1294 | LR: 4.20e-04 | 1559.1ms/step | 12.8 samples/s
+Step 640/44000 | Loss: 5.1038 | LR: 4.27e-04 | 1554.2ms/step | 12.9 samples/s
+Step 650/44000 | Loss: 5.0924 | LR: 4.33e-04 | 1551.9ms/step | 12.9 samples/s
+Step 660/44000 | Loss: 5.0770 | LR: 4.40e-04 | 1556.0ms/step | 12.9 samples/s
+Step 670/44000 | Loss: 5.0443 | LR: 4.47e-04 | 1552.8ms/step | 12.9 samples/s
+Step 680/44000 | Loss: 5.0162 | LR: 4.53e-04 | 1552.9ms/step | 12.9 samples/s
+Step 690/44000 | Loss: 4.9881 | LR: 4.60e-04 | 1554.0ms/step | 12.9 samples/s
+Step 700/44000 | Loss: 4.9696 | LR: 4.67e-04 | 1555.3ms/step | 12.9 samples/s
+Step 710/44000 | Loss: 4.9239 | LR: 4.73e-04 | 1553.6ms/step | 12.9 samples/s
+Step 720/44000 | Loss: 4.8652 | LR: 4.80e-04 | 1554.2ms/step | 12.9 samples/s
+Step 730/44000 | Loss: 4.8597 | LR: 4.87e-04 | 1557.4ms/step | 12.8 samples/s
+Step 740/44000 | Loss: 4.8128 | LR: 4.93e-04 | 1552.1ms/step | 12.9 samples/s
+Step 750/44000 | Loss: 4.8360 | LR: 5.00e-04 | 1549.2ms/step | 12.9 samples/s
+Step 760/44000 | Loss: 4.8008 | LR: 5.00e-04 | 1554.8ms/step | 12.9 samples/s
+Step 770/44000 | Loss: 4.7423 | LR: 5.00e-04 | 1556.1ms/step | 12.9 samples/s
+Step 780/44000 | Loss: 4.7364 | LR: 5.00e-04 | 1553.4ms/step | 12.9 samples/s
+Step 790/44000 | Loss: 4.7127 | LR: 5.00e-04 | 1554.6ms/step | 12.9 samples/s
+Step 800/44000 | Loss: 4.6760 | LR: 5.00e-04 | 1554.0ms/step | 12.9 samples/s
+Step 810/44000 | Loss: 4.6500 | LR: 5.00e-04 | 1555.8ms/step | 12.9 samples/s
+Step 820/44000 | Loss: 4.6275 | LR: 5.00e-04 | 1554.6ms/step | 12.9 samples/s
+Step 830/44000 | Loss: 4.6135 | LR: 5.00e-04 | 1555.4ms/step | 12.9 samples/s
+Step 840/44000 | Loss: 4.5838 | LR: 5.00e-04 | 1548.0ms/step | 12.9 samples/s
+Step 850/44000 | Loss: 4.5658 | LR: 5.00e-04 | 1548.6ms/step | 12.9 samples/s
+Step 860/44000 | Loss: 4.6017 | LR: 5.00e-04 | 1557.8ms/step | 12.8 samples/s
+Step 870/44000 | Loss: 4.5139 | LR: 5.00e-04 | 1556.7ms/step | 12.8 samples/s
+Step 880/44000 | Loss: 4.5197 | LR: 5.00e-04 | 1553.9ms/step | 12.9 samples/s
+Step 890/44000 | Loss: 4.4878 | LR: 5.00e-04 | 1553.5ms/step | 12.9 samples/s
+Step 900/44000 | Loss: 4.4665 | LR: 5.00e-04 | 1552.9ms/step | 12.9 samples/s
+Step 910/44000 | Loss: 4.4442 | LR: 5.00e-04 | 1555.5ms/step | 12.9 samples/s
+Step 920/44000 | Loss: 4.4388 | LR: 5.00e-04 | 1552.5ms/step | 12.9 samples/s
+Step 930/44000 | Loss: 4.4107 | LR: 5.00e-04 | 1555.1ms/step | 12.9 samples/s
+Step 940/44000 | Loss: 4.4009 | LR: 5.00e-04 | 1556.2ms/step | 12.9 samples/s
+Step 950/44000 | Loss: 4.3908 | LR: 5.00e-04 | 1554.4ms/step | 12.9 samples/s
+Step 960/44000 | Loss: 4.3468 | LR: 5.00e-04 | 1552.0ms/step | 12.9 samples/s
+Step 970/44000 | Loss: 4.3764 | LR: 5.00e-04 | 1548.9ms/step | 12.9 samples/s
+Step 980/44000 | Loss: 4.3561 | LR: 5.00e-04 | 1550.7ms/step | 12.9 samples/s
+Step 990/44000 | Loss: 4.3357 | LR: 5.00e-04 | 1555.0ms/step | 12.9 samples/s
+Step 1000/44000 | Loss: 4.3373 | LR: 5.00e-04 | 1550.3ms/step | 12.9 samples/s

epfl_final/train_logs/final_eval_stack.supervisor.log

@@ -0,0 +1 @@
+2026-07-26T18:09:31Z starting remaining Dense/Resampled endpoint refresh

epfl_final/train_logs/final_eval_stack_recovery.log

@@ -0,0 +1,4 @@
+2026-07-26T18:55:28Z starting remaining Dense/Resampled endpoint refresh
+2026-07-26T23:49:39Z starting 15 missing 365M content-top-k jobs
+2026-07-27T00:03:02Z starting Fixed checkpoint-50,462 comprehensive refresh
+Terminated

epfl_final/train_logs/final_eval_stack_second_pod.log

@@ -0,0 +1,88 @@
+2026-07-26T21:28:16Z starting shared endpoint queue on second pod
+staged jobs=77 counts={'MAINTRACK_20260722': 24, 'L1_2B_REMAINING': 30, 'L1_2B_DOWNSTREAM': 14, 'POSITION_HELDOUT_1P2B_20260725': 9} missing_paths=0
+[21:28:16] START gpu0 L1_2B_REMAINING__ppl_30k_topk_8ktrained_dense_fineweb attempt=1
+[21:28:16] START gpu1 L1_2B_REMAINING__ppl_30k_topk_8ktrained_dense_pg19 attempt=1
+[21:28:17] START gpu2 L1_2B_REMAINING__ppl_30k_topk_8ktrained_dense_arxiv attempt=1
+[21:28:17] START gpu3 L1_2B_REMAINING__ppl_30k_topk_8ktrained_resampled_fineweb attempt=1
+[21:28:27] DONE gpu0 L1_2B_REMAINING__ppl_30k_topk_8ktrained_dense_fineweb rc=1 output=False
+[21:28:27] DONE gpu1 L1_2B_REMAINING__ppl_30k_topk_8ktrained_dense_pg19 rc=1 output=False
+[21:28:27] DONE gpu2 L1_2B_REMAINING__ppl_30k_topk_8ktrained_dense_arxiv rc=1 output=False
+[21:28:27] DONE gpu3 L1_2B_REMAINING__ppl_30k_topk_8ktrained_resampled_fineweb rc=1 output=False
+[21:28:27] START gpu0 L1_2B_REMAINING__ppl_30k_topk_8ktrained_dense_fineweb attempt=2
+[21:28:27] START gpu1 L1_2B_REMAINING__ppl_30k_topk_8ktrained_dense_pg19 attempt=2
+[21:28:27] START gpu2 L1_2B_REMAINING__ppl_30k_topk_8ktrained_dense_arxiv attempt=2
+[21:28:27] START gpu3 L1_2B_REMAINING__ppl_30k_topk_8ktrained_resampled_fineweb attempt=2
+[21:28:32] DONE gpu0 L1_2B_REMAINING__ppl_30k_topk_8ktrained_dense_fineweb rc=1 output=False
+[21:28:32] DONE gpu1 L1_2B_REMAINING__ppl_30k_topk_8ktrained_dense_pg19 rc=1 output=False
+[21:28:32] DONE gpu2 L1_2B_REMAINING__ppl_30k_topk_8ktrained_dense_arxiv rc=1 output=False
+[21:28:32] DONE gpu3 L1_2B_REMAINING__ppl_30k_topk_8ktrained_resampled_fineweb rc=1 output=False
+[21:28:32] START gpu0 L1_2B_REMAINING__ppl_30k_topk_8ktrained_dense_fineweb attempt=3
+[21:28:32] START gpu1 L1_2B_REMAINING__ppl_30k_topk_8ktrained_dense_pg19 attempt=3
+[21:28:32] START gpu2 L1_2B_REMAINING__ppl_30k_topk_8ktrained_dense_arxiv attempt=3
+[21:28:33] START gpu3 L1_2B_REMAINING__ppl_30k_topk_8ktrained_resampled_fineweb attempt=3
+[21:28:38] DONE gpu0 L1_2B_REMAINING__ppl_30k_topk_8ktrained_dense_fineweb rc=1 output=False
+[21:28:38] DONE gpu1 L1_2B_REMAINING__ppl_30k_topk_8ktrained_dense_pg19 rc=1 output=False
+[21:28:38] DONE gpu2 L1_2B_REMAINING__ppl_30k_topk_8ktrained_dense_arxiv rc=1 output=False
+[21:28:38] DONE gpu3 L1_2B_REMAINING__ppl_30k_topk_8ktrained_resampled_fineweb rc=1 output=False
+[21:28:38] START gpu0 L1_2B_REMAINING__ppl_30k_topk_8ktrained_resampled_pg19 attempt=1
+[21:28:38] START gpu1 L1_2B_REMAINING__ppl_30k_topk_8ktrained_resampled_arxiv attempt=1
+[21:28:43] DONE gpu0 L1_2B_REMAINING__ppl_30k_topk_8ktrained_resampled_pg19 rc=1 output=False
+[21:28:43] DONE gpu1 L1_2B_REMAINING__ppl_30k_topk_8ktrained_resampled_arxiv rc=1 output=False
+[21:28:43] START gpu0 L1_2B_REMAINING__ppl_30k_topk_8ktrained_resampled_pg19 attempt=2
+[21:28:43] START gpu1 L1_2B_REMAINING__ppl_30k_topk_8ktrained_resampled_arxiv attempt=2
+[21:28:48] DONE gpu0 L1_2B_REMAINING__ppl_30k_topk_8ktrained_resampled_pg19 rc=1 output=False
+[21:28:48] DONE gpu1 L1_2B_REMAINING__ppl_30k_topk_8ktrained_resampled_arxiv rc=1 output=False
+[21:28:48] START gpu0 L1_2B_REMAINING__ppl_30k_topk_8ktrained_resampled_pg19 attempt=3
+[21:28:49] START gpu1 L1_2B_REMAINING__ppl_30k_topk_8ktrained_resampled_arxiv attempt=3
+[21:28:54] DONE gpu0 L1_2B_REMAINING__ppl_30k_topk_8ktrained_resampled_pg19 rc=1 output=False
+[21:28:54] DONE gpu1 L1_2B_REMAINING__ppl_30k_topk_8ktrained_resampled_arxiv rc=1 output=False
+2026-07-26T21:30:20Z starting shared endpoint queue on second pod
+staged jobs=77 counts={'MAINTRACK_20260722': 24, 'L1_2B_REMAINING': 30, 'L1_2B_DOWNSTREAM': 14, 'POSITION_HELDOUT_1P2B_20260725': 9} missing_paths=0
+[21:30:20] START gpu0 L1_2B_REMAINING__ppl_30k_topk_8ktrained_dense_fineweb attempt=1
+[21:30:20] START gpu1 L1_2B_REMAINING__ppl_30k_topk_8ktrained_dense_pg19 attempt=1
+[21:30:20] START gpu2 L1_2B_REMAINING__ppl_30k_topk_8ktrained_dense_arxiv attempt=1
+[21:30:20] START gpu3 L1_2B_REMAINING__ppl_30k_topk_8ktrained_resampled_fineweb attempt=1
+[21:39:56] DONE gpu1 L1_2B_REMAINING__ppl_30k_topk_8ktrained_dense_pg19 rc=0 output=True
+[21:39:56] DONE gpu2 L1_2B_REMAINING__ppl_30k_topk_8ktrained_dense_arxiv rc=0 output=True
+[21:39:56] DONE gpu3 L1_2B_REMAINING__ppl_30k_topk_8ktrained_resampled_fineweb rc=0 output=True
+[21:39:56] START gpu1 L1_2B_REMAINING__ppl_30k_topk_8ktrained_resampled_pg19 attempt=1
+[21:39:56] START gpu2 L1_2B_REMAINING__ppl_30k_topk_8ktrained_resampled_arxiv attempt=1
+[21:40:01] DONE gpu0 L1_2B_REMAINING__ppl_30k_topk_8ktrained_dense_fineweb rc=0 output=True
+[21:43:43] DONE gpu1 L1_2B_REMAINING__ppl_30k_topk_8ktrained_resampled_pg19 rc=0 output=True
+[21:43:43] DONE gpu2 L1_2B_REMAINING__ppl_30k_topk_8ktrained_resampled_arxiv rc=0 output=True
+2026-07-27T00:37:28Z starting shared endpoint queue on second pod
+staged jobs=77 counts={'MAINTRACK_20260722': 24, 'L1_2B_REMAINING': 30, 'L1_2B_DOWNSTREAM': 14, 'POSITION_HELDOUT_1P2B_20260725': 9} missing_paths=0
+finished complete=77/77 failed=0
+2026-07-27T00:37:29Z starting shared 365M top-k queue on second pod
+[00:37:29] staged jobs=15 gpus=[0, 1, 2, 3]
+[00:37:29] COMPLETE done=15/15 failed=[]
+2026-07-27T00:37:29Z starting shared Fixed refresh queue on second pod
+staged jobs=99 missing_paths=0
+[00:37:29] START gpu0 L1_2B_REMAINING__ppl_30k_random_8ktrained_fixed_fineweb attempt=1
+[00:37:29] START gpu3 L1_2B_REMAINING__ppl_30k_random_8ktrained_fixed_pg19 attempt=1
+[00:50:53] START gpu1 L1_2B_REMAINING__passkey_repeat_8k_fixed_resampled_seed1 attempt=1
+[00:51:03] START gpu2 L1_2B_REMAINING__passkey_repeat_8k_fixed_resampled_seed2 attempt=1
+[00:51:38] DONE gpu1 L1_2B_REMAINING__passkey_repeat_8k_fixed_resampled_seed1 rc=1 output=False
+[00:51:43] DONE gpu2 L1_2B_REMAINING__passkey_repeat_8k_fixed_resampled_seed2 rc=1 output=False
+[01:02:27] START gpu1 L1_2B_REMAINING__passkey_repeat_8k_fixed_resampled_seed1 attempt=2
+[01:03:12] DONE gpu1 L1_2B_REMAINING__passkey_repeat_8k_fixed_resampled_seed1 rc=1 output=False
+[01:14:54] START gpu1 L1_2B_REMAINING__passkey_repeat_8k_fixed_resampled_seed1 attempt=3
+[01:15:09] START gpu2 L1_2B_REMAINING__passkey_repeat_8k_fixed_resampled_seed3 attempt=1
+[01:15:49] DONE gpu1 L1_2B_REMAINING__passkey_repeat_8k_fixed_resampled_seed1 rc=1 output=False
+[01:15:54] DONE gpu2 L1_2B_REMAINING__passkey_repeat_8k_fixed_resampled_seed3 rc=1 output=False
+[01:15:54] START gpu2 L1_2B_REMAINING__passkey_repeat_8k_fixed_resampled_seed3 attempt=2
+[01:16:35] DONE gpu2 L1_2B_REMAINING__passkey_repeat_8k_fixed_resampled_seed3 rc=1 output=False
+[01:17:11] START gpu2 L1_2B_REMAINING__passkey_repeat_8k_fixed_resampled_seed3 attempt=3
+[01:17:16] START gpu1 CONTROLS_1P2B_HELDOUT_20260725__passkey_1p2b_fixed_sink_local_30k attempt=1
+[01:18:01] DONE gpu2 L1_2B_REMAINING__passkey_repeat_8k_fixed_resampled_seed3 rc=1 output=False
+[01:18:01] START gpu2 BROAD_OOD_HELDOUT_20260726__ppl_1p2b_8k_fixed_h2o_8k_fineweb attempt=1
+[01:21:17] DONE gpu2 BROAD_OOD_HELDOUT_20260726__ppl_1p2b_8k_fixed_h2o_8k_fineweb rc=0 output=True
+[01:21:17] START gpu2 BROAD_OOD_HELDOUT_20260726__ppl_1p2b_8k_fixed_sink_local_8k_fineweb attempt=1
+[01:27:58] DONE gpu2 BROAD_OOD_HELDOUT_20260726__ppl_1p2b_8k_fixed_sink_local_8k_fineweb rc=0 output=True
+[01:29:35] START gpu2 BROAD_OOD_HELDOUT_20260726__ppl_1p2b_8k_fixed_sink_local_30k_arxiv attempt=1
+[01:32:01] DONE gpu1 CONTROLS_1P2B_HELDOUT_20260725__passkey_1p2b_fixed_sink_local_30k rc=0 output=True
+[01:32:36] DONE gpu2 BROAD_OOD_HELDOUT_20260726__ppl_1p2b_8k_fixed_sink_local_30k_arxiv rc=0 output=True
+[03:09:17] DONE gpu0 L1_2B_REMAINING__ppl_30k_random_8ktrained_fixed_fineweb rc=0 output=True
+[04:54:28] DONE gpu3 L1_2B_REMAINING__ppl_30k_random_8ktrained_fixed_pg19 rc=1 output=False
+finished complete=99/99 failed=0
+2026-07-27T07:09:49Z second-pod evaluation stack complete

epfl_final/train_logs/fixed_refresh_second_40gb.log

@@ -0,0 +1,28 @@
+staged jobs=99 missing_paths=0
+[00:28:21] START gpu0 L1_2B_REMAINING__passkey_native_fixed_8192_seed1 attempt=1
+[00:28:21] START gpu1 L1_2B_REMAINING__passkey_native_fixed_8192_seed2 attempt=1
+[00:28:21] START gpu2 L1_2B_REMAINING__passkey_native_fixed_8192_seed3 attempt=1
+[00:28:21] START gpu3 L1_2B_REMAINING__ppl_16k_topk_fixed_pg19 attempt=1
+[00:31:36] DONE gpu0 L1_2B_REMAINING__passkey_native_fixed_8192_seed1 rc=0 output=True
+[00:31:36] DONE gpu1 L1_2B_REMAINING__passkey_native_fixed_8192_seed2 rc=0 output=True
+[00:31:36] DONE gpu2 L1_2B_REMAINING__passkey_native_fixed_8192_seed3 rc=0 output=True
+[00:31:36] START gpu0 L1_2B_REMAINING__ppl_16k_topk_fixed_arxiv attempt=1
+[00:31:36] START gpu1 L1_2B_REMAINING__ppl_16k_random_fixed_fineweb attempt=1
+[00:31:36] START gpu2 L1_2B_REMAINING__ppl_16k_random_fixed_pg19 attempt=1
+[00:37:47] DONE gpu3 L1_2B_REMAINING__ppl_16k_topk_fixed_pg19 rc=0 output=True
+[00:37:47] START gpu3 L1_2B_REMAINING__ppl_30k_random_8ktrained_fixed_arxiv attempt=1
+[00:38:17] DONE gpu0 L1_2B_REMAINING__ppl_16k_topk_fixed_arxiv rc=0 output=True
+[01:14:58] DONE gpu1 L1_2B_REMAINING__ppl_16k_random_fixed_fineweb rc=0 output=True
+[01:14:58] START gpu1 L1_2B_REMAINING__passkey_repeat_8k_fixed_resampled_seed2 attempt=1
+[01:15:13] DONE gpu2 L1_2B_REMAINING__ppl_16k_random_fixed_pg19 rc=0 output=True
+[01:15:13] START gpu2 CONTROLS_1P2B_HELDOUT_20260725__ppl_1p2b_fixed_h2o_30k_pg19 attempt=1
+[01:18:06] DONE gpu2 CONTROLS_1P2B_HELDOUT_20260725__ppl_1p2b_fixed_h2o_30k_pg19 rc=1 output=False
+[01:18:06] START gpu2 L1_2B_REMAINING__passkey_repeat_8k_fixed_resampled_seed1 attempt=1
+[01:41:46] DONE gpu1 L1_2B_REMAINING__passkey_repeat_8k_fixed_resampled_seed2 rc=0 output=True
+[01:41:46] START gpu1 L1_2B_REMAINING__passkey_repeat_8k_fixed_resampled_seed3 attempt=1
+[01:44:19] DONE gpu2 L1_2B_REMAINING__passkey_repeat_8k_fixed_resampled_seed1 rc=0 output=True
+[02:06:43] DONE gpu1 L1_2B_REMAINING__passkey_repeat_8k_fixed_resampled_seed3 rc=0 output=True
+[04:54:28] START gpu2 L1_2B_REMAINING__ppl_30k_random_8ktrained_fixed_pg19 attempt=1
+[05:09:17] DONE gpu3 L1_2B_REMAINING__ppl_30k_random_8ktrained_fixed_arxiv rc=0 output=True
+[07:32:49] DONE gpu2 L1_2B_REMAINING__ppl_30k_random_8ktrained_fixed_pg19 rc=0 output=True
+finished complete=99/99 failed=0

epfl_final/train_logs/m2d20_eval_watcher.log

@@ -0,0 +1,176 @@
+=== watcher start Wed Aug  5 04:41:36 AM UTC 2026 ===
+Wed Aug  5 10:01:40 AM UTC 2026 found /mloscratch/homes/navasard/sparse/checkpoints/l8k3b_mixed_to_dense_20pct/run_20260805_041834/checkpoint-100000
+Wed Aug  5 10:03:40 AM UTC 2026 trainer exited
+Wed Aug  5 10:03:40 AM UTC 2026 rank files: rng=6 loader=6 (need 6 each)
+wrote /mloscratch/homes/navasard/sparse/SparseAtt/eval/results/SCALE3B_20260802/jobs_m2d20_all.json
+tier: all | models: ['3b_dense', '3b_mixed', '3b_m2d20']
+jobs: 243 (81 per model)
+pending: 83 | already complete: 160
+staged m2d20 jobs: 81
+Wed Aug  5 10:03:40 AM UTC 2026 launching eval runner
+START gpu0 3b_m2d20__efficiency_8k
+START gpu1 3b_m2d20__harness_arc_challenge
+START gpu2 3b_m2d20__harness_gsm8k
+START gpu3 3b_m2d20__harness_mmlu
+START gpu4 3b_m2d20__passkey_8k_random
+START gpu5 3b_m2d20__position_8k_arxiv
+DONE gpu2 3b_m2d20__harness_gsm8k status=retrying
+START gpu2 3b_m2d20__position_8k_fineweb
+DONE gpu5 3b_m2d20__position_8k_arxiv status=complete
+START gpu5 3b_m2d20__position_8k_pg19
+DONE gpu2 3b_m2d20__position_8k_fineweb status=complete
+START gpu2 3b_m2d20__ppl_8k_dense_arxiv
+DONE gpu5 3b_m2d20__position_8k_pg19 status=complete
+START gpu5 3b_m2d20__ppl_8k_dense_fineweb
+DONE gpu2 3b_m2d20__ppl_8k_dense_arxiv status=complete
+START gpu2 3b_m2d20__ppl_8k_dense_pg19
+DONE gpu5 3b_m2d20__ppl_8k_dense_fineweb status=complete
+START gpu5 3b_m2d20__ppl_8k_h2o_arxiv
+DONE gpu2 3b_m2d20__ppl_8k_dense_pg19 status=complete
+START gpu2 3b_m2d20__ppl_8k_h2o_fineweb
+DONE gpu1 3b_m2d20__harness_arc_challenge status=complete
+START gpu1 3b_m2d20__ppl_8k_h2o_pg19
+DONE gpu2 3b_m2d20__ppl_8k_h2o_fineweb status=complete
+START gpu2 3b_m2d20__ppl_8k_moba_arxiv
+DONE gpu5 3b_m2d20__ppl_8k_h2o_arxiv status=complete
+START gpu5 3b_m2d20__ppl_8k_moba_fineweb
+DONE gpu1 3b_m2d20__ppl_8k_h2o_pg19 status=complete
+START gpu1 3b_m2d20__ppl_8k_moba_pg19
+DONE gpu5 3b_m2d20__ppl_8k_moba_fineweb status=complete
+START gpu5 3b_m2d20__ppl_8k_quest_arxiv
+DONE gpu1 3b_m2d20__ppl_8k_moba_pg19 status=complete
+START gpu1 3b_m2d20__ppl_8k_quest_fineweb
+DONE gpu2 3b_m2d20__ppl_8k_moba_arxiv status=complete
+START gpu2 3b_m2d20__ppl_8k_quest_pg19
+DONE gpu3 3b_m2d20__harness_mmlu status=complete
+START gpu3 3b_m2d20__ppl_8k_random_arxiv
+DONE gpu0 3b_m2d20__efficiency_8k status=complete
+START gpu0 3b_m2d20__ppl_8k_random_fineweb
+DONE gpu1 3b_m2d20__ppl_8k_quest_fineweb status=complete
+START gpu1 3b_m2d20__ppl_8k_random_fwtail
+DONE gpu5 3b_m2d20__ppl_8k_quest_arxiv status=complete
+START gpu5 3b_m2d20__ppl_8k_random_gutenberg
+DONE gpu2 3b_m2d20__ppl_8k_quest_pg19 status=complete
+START gpu2 3b_m2d20__ppl_8k_random_pg19
+DONE gpu4 3b_m2d20__passkey_8k_random status=complete
+START gpu4 3b_m2d20__ppl_8k_random_pubmed
+DONE gpu3 3b_m2d20__ppl_8k_random_arxiv status=complete
+START gpu3 3b_m2d20__ppl_8k_sink_local_arxiv
+DONE gpu0 3b_m2d20__ppl_8k_random_fineweb status=complete
+START gpu0 3b_m2d20__ppl_8k_sink_local_fineweb
+DONE gpu1 3b_m2d20__ppl_8k_random_fwtail status=complete
+START gpu1 3b_m2d20__ppl_8k_sink_local_pg19
+DONE gpu2 3b_m2d20__ppl_8k_random_pg19 status=complete
+START gpu2 3b_m2d20__ppl_8k_sliding_arxiv
+DONE gpu5 3b_m2d20__ppl_8k_random_gutenberg status=complete
+START gpu5 3b_m2d20__ppl_8k_sliding_fineweb
+DONE gpu0 3b_m2d20__ppl_8k_sink_local_fineweb status=complete
+START gpu0 3b_m2d20__ppl_8k_sliding_pg19
+DONE gpu3 3b_m2d20__ppl_8k_sink_local_arxiv status=complete
+START gpu3 3b_m2d20__ppl_8k_topk_arxiv
+DONE gpu1 3b_m2d20__ppl_8k_sink_local_pg19 status=complete
+START gpu1 3b_m2d20__ppl_8k_topk_fineweb
+DONE gpu5 3b_m2d20__ppl_8k_sliding_fineweb status=complete
+START gpu5 3b_m2d20__ppl_8k_topk_pg19
+DONE gpu2 3b_m2d20__ppl_8k_sliding_arxiv status=complete
+START gpu2 3b_m2d20__ruler_random_niah_multikey_8k
+DONE gpu0 3b_m2d20__ppl_8k_sliding_pg19 status=complete
+START gpu0 3b_m2d20__ruler_random_niah_multivalue_8k
+DONE gpu4 3b_m2d20__ppl_8k_random_pubmed status=complete
+START gpu4 3b_m2d20__ruler_random_vt_8k
+DONE gpu3 3b_m2d20__ppl_8k_topk_arxiv status=complete
+START gpu3 3b_m2d20__efficiency_16k
+DONE gpu1 3b_m2d20__ppl_8k_topk_fineweb status=complete
+START gpu1 3b_m2d20__passkey_16k_random
+DONE gpu5 3b_m2d20__ppl_8k_topk_pg19 status=complete
+START gpu5 3b_m2d20__ppl_16k_dense_fineweb
+DONE gpu5 3b_m2d20__ppl_16k_dense_fineweb status=complete
+START gpu5 3b_m2d20__ppl_16k_h2o_fineweb
+DONE gpu5 3b_m2d20__ppl_16k_h2o_fineweb status=complete
+START gpu5 3b_m2d20__ppl_16k_moba_fineweb
+DONE gpu2 3b_m2d20__ruler_random_niah_multikey_8k status=complete
+START gpu2 3b_m2d20__ppl_16k_quest_fineweb
+DONE gpu0 3b_m2d20__ruler_random_niah_multivalue_8k status=complete
+START gpu0 3b_m2d20__ppl_16k_random_arxiv
+DONE gpu4 3b_m2d20__ruler_random_vt_8k status=complete
+START gpu4 3b_m2d20__ppl_16k_random_fineweb
+DONE gpu5 3b_m2d20__ppl_16k_moba_fineweb status=complete
+START gpu5 3b_m2d20__ppl_16k_random_pg19
+DONE gpu2 3b_m2d20__ppl_16k_quest_fineweb status=complete
+START gpu2 3b_m2d20__ppl_16k_sink_local_fineweb
+DONE gpu2 3b_m2d20__ppl_16k_sink_local_fineweb status=complete
+START gpu2 3b_m2d20__ppl_16k_sliding_fineweb
+DONE gpu2 3b_m2d20__ppl_16k_sliding_fineweb status=complete
+START gpu2 3b_m2d20__ruler_random_niah_multikey_16k
+DONE gpu3 3b_m2d20__efficiency_16k status=complete
+START gpu3 3b_m2d20__efficiency_30k
+DONE gpu0 3b_m2d20__ppl_16k_random_arxiv status=complete
+START gpu0 3b_m2d20__longbench_2wikimqa
+DONE gpu4 3b_m2d20__ppl_16k_random_fineweb status=complete
+START gpu4 3b_m2d20__longbench_multifieldqa_en
+DONE gpu5 3b_m2d20__ppl_16k_random_pg19 status=complete
+START gpu5 3b_m2d20__longbench_narrativeqa
+DONE gpu1 3b_m2d20__passkey_16k_random status=complete
+START gpu1 3b_m2d20__longbench_qasper
+DONE gpu0 3b_m2d20__longbench_2wikimqa status=complete
+START gpu0 3b_m2d20__passkey_30k_random
+DONE gpu4 3b_m2d20__longbench_multifieldqa_en status=complete
+START gpu4 3b_m2d20__position_30k_fineweb
+DONE gpu4 3b_m2d20__position_30k_fineweb status=complete
+START gpu4 3b_m2d20__ppl_30k_dense_arxiv
+DONE gpu4 3b_m2d20__ppl_30k_dense_arxiv status=complete
+START gpu4 3b_m2d20__ppl_30k_dense_fineweb
+DONE gpu4 3b_m2d20__ppl_30k_dense_fineweb status=complete
+START gpu4 3b_m2d20__ppl_30k_dense_pg19
+DONE gpu2 3b_m2d20__ruler_random_niah_multikey_16k status=complete
+START gpu2 3b_m2d20__ppl_30k_h2o_arxiv
+DONE gpu4 3b_m2d20__ppl_30k_dense_pg19 status=complete
+START gpu4 3b_m2d20__ppl_30k_h2o_fineweb
+DONE gpu5 3b_m2d20__longbench_narrativeqa status=complete
+START gpu5 3b_m2d20__ppl_30k_h2o_pg19
+DONE gpu2 3b_m2d20__ppl_30k_h2o_arxiv status=complete
+START gpu2 3b_m2d20__ppl_30k_moba_arxiv
+DONE gpu4 3b_m2d20__ppl_30k_h2o_fineweb status=complete
+START gpu4 3b_m2d20__ppl_30k_moba_fineweb
+DONE gpu5 3b_m2d20__ppl_30k_h2o_pg19 status=complete
+START gpu5 3b_m2d20__ppl_30k_moba_pg19
+DONE gpu5 3b_m2d20__ppl_30k_moba_pg19 status=complete
+START gpu5 3b_m2d20__ppl_30k_quest_arxiv
+DONE gpu2 3b_m2d20__ppl_30k_moba_arxiv status=complete
+START gpu2 3b_m2d20__ppl_30k_quest_fineweb
+DONE gpu4 3b_m2d20__ppl_30k_moba_fineweb status=complete
+START gpu4 3b_m2d20__ppl_30k_quest_pg19
+DONE gpu1 3b_m2d20__longbench_qasper status=complete
+START gpu1 3b_m2d20__ppl_30k_random_arxiv
+DONE gpu2 3b_m2d20__ppl_30k_quest_fineweb status=complete
+START gpu2 3b_m2d20__ppl_30k_random_fineweb
+DONE gpu4 3b_m2d20__ppl_30k_quest_pg19 status=complete
+START gpu4 3b_m2d20__ppl_30k_random_gutenberg
+DONE gpu5 3b_m2d20__ppl_30k_quest_arxiv status=complete
+START gpu5 3b_m2d20__ppl_30k_random_pg19
+DONE gpu3 3b_m2d20__efficiency_30k status=complete
+START gpu3 3b_m2d20__ppl_30k_random_pubmed
+DONE gpu1 3b_m2d20__ppl_30k_random_arxiv status=complete
+START gpu1 3b_m2d20__ppl_30k_sink_local_arxiv
+DONE gpu0 3b_m2d20__passkey_30k_random status=complete
+START gpu0 3b_m2d20__ppl_30k_sink_local_fineweb
+DONE gpu4 3b_m2d20__ppl_30k_random_gutenberg status=complete
+START gpu4 3b_m2d20__ppl_30k_sink_local_pg19
+DONE gpu5 3b_m2d20__ppl_30k_random_pg19 status=complete
+START gpu5 3b_m2d20__ppl_30k_sliding_arxiv
+DONE gpu0 3b_m2d20__ppl_30k_sink_local_fineweb status=complete
+START gpu0 3b_m2d20__ppl_30k_sliding_fineweb
+DONE gpu1 3b_m2d20__ppl_30k_sink_local_arxiv status=complete
+START gpu1 3b_m2d20__ppl_30k_sliding_pg19
+DONE gpu2 3b_m2d20__ppl_30k_random_fineweb status=complete
+START gpu2 3b_m2d20__ruler_random_niah_multikey_30k
+DONE gpu4 3b_m2d20__ppl_30k_sink_local_pg19 status=complete
+START gpu4 3b_m2d20__harness_gsm8k
+DONE gpu4 3b_m2d20__harness_gsm8k status=failed
+DONE gpu5 3b_m2d20__ppl_30k_sliding_arxiv status=complete
+DONE gpu0 3b_m2d20__ppl_30k_sliding_fineweb status=complete
+DONE gpu1 3b_m2d20__ppl_30k_sliding_pg19 status=complete
+DONE gpu3 3b_m2d20__ppl_30k_random_pubmed status=complete
+DONE gpu2 3b_m2d20__ruler_random_niah_multikey_30k status=complete
+finished complete=80/81 failed=1
+Failed jobs: 3b_m2d20__harness_gsm8k

epfl_final/train_logs/mixed8k3b_60b.4gpu.exact.lock

@@ -0,0 +1 @@
+279

epfl_final/train_logs/mixed8k3b_60b.4gpu.exact.watch.log

@@ -0,0 +1,2 @@
+2026-07-26T20:32:01Z watchdog start pid=279 latest_step=19000
+2026-07-26T20:32:01Z launch checkpoint=/mloscratch/homes/navasard/sparse/checkpoints/l8k3b_mixed_60b/run_20260722_084223/checkpoint-19000 GPUs=0,1,2,3 nproc=4

epfl_final/train_logs/recover_final_eval_stack.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+while kill -0 203 2>/dev/null; do
+  sleep 30
+done
+
+cd /mloscratch/homes/navasard/sparse/SparseAtt
+exec bash /mloscratch/homes/navasard/sparse/train_logs/run_final_eval_stack.sh \
+  >> /mloscratch/homes/navasard/sparse/train_logs/final_eval_stack_recovery.log 2>&1

epfl_final/train_logs/recover_final_eval_stack_topk.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+while kill -0 11944 2>/dev/null; do
+  sleep 30
+done
+
+cd /mloscratch/homes/navasard/sparse/SparseAtt
+exec bash /mloscratch/homes/navasard/sparse/train_logs/run_final_eval_stack.sh \
+  >> /mloscratch/homes/navasard/sparse/train_logs/final_eval_stack_topk_recovery.log 2>&1

epfl_final/train_logs/run_final_eval_stack_second_pod.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -euo pipefail
+
+BASE=/mloscratch/homes/navasard/sparse
+REPO="$BASE/SparseAtt"
+PYTHON=/mloscratch/homes/navasard/conda/bin/python3.13
+LOG="$BASE/train_logs/final_eval_stack_second_pod.log"
+
+log() {
+  echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*" | tee -a "$LOG"
+}
+
+cd "$REPO"
+log "starting shared endpoint queue on second pod"
+"$PYTHON" eval/run_1p2b_final_endpoint_refresh.py \
+  --gpus 0,1,2,3 --stale-min 720 >> "$LOG" 2>&1
+
+log "starting shared 365M top-k queue on second pod"
+"$PYTHON" eval/run_complete_365m_topk_row.py \
+  --gpus 0,1,2,3 >> "$LOG" 2>&1
+
+log "starting shared Fixed refresh queue on second pod"
+"$PYTHON" eval/run_fixed_1p2b_final_refresh.py \
+  --gpus 0,1,2,3 --stale-min 720 >> "$LOG" 2>&1
+
+log "second-pod evaluation stack complete"

epfl_final/train_logs/scale3b_40g_a.log

@@ -0,0 +1,25 @@
+START gpu0 3b_dense__ppl_30k_random_pg19
+START gpu1 3b_mixed__ppl_30k_random_pg19
+START gpu2 3b_mixed__ppl_16k_random_pg19
+START gpu3 3b_dense__ppl_8k_moba_pg19
+DONE gpu3 3b_dense__ppl_8k_moba_pg19 status=complete
+START gpu3 3b_dense__ppl_8k_sliding_pg19
+DONE gpu3 3b_dense__ppl_8k_sliding_pg19 status=complete
+START gpu3 3b_dense__ppl_8k_h2o_pg19
+DONE gpu3 3b_dense__ppl_8k_h2o_pg19 status=complete
+START gpu3 3b_dense__position_8k_pg19
+DONE gpu3 3b_dense__position_8k_pg19 status=complete
+START gpu3 3b_dense__ruler_random_vt_8k
+DONE gpu2 3b_mixed__ppl_16k_random_pg19 status=complete
+START gpu2 3b_mixed__ppl_8k_moba_arxiv
+DONE gpu3 3b_dense__ruler_random_vt_8k status=complete
+START gpu3 3b_mixed__ppl_8k_sliding_arxiv
+DONE gpu2 3b_mixed__ppl_8k_moba_arxiv status=complete
+START gpu2 3b_mixed__ppl_8k_h2o_arxiv
+DONE gpu3 3b_mixed__ppl_8k_sliding_arxiv status=complete
+START gpu3 3b_mixed__position_8k_arxiv
+DONE gpu3 3b_mixed__position_8k_arxiv status=complete
+DONE gpu2 3b_mixed__ppl_8k_h2o_arxiv status=complete
+DONE gpu1 3b_mixed__ppl_30k_random_pg19 status=complete
+DONE gpu0 3b_dense__ppl_30k_random_pg19 status=complete
+finished complete=12/12 failed=0

epfl_final/train_logs/scale3b_40g_b.log

@@ -0,0 +1,25 @@
+START gpu0 3b_dense__ppl_30k_random_arxiv
+START gpu1 3b_mixed__ppl_30k_random_arxiv
+START gpu2 3b_mixed__ppl_16k_random_arxiv
+START gpu3 3b_dense__ppl_8k_moba_arxiv
+DONE gpu3 3b_dense__ppl_8k_moba_arxiv status=complete
+START gpu3 3b_dense__ppl_8k_sliding_arxiv
+DONE gpu3 3b_dense__ppl_8k_sliding_arxiv status=complete
+START gpu3 3b_dense__ppl_8k_h2o_arxiv
+DONE gpu3 3b_dense__ppl_8k_h2o_arxiv status=complete
+START gpu3 3b_dense__position_8k_arxiv
+DONE gpu3 3b_dense__position_8k_arxiv status=complete
+START gpu3 3b_mixed__ppl_8k_moba_fineweb
+DONE gpu3 3b_mixed__ppl_8k_moba_fineweb status=complete
+START gpu3 3b_mixed__ppl_8k_sliding_fineweb
+DONE gpu2 3b_mixed__ppl_16k_random_arxiv status=complete
+START gpu2 3b_mixed__ppl_8k_h2o_fineweb
+DONE gpu2 3b_mixed__ppl_8k_h2o_fineweb status=complete
+START gpu2 3b_mixed__position_8k_fineweb
+DONE gpu2 3b_mixed__position_8k_fineweb status=complete
+START gpu2 3b_mixed__ruler_random_niah_multivalue_8k
+DONE gpu3 3b_mixed__ppl_8k_sliding_fineweb status=complete
+DONE gpu2 3b_mixed__ruler_random_niah_multivalue_8k status=complete
+DONE gpu0 3b_dense__ppl_30k_random_arxiv status=complete
+DONE gpu1 3b_mixed__ppl_30k_random_arxiv status=complete
+finished complete=12/12 failed=0

epfl_final/train_logs/scale3b_80g_c.log

@@ -0,0 +1,25 @@
+START gpu0 3b_mixed__ppl_30k_random_fineweb
+START gpu1 3b_mixed__ppl_16k_random_fineweb
+START gpu2 3b_dense__ppl_8k_moba_fineweb
+START gpu3 3b_dense__ppl_8k_sliding_fineweb
+DONE gpu2 3b_dense__ppl_8k_moba_fineweb status=complete
+START gpu2 3b_dense__ppl_8k_h2o_fineweb
+DONE gpu3 3b_dense__ppl_8k_sliding_fineweb status=complete
+START gpu3 3b_dense__position_8k_fineweb
+DONE gpu2 3b_dense__ppl_8k_h2o_fineweb status=complete
+START gpu2 3b_dense__ruler_random_niah_multivalue_8k
+DONE gpu3 3b_dense__position_8k_fineweb status=complete
+START gpu3 3b_mixed__ppl_8k_moba_pg19
+DONE gpu3 3b_mixed__ppl_8k_moba_pg19 status=complete
+START gpu3 3b_mixed__ppl_8k_sliding_pg19
+DONE gpu2 3b_dense__ruler_random_niah_multivalue_8k status=complete
+START gpu2 3b_mixed__ppl_8k_h2o_pg19
+DONE gpu2 3b_mixed__ppl_8k_h2o_pg19 status=complete
+START gpu2 3b_mixed__position_8k_pg19
+DONE gpu2 3b_mixed__position_8k_pg19 status=complete
+START gpu2 3b_mixed__ruler_random_vt_8k
+DONE gpu3 3b_mixed__ppl_8k_sliding_pg19 status=complete
+DONE gpu1 3b_mixed__ppl_16k_random_fineweb status=complete
+DONE gpu2 3b_mixed__ruler_random_vt_8k status=complete
+DONE gpu0 3b_mixed__ppl_30k_random_fineweb status=complete
+finished complete=12/12 failed=0

epfl_final/train_logs/scale3b_a100_8k.log

@@ -0,0 +1,110 @@
+START gpu0 3b_dense__position_8k_arxiv
+START gpu1 3b_dense__position_8k_fineweb
+START gpu2 3b_dense__position_8k_pg19
+START gpu3 3b_dense__ppl_8k_dense_arxiv
+DONE gpu1 3b_dense__position_8k_fineweb status=complete
+START gpu1 3b_dense__ppl_8k_dense_fineweb
+DONE gpu0 3b_dense__position_8k_arxiv status=complete
+START gpu0 3b_dense__ppl_8k_dense_pg19
+DONE gpu2 3b_dense__position_8k_pg19 status=complete
+START gpu2 3b_dense__ppl_8k_h2o_arxiv
+DONE gpu3 3b_dense__ppl_8k_dense_arxiv status=complete
+START gpu3 3b_dense__ppl_8k_h2o_fineweb
+DONE gpu1 3b_dense__ppl_8k_dense_fineweb status=complete
+START gpu1 3b_dense__ppl_8k_h2o_pg19
+DONE gpu0 3b_dense__ppl_8k_dense_pg19 status=complete
+START gpu0 3b_dense__ppl_8k_moba_arxiv
+DONE gpu3 3b_dense__ppl_8k_h2o_fineweb status=complete
+START gpu3 3b_dense__ppl_8k_moba_fineweb
+DONE gpu2 3b_dense__ppl_8k_h2o_arxiv status=complete
+START gpu2 3b_dense__ppl_8k_moba_pg19
+DONE gpu1 3b_dense__ppl_8k_h2o_pg19 status=complete
+START gpu1 3b_dense__ppl_8k_quest_arxiv
+DONE gpu0 3b_dense__ppl_8k_moba_arxiv status=complete
+START gpu0 3b_dense__ppl_8k_quest_fineweb
+DONE gpu2 3b_dense__ppl_8k_moba_pg19 status=complete
+START gpu2 3b_dense__ppl_8k_quest_pg19
+DONE gpu3 3b_dense__ppl_8k_moba_fineweb status=complete
+START gpu3 3b_dense__ppl_8k_sink_local_arxiv
+DONE gpu1 3b_dense__ppl_8k_quest_arxiv status=complete
+START gpu1 3b_dense__ppl_8k_sink_local_fineweb
+DONE gpu0 3b_dense__ppl_8k_quest_fineweb status=complete
+START gpu0 3b_dense__ppl_8k_sink_local_pg19
+DONE gpu2 3b_dense__ppl_8k_quest_pg19 status=complete
+START gpu2 3b_dense__ppl_8k_sliding_arxiv
+DONE gpu0 3b_dense__ppl_8k_sink_local_pg19 status=complete
+START gpu0 3b_dense__ppl_8k_sliding_fineweb
+DONE gpu3 3b_dense__ppl_8k_sink_local_arxiv status=complete
+START gpu3 3b_dense__ppl_8k_sliding_pg19
+DONE gpu1 3b_dense__ppl_8k_sink_local_fineweb status=complete
+START gpu1 3b_dense__ppl_8k_topk_arxiv
+DONE gpu1 3b_dense__ppl_8k_topk_arxiv status=retrying
+START gpu1 3b_dense__ppl_8k_topk_fineweb
+DONE gpu1 3b_dense__ppl_8k_topk_fineweb status=retrying
+START gpu1 3b_dense__ppl_8k_topk_pg19
+DONE gpu1 3b_dense__ppl_8k_topk_pg19 status=retrying
+START gpu1 3b_mixed__position_8k_arxiv
+DONE gpu1 3b_mixed__position_8k_arxiv status=complete
+START gpu1 3b_mixed__position_8k_fineweb
+DONE gpu1 3b_mixed__position_8k_fineweb status=complete
+START gpu1 3b_mixed__position_8k_pg19
+DONE gpu2 3b_dense__ppl_8k_sliding_arxiv status=complete
+START gpu2 3b_mixed__ppl_8k_dense_arxiv
+DONE gpu1 3b_mixed__position_8k_pg19 status=complete
+START gpu1 3b_mixed__ppl_8k_dense_fineweb
+DONE gpu3 3b_dense__ppl_8k_sliding_pg19 status=complete
+START gpu3 3b_mixed__ppl_8k_dense_pg19
+DONE gpu0 3b_dense__ppl_8k_sliding_fineweb status=complete
+START gpu0 3b_mixed__ppl_8k_h2o_arxiv
+DONE gpu2 3b_mixed__ppl_8k_dense_arxiv status=complete
+START gpu2 3b_mixed__ppl_8k_h2o_fineweb
+DONE gpu1 3b_mixed__ppl_8k_dense_fineweb status=complete
+START gpu1 3b_mixed__ppl_8k_h2o_pg19
+DONE gpu3 3b_mixed__ppl_8k_dense_pg19 status=complete
+START gpu3 3b_mixed__ppl_8k_moba_arxiv
+DONE gpu0 3b_mixed__ppl_8k_h2o_arxiv status=complete
+START gpu0 3b_mixed__ppl_8k_moba_fineweb
+DONE gpu1 3b_mixed__ppl_8k_h2o_pg19 status=complete
+START gpu1 3b_mixed__ppl_8k_moba_pg19
+DONE gpu2 3b_mixed__ppl_8k_h2o_fineweb status=complete
+START gpu2 3b_mixed__ppl_8k_quest_arxiv
+DONE gpu3 3b_mixed__ppl_8k_moba_arxiv status=complete
+START gpu3 3b_mixed__ppl_8k_quest_fineweb
+DONE gpu0 3b_mixed__ppl_8k_moba_fineweb status=complete
+START gpu0 3b_mixed__ppl_8k_quest_pg19
+DONE gpu1 3b_mixed__ppl_8k_moba_pg19 status=complete
+START gpu1 3b_mixed__ppl_8k_sink_local_arxiv
+DONE gpu2 3b_mixed__ppl_8k_quest_arxiv status=complete
+START gpu2 3b_mixed__ppl_8k_sink_local_fineweb
+DONE gpu3 3b_mixed__ppl_8k_quest_fineweb status=complete
+START gpu3 3b_mixed__ppl_8k_sink_local_pg19
+DONE gpu0 3b_mixed__ppl_8k_quest_pg19 status=complete
+START gpu0 3b_mixed__ppl_8k_sliding_arxiv
+DONE gpu1 3b_mixed__ppl_8k_sink_local_arxiv status=complete
+START gpu1 3b_mixed__ppl_8k_sliding_fineweb
+DONE gpu2 3b_mixed__ppl_8k_sink_local_fineweb status=complete
+START gpu2 3b_mixed__ppl_8k_sliding_pg19
+DONE gpu3 3b_mixed__ppl_8k_sink_local_pg19 status=complete
+START gpu3 3b_mixed__ppl_8k_topk_arxiv
+DONE gpu3 3b_mixed__ppl_8k_topk_arxiv status=retrying
+START gpu3 3b_mixed__ppl_8k_topk_fineweb
+DONE gpu3 3b_mixed__ppl_8k_topk_fineweb status=retrying
+START gpu3 3b_mixed__ppl_8k_topk_pg19
+DONE gpu3 3b_mixed__ppl_8k_topk_pg19 status=retrying
+START gpu3 3b_dense__ppl_8k_topk_arxiv
+DONE gpu3 3b_dense__ppl_8k_topk_arxiv status=failed
+START gpu3 3b_dense__ppl_8k_topk_fineweb
+DONE gpu3 3b_dense__ppl_8k_topk_fineweb status=failed
+START gpu3 3b_dense__ppl_8k_topk_pg19
+DONE gpu3 3b_dense__ppl_8k_topk_pg19 status=failed
+START gpu3 3b_mixed__ppl_8k_topk_arxiv
+DONE gpu3 3b_mixed__ppl_8k_topk_arxiv status=failed
+START gpu3 3b_mixed__ppl_8k_topk_fineweb
+DONE gpu3 3b_mixed__ppl_8k_topk_fineweb status=failed
+START gpu3 3b_mixed__ppl_8k_topk_pg19
+DONE gpu3 3b_mixed__ppl_8k_topk_pg19 status=failed
+DONE gpu0 3b_mixed__ppl_8k_sliding_arxiv status=complete
+DONE gpu1 3b_mixed__ppl_8k_sliding_fineweb status=complete
+DONE gpu2 3b_mixed__ppl_8k_sliding_pg19 status=complete
+finished complete=42/48 failed=6
+Failed jobs: 3b_dense__ppl_8k_topk_arxiv, 3b_dense__ppl_8k_topk_fineweb, 3b_dense__ppl_8k_topk_pg19, 3b_mixed__ppl_8k_topk_arxiv, 3b_mixed__ppl_8k_topk_fineweb, 3b_mixed__ppl_8k_topk_pg19

epfl_final/train_logs/scale3b_eval_20260802.log

@@ -0,0 +1,27 @@
+START gpu0 3b_dense__ppl_8k_random_fineweb
+START gpu1 3b_dense__ppl_8k_random_pg19
+START gpu2 3b_dense__ppl_8k_random_arxiv
+START gpu3 3b_dense__ppl_8k_quest_fineweb
+START gpu4 3b_dense__ppl_8k_quest_pg19
+START gpu5 3b_dense__ppl_8k_quest_arxiv
+DONE gpu4 3b_dense__ppl_8k_quest_pg19 status=complete
+START gpu4 3b_dense__ppl_8k_sink_local_fineweb
+DONE gpu3 3b_dense__ppl_8k_quest_fineweb status=complete
+START gpu3 3b_dense__ppl_8k_sink_local_pg19
+DONE gpu5 3b_dense__ppl_8k_quest_arxiv status=complete
+START gpu5 3b_dense__ppl_8k_sink_local_arxiv
+DONE gpu2 3b_dense__ppl_8k_random_arxiv status=complete
+START gpu2 3b_dense__ppl_8k_dense_pg19
+DONE gpu1 3b_dense__ppl_8k_random_pg19 status=complete
+START gpu1 3b_dense__ppl_8k_dense_arxiv
+DONE gpu0 3b_dense__ppl_8k_random_fineweb status=complete
+START gpu0 3b_dense__ruler_random_niah_multikey_8k
+DONE gpu2 3b_dense__ppl_8k_dense_pg19 status=complete
+START gpu2 3b_dense__passkey_8k_random
+DONE gpu1 3b_dense__ppl_8k_dense_arxiv status=complete
+DONE gpu3 3b_dense__ppl_8k_sink_local_pg19 status=complete
+DONE gpu4 3b_dense__ppl_8k_sink_local_fineweb status=complete
+DONE gpu5 3b_dense__ppl_8k_sink_local_arxiv status=complete
+DONE gpu0 3b_dense__ruler_random_niah_multikey_8k status=complete
+DONE gpu2 3b_dense__passkey_8k_random status=complete
+finished complete=14/14 failed=0

epfl_final/train_logs/scale3b_full_a100_20260804.log

@@ -0,0 +1,4 @@
+START gpu0 3b_dense__ppl_16k_random_fineweb
+START gpu1 3b_dense__ppl_16k_random_pg19
+START gpu2 3b_dense__ppl_16k_random_arxiv
+START gpu3 3b_dense__ppl_30k_random_fineweb

epfl_final/train_logs/scale3b_gsm8k_all.log

@@ -0,0 +1,7 @@
+START gpu0 3b_dense__harness_gsm8k
+START gpu1 3b_mixed__harness_gsm8k
+START gpu2 3b_m2d20__harness_gsm8k
+DONE gpu0 3b_dense__harness_gsm8k status=complete
+DONE gpu2 3b_m2d20__harness_gsm8k status=complete
+DONE gpu1 3b_mixed__harness_gsm8k status=complete
+finished complete=3/3 failed=0

epfl_final/train_logs/scale3b_h100_long.log

@@ -0,0 +1,101 @@
+START gpu0 3b_dense__ppl_16k_dense_fineweb
+START gpu1 3b_dense__ppl_16k_h2o_fineweb
+DONE gpu0 3b_dense__ppl_16k_dense_fineweb status=complete
+START gpu0 3b_dense__ppl_16k_moba_fineweb
+DONE gpu1 3b_dense__ppl_16k_h2o_fineweb status=complete
+START gpu1 3b_dense__ppl_16k_quest_fineweb
+DONE gpu0 3b_dense__ppl_16k_moba_fineweb status=complete
+START gpu0 3b_dense__ppl_16k_sink_local_fineweb
+DONE gpu1 3b_dense__ppl_16k_quest_fineweb status=complete
+START gpu1 3b_dense__ppl_16k_sliding_fineweb
+DONE gpu0 3b_dense__ppl_16k_sink_local_fineweb status=complete
+START gpu0 3b_mixed__ppl_16k_dense_fineweb
+DONE gpu0 3b_mixed__ppl_16k_dense_fineweb status=complete
+START gpu0 3b_mixed__ppl_16k_h2o_fineweb
+DONE gpu1 3b_dense__ppl_16k_sliding_fineweb status=complete
+START gpu1 3b_mixed__ppl_16k_moba_fineweb
+DONE gpu0 3b_mixed__ppl_16k_h2o_fineweb status=complete
+START gpu0 3b_mixed__ppl_16k_quest_fineweb
+DONE gpu1 3b_mixed__ppl_16k_moba_fineweb status=complete
+START gpu1 3b_mixed__ppl_16k_sink_local_fineweb
+DONE gpu0 3b_mixed__ppl_16k_quest_fineweb status=complete
+START gpu0 3b_mixed__ppl_16k_sliding_fineweb
+DONE gpu1 3b_mixed__ppl_16k_sink_local_fineweb status=complete
+START gpu1 3b_dense__position_30k_fineweb
+DONE gpu1 3b_dense__position_30k_fineweb status=complete
+START gpu1 3b_dense__ppl_30k_dense_arxiv
+DONE gpu1 3b_dense__ppl_30k_dense_arxiv status=complete
+START gpu1 3b_dense__ppl_30k_dense_fineweb
+DONE gpu0 3b_mixed__ppl_16k_sliding_fineweb status=complete
+START gpu0 3b_dense__ppl_30k_dense_pg19
+DONE gpu1 3b_dense__ppl_30k_dense_fineweb status=complete
+START gpu1 3b_dense__ppl_30k_h2o_arxiv
+DONE gpu0 3b_dense__ppl_30k_dense_pg19 status=complete
+START gpu0 3b_dense__ppl_30k_h2o_fineweb
+DONE gpu1 3b_dense__ppl_30k_h2o_arxiv status=complete
+START gpu1 3b_dense__ppl_30k_h2o_pg19
+DONE gpu0 3b_dense__ppl_30k_h2o_fineweb status=complete
+START gpu0 3b_dense__ppl_30k_moba_arxiv
+DONE gpu1 3b_dense__ppl_30k_h2o_pg19 status=complete
+START gpu1 3b_dense__ppl_30k_moba_fineweb
+DONE gpu0 3b_dense__ppl_30k_moba_arxiv status=complete
+START gpu0 3b_dense__ppl_30k_moba_pg19
+DONE gpu1 3b_dense__ppl_30k_moba_fineweb status=complete
+START gpu1 3b_dense__ppl_30k_quest_arxiv
+DONE gpu0 3b_dense__ppl_30k_moba_pg19 status=complete
+START gpu0 3b_dense__ppl_30k_quest_fineweb
+DONE gpu1 3b_dense__ppl_30k_quest_arxiv status=complete
+START gpu1 3b_dense__ppl_30k_quest_pg19
+DONE gpu0 3b_dense__ppl_30k_quest_fineweb status=complete
+START gpu0 3b_dense__ppl_30k_sink_local_arxiv
+DONE gpu1 3b_dense__ppl_30k_quest_pg19 status=complete
+START gpu1 3b_dense__ppl_30k_sink_local_fineweb
+DONE gpu0 3b_dense__ppl_30k_sink_local_arxiv status=complete
+START gpu0 3b_dense__ppl_30k_sink_local_pg19
+DONE gpu1 3b_dense__ppl_30k_sink_local_fineweb status=complete
+START gpu1 3b_dense__ppl_30k_sliding_arxiv
+DONE gpu0 3b_dense__ppl_30k_sink_local_pg19 status=complete
+START gpu0 3b_dense__ppl_30k_sliding_fineweb
+DONE gpu1 3b_dense__ppl_30k_sliding_arxiv status=complete
+START gpu1 3b_dense__ppl_30k_sliding_pg19
+DONE gpu0 3b_dense__ppl_30k_sliding_fineweb status=complete
+START gpu0 3b_mixed__position_30k_fineweb
+DONE gpu0 3b_mixed__position_30k_fineweb status=complete
+START gpu0 3b_mixed__ppl_30k_dense_arxiv
+DONE gpu1 3b_dense__ppl_30k_sliding_pg19 status=complete
+START gpu1 3b_mixed__ppl_30k_dense_fineweb
+DONE gpu0 3b_mixed__ppl_30k_dense_arxiv status=complete
+START gpu0 3b_mixed__ppl_30k_dense_pg19
+DONE gpu1 3b_mixed__ppl_30k_dense_fineweb status=complete
+START gpu1 3b_mixed__ppl_30k_h2o_arxiv
+DONE gpu0 3b_mixed__ppl_30k_dense_pg19 status=complete
+START gpu0 3b_mixed__ppl_30k_h2o_fineweb
+DONE gpu1 3b_mixed__ppl_30k_h2o_arxiv status=complete
+START gpu1 3b_mixed__ppl_30k_h2o_pg19
+DONE gpu0 3b_mixed__ppl_30k_h2o_fineweb status=complete
+START gpu0 3b_mixed__ppl_30k_moba_arxiv
+DONE gpu1 3b_mixed__ppl_30k_h2o_pg19 status=complete
+START gpu1 3b_mixed__ppl_30k_moba_fineweb
+DONE gpu0 3b_mixed__ppl_30k_moba_arxiv status=complete
+START gpu0 3b_mixed__ppl_30k_moba_pg19
+DONE gpu1 3b_mixed__ppl_30k_moba_fineweb status=complete
+START gpu1 3b_mixed__ppl_30k_quest_arxiv
+DONE gpu0 3b_mixed__ppl_30k_moba_pg19 status=complete
+START gpu0 3b_mixed__ppl_30k_quest_fineweb
+DONE gpu1 3b_mixed__ppl_30k_quest_arxiv status=complete
+START gpu1 3b_mixed__ppl_30k_quest_pg19
+DONE gpu0 3b_mixed__ppl_30k_quest_fineweb status=complete
+START gpu0 3b_mixed__ppl_30k_sink_local_arxiv
+DONE gpu1 3b_mixed__ppl_30k_quest_pg19 status=complete
+START gpu1 3b_mixed__ppl_30k_sink_local_fineweb
+DONE gpu0 3b_mixed__ppl_30k_sink_local_arxiv status=complete
+START gpu0 3b_mixed__ppl_30k_sink_local_pg19
+DONE gpu1 3b_mixed__ppl_30k_sink_local_fineweb status=complete
+START gpu1 3b_mixed__ppl_30k_sliding_arxiv
+DONE gpu0 3b_mixed__ppl_30k_sink_local_pg19 status=complete
+START gpu0 3b_mixed__ppl_30k_sliding_fineweb
+DONE gpu1 3b_mixed__ppl_30k_sliding_arxiv status=complete
+START gpu1 3b_mixed__ppl_30k_sliding_pg19
+DONE gpu0 3b_mixed__ppl_30k_sliding_fineweb status=complete
+DONE gpu1 3b_mixed__ppl_30k_sliding_pg19 status=complete
+finished complete=50/50 failed=0

epfl_final/train_logs/scale3b_h1gpu.log

@@ -0,0 +1,10 @@
+START gpu0 3b_dense__ppl_16k_quest_fineweb
+DONE gpu0 3b_dense__ppl_16k_quest_fineweb status=retrying
+START gpu0 3b_dense__ppl_16k_sink_local_fineweb
+DONE gpu0 3b_dense__ppl_16k_sink_local_fineweb status=retrying
+START gpu0 3b_dense__ppl_16k_quest_fineweb
+DONE gpu0 3b_dense__ppl_16k_quest_fineweb status=failed
+START gpu0 3b_dense__ppl_16k_sink_local_fineweb
+DONE gpu0 3b_dense__ppl_16k_sink_local_fineweb status=failed
+finished complete=0/2 failed=2
+Failed jobs: 3b_dense__ppl_16k_quest_fineweb, 3b_dense__ppl_16k_sink_local_fineweb

epfl_final/train_logs/scale3b_h2gpu.log

@@ -0,0 +1,66 @@
+START gpu0 3b_dense__ppl_30k_random_gutenberg
+START gpu1 3b_mixed__ppl_30k_random_pubmed
+DONE gpu1 3b_mixed__ppl_30k_random_pubmed status=complete
+START gpu1 3b_dense__ppl_16k_moba_fineweb
+DONE gpu0 3b_dense__ppl_30k_random_gutenberg status=complete
+START gpu0 3b_dense__ppl_16k_sliding_fineweb
+DONE gpu1 3b_dense__ppl_16k_moba_fineweb status=retrying
+START gpu1 3b_dense__passkey_16k_random
+DONE gpu0 3b_dense__ppl_16k_sliding_fineweb status=retrying
+START gpu0 3b_mixed__ppl_16k_quest_fineweb
+DONE gpu0 3b_mixed__ppl_16k_quest_fineweb status=retrying
+START gpu0 3b_mixed__ppl_16k_sink_local_fineweb
+DONE gpu0 3b_mixed__ppl_16k_sink_local_fineweb status=retrying
+START gpu0 3b_mixed__ppl_16k_moba_fineweb
+DONE gpu0 3b_mixed__ppl_16k_moba_fineweb status=retrying
+START gpu0 3b_mixed__ppl_16k_sliding_fineweb
+DONE gpu0 3b_mixed__ppl_16k_sliding_fineweb status=retrying
+START gpu0 3b_mixed__passkey_16k_random
+DONE gpu1 3b_dense__passkey_16k_random status=complete
+START gpu1 3b_dense__ruler_random_niah_multikey_30k
+DONE gpu0 3b_mixed__passkey_16k_random status=complete
+START gpu0 3b_dense__passkey_30k_random
+DONE gpu1 3b_dense__ruler_random_niah_multikey_30k status=complete
+START gpu1 3b_dense__efficiency_30k
+DONE gpu0 3b_dense__passkey_30k_random status=complete
+START gpu0 3b_dense__longbench_qasper
+DONE gpu0 3b_dense__longbench_qasper status=complete
+START gpu0 3b_dense__longbench_multifieldqa_en
+DONE gpu0 3b_dense__longbench_multifieldqa_en status=complete
+START gpu0 3b_dense__longbench_2wikimqa
+DONE gpu0 3b_dense__longbench_2wikimqa status=complete
+START gpu0 3b_mixed__ruler_random_niah_multikey_30k
+DONE gpu1 3b_dense__efficiency_30k status=complete
+START gpu1 3b_mixed__passkey_30k_random
+DONE gpu0 3b_mixed__ruler_random_niah_multikey_30k status=complete
+START gpu0 3b_mixed__efficiency_30k
+DONE gpu1 3b_mixed__passkey_30k_random status=complete
+START gpu1 3b_mixed__longbench_narrativeqa
+DONE gpu1 3b_mixed__longbench_narrativeqa status=complete
+START gpu1 3b_mixed__longbench_multifieldqa_en
+DONE gpu1 3b_mixed__longbench_multifieldqa_en status=complete
+START gpu1 3b_mixed__longbench_2wikimqa
+DONE gpu1 3b_mixed__longbench_2wikimqa status=complete
+START gpu1 3b_dense__harness_gsm8k
+DONE gpu1 3b_dense__harness_gsm8k status=retrying
+START gpu1 3b_mixed__harness_gsm8k
+DONE gpu1 3b_mixed__harness_gsm8k status=retrying
+START gpu1 3b_dense__ppl_16k_moba_fineweb
+DONE gpu1 3b_dense__ppl_16k_moba_fineweb status=failed
+START gpu1 3b_dense__ppl_16k_sliding_fineweb
+DONE gpu1 3b_dense__ppl_16k_sliding_fineweb status=failed
+START gpu1 3b_mixed__ppl_16k_quest_fineweb
+DONE gpu1 3b_mixed__ppl_16k_quest_fineweb status=failed
+START gpu1 3b_mixed__ppl_16k_sink_local_fineweb
+DONE gpu1 3b_mixed__ppl_16k_sink_local_fineweb status=failed
+START gpu1 3b_mixed__ppl_16k_moba_fineweb
+DONE gpu1 3b_mixed__ppl_16k_moba_fineweb status=failed
+START gpu1 3b_mixed__ppl_16k_sliding_fineweb
+DONE gpu1 3b_mixed__ppl_16k_sliding_fineweb status=failed
+START gpu1 3b_dense__harness_gsm8k
+DONE gpu1 3b_dense__harness_gsm8k status=failed
+START gpu1 3b_mixed__harness_gsm8k
+DONE gpu1 3b_mixed__harness_gsm8k status=failed
+DONE gpu0 3b_mixed__efficiency_30k status=complete
+finished complete=16/24 failed=8
+Failed jobs: 3b_dense__ppl_16k_moba_fineweb, 3b_dense__ppl_16k_sliding_fineweb, 3b_mixed__ppl_16k_quest_fineweb, 3b_mixed__ppl_16k_sink_local_fineweb, 3b_mixed__ppl_16k_moba_fineweb, 3b_mixed__ppl_16k_sliding_fineweb, 3b_dense__harness_gsm8k, 3b_mixed__harness_gsm8k

epfl_final/train_logs/scale3b_mixed_evals_20260804.log

@@ -0,0 +1,29 @@
+START gpu0 3b_mixed__ppl_8k_random_fineweb
+DONE gpu0 3b_mixed__ppl_8k_random_fineweb status=complete
+START gpu0 3b_mixed__ppl_8k_random_pg19
+DONE gpu0 3b_mixed__ppl_8k_random_pg19 status=complete
+START gpu0 3b_mixed__ppl_8k_random_arxiv
+DONE gpu0 3b_mixed__ppl_8k_random_arxiv status=complete
+START gpu0 3b_mixed__ppl_8k_quest_fineweb
+DONE gpu0 3b_mixed__ppl_8k_quest_fineweb status=complete
+START gpu0 3b_mixed__ppl_8k_quest_pg19
+DONE gpu0 3b_mixed__ppl_8k_quest_pg19 status=complete
+START gpu0 3b_mixed__ppl_8k_quest_arxiv
+DONE gpu0 3b_mixed__ppl_8k_quest_arxiv status=complete
+START gpu0 3b_mixed__ppl_8k_sink_local_fineweb
+DONE gpu0 3b_mixed__ppl_8k_sink_local_fineweb status=complete
+START gpu0 3b_mixed__ppl_8k_sink_local_pg19
+DONE gpu0 3b_mixed__ppl_8k_sink_local_pg19 status=complete
+START gpu0 3b_mixed__ppl_8k_sink_local_arxiv
+DONE gpu0 3b_mixed__ppl_8k_sink_local_arxiv status=complete
+START gpu0 3b_mixed__ppl_8k_dense_fineweb
+DONE gpu0 3b_mixed__ppl_8k_dense_fineweb status=complete
+START gpu0 3b_mixed__ppl_8k_dense_pg19
+DONE gpu0 3b_mixed__ppl_8k_dense_pg19 status=complete
+START gpu0 3b_mixed__ppl_8k_dense_arxiv
+DONE gpu0 3b_mixed__ppl_8k_dense_arxiv status=complete
+START gpu0 3b_mixed__ruler_random_niah_multikey_8k
+DONE gpu0 3b_mixed__ruler_random_niah_multikey_8k status=complete
+START gpu0 3b_mixed__passkey_8k_random
+DONE gpu0 3b_mixed__passkey_8k_random status=complete
+finished complete=28/28 failed=0

epfl_final/train_logs/scale3b_topkfix.log

@@ -0,0 +1,13 @@
+START gpu0 3b_dense__ppl_8k_topk_arxiv
+START gpu1 3b_dense__ppl_8k_topk_fineweb
+DONE gpu1 3b_dense__ppl_8k_topk_fineweb status=complete
+START gpu1 3b_dense__ppl_8k_topk_pg19
+DONE gpu0 3b_dense__ppl_8k_topk_arxiv status=complete
+START gpu0 3b_mixed__ppl_8k_topk_arxiv
+DONE gpu0 3b_mixed__ppl_8k_topk_arxiv status=complete
+START gpu0 3b_mixed__ppl_8k_topk_fineweb
+DONE gpu1 3b_dense__ppl_8k_topk_pg19 status=complete
+START gpu1 3b_mixed__ppl_8k_topk_pg19
+DONE gpu1 3b_mixed__ppl_8k_topk_pg19 status=complete
+DONE gpu0 3b_mixed__ppl_8k_topk_fineweb status=complete
+finished complete=6/6 failed=0

epfl_final/train_logs/wait_1p2b_then_365m_native30k.log

@@ -0,0 +1,2 @@
+2026-07-27T03:11:06Z waiter start latest_1p2b_step=41000
+2026-07-27T19:18:33Z waiter start latest_1p2b_step=46000

epfl_final/train_logs/wait_1p2b_then_365m_native30k.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+set -u
+
+BASE=/mloscratch/homes/navasard/sparse
+REPO="$BASE/SparseAtt-baselines"
+PYTHON=/mloscratch/homes/navasard/conda/bin/python3.13
+WATCHDOG="$BASE/train_logs/watchdog_365m_native30k_exact.sh"
+LOG="$BASE/train_logs/wait_1p2b_then_365m_native30k.log"
+ONEP2B="$BASE/checkpoints/l8k1_2b_mixed_seed43"
+
+log() {
+  echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*" >> "$LOG"
+}
+
+latest_1p2b_step() {
+  "$PYTHON" - "$ONEP2B" <<'PY'
+from pathlib import Path
+import sys
+
+steps = []
+for checkpoint in Path(sys.argv[1]).glob("run_*/checkpoint-*"):
+    try:
+        step = int(checkpoint.name.rsplit("-", 1)[1])
+    except ValueError:
+        continue
+    required = [checkpoint / "model.pt", checkpoint / "training_state.pt"]
+    required.extend(checkpoint / f"rng_state_rank{rank}.pt" for rank in range(4))
+    required.extend(
+        checkpoint / f"loader_gen_state_rank{rank}.pt" for rank in range(4)
+    )
+    if all(path.is_file() and path.stat().st_size > 0 for path in required):
+        steps.append(step)
+print(max(steps) if steps else 0)
+PY
+}
+
+log "waiter start latest_1p2b_step=$(latest_1p2b_step)"
+while [ "$(latest_1p2b_step)" -lt 50462 ]; do
+  sleep 60
+done
+
+while pgrep -f "[t]rain_resume_exact.py --config configs/1p2b_mixed_8k_seed43_4gpu.runtime.json" >/dev/null; do
+  sleep 30
+done
+
+log "1.2B complete; launching four native-30k 365M seed watchdogs"
+nohup bash "$WATCHDOG" \
+  dense_365m_30k_seed43 \
+  "$REPO/configs/multiseed/dense_365m_30k_seed43.json" \
+  "$BASE/checkpoints/l30k365m_dense_seed43" 0 \
+  >> "$BASE/train_logs/dense_365m_30k_seed43.exact.supervisor.log" 2>&1 &
+nohup bash "$WATCHDOG" \
+  dense_365m_30k_seed44 \
+  "$REPO/configs/multiseed/dense_365m_30k_seed44.json" \
+  "$BASE/checkpoints/l30k365m_dense_seed44" 1 \
+  >> "$BASE/train_logs/dense_365m_30k_seed44.exact.supervisor.log" 2>&1 &
+nohup bash "$WATCHDOG" \
+  mixed_365m_30k_seed43 \
+  "$REPO/configs/multiseed/mixed_365m_30k_seed43.json" \
+  "$BASE/checkpoints/l30k365m_mixed_seed43" 2 \
+  >> "$BASE/train_logs/mixed_365m_30k_seed43.exact.supervisor.log" 2>&1 &
+nohup bash "$WATCHDOG" \
+  mixed_365m_30k_seed44 \
+  "$REPO/configs/multiseed/mixed_365m_30k_seed44.json" \
+  "$BASE/checkpoints/l30k365m_mixed_seed44" 3 \
+  >> "$BASE/train_logs/mixed_365m_30k_seed44.exact.supervisor.log" 2>&1 &
+log "all four seed watchdogs launched"

epfl_final/train_logs/wait_1p2b_then_dense_ruler_then_365m.log

@@ -0,0 +1,2 @@
+2026-07-27T22:07:44Z waiter start latest_1p2b_step=47000
+2026-07-28T04:29:09Z launching failed Dense 30k random-RULER retry on GPU 0

epfl_final/train_logs/wait_1p2b_then_dense_ruler_then_365m.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+set -u
+
+BASE=/mloscratch/homes/navasard/sparse
+EVAL_REPO="$BASE/SparseAtt"
+PYTHON=/mloscratch/homes/navasard/conda/bin/python3.13
+ONEP2B="$BASE/checkpoints/l8k1_2b_mixed_seed43"
+SEED_HANDOFF="$BASE/train_logs/wait_1p2b_then_365m_native30k.sh"
+RESULT="$EVAL_REPO/eval/results/FINAL_DENSE_RULER_1P2B_20260727/ruler/ruler_1p2b_dense_random_niah_multikey_30k.tsv"
+LOG="$BASE/train_logs/wait_1p2b_then_dense_ruler_then_365m.log"
+
+log() {
+  echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*" >> "$LOG"
+}
+
+latest_1p2b_step() {
+  "$PYTHON" - "$ONEP2B" <<'PY'
+from pathlib import Path
+import sys
+
+steps = []
+for checkpoint in Path(sys.argv[1]).glob("run_*/checkpoint-*"):
+    try:
+        step = int(checkpoint.name.rsplit("-", 1)[1])
+    except ValueError:
+        continue
+    required = [checkpoint / "model.pt", checkpoint / "training_state.pt"]
+    required.extend(checkpoint / f"rng_state_rank{rank}.pt" for rank in range(4))
+    required.extend(
+        checkpoint / f"loader_gen_state_rank{rank}.pt" for rank in range(4)
+    )
+    if all(path.is_file() and path.stat().st_size > 0 for path in required):
+        steps.append(step)
+print(max(steps) if steps else 0)
+PY
+}
+
+result_complete() {
+  [ -s "$RESULT" ] && [ "$(grep -cve '^[[:space:]]*$' "$RESULT")" -eq 16 ]
+}
+
+log "waiter start latest_1p2b_step=$(latest_1p2b_step)"
+while [ "$(latest_1p2b_step)" -lt 50462 ]; do
+  sleep 60
+done
+
+while pgrep -f "[t]rain_resume_exact.py --config configs/1p2b_mixed_8k_seed43_4gpu.runtime.json" >/dev/null; do
+  sleep 30
+done
+while nvidia-smi --query-compute-apps=pid --format=csv,noheader 2>/dev/null | grep -q '[0-9]'; do
+  sleep 30
+done
+
+while ! result_complete; do
+  log "launching failed Dense 30k random-RULER retry on GPU 0"
+  (
+    cd "$EVAL_REPO" &&
+    CUDA_VISIBLE_DEVICES=0 \
+    PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
+      "$PYTHON" eval/run_final_dense_1p2b_ruler_refresh.py --gpus 0
+  ) >> "$BASE/train_logs/final_dense_ruler_retry.log" 2>&1
+  if ! result_complete; then
+    log "Dense RULER retry incomplete; retrying in 60 seconds"
+    sleep 60
+  fi
+done
+
+log "Dense RULER complete; launching native-30k 365M handoff"
+exec bash "$SEED_HANDOFF"

epfl_final/train_logs/wait_random_ruler_then_1p2b.log

@@ -0,0 +1 @@
+2026-07-27T16:57:40Z waiter start

epfl_final/train_logs/wait_random_ruler_then_1p2b.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+set -u
+
+BASE=/mloscratch/homes/navasard/sparse
+LOG="$BASE/train_logs/wait_random_ruler_then_1p2b.log"
+ONEP2B_WATCHDOG="$BASE/train_logs/watchdog_1p2b_mixed_seed43_exact.sh"
+SEED_HANDOFF="$BASE/train_logs/wait_1p2b_then_365m_native30k.sh"
+ONEP2B_LOCK="$BASE/train_logs/mixed8k1p2b_s43.exact.lock"
+
+log() {
+  echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*" >> "$LOG"
+}
+
+log "waiter start"
+while pgrep -f "[r]un_final_random_1p2b_ruler_30k.py|[r]uler_eval.py.*FINAL_RANDOM_RULER_30K_1P2B_20260727" >/dev/null; do
+  sleep 60
+done
+
+while nvidia-smi --query-compute-apps=pid --format=csv,noheader 2>/dev/null | grep -q '[0-9]'; do
+  sleep 30
+done
+
+rm -f "$ONEP2B_LOCK"
+log "random-RULER complete and GPUs clear; launching 1.2B exact-resume watchdog"
+nohup bash "$ONEP2B_WATCHDOG" \
+  >> "$BASE/train_logs/mixed8k1p2b_s43.exact.supervisor.log" 2>&1 &
+
+log "launching post-1.2B native-30k 365M handoff waiter"
+nohup bash "$SEED_HANDOFF" \
+  >> "$BASE/train_logs/wait_1p2b_then_365m_native30k.supervisor.log" 2>&1 &
+log "handoff chain installed"

epfl_final/train_logs/watchdog_365m_native30k_exact.sh

@@ -0,0 +1,130 @@
+#!/bin/bash
+set -u
+
+JOB="$1"
+SOURCE_CFG="$2"
+OUTDIR="$3"
+GPU="$4"
+
+BASE=/mloscratch/homes/navasard/sparse
+REPO="$BASE/SparseAtt-baselines"
+PYTHON=/mloscratch/homes/navasard/conda/bin/python3.13
+TORCHRUN=/mloscratch/homes/navasard/conda/bin/torchrun
+TRAINER=train_resume_exact.py
+RUNTIME_CFG="$REPO/configs/multiseed/${JOB}.runtime.json"
+TOTAL=11700
+WLOG="$BASE/train_logs/${JOB}.exact.watch.log"
+RUNLOG="$BASE/train_logs/${JOB}.exact.run.log"
+LOCK="$BASE/train_logs/${JOB}.exact.lock"
+PROC_PAT="[t]rain_resume_exact.py --config configs/multiseed/${JOB}.runtime.json"
+
+export HF_HOME=/mloscratch/homes/navasard/hf_cache
+
+log() {
+  echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*" >> "$WLOG"
+}
+
+latest_checkpoint() {
+  "$PYTHON" - "$OUTDIR" <<'PY'
+from pathlib import Path
+import sys
+
+root = Path(sys.argv[1])
+candidates = sorted(
+    root.glob("run_*/checkpoint-*"),
+    key=lambda path: int(path.name.rsplit("-", 1)[1]),
+    reverse=True,
+)
+for checkpoint in candidates:
+    required = [
+        checkpoint / "model.pt",
+        checkpoint / "training_state.pt",
+        checkpoint / "rng_state_rank0.pt",
+        checkpoint / "loader_gen_state_rank0.pt",
+    ]
+    if all(path.is_file() and path.stat().st_size > 0 for path in required):
+        print(checkpoint)
+        break
+PY
+}
+
+latest_step() {
+  local checkpoint
+  checkpoint=$(latest_checkpoint)
+  if [ -n "${checkpoint:-}" ]; then
+    basename "$checkpoint" | sed 's/checkpoint-//'
+  fi
+}
+
+process_count() {
+  pgrep -f "$PROC_PAT" 2>/dev/null | grep -c '[0-9]'
+}
+
+write_runtime_config() {
+  local checkpoint="$1"
+  "$PYTHON" - "$SOURCE_CFG" "$RUNTIME_CFG" "$checkpoint" <<'PY'
+import json
+from pathlib import Path
+import sys
+
+source, destination, checkpoint = map(Path, sys.argv[1:])
+config = json.loads(source.read_text())
+training = config["training"]
+training["resume_from_checkpoint"] = str(checkpoint)
+training["preserve_checkpoint_lr"] = True
+destination.write_text(json.dumps(config, indent=2) + "\n")
+PY
+}
+
+launch() {
+  local checkpoint
+  checkpoint=$(latest_checkpoint)
+  if [ -z "${checkpoint:-}" ]; then
+    log "no complete rank-0 checkpoint found; refusing launch"
+    return 1
+  fi
+  write_runtime_config "$checkpoint" || return 1
+  cd "$REPO" || return 1
+  log "launch checkpoint=$checkpoint GPU=$GPU"
+  setsid nohup env \
+    CUDA_VISIBLE_DEVICES="$GPU" \
+    PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
+    TORCHINDUCTOR_CACHE_DIR="$BASE/.torchinductor_${JOB}" \
+    "$TORCHRUN" --standalone --nproc_per_node=1 \
+    "$TRAINER" --config "configs/multiseed/${JOB}.runtime.json" \
+    >> "$RUNLOG" 2>&1 < /dev/null &
+}
+
+if [ -f "$LOCK" ]; then
+  old_pid=$(cat "$LOCK" 2>/dev/null || true)
+  if [ -n "${old_pid:-}" ] && kill -0 "$old_pid" 2>/dev/null; then
+    exit 0
+  fi
+fi
+
+echo $$ > "$LOCK"
+log "watchdog start pid=$$ latest_step=$(latest_step)"
+
+if [ "$(process_count)" -eq 0 ] && [ "$(latest_step)" -lt "$TOTAL" ]; then
+  launch
+fi
+
+dead_checks=0
+while true; do
+  if [ "$(latest_step)" -ge "$TOTAL" ] 2>/dev/null; then
+    log "complete latest_step=$(latest_step)"
+    rm -f "$LOCK"
+    exit 0
+  fi
+  if [ "$(process_count)" -eq 0 ]; then
+    dead_checks=$((dead_checks + 1))
+  else
+    dead_checks=0
+  fi
+  if [ "$dead_checks" -ge 10 ]; then
+    log "process absent for 5 minutes; relaunching latest complete checkpoint"
+    launch
+    dead_checks=0
+  fi
+  sleep 30
+done

epfl_final/train_logs/watchdog_3b_dense_4gpu_exact.sh

@@ -0,0 +1,135 @@
+#!/bin/bash
+set -u
+
+BASE=/mloscratch/homes/navasard/sparse
+REPO="$BASE/SparseAtt-baselines"
+OUTDIR="$BASE/checkpoints/l8k3b_dense_60b"
+SOURCE_CFG="$REPO/configs/ladder8k/dense_3b_60b_4gpu_exact.json"
+RUNTIME_CFG="$REPO/configs/ladder8k/dense_3b_60b_4gpu_exact.runtime.json"
+TRAINER=train_resume_exact.py
+TORCHRUN=/mloscratch/homes/navasard/conda/bin/torchrun
+PYTHON=/mloscratch/homes/navasard/conda/bin/python3.13
+TOTAL=122070
+WLOG="$BASE/train_logs/dense8k3b_60b.4gpu.watch.log"
+RUNLOG="$BASE/train_logs/dense8k3b_60b.4gpu.run.log"
+LOCK="$BASE/train_logs/dense8k3b_60b.4gpu.lock"
+PROC_PAT="[t]rain_resume_exact.py --config configs/ladder8k/dense_3b_60b_4gpu_exact.runtime.json"
+
+export HF_HOME=/mloscratch/homes/navasard/hf_cache
+
+log() {
+  echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*" >> "$WLOG"
+}
+
+latest_checkpoint() {
+  "$PYTHON" - "$OUTDIR" <<'PY'
+from pathlib import Path
+import sys
+
+root = Path(sys.argv[1])
+candidates = sorted(
+    root.glob("run_*/checkpoint-*"),
+    key=lambda path: int(path.name.rsplit("-", 1)[1]),
+    reverse=True,
+)
+for checkpoint in candidates:
+    required = [checkpoint / "model.pt", checkpoint / "training_state.pt"]
+    required.extend(checkpoint / f"rng_state_rank{rank}.pt" for rank in range(4))
+    required.extend(
+        checkpoint / f"loader_gen_state_rank{rank}.pt" for rank in range(4)
+    )
+    if all(path.is_file() and path.stat().st_size > 0 for path in required):
+        print(checkpoint)
+        break
+PY
+}
+
+latest_step() {
+  local checkpoint
+  checkpoint=$(latest_checkpoint)
+  if [ -n "${checkpoint:-}" ]; then
+    basename "$checkpoint" | sed 's/checkpoint-//'
+  fi
+}
+
+is_complete() {
+  local step
+  step=$(latest_step)
+  [ -n "${step:-}" ] && [ "$step" -ge "$TOTAL" ] 2>/dev/null
+}
+
+process_count() {
+  pgrep -f "$PROC_PAT" 2>/dev/null | grep -c '[0-9]'
+}
+
+write_runtime_config() {
+  local checkpoint="$1"
+  "$PYTHON" - "$SOURCE_CFG" "$RUNTIME_CFG" "$checkpoint" <<'PY'
+import json
+from pathlib import Path
+import sys
+
+source, destination, checkpoint = map(Path, sys.argv[1:])
+config = json.loads(source.read_text())
+training = config["training"]
+training["resume_from_checkpoint"] = str(checkpoint)
+training["preserve_checkpoint_lr"] = True
+training["num_epochs"] = max(3, int(training.get("num_epochs", 0)))
+destination.write_text(json.dumps(config, indent=2) + "\n")
+PY
+}
+
+launch() {
+  local checkpoint
+  checkpoint=$(latest_checkpoint)
+  if [ -z "${checkpoint:-}" ]; then
+    log "no complete four-rank checkpoint found; refusing launch"
+    return 1
+  fi
+  write_runtime_config "$checkpoint" || return 1
+  cd "$REPO" || return 1
+  log "launch checkpoint=$checkpoint GPUs=0,1,2,3 nproc=4"
+  setsid nohup env \
+    CUDA_VISIBLE_DEVICES=0,1,2,3 \
+    PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
+    TORCHINDUCTOR_CACHE_DIR="$BASE/.torchinductor_3b_dense_4gpu" \
+    "$TORCHRUN" --standalone --nproc_per_node=4 \
+    "$TRAINER" --config configs/ladder8k/dense_3b_60b_4gpu_exact.runtime.json \
+    >> "$RUNLOG" 2>&1 < /dev/null &
+}
+
+if [ -f "$LOCK" ]; then
+  old_pid=$(cat "$LOCK" 2>/dev/null || true)
+  if [ -n "${old_pid:-}" ] && kill -0 "$old_pid" 2>/dev/null; then
+    exit 0
+  fi
+fi
+
+echo $$ > "$LOCK"
+log "watchdog start pid=$$ latest_step=$(latest_step)"
+
+if [ "$(process_count)" -eq 0 ] && ! is_complete; then
+  launch
+fi
+
+dead_checks=0
+while true; do
+  if is_complete; then
+    log "complete latest_step=$(latest_step)"
+    rm -f "$LOCK"
+    exit 0
+  fi
+
+  if [ "$(process_count)" -eq 0 ]; then
+    dead_checks=$((dead_checks + 1))
+  else
+    dead_checks=0
+  fi
+
+  if [ "$dead_checks" -ge 10 ]; then
+    log "process absent for 5 minutes; relaunching latest complete checkpoint"
+    launch
+    dead_checks=0
+  fi
+  sleep 30
+done

epfl_final/train_logs/watchdog_3b_dense_exact.sh

@@ -0,0 +1,151 @@
+#!/bin/bash
+set -u
+
+BASE=/mloscratch/homes/navasard/sparse
+REPO="$BASE/SparseAtt-baselines"
+OUTDIR="$BASE/checkpoints/l8k3b_dense_60b"
+SOURCE_CFG="$REPO/configs/ladder8k/dense_3b_60b_6gpu.json"
+RUNTIME_CFG="$REPO/configs/ladder8k/dense_3b_60b_6gpu.runtime.json"
+TRAINER=train_resume_exact.py
+TORCHRUN=/mloscratch/homes/navasard/conda/bin/torchrun
+PYTHON=/mloscratch/homes/navasard/conda/bin/python3.13
+TOTAL=122070
+MIGRATION_STEP=78000
+WLOG="$BASE/train_logs/dense8k3b_60b.watch.log"
+RUNLOG="$BASE/train_logs/dense8k3b_60b.run.log"
+LOCK="$BASE/train_logs/dense8k3b_60b.lock"
+PROC_PAT="[t]rain_resume_exact.py --config configs/ladder8k/dense_3b_60b_6gpu.runtime.json"
+
+export HF_HOME=/mloscratch/homes/navasard/hf_cache
+
+log() {
+  echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*" >> "$WLOG"
+}
+
+latest_checkpoint() {
+  "$PYTHON" - "$OUTDIR" "$MIGRATION_STEP" <<'PY'
+from pathlib import Path
+import sys
+
+root = Path(sys.argv[1])
+migration_step = int(sys.argv[2])
+candidates = sorted(
+    root.glob("run_*/checkpoint-*"),
+    key=lambda path: int(path.name.rsplit("-", 1)[1]),
+    reverse=True,
+)
+for checkpoint in candidates:
+    step = int(checkpoint.name.rsplit("-", 1)[1])
+    core = [checkpoint / "model.pt", checkpoint / "training_state.pt"]
+    six_rank = [
+        checkpoint / f"rng_state_rank{rank}.pt" for rank in range(6)
+    ]
+    six_rank.extend(
+        checkpoint / f"loader_gen_state_rank{rank}.pt" for rank in range(6)
+    )
+    four_rank = [
+        checkpoint / f"rng_state_rank{rank}.pt" for rank in range(4)
+    ]
+    four_rank.extend(
+        checkpoint / f"loader_gen_state_rank{rank}.pt" for rank in range(4)
+    )
+    complete = lambda paths: all(
+        path.is_file() and path.stat().st_size > 0 for path in paths
+    )
+    if complete(core) and (
+        complete(six_rank) or (step == migration_step and complete(four_rank))
+    ):
+        print(checkpoint)
+        break
+PY
+}
+
+latest_step() {
+  local checkpoint
+  checkpoint=$(latest_checkpoint)
+  if [ -n "${checkpoint:-}" ]; then
+    basename "$checkpoint" | sed 's/checkpoint-//'
+  fi
+}
+
+is_complete() {
+  local step
+  step=$(latest_step)
+  [ -n "${step:-}" ] && [ "$step" -ge "$TOTAL" ] 2>/dev/null
+}
+
+process_count() {
+  pgrep -f "$PROC_PAT" 2>/dev/null | grep -c '[0-9]'
+}
+
+write_runtime_config() {
+  local checkpoint="$1"
+  "$PYTHON" - "$SOURCE_CFG" "$RUNTIME_CFG" "$checkpoint" <<'PY'
+import json
+from pathlib import Path
+import sys
+
+source, destination, checkpoint = map(Path, sys.argv[1:])
+config = json.loads(source.read_text())
+training = config["training"]
+training["resume_from_checkpoint"] = str(checkpoint)
+training["preserve_checkpoint_lr"] = True
+training["num_epochs"] = max(3, int(training.get("num_epochs", 0)))
+destination.write_text(json.dumps(config, indent=2) + "\n")
+PY
+}
+
+launch() {
+  local checkpoint
+  checkpoint=$(latest_checkpoint)
+  if [ -z "${checkpoint:-}" ]; then
+    log "no complete six-rank or approved migration checkpoint found; refusing launch"
+    return 1
+  fi
+  write_runtime_config "$checkpoint" || return 1
+  cd "$REPO" || return 1
+  log "launch checkpoint=$checkpoint GPUs=0,1,2,3,4,5 nproc=6"
+  setsid nohup env \
+    CUDA_VISIBLE_DEVICES=0,1,2,3,4,5 \
+    PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
+    TORCHINDUCTOR_CACHE_DIR="$BASE/.torchinductor_3b_dense" \
+    "$TORCHRUN" --standalone --nproc_per_node=6 \
+    "$TRAINER" --config configs/ladder8k/dense_3b_60b_6gpu.runtime.json \
+    >> "$RUNLOG" 2>&1 < /dev/null &
+}
+
+if [ -f "$LOCK" ]; then
+  old_pid=$(cat "$LOCK" 2>/dev/null || true)
+  if [ -n "${old_pid:-}" ] && kill -0 "$old_pid" 2>/dev/null; then
+    exit 0
+  fi
+fi
+
+echo $$ > "$LOCK"
+log "watchdog start pid=$$ latest_step=$(latest_step)"
+
+if [ "$(process_count)" -eq 0 ] && ! is_complete; then
+  launch
+fi
+
+dead_checks=0
+while true; do
+  if is_complete; then
+    log "complete latest_step=$(latest_step)"
+    rm -f "$LOCK"
+    exit 0
+  fi
+
+  if [ "$(process_count)" -eq 0 ]; then
+    dead_checks=$((dead_checks + 1))
+  else
+    dead_checks=0
+  fi
+
+  if [ "$dead_checks" -ge 10 ]; then
+    log "process absent for 5 minutes; relaunching latest complete checkpoint"
+    launch
+    dead_checks=0
+  fi
+  sleep 30
+done

epfl_final/train_logs/watchdog_3b_mixed_4gpu_exact.sh

@@ -0,0 +1,134 @@
+#!/bin/bash
+set -u
+
+BASE=/mloscratch/homes/navasard/sparse
+REPO="$BASE/SparseAtt-baselines"
+OUTDIR="$BASE/checkpoints/l8k3b_mixed_60b"
+SOURCE_CFG="$REPO/configs/ladder8k/mixed_3b_60b_4gpu_exact.json"
+RUNTIME_CFG="$REPO/configs/ladder8k/mixed_3b_60b_4gpu_exact.runtime.json"
+TRAINER=train_resume_exact.py
+TORCHRUN=/mloscratch/homes/navasard/conda/bin/torchrun
+PYTHON=/mloscratch/homes/navasard/conda/bin/python3.13
+TOTAL=122070
+WLOG="$BASE/train_logs/mixed8k3b_60b.4gpu.exact.watch.log"
+RUNLOG="$BASE/train_logs/mixed8k3b_60b.4gpu.exact.run.log"
+LOCK="$BASE/train_logs/mixed8k3b_60b.4gpu.exact.lock"
+PROC_PAT="[t]rain_resume_exact.py --config configs/ladder8k/mixed_3b_60b_4gpu_exact.runtime.json"
+
+export HF_HOME=/mloscratch/homes/navasard/hf_cache
+
+log() {
+  echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*" >> "$WLOG"
+}
+
+latest_checkpoint() {
+  "$PYTHON" - "$OUTDIR" <<'PY'
+from pathlib import Path
+import sys
+
+root = Path(sys.argv[1])
+candidates = sorted(
+    root.glob("run_*/checkpoint-*"),
+    key=lambda path: int(path.name.rsplit("-", 1)[1]),
+    reverse=True,
+)
+for checkpoint in candidates:
+    required = [checkpoint / "model.pt", checkpoint / "training_state.pt"]
+    required.extend(checkpoint / f"rng_state_rank{rank}.pt" for rank in range(4))
+    required.extend(
+        checkpoint / f"loader_gen_state_rank{rank}.pt" for rank in range(4)
+    )
+    if all(path.is_file() and path.stat().st_size > 0 for path in required):
+        print(checkpoint)
+        break
+PY
+}
+
+latest_step() {
+  local checkpoint
+  checkpoint=$(latest_checkpoint)
+  if [ -n "${checkpoint:-}" ]; then
+    basename "$checkpoint" | sed 's/checkpoint-//'
+  fi
+}
+
+is_complete() {
+  local step
+  step=$(latest_step)
+  [ -n "${step:-}" ] && [ "$step" -ge "$TOTAL" ] 2>/dev/null
+}
+
+process_count() {
+  pgrep -f "$PROC_PAT" 2>/dev/null | grep -c '[0-9]'
+}
+
+write_runtime_config() {
+  local checkpoint="$1"
+  "$PYTHON" - "$SOURCE_CFG" "$RUNTIME_CFG" "$checkpoint" <<'PY'
+import json
+from pathlib import Path
+import sys
+
+source, destination, checkpoint = map(Path, sys.argv[1:])
+config = json.loads(source.read_text())
+training = config["training"]
+training["resume_from_checkpoint"] = str(checkpoint)
+training["preserve_checkpoint_lr"] = True
+destination.write_text(json.dumps(config, indent=2) + "\n")
+PY
+}
+
+launch() {
+  local checkpoint
+  checkpoint=$(latest_checkpoint)
+  if [ -z "${checkpoint:-}" ]; then
+    log "no complete four-rank checkpoint found; refusing launch"
+    return 1
+  fi
+  write_runtime_config "$checkpoint" || return 1
+  cd "$REPO" || return 1
+  log "launch checkpoint=$checkpoint GPUs=0,1,2,3 nproc=4"
+  setsid nohup env \
+    CUDA_VISIBLE_DEVICES=0,1,2,3 \
+    PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
+    TORCHINDUCTOR_CACHE_DIR="$BASE/.torchinductor_3b_mixed_4gpu" \
+    "$TORCHRUN" --standalone --nproc_per_node=4 \
+    "$TRAINER" --config configs/ladder8k/mixed_3b_60b_4gpu_exact.runtime.json \
+    >> "$RUNLOG" 2>&1 < /dev/null &
+}
+
+if [ -f "$LOCK" ]; then
+  old_pid=$(cat "$LOCK" 2>/dev/null || true)
+  if [ -n "${old_pid:-}" ] && kill -0 "$old_pid" 2>/dev/null; then
+    exit 0
+  fi
+fi
+
+echo $$ > "$LOCK"
+log "watchdog start pid=$$ latest_step=$(latest_step)"
+
+if [ "$(process_count)" -eq 0 ] && ! is_complete; then
+  launch
+fi
+
+dead_checks=0
+while true; do
+  if is_complete; then
+    log "complete latest_step=$(latest_step)"
+    rm -f "$LOCK"
+    exit 0
+  fi
+
+  if [ "$(process_count)" -eq 0 ]; then
+    dead_checks=$((dead_checks + 1))
+  else
+    dead_checks=0
+  fi
+
+  if [ "$dead_checks" -ge 10 ]; then
+    log "process absent for 5 minutes; relaunching latest complete checkpoint"
+    launch
+    dead_checks=0
+  fi
+  sleep 30
+done