EPFL final checkpoints and W&B history 951-1000 of 1463

.gitattributes

@@ -124,3 +124,10 @@ epfl_final/SparseAtt/wandb/run-20260822_043611-xvizh3ih/run-xvizh3ih.wandb filte
 epfl_final/SparseAtt/wandb/run-20260822_043614-pq6ov2y3/run-pq6ov2y3.wandb filter=lfs diff=lfs merge=lfs -text
 epfl_final/SparseAtt/wandb/run-20260822_043615-8naq20vs/run-8naq20vs.wandb filter=lfs diff=lfs merge=lfs -text
 epfl_final/SparseAtt/wandb/run-20260822_115228-8xh3oikj/run-8xh3oikj.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260822_133517-81uee6bn/run-81uee6bn.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260822_143443-9nppzrru/run-9nppzrru.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260822_161845-8xh3oikj/run-8xh3oikj.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260822_161907-9nppzrru/run-9nppzrru.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260822_210348-i8ewyy95/run-i8ewyy95.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260823_041148-i8ewyy95/run-i8ewyy95.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260823_043841-rgmkfpnl/run-rgmkfpnl.wandb filter=lfs diff=lfs merge=lfs -text

epfl_final/SparseAtt/wandb/run-20260822_133517-81uee6bn/run-81uee6bn.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f886d9fee718c3a5206f5e63e80163d3692fc8380c825938e3ce3b290836524
+size 10023668

epfl_final/SparseAtt/wandb/run-20260822_143443-9nppzrru/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260822_143443-9nppzrru/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-08-22T14:34:43.097867Z",
+  "args":  [
+    "--config",
+    "configs/p1_scheduled_h1004_20260822/l8k365m_resampled_nrb35_seed45.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "5637886f0cc760009989b14e68c6ae06d4827418"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-workspace-4-gpu-3-0-8",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "9371709865984",
+      "used":  "6533931311104"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-37a1c6c2-d3df-3064-44df-33fa82304bd0"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-9a903f43-9348-6fa6-bcf7-e12bd01803a7"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-24b8d952-dc08-33b9-d5df-54ce871fd288"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-b87e8111-63b6-b364-28b9-c4ce747aec08"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "cbaqtxrs3at22jmk9fd4mv41hetb6rl6"
+}

epfl_final/SparseAtt/wandb/run-20260822_143443-9nppzrru/logs/debug.log

@@ -0,0 +1,19 @@
+2026-08-22 14:34:43,154 INFO    MainThread:11913 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-08-22 14:34:43,154 INFO    MainThread:11913 [wandb_setup.py:_flush():81] Configure stats pid to 11913
+2026-08-22 14:34:43,155 INFO    MainThread:11913 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-22 14:34:43,155 INFO    MainThread:11913 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_143443-9nppzrru/logs/debug.log
+2026-08-22 14:34:43,155 INFO    MainThread:11913 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_143443-9nppzrru/logs/debug-internal.log
+2026-08-22 14:34:43,155 INFO    MainThread:11913 [wandb_init.py:init():850] calling init triggers
+2026-08-22 14:34:43,155 INFO    MainThread:11913 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256', 'resume_from_checkpoint': None, 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 5, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 45, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'P1 DDP4 365m resampled 8k nrb35 seed45', 'wandb_tags': ['p1', 'structured-control', 'qwen3', '365m', '8k', 'resampled', 'nrb35', 'seed45', 'ddp4', 'h1004', 'scheduled'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_resampled_nrb35_s0', 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-08-22 14:34:43,155 INFO    MainThread:11913 [wandb_init.py:init():898] starting backend
+2026-08-22 14:34:43,466 INFO    MainThread:11913 [wandb_init.py:init():913] sending inform_init request
+2026-08-22 14:34:43,899 INFO    MainThread:11913 [wandb_init.py:init():918] backend started and connected
+2026-08-22 14:34:43,900 INFO    MainThread:11913 [wandb_init.py:init():988] updated telemetry
+2026-08-22 14:34:43,919 INFO    MainThread:11913 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-08-22 14:34:44,299 INFO    MainThread:11913 [wandb_init.py:init():1056] starting run threads in backend
+2026-08-22 14:34:44,575 INFO    MainThread:11913 [wandb_run.py:_console_start():2554] atexit reg
+2026-08-22 14:34:44,575 INFO    MainThread:11913 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-08-22 14:34:44,575 INFO    MainThread:11913 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-08-22 14:34:44,575 INFO    MainThread:11913 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-08-22 14:34:44,581 INFO    MainThread:11913 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/run-20260822_143443-9nppzrru/run-9nppzrru.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2f3ac6042b7f5e466166826d75c402798be8c6bb467d4cd8fe6280b2d6c6f24
+size 3047424

epfl_final/SparseAtt/wandb/run-20260822_161845-8xh3oikj/files/config.yaml

@@ -0,0 +1,190 @@
+_wandb:
+    value:
+        cli_version: 0.26.1
+        e:
+            0vqc90e8r076jfrxgonbbfpa2krs4vxw:
+                args:
+                    - --config
+                    - configs/p1_scheduled_h1002_20260822/l8k365m_resampled_nrb35_seed46.json
+                codePath: train.py
+                codePathLocal: train.py
+                cpu_count: 96
+                cpu_count_logical: 192
+                cudaVersion: "13.0"
+                disk:
+                    /:
+                        total: "9374931091456"
+                        used: "6412527095808"
+                email: ruben.navasardyan@mail.utoronto.ca
+                executable: /mloscratch/homes/navasard/conda/bin/python
+                git:
+                    commit: 5637886f0cc760009989b14e68c6ae06d4827418
+                    remote: https://github.com/rubentium/TokenPrune.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 2
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-af79466d-4300-4171-b5bd-7697eca27264
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-0460dc7d-d83e-fd0f-7548-a3079803b79e
+                host: h100-workspace-2-gpu-0-9
+                memory:
+                    total: "1623005732864"
+                os: Linux-6.8.0-124-generic-x86_64-with-glibc2.35
+                program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
+                python: CPython 3.13.13
+                root: /mloscratch/homes/navasard/sparse/SparseAtt
+                startedAt: "2026-08-22T16:18:45.521663Z"
+                writerId: 0vqc90e8r076jfrxgonbbfpa2krs4vxw
+        m: []
+        python_version: 3.13.13
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 5
+                - 13
+                - 14
+                - 15
+                - 16
+                - 61
+                - 62
+            "4": 3.13.13
+            "5": 0.26.1
+            "6": 5.7.0
+            "12": 0.26.1
+            "13": linux-x86_64
+attention_dropout:
+    value: null
+attention_type:
+    value: sparse_stochastic
+checkpoint_activations:
+    value: false
+dense_mode:
+    value: false
+dense_step_intervals:
+    value: null
+eval_batches:
+    value: 50
+eval_steps:
+    value: 250
+force_new_run:
+    value: false
+gradient_accumulation_steps:
+    value: 5
+head_dim:
+    value: null
+hidden_size:
+    value: 768
+init_from_scratch:
+    value: true
+intermediate_size:
+    value: 3072
+learning_rate:
+    value: 0.0005
+logging_steps:
+    value: 10
+lr_scheduler:
+    value: cosine
+max_grad_norm:
+    value: 1
+max_seq_length:
+    value: 8192
+max_steps:
+    value: 44000
+mixed_precision:
+    value: bf16
+model_dtype:
+    value: bfloat16
+model_path:
+    value: /mloscratch/homes/navasard/qwen3_06b_base
+num_attention_heads:
+    value: 6
+num_epochs:
+    value: 1
+num_hidden_layers:
+    value: 28
+num_key_value_heads:
+    value: 2
+num_workers:
+    value: 8
+output_dir:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035
+per_device_batch_size:
+    value: 2
+resume_from_checkpoint:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-21000
+rope_train_position_scale_set:
+    value: null
+save_steps:
+    value: 1000
+seed:
+    value: 46
+sparse_block_size:
+    value: 128
+sparse_layer_indices:
+    value: null
+sparse_mask_cache_dir:
+    value: /mloscratch/homes/navasard/sparse/mask_cache_8k_resampled_nrb35_s0
+sparse_mask_pool_size:
+    value: 10000
+sparse_mask_seed:
+    value: 0
+sparse_mixed_nrb_set:
+    value: null
+sparse_num_random_blocks:
+    value: 35
+sparse_val_dense_compare:
+    value: false
+strategy:
+    value: ddp
+train_path:
+    value: /mloscratch/homes/navasard/data/fineweb_large/tokenized
+val_path:
+    value: null
+verbose:
+    value: true
+wandb_enabled:
+    value: true
+wandb_name:
+    value: P1 DDP2 365m resampled 8k nrb35 seed46
+wandb_project:
+    value: stochastic_sparsity
+wandb_tags:
+    value:
+        - p1
+        - structured-control
+        - qwen3
+        - 365m
+        - 8k
+        - resampled
+        - nrb35
+        - seed46
+        - ddp2
+        - h1002
+        - scheduled
+warmup_steps:
+    value: 750
+weight_decay:
+    value: 0.01

epfl_final/SparseAtt/wandb/run-20260822_161845-8xh3oikj/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260822_161845-8xh3oikj/files/wandb-metadata.json

@@ -0,0 +1,51 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-08-22T16:18:45.521663Z",
+  "args":  [
+    "--config",
+    "configs/p1_scheduled_h1002_20260822/l8k365m_resampled_nrb35_seed46.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "5637886f0cc760009989b14e68c6ae06d4827418"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-workspace-2-gpu-0-9",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  2,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "6412527095808"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-af79466d-4300-4171-b5bd-7697eca27264"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-0460dc7d-d83e-fd0f-7548-a3079803b79e"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "0vqc90e8r076jfrxgonbbfpa2krs4vxw"
+}

epfl_final/SparseAtt/wandb/run-20260822_161845-8xh3oikj/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":32380},"_step":44000,"eval/uses_dense_attention":0,"speed/ms_per_step":691.0484075546265,"speed/steps_per_sec":1.4470766288842816,"eval/dense_interval_end":-1,"_wandb.runtime":15524,"train/learning_rate":0,"eval/loss":2.9298389625549315,"eval/perplexity":18.72461488972488,"train/epoch":0.2906439083679023,"eval/dense_interval_start":-1,"_runtime":32380.649742912,"eval/uses_sparse_attention":1,"eval/dense_interval_active":0,"train/global_step":44000,"train/loss":2.859005928039551,"_timestamp":1.787432377748786e+09,"speed/samples_per_sec":28.94153257768563}

epfl_final/SparseAtt/wandb/run-20260822_161845-8xh3oikj/logs/debug.log

@@ -0,0 +1,25 @@
+2026-08-22 16:18:45,554 INFO    MainThread:207 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-08-22 16:18:45,554 INFO    MainThread:207 [wandb_setup.py:_flush():81] Configure stats pid to 207
+2026-08-22 16:18:45,554 INFO    MainThread:207 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-22 16:18:45,555 INFO    MainThread:207 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_161845-8xh3oikj/logs/debug.log
+2026-08-22 16:18:45,558 INFO    MainThread:207 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_161845-8xh3oikj/logs/debug-internal.log
+2026-08-22 16:18:45,558 INFO    MainThread:207 [wandb_init.py:init():850] calling init triggers
+2026-08-22 16:18:45,558 INFO    MainThread:207 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed46/run_20260822_115035/checkpoint-21000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 5, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 46, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'P1 DDP2 365m resampled 8k nrb35 seed46', 'wandb_tags': ['p1', 'structured-control', 'qwen3', '365m', '8k', 'resampled', 'nrb35', 'seed46', 'ddp2', 'h1002', 'scheduled'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_resampled_nrb35_s0', 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-08-22 16:18:45,558 INFO    MainThread:207 [wandb_init.py:init():898] starting backend
+2026-08-22 16:18:46,488 INFO    MainThread:207 [wandb_init.py:init():913] sending inform_init request
+2026-08-22 16:18:46,923 INFO    MainThread:207 [wandb_init.py:init():918] backend started and connected
+2026-08-22 16:18:46,926 INFO    MainThread:207 [wandb_init.py:init():988] updated telemetry
+2026-08-22 16:18:46,954 INFO    MainThread:207 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-08-22 16:18:47,483 INFO    MainThread:207 [wandb_init.py:init():1051] run resumed
+2026-08-22 16:18:47,485 INFO    MainThread:207 [wandb_init.py:init():1056] starting run threads in backend
+2026-08-22 16:18:47,694 INFO    MainThread:207 [wandb_run.py:_console_start():2554] atexit reg
+2026-08-22 16:18:47,694 INFO    MainThread:207 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-08-22 16:18:47,694 INFO    MainThread:207 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-08-22 16:18:47,694 INFO    MainThread:207 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-08-22 16:18:47,701 INFO    MainThread:207 [wandb_init.py:init():1094] run started, returning control to user process
+2026-08-22 20:59:44,133 INFO    MainThread:207 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/8xh3oikj
+2026-08-22 20:59:44,134 INFO    MainThread:207 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
+2026-08-22 20:59:44,135 INFO    MainThread:207 [wandb_run.py:_restore():2502] restore
+2026-08-22 20:59:44,135 INFO    MainThread:207 [wandb_run.py:_restore():2508] restore done
+2026-08-22 20:59:45,701 INFO    MainThread:207 [wandb_run.py:_footer_sync_info():3889] logging synced files

epfl_final/SparseAtt/wandb/run-20260822_161845-8xh3oikj/run-8xh3oikj.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69fc801958838060823721d09222e69e48d51c01d41f93846895634013ab924c
+size 5481444

epfl_final/SparseAtt/wandb/run-20260822_161907-9nppzrru/files/config.yaml

@@ -0,0 +1,200 @@
+_wandb:
+    value:
+        cli_version: 0.26.1
+        e:
+            2julent36u73idzfye4y5gn025rll495:
+                args:
+                    - --config
+                    - configs/p1_scheduled_h1004_20260822/l8k365m_resampled_nrb35_seed45.json
+                codePath: train.py
+                codePathLocal: train.py
+                cpu_count: 96
+                cpu_count_logical: 192
+                cudaVersion: "13.0"
+                disk:
+                    /:
+                        total: "9371709865984"
+                        used: "6533732798464"
+                email: ruben.navasardyan@mail.utoronto.ca
+                executable: /mloscratch/homes/navasard/conda/bin/python
+                git:
+                    commit: 5637886f0cc760009989b14e68c6ae06d4827418
+                    remote: https://github.com/rubentium/TokenPrune.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 4
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-9a903f43-9348-6fa6-bcf7-e12bd01803a7
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-d0294574-a180-e984-6ca3-0573e9724f2e
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-24b8d952-dc08-33b9-d5df-54ce871fd288
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-b87e8111-63b6-b364-28b9-c4ce747aec08
+                host: h100-workspace-4-gpu-3-0-9
+                memory:
+                    total: "1623005732864"
+                os: Linux-6.8.0-124-generic-x86_64-with-glibc2.35
+                program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
+                python: CPython 3.13.13
+                root: /mloscratch/homes/navasard/sparse/SparseAtt
+                startedAt: "2026-08-22T16:19:07.033644Z"
+                writerId: 2julent36u73idzfye4y5gn025rll495
+        m: []
+        python_version: 3.13.13
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 5
+                - 13
+                - 14
+                - 15
+                - 16
+                - 61
+                - 62
+            "4": 3.13.13
+            "5": 0.26.1
+            "6": 5.7.0
+            "12": 0.26.1
+            "13": linux-x86_64
+attention_dropout:
+    value: null
+attention_type:
+    value: sparse_stochastic
+checkpoint_activations:
+    value: false
+dense_mode:
+    value: false
+dense_step_intervals:
+    value: null
+eval_batches:
+    value: 50
+eval_steps:
+    value: 250
+force_new_run:
+    value: false
+gradient_accumulation_steps:
+    value: 5
+head_dim:
+    value: null
+hidden_size:
+    value: 768
+init_from_scratch:
+    value: true
+intermediate_size:
+    value: 3072
+learning_rate:
+    value: 0.0005
+logging_steps:
+    value: 10
+lr_scheduler:
+    value: cosine
+max_grad_norm:
+    value: 1
+max_seq_length:
+    value: 8192
+max_steps:
+    value: 44000
+mixed_precision:
+    value: bf16
+model_dtype:
+    value: bfloat16
+model_path:
+    value: /mloscratch/homes/navasard/qwen3_06b_base
+num_attention_heads:
+    value: 6
+num_epochs:
+    value: 1
+num_hidden_layers:
+    value: 28
+num_key_value_heads:
+    value: 2
+num_workers:
+    value: 8
+output_dir:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256
+per_device_batch_size:
+    value: 1
+resume_from_checkpoint:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-13000
+rope_train_position_scale_set:
+    value: null
+save_steps:
+    value: 1000
+seed:
+    value: 45
+sparse_block_size:
+    value: 128
+sparse_layer_indices:
+    value: null
+sparse_mask_cache_dir:
+    value: /mloscratch/homes/navasard/sparse/mask_cache_8k_resampled_nrb35_s0
+sparse_mask_pool_size:
+    value: 10000
+sparse_mask_seed:
+    value: 0
+sparse_mixed_nrb_set:
+    value: null
+sparse_num_random_blocks:
+    value: 35
+sparse_val_dense_compare:
+    value: false
+strategy:
+    value: ddp
+train_path:
+    value: /mloscratch/homes/navasard/data/fineweb_large/tokenized
+val_path:
+    value: null
+verbose:
+    value: true
+wandb_enabled:
+    value: true
+wandb_name:
+    value: P1 DDP4 365m resampled 8k nrb35 seed45
+wandb_project:
+    value: stochastic_sparsity
+wandb_tags:
+    value:
+        - p1
+        - structured-control
+        - qwen3
+        - 365m
+        - 8k
+        - resampled
+        - nrb35
+        - seed45
+        - ddp4
+        - h1004
+        - scheduled
+warmup_steps:
+    value: 750
+weight_decay:
+    value: 0.01

epfl_final/SparseAtt/wandb/run-20260822_161907-9nppzrru/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260822_161907-9nppzrru/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-08-22T16:19:07.033644Z",
+  "args":  [
+    "--config",
+    "configs/p1_scheduled_h1004_20260822/l8k365m_resampled_nrb35_seed45.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "5637886f0cc760009989b14e68c6ae06d4827418"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-workspace-4-gpu-3-0-9",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "9371709865984",
+      "used":  "6533732798464"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-9a903f43-9348-6fa6-bcf7-e12bd01803a7"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-d0294574-a180-e984-6ca3-0573e9724f2e"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-24b8d952-dc08-33b9-d5df-54ce871fd288"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-b87e8111-63b6-b364-28b9-c4ce747aec08"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "2julent36u73idzfye4y5gn025rll495"
+}

epfl_final/SparseAtt/wandb/run-20260822_161907-9nppzrru/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":18651},"eval/uses_dense_attention":0,"eval/perplexity":18.510780300655572,"_wandb.runtime":5588,"train/epoch":0.2906439083679023,"speed/steps_per_sec":2.4873652604306247,"_runtime":18651.342134185,"eval/loss":2.918353281323872,"_timestamp":1.7874286057213173e+09,"train/global_step":44000,"train/learning_rate":0,"eval/uses_sparse_attention":1,"speed/samples_per_sec":49.747305208612495,"eval/dense_interval_start":-1,"eval/dense_interval_end":-1,"speed/ms_per_step":402.0318269729614,"eval/dense_interval_active":0,"_step":44000,"train/loss":2.8272599625587462}

epfl_final/SparseAtt/wandb/run-20260822_161907-9nppzrru/logs/debug.log

@@ -0,0 +1,25 @@
+2026-08-22 16:19:07,062 INFO    MainThread:236 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-08-22 16:19:07,063 INFO    MainThread:236 [wandb_setup.py:_flush():81] Configure stats pid to 236
+2026-08-22 16:19:07,063 INFO    MainThread:236 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-22 16:19:07,063 INFO    MainThread:236 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_161907-9nppzrru/logs/debug.log
+2026-08-22 16:19:07,063 INFO    MainThread:236 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_161907-9nppzrru/logs/debug-internal.log
+2026-08-22 16:19:07,063 INFO    MainThread:236 [wandb_init.py:init():850] calling init triggers
+2026-08-22 16:19:07,063 INFO    MainThread:236 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed45/run_20260822_143256/checkpoint-13000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 5, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 45, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'P1 DDP4 365m resampled 8k nrb35 seed45', 'wandb_tags': ['p1', 'structured-control', 'qwen3', '365m', '8k', 'resampled', 'nrb35', 'seed45', 'ddp4', 'h1004', 'scheduled'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_resampled_nrb35_s0', 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-08-22 16:19:07,063 INFO    MainThread:236 [wandb_init.py:init():898] starting backend
+2026-08-22 16:19:07,879 INFO    MainThread:236 [wandb_init.py:init():913] sending inform_init request
+2026-08-22 16:19:08,330 INFO    MainThread:236 [wandb_init.py:init():918] backend started and connected
+2026-08-22 16:19:08,332 INFO    MainThread:236 [wandb_init.py:init():988] updated telemetry
+2026-08-22 16:19:08,361 INFO    MainThread:236 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-08-22 16:19:08,831 INFO    MainThread:236 [wandb_init.py:init():1051] run resumed
+2026-08-22 16:19:08,834 INFO    MainThread:236 [wandb_init.py:init():1056] starting run threads in backend
+2026-08-22 16:19:09,131 INFO    MainThread:236 [wandb_run.py:_console_start():2554] atexit reg
+2026-08-22 16:19:09,131 INFO    MainThread:236 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-08-22 16:19:09,131 INFO    MainThread:236 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-08-22 16:19:09,131 INFO    MainThread:236 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-08-22 16:19:09,140 INFO    MainThread:236 [wandb_init.py:init():1094] run started, returning control to user process
+2026-08-22 19:56:52,174 INFO    MainThread:236 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/9nppzrru
+2026-08-22 19:56:52,175 INFO    MainThread:236 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
+2026-08-22 19:56:52,176 INFO    MainThread:236 [wandb_run.py:_restore():2502] restore
+2026-08-22 19:56:52,176 INFO    MainThread:236 [wandb_run.py:_restore():2508] restore done
+2026-08-22 19:56:53,905 INFO    MainThread:236 [wandb_run.py:_footer_sync_info():3889] logging synced files

epfl_final/SparseAtt/wandb/run-20260822_161907-9nppzrru/run-9nppzrru.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57dd155d40f0ecc09d02e1674bbd90cd2df21a4161f2a85a2af7b06ce6b04e47
+size 7137612

epfl_final/SparseAtt/wandb/run-20260822_210348-i8ewyy95/files/output.log

@@ -0,0 +1,350 @@
+Starting training...
+Switching attention mode for step 1: sparse_stochastic (configured sparse attention)
+Step 10/50462 | Loss: 12.2285 | LR: 6.67e-06 | 13558.2ms/step | 4.4 samples/s
+Step 20/50462 | Loss: 11.9217 | LR: 1.33e-05 | 7347.1ms/step | 8.2 samples/s
+Step 30/50462 | Loss: 11.0220 | LR: 2.00e-05 | 7347.0ms/step | 8.2 samples/s
+Step 40/50462 | Loss: 10.1514 | LR: 2.67e-05 | 7315.7ms/step | 8.2 samples/s
+Step 50/50462 | Loss: 9.7299 | LR: 3.33e-05 | 7338.4ms/step | 8.2 samples/s
+Step 60/50462 | Loss: 9.2932 | LR: 4.00e-05 | 7295.1ms/step | 8.2 samples/s
+Step 70/50462 | Loss: 8.7279 | LR: 4.67e-05 | 7344.8ms/step | 8.2 samples/s
+Step 80/50462 | Loss: 8.2714 | LR: 5.33e-05 | 7343.8ms/step | 8.2 samples/s
+Step 90/50462 | Loss: 7.8736 | LR: 6.00e-05 | 7312.5ms/step | 8.2 samples/s
+Step 100/50462 | Loss: 7.4990 | LR: 6.67e-05 | 7328.6ms/step | 8.2 samples/s
+Step 110/50462 | Loss: 7.2479 | LR: 7.33e-05 | 7341.2ms/step | 8.2 samples/s
+Step 120/50462 | Loss: 7.0941 | LR: 8.00e-05 | 7335.0ms/step | 8.2 samples/s
+Step 130/50462 | Loss: 6.9836 | LR: 8.67e-05 | 7345.7ms/step | 8.2 samples/s
+Step 140/50462 | Loss: 6.8738 | LR: 9.33e-05 | 7361.3ms/step | 8.2 samples/s
+Step 150/50462 | Loss: 6.7698 | LR: 1.00e-04 | 7359.0ms/step | 8.2 samples/s
+Step 160/50462 | Loss: 6.6554 | LR: 1.07e-04 | 7384.1ms/step | 8.1 samples/s
+Step 170/50462 | Loss: 6.5238 | LR: 1.13e-04 | 7386.8ms/step | 8.1 samples/s
+Step 180/50462 | Loss: 6.4588 | LR: 1.20e-04 | 7359.1ms/step | 8.2 samples/s
+Step 190/50462 | Loss: 6.3743 | LR: 1.27e-04 | 7343.1ms/step | 8.2 samples/s
+Step 200/50462 | Loss: 6.2806 | LR: 1.33e-04 | 7351.1ms/step | 8.2 samples/s
+Step 210/50462 | Loss: 6.2474 | LR: 1.40e-04 | 7363.2ms/step | 8.1 samples/s
+Step 220/50462 | Loss: 6.2108 | LR: 1.47e-04 | 7324.9ms/step | 8.2 samples/s
+Step 230/50462 | Loss: 6.1300 | LR: 1.53e-04 | 7372.7ms/step | 8.1 samples/s
+Step 240/50462 | Loss: 6.0595 | LR: 1.60e-04 | 7399.2ms/step | 8.1 samples/s
+Step 250/50462 | Loss: 6.0046 | LR: 1.67e-04 | 7362.8ms/step | 8.1 samples/s
+Validation at step 250: Loss = 5.9596, Perplexity = 387.45
+Step 260/50462 | Loss: 5.9345 | LR: 1.73e-04 | 15108.8ms/step | 4.0 samples/s
+Step 270/50462 | Loss: 5.8871 | LR: 1.80e-04 | 7355.1ms/step | 8.2 samples/s
+Step 280/50462 | Loss: 5.8307 | LR: 1.87e-04 | 7401.4ms/step | 8.1 samples/s
+Step 290/50462 | Loss: 5.7848 | LR: 1.93e-04 | 7368.6ms/step | 8.1 samples/s
+Step 300/50462 | Loss: 5.7138 | LR: 2.00e-04 | 7369.7ms/step | 8.1 samples/s
+Step 310/50462 | Loss: 5.6581 | LR: 2.07e-04 | 7373.0ms/step | 8.1 samples/s
+Step 320/50462 | Loss: 5.6094 | LR: 2.13e-04 | 7385.3ms/step | 8.1 samples/s
+Step 330/50462 | Loss: 5.5933 | LR: 2.20e-04 | 7341.1ms/step | 8.2 samples/s
+Step 340/50462 | Loss: 5.5433 | LR: 2.27e-04 | 7334.3ms/step | 8.2 samples/s
+Step 350/50462 | Loss: 5.4962 | LR: 2.33e-04 | 7377.7ms/step | 8.1 samples/s
+Step 360/50462 | Loss: 5.4527 | LR: 2.40e-04 | 7381.3ms/step | 8.1 samples/s
+Step 370/50462 | Loss: 5.3898 | LR: 2.47e-04 | 7373.7ms/step | 8.1 samples/s
+Step 380/50462 | Loss: 5.3137 | LR: 2.53e-04 | 7353.7ms/step | 8.2 samples/s
+Step 390/50462 | Loss: 5.2732 | LR: 2.60e-04 | 7360.9ms/step | 8.2 samples/s
+Step 400/50462 | Loss: 5.2140 | LR: 2.67e-04 | 7363.3ms/step | 8.1 samples/s
+Step 410/50462 | Loss: 5.2023 | LR: 2.73e-04 | 7377.4ms/step | 8.1 samples/s
+Step 420/50462 | Loss: 5.1795 | LR: 2.80e-04 | 7380.2ms/step | 8.1 samples/s
+Step 430/50462 | Loss: 5.1245 | LR: 2.87e-04 | 7372.2ms/step | 8.1 samples/s
+Step 440/50462 | Loss: 5.0656 | LR: 2.93e-04 | 7356.0ms/step | 8.2 samples/s
+Step 450/50462 | Loss: 5.0408 | LR: 3.00e-04 | 7371.9ms/step | 8.1 samples/s
+Step 460/50462 | Loss: 5.0109 | LR: 3.07e-04 | 7383.0ms/step | 8.1 samples/s
+Step 470/50462 | Loss: 5.0111 | LR: 3.13e-04 | 7364.5ms/step | 8.1 samples/s
+Step 480/50462 | Loss: 4.9881 | LR: 3.20e-04 | 7383.8ms/step | 8.1 samples/s
+Step 490/50462 | Loss: 4.8550 | LR: 3.27e-04 | 7361.0ms/step | 8.2 samples/s
+Step 500/50462 | Loss: 4.8979 | LR: 3.33e-04 | 7376.8ms/step | 8.1 samples/s
+Validation at step 500: Loss = 4.9163, Perplexity = 136.50
+Step 510/50462 | Loss: 4.9215 | LR: 3.40e-04 | 11221.6ms/step | 5.3 samples/s
+Step 520/50462 | Loss: 4.8483 | LR: 3.47e-04 | 7382.2ms/step | 8.1 samples/s
+Step 530/50462 | Loss: 4.7789 | LR: 3.53e-04 | 7335.3ms/step | 8.2 samples/s
+Step 540/50462 | Loss: 4.7686 | LR: 3.60e-04 | 7365.3ms/step | 8.1 samples/s
+Step 550/50462 | Loss: 4.8117 | LR: 3.67e-04 | 7364.1ms/step | 8.1 samples/s
+Step 560/50462 | Loss: 4.7064 | LR: 3.73e-04 | 7379.0ms/step | 8.1 samples/s
+Step 570/50462 | Loss: 4.6854 | LR: 3.80e-04 | 7355.6ms/step | 8.2 samples/s
+Step 580/50462 | Loss: 4.6726 | LR: 3.87e-04 | 7331.4ms/step | 8.2 samples/s
+Step 590/50462 | Loss: 4.5973 | LR: 3.93e-04 | 7381.6ms/step | 8.1 samples/s
+Step 600/50462 | Loss: 4.5529 | LR: 4.00e-04 | 7349.9ms/step | 8.2 samples/s
+Step 610/50462 | Loss: 4.6087 | LR: 4.07e-04 | 7334.5ms/step | 8.2 samples/s
+Step 620/50462 | Loss: 4.6407 | LR: 4.13e-04 | 7371.9ms/step | 8.1 samples/s
+Step 630/50462 | Loss: 4.5022 | LR: 4.20e-04 | 7390.2ms/step | 8.1 samples/s
+Step 640/50462 | Loss: 4.4227 | LR: 4.27e-04 | 7391.0ms/step | 8.1 samples/s
+Step 650/50462 | Loss: 4.4289 | LR: 4.33e-04 | 7363.4ms/step | 8.1 samples/s
+Step 660/50462 | Loss: 4.4917 | LR: 4.40e-04 | 7351.9ms/step | 8.2 samples/s
+Step 670/50462 | Loss: 4.4139 | LR: 4.47e-04 | 7390.2ms/step | 8.1 samples/s
+Step 680/50462 | Loss: 4.3439 | LR: 4.53e-04 | 7400.2ms/step | 8.1 samples/s
+Step 690/50462 | Loss: 4.2979 | LR: 4.60e-04 | 7343.6ms/step | 8.2 samples/s
+Step 700/50462 | Loss: 4.3337 | LR: 4.67e-04 | 7367.1ms/step | 8.1 samples/s
+Step 710/50462 | Loss: 4.3510 | LR: 4.73e-04 | 7343.7ms/step | 8.2 samples/s
+Step 720/50462 | Loss: 4.3167 | LR: 4.80e-04 | 7353.9ms/step | 8.2 samples/s
+Step 730/50462 | Loss: 4.2202 | LR: 4.87e-04 | 7370.9ms/step | 8.1 samples/s
+Step 740/50462 | Loss: 4.1858 | LR: 4.93e-04 | 7374.4ms/step | 8.1 samples/s
+Step 750/50462 | Loss: 4.2042 | LR: 5.00e-04 | 7381.0ms/step | 8.1 samples/s
+Validation at step 750: Loss = 4.1897, Perplexity = 66.01
+Step 760/50462 | Loss: 4.1547 | LR: 5.00e-04 | 11212.7ms/step | 5.4 samples/s
+Step 770/50462 | Loss: 4.1554 | LR: 5.00e-04 | 7374.2ms/step | 8.1 samples/s
+Step 780/50462 | Loss: 4.1212 | LR: 5.00e-04 | 7378.7ms/step | 8.1 samples/s
+Step 790/50462 | Loss: 4.0891 | LR: 5.00e-04 | 7378.7ms/step | 8.1 samples/s
+Step 800/50462 | Loss: 4.0808 | LR: 5.00e-04 | 7383.1ms/step | 8.1 samples/s
+Step 810/50462 | Loss: 4.0226 | LR: 5.00e-04 | 7397.9ms/step | 8.1 samples/s
+Step 820/50462 | Loss: 4.0400 | LR: 5.00e-04 | 7372.5ms/step | 8.1 samples/s
+Step 830/50462 | Loss: 4.0047 | LR: 5.00e-04 | 7357.1ms/step | 8.2 samples/s
+Step 840/50462 | Loss: 4.0164 | LR: 5.00e-04 | 7380.1ms/step | 8.1 samples/s
+Step 850/50462 | Loss: 3.9565 | LR: 5.00e-04 | 7381.0ms/step | 8.1 samples/s
+Step 860/50462 | Loss: 3.9267 | LR: 5.00e-04 | 7365.3ms/step | 8.1 samples/s
+Step 870/50462 | Loss: 3.9078 | LR: 5.00e-04 | 7375.7ms/step | 8.1 samples/s
+Step 880/50462 | Loss: 3.9065 | LR: 5.00e-04 | 7360.9ms/step | 8.2 samples/s
+Step 890/50462 | Loss: 3.9025 | LR: 5.00e-04 | 7364.2ms/step | 8.1 samples/s
+Step 900/50462 | Loss: 3.8937 | LR: 5.00e-04 | 7341.4ms/step | 8.2 samples/s
+Step 910/50462 | Loss: 3.8811 | LR: 5.00e-04 | 7388.4ms/step | 8.1 samples/s
+Step 920/50462 | Loss: 3.8701 | LR: 5.00e-04 | 7363.6ms/step | 8.1 samples/s
+Step 930/50462 | Loss: 3.8368 | LR: 5.00e-04 | 7377.5ms/step | 8.1 samples/s
+Step 940/50462 | Loss: 3.8551 | LR: 5.00e-04 | 7390.4ms/step | 8.1 samples/s
+Step 950/50462 | Loss: 3.8354 | LR: 5.00e-04 | 7345.6ms/step | 8.2 samples/s
+Step 960/50462 | Loss: 3.7922 | LR: 5.00e-04 | 7367.2ms/step | 8.1 samples/s
+Step 970/50462 | Loss: 3.8038 | LR: 5.00e-04 | 7372.8ms/step | 8.1 samples/s
+Step 980/50462 | Loss: 3.7930 | LR: 5.00e-04 | 7374.7ms/step | 8.1 samples/s
+Step 990/50462 | Loss: 3.7858 | LR: 5.00e-04 | 7361.0ms/step | 8.2 samples/s
+Step 1000/50462 | Loss: 3.7586 | LR: 5.00e-04 | 7394.0ms/step | 8.1 samples/s
+Validation at step 1000: Loss = 3.7727, Perplexity = 43.50
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-1000
+Step 1010/50462 | Loss: 3.7469 | LR: 5.00e-04 | 12122.1ms/step | 4.9 samples/s
+Step 1020/50462 | Loss: 3.7524 | LR: 5.00e-04 | 7364.7ms/step | 8.1 samples/s
+Step 1030/50462 | Loss: 3.7150 | LR: 5.00e-04 | 7356.9ms/step | 8.2 samples/s
+Step 1040/50462 | Loss: 3.7204 | LR: 5.00e-04 | 7359.6ms/step | 8.2 samples/s
+Step 1050/50462 | Loss: 3.7048 | LR: 5.00e-04 | 7369.4ms/step | 8.1 samples/s
+Step 1060/50462 | Loss: 3.7026 | LR: 5.00e-04 | 7397.3ms/step | 8.1 samples/s
+Step 1070/50462 | Loss: 3.6855 | LR: 5.00e-04 | 7386.1ms/step | 8.1 samples/s
+Step 1080/50462 | Loss: 3.6904 | LR: 5.00e-04 | 7365.9ms/step | 8.1 samples/s
+Step 1090/50462 | Loss: 3.6655 | LR: 5.00e-04 | 7377.9ms/step | 8.1 samples/s
+Step 1100/50462 | Loss: 3.6564 | LR: 5.00e-04 | 7375.4ms/step | 8.1 samples/s
+Step 1110/50462 | Loss: 3.6768 | LR: 5.00e-04 | 7365.5ms/step | 8.1 samples/s
+Step 1120/50462 | Loss: 3.6380 | LR: 5.00e-04 | 7389.6ms/step | 8.1 samples/s
+Step 1130/50462 | Loss: 3.6196 | LR: 5.00e-04 | 7389.6ms/step | 8.1 samples/s
+Step 1140/50462 | Loss: 3.6203 | LR: 5.00e-04 | 7380.5ms/step | 8.1 samples/s
+Step 1150/50462 | Loss: 3.6380 | LR: 5.00e-04 | 7374.1ms/step | 8.1 samples/s
+Step 1160/50462 | Loss: 3.6174 | LR: 5.00e-04 | 7363.0ms/step | 8.1 samples/s
+Step 1170/50462 | Loss: 3.6118 | LR: 5.00e-04 | 7369.1ms/step | 8.1 samples/s
+Step 1180/50462 | Loss: 3.5977 | LR: 5.00e-04 | 7338.4ms/step | 8.2 samples/s
+Step 1190/50462 | Loss: 3.6211 | LR: 5.00e-04 | 7363.7ms/step | 8.1 samples/s
+Step 1200/50462 | Loss: 3.5694 | LR: 5.00e-04 | 7369.7ms/step | 8.1 samples/s
+Step 1210/50462 | Loss: 3.5754 | LR: 5.00e-04 | 7364.3ms/step | 8.1 samples/s
+Step 1220/50462 | Loss: 3.6185 | LR: 5.00e-04 | 7361.4ms/step | 8.2 samples/s
+Step 1230/50462 | Loss: 3.5867 | LR: 5.00e-04 | 7368.8ms/step | 8.1 samples/s
+Step 1240/50462 | Loss: 3.5624 | LR: 5.00e-04 | 7343.2ms/step | 8.2 samples/s
+Step 1250/50462 | Loss: 3.5676 | LR: 5.00e-04 | 7344.9ms/step | 8.2 samples/s
+Validation at step 1250: Loss = 3.5575, Perplexity = 35.08
+Step 1260/50462 | Loss: 3.5516 | LR: 5.00e-04 | 11209.6ms/step | 5.4 samples/s
+Step 1270/50462 | Loss: 3.5104 | LR: 5.00e-04 | 7370.2ms/step | 8.1 samples/s
+Step 1280/50462 | Loss: 3.5620 | LR: 5.00e-04 | 7376.4ms/step | 8.1 samples/s
+Step 1290/50462 | Loss: 3.5158 | LR: 5.00e-04 | 7349.9ms/step | 8.2 samples/s
+Step 1300/50462 | Loss: 3.5342 | LR: 5.00e-04 | 7373.2ms/step | 8.1 samples/s
+Step 1310/50462 | Loss: 3.5230 | LR: 5.00e-04 | 7391.4ms/step | 8.1 samples/s
+Step 1320/50462 | Loss: 3.5181 | LR: 5.00e-04 | 7381.5ms/step | 8.1 samples/s
+Step 1330/50462 | Loss: 3.4818 | LR: 5.00e-04 | 7386.5ms/step | 8.1 samples/s
+Step 1340/50462 | Loss: 3.4871 | LR: 5.00e-04 | 7382.4ms/step | 8.1 samples/s
+Step 1350/50462 | Loss: 3.4923 | LR: 5.00e-04 | 7370.9ms/step | 8.1 samples/s
+Step 1360/50462 | Loss: 3.4964 | LR: 5.00e-04 | 7355.6ms/step | 8.2 samples/s
+Step 1370/50462 | Loss: 3.5066 | LR: 5.00e-04 | 7372.6ms/step | 8.1 samples/s
+Step 1380/50462 | Loss: 3.4892 | LR: 5.00e-04 | 7384.9ms/step | 8.1 samples/s
+Step 1390/50462 | Loss: 3.4640 | LR: 5.00e-04 | 7387.6ms/step | 8.1 samples/s
+Step 1400/50462 | Loss: 3.4667 | LR: 5.00e-04 | 7355.3ms/step | 8.2 samples/s
+Step 1410/50462 | Loss: 3.4724 | LR: 5.00e-04 | 7371.0ms/step | 8.1 samples/s
+Step 1420/50462 | Loss: 3.4577 | LR: 5.00e-04 | 7366.0ms/step | 8.1 samples/s
+Step 1430/50462 | Loss: 3.4689 | LR: 5.00e-04 | 7380.9ms/step | 8.1 samples/s
+Step 1440/50462 | Loss: 3.4861 | LR: 5.00e-04 | 7367.2ms/step | 8.1 samples/s
+Step 1450/50462 | Loss: 3.4154 | LR: 5.00e-04 | 7378.6ms/step | 8.1 samples/s
+Step 1460/50462 | Loss: 3.4327 | LR: 5.00e-04 | 7386.3ms/step | 8.1 samples/s
+Step 1470/50462 | Loss: 3.4261 | LR: 5.00e-04 | 7363.5ms/step | 8.1 samples/s
+Step 1480/50462 | Loss: 3.4263 | LR: 5.00e-04 | 7359.7ms/step | 8.2 samples/s
+Step 1490/50462 | Loss: 3.4337 | LR: 5.00e-04 | 7356.8ms/step | 8.2 samples/s
+Step 1500/50462 | Loss: 3.4186 | LR: 5.00e-04 | 7358.8ms/step | 8.2 samples/s
+Validation at step 1500: Loss = 3.4334, Perplexity = 30.98
+Step 1510/50462 | Loss: 3.3993 | LR: 5.00e-04 | 11241.4ms/step | 5.3 samples/s
+Step 1520/50462 | Loss: 3.3938 | LR: 5.00e-04 | 7390.4ms/step | 8.1 samples/s
+Step 1530/50462 | Loss: 3.4041 | LR: 5.00e-04 | 7363.7ms/step | 8.1 samples/s
+Step 1540/50462 | Loss: 3.4045 | LR: 5.00e-04 | 7373.7ms/step | 8.1 samples/s
+Step 1550/50462 | Loss: 3.4000 | LR: 5.00e-04 | 7353.9ms/step | 8.2 samples/s
+Step 1560/50462 | Loss: 3.4077 | LR: 5.00e-04 | 7359.4ms/step | 8.2 samples/s
+Step 1570/50462 | Loss: 3.4002 | LR: 5.00e-04 | 7372.3ms/step | 8.1 samples/s
+Step 1580/50462 | Loss: 3.3963 | LR: 5.00e-04 | 7343.2ms/step | 8.2 samples/s
+Step 1590/50462 | Loss: 3.3845 | LR: 5.00e-04 | 7337.8ms/step | 8.2 samples/s
+Step 1600/50462 | Loss: 3.4028 | LR: 5.00e-04 | 7377.2ms/step | 8.1 samples/s
+Step 1610/50462 | Loss: 3.3927 | LR: 5.00e-04 | 7372.9ms/step | 8.1 samples/s
+Step 1620/50462 | Loss: 3.3785 | LR: 5.00e-04 | 7391.5ms/step | 8.1 samples/s
+Step 1630/50462 | Loss: 3.3889 | LR: 5.00e-04 | 7370.4ms/step | 8.1 samples/s
+Step 1640/50462 | Loss: 3.3532 | LR: 5.00e-04 | 7387.5ms/step | 8.1 samples/s
+Step 1650/50462 | Loss: 3.3588 | LR: 5.00e-04 | 7345.7ms/step | 8.2 samples/s
+Step 1660/50462 | Loss: 3.3696 | LR: 5.00e-04 | 7386.8ms/step | 8.1 samples/s
+Step 1670/50462 | Loss: 3.3443 | LR: 5.00e-04 | 7360.3ms/step | 8.2 samples/s
+Step 1680/50462 | Loss: 3.3622 | LR: 5.00e-04 | 7370.5ms/step | 8.1 samples/s
+Step 1690/50462 | Loss: 3.3558 | LR: 5.00e-04 | 7360.3ms/step | 8.2 samples/s
+Step 1700/50462 | Loss: 3.3337 | LR: 5.00e-04 | 7360.3ms/step | 8.2 samples/s
+Step 1710/50462 | Loss: 3.3476 | LR: 5.00e-04 | 7362.4ms/step | 8.1 samples/s
+Step 1720/50462 | Loss: 3.3274 | LR: 5.00e-04 | 7381.6ms/step | 8.1 samples/s
+Step 1730/50462 | Loss: 3.3361 | LR: 5.00e-04 | 7371.6ms/step | 8.1 samples/s
+Step 1740/50462 | Loss: 3.3073 | LR: 5.00e-04 | 7363.3ms/step | 8.1 samples/s
+Step 1750/50462 | Loss: 3.3367 | LR: 5.00e-04 | 7366.8ms/step | 8.1 samples/s
+Validation at step 1750: Loss = 3.3428, Perplexity = 28.30
+Step 1760/50462 | Loss: 3.3488 | LR: 4.99e-04 | 11240.3ms/step | 5.3 samples/s
+Step 1770/50462 | Loss: 3.3295 | LR: 4.99e-04 | 7368.6ms/step | 8.1 samples/s
+Step 1780/50462 | Loss: 3.3044 | LR: 4.99e-04 | 7381.3ms/step | 8.1 samples/s
+Step 1790/50462 | Loss: 3.3265 | LR: 4.99e-04 | 7351.5ms/step | 8.2 samples/s
+Step 1800/50462 | Loss: 3.3426 | LR: 4.99e-04 | 7354.6ms/step | 8.2 samples/s
+Step 1810/50462 | Loss: 3.3453 | LR: 4.99e-04 | 7364.2ms/step | 8.1 samples/s
+Step 1820/50462 | Loss: 3.3235 | LR: 4.99e-04 | 7377.3ms/step | 8.1 samples/s
+Step 1830/50462 | Loss: 3.3117 | LR: 4.99e-04 | 7371.0ms/step | 8.1 samples/s
+Step 1840/50462 | Loss: 3.2834 | LR: 4.99e-04 | 7374.8ms/step | 8.1 samples/s
+Step 1850/50462 | Loss: 3.3039 | LR: 4.99e-04 | 7383.4ms/step | 8.1 samples/s
+Step 1860/50462 | Loss: 3.2982 | LR: 4.99e-04 | 7386.3ms/step | 8.1 samples/s
+Step 1870/50462 | Loss: 3.3197 | LR: 4.99e-04 | 7362.8ms/step | 8.1 samples/s
+Step 1880/50462 | Loss: 3.3018 | LR: 4.99e-04 | 7345.8ms/step | 8.2 samples/s
+Step 1890/50462 | Loss: 3.2601 | LR: 4.99e-04 | 7371.6ms/step | 8.1 samples/s
+Step 1900/50462 | Loss: 3.2972 | LR: 4.99e-04 | 7377.8ms/step | 8.1 samples/s
+Step 1910/50462 | Loss: 3.2853 | LR: 4.99e-04 | 7369.0ms/step | 8.1 samples/s
+Step 1920/50462 | Loss: 3.2891 | LR: 4.99e-04 | 7334.5ms/step | 8.2 samples/s
+Step 1930/50462 | Loss: 3.2646 | LR: 4.99e-04 | 7325.2ms/step | 8.2 samples/s
+Step 1940/50462 | Loss: 3.2892 | LR: 4.99e-04 | 7362.3ms/step | 8.1 samples/s
+Step 1950/50462 | Loss: 3.2870 | LR: 4.99e-04 | 7350.6ms/step | 8.2 samples/s
+Step 1960/50462 | Loss: 3.2640 | LR: 4.99e-04 | 7379.5ms/step | 8.1 samples/s
+Step 1970/50462 | Loss: 3.2668 | LR: 4.99e-04 | 7381.9ms/step | 8.1 samples/s
+Step 1980/50462 | Loss: 3.2723 | LR: 4.99e-04 | 7371.1ms/step | 8.1 samples/s
+Step 1990/50462 | Loss: 3.2744 | LR: 4.99e-04 | 7357.0ms/step | 8.2 samples/s
+Step 2000/50462 | Loss: 3.2649 | LR: 4.99e-04 | 7394.0ms/step | 8.1 samples/s
+Validation at step 2000: Loss = 3.2750, Perplexity = 26.44
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-2000
+Step 2010/50462 | Loss: 3.2365 | LR: 4.99e-04 | 12182.3ms/step | 4.9 samples/s
+Step 2020/50462 | Loss: 3.2684 | LR: 4.99e-04 | 7350.3ms/step | 8.2 samples/s
+Step 2030/50462 | Loss: 3.2474 | LR: 4.99e-04 | 7358.6ms/step | 8.2 samples/s
+Step 2040/50462 | Loss: 3.2492 | LR: 4.99e-04 | 7387.8ms/step | 8.1 samples/s
+Step 2050/50462 | Loss: 3.2074 | LR: 4.99e-04 | 7398.8ms/step | 8.1 samples/s
+Step 2060/50462 | Loss: 3.2730 | LR: 4.99e-04 | 7362.4ms/step | 8.1 samples/s
+Step 2070/50462 | Loss: 3.2711 | LR: 4.99e-04 | 7352.6ms/step | 8.2 samples/s
+Step 2080/50462 | Loss: 3.2437 | LR: 4.99e-04 | 7358.6ms/step | 8.2 samples/s
+Step 2090/50462 | Loss: 3.2618 | LR: 4.99e-04 | 7356.2ms/step | 8.2 samples/s
+Step 2100/50462 | Loss: 3.2509 | LR: 4.99e-04 | 7385.5ms/step | 8.1 samples/s
+Step 2110/50462 | Loss: 3.2308 | LR: 4.99e-04 | 7381.3ms/step | 8.1 samples/s
+Step 2120/50462 | Loss: 3.2274 | LR: 4.99e-04 | 7387.9ms/step | 8.1 samples/s
+Step 2130/50462 | Loss: 3.2482 | LR: 4.99e-04 | 7376.4ms/step | 8.1 samples/s
+Step 2140/50462 | Loss: 3.2243 | LR: 4.99e-04 | 7374.2ms/step | 8.1 samples/s
+Step 2150/50462 | Loss: 3.2249 | LR: 4.99e-04 | 7367.1ms/step | 8.1 samples/s
+Step 2160/50462 | Loss: 3.2319 | LR: 4.99e-04 | 7336.0ms/step | 8.2 samples/s
+Step 2170/50462 | Loss: 3.2290 | LR: 4.99e-04 | 7345.3ms/step | 8.2 samples/s
+Step 2180/50462 | Loss: 3.2386 | LR: 4.99e-04 | 7376.9ms/step | 8.1 samples/s
+Step 2190/50462 | Loss: 3.2052 | LR: 4.99e-04 | 7368.0ms/step | 8.1 samples/s
+Step 2200/50462 | Loss: 3.2152 | LR: 4.99e-04 | 7360.8ms/step | 8.2 samples/s
+Step 2210/50462 | Loss: 3.2154 | LR: 4.99e-04 | 7399.2ms/step | 8.1 samples/s
+Step 2220/50462 | Loss: 3.2344 | LR: 4.99e-04 | 7379.5ms/step | 8.1 samples/s
+Step 2230/50462 | Loss: 3.2164 | LR: 4.99e-04 | 7384.7ms/step | 8.1 samples/s
+Step 2240/50462 | Loss: 3.2290 | LR: 4.99e-04 | 7360.1ms/step | 8.2 samples/s
+Step 2250/50462 | Loss: 3.2164 | LR: 4.99e-04 | 7361.5ms/step | 8.2 samples/s
+Validation at step 2250: Loss = 3.2215, Perplexity = 25.07
+Step 2260/50462 | Loss: 3.2159 | LR: 4.99e-04 | 11245.4ms/step | 5.3 samples/s
+Step 2270/50462 | Loss: 3.2168 | LR: 4.99e-04 | 7374.4ms/step | 8.1 samples/s
+Step 2280/50462 | Loss: 3.2215 | LR: 4.99e-04 | 7384.3ms/step | 8.1 samples/s
+Step 2290/50462 | Loss: 3.2073 | LR: 4.99e-04 | 7379.9ms/step | 8.1 samples/s
+Step 2300/50462 | Loss: 3.2177 | LR: 4.99e-04 | 7398.5ms/step | 8.1 samples/s
+Step 2310/50462 | Loss: 3.2115 | LR: 4.99e-04 | 7360.8ms/step | 8.2 samples/s
+Step 2320/50462 | Loss: 3.2178 | LR: 4.99e-04 | 7378.7ms/step | 8.1 samples/s
+Step 2330/50462 | Loss: 3.1953 | LR: 4.99e-04 | 7378.5ms/step | 8.1 samples/s
+Step 2340/50462 | Loss: 3.2209 | LR: 4.99e-04 | 7343.1ms/step | 8.2 samples/s
+Step 2350/50462 | Loss: 3.1798 | LR: 4.99e-04 | 7357.7ms/step | 8.2 samples/s
+Step 2360/50462 | Loss: 3.1912 | LR: 4.99e-04 | 7357.5ms/step | 8.2 samples/s
+Step 2370/50462 | Loss: 3.2037 | LR: 4.99e-04 | 7364.4ms/step | 8.1 samples/s
+Step 2380/50462 | Loss: 3.1862 | LR: 4.99e-04 | 7359.4ms/step | 8.2 samples/s
+Step 2390/50462 | Loss: 3.2056 | LR: 4.99e-04 | 7380.6ms/step | 8.1 samples/s
+Step 2400/50462 | Loss: 3.2027 | LR: 4.99e-04 | 7377.0ms/step | 8.1 samples/s
+Step 2410/50462 | Loss: 3.1688 | LR: 4.99e-04 | 7356.0ms/step | 8.2 samples/s
+Step 2420/50462 | Loss: 3.1642 | LR: 4.99e-04 | 7364.2ms/step | 8.1 samples/s
+Step 2430/50462 | Loss: 3.1793 | LR: 4.99e-04 | 7400.6ms/step | 8.1 samples/s
+Step 2440/50462 | Loss: 3.1719 | LR: 4.99e-04 | 7396.7ms/step | 8.1 samples/s
+Step 2450/50462 | Loss: 3.1869 | LR: 4.99e-04 | 7373.8ms/step | 8.1 samples/s
+Step 2460/50462 | Loss: 3.1767 | LR: 4.99e-04 | 7387.4ms/step | 8.1 samples/s
+Step 2470/50462 | Loss: 3.1712 | LR: 4.99e-04 | 7355.1ms/step | 8.2 samples/s
+Step 2480/50462 | Loss: 3.1618 | LR: 4.99e-04 | 7382.6ms/step | 8.1 samples/s
+Step 2490/50462 | Loss: 3.1676 | LR: 4.98e-04 | 7389.2ms/step | 8.1 samples/s
+Step 2500/50462 | Loss: 3.1875 | LR: 4.98e-04 | 7374.6ms/step | 8.1 samples/s
+Validation at step 2500: Loss = 3.1794, Perplexity = 24.03
+Step 2510/50462 | Loss: 3.1607 | LR: 4.98e-04 | 11242.0ms/step | 5.3 samples/s
+Step 2520/50462 | Loss: 3.1564 | LR: 4.98e-04 | 7383.3ms/step | 8.1 samples/s
+Step 2530/50462 | Loss: 3.1756 | LR: 4.98e-04 | 7379.6ms/step | 8.1 samples/s
+Step 2540/50462 | Loss: 3.1665 | LR: 4.98e-04 | 7360.2ms/step | 8.2 samples/s
+Step 2550/50462 | Loss: 3.1647 | LR: 4.98e-04 | 7362.0ms/step | 8.2 samples/s
+Step 2560/50462 | Loss: 3.1507 | LR: 4.98e-04 | 7372.0ms/step | 8.1 samples/s
+Step 2570/50462 | Loss: 3.1585 | LR: 4.98e-04 | 7358.7ms/step | 8.2 samples/s
+Step 2580/50462 | Loss: 3.1548 | LR: 4.98e-04 | 7358.4ms/step | 8.2 samples/s
+Step 2590/50462 | Loss: 3.1384 | LR: 4.98e-04 | 7380.4ms/step | 8.1 samples/s
+Step 2600/50462 | Loss: 3.1661 | LR: 4.98e-04 | 7371.2ms/step | 8.1 samples/s
+Step 2610/50462 | Loss: 3.1530 | LR: 4.98e-04 | 7399.3ms/step | 8.1 samples/s
+Step 2620/50462 | Loss: 3.1696 | LR: 4.98e-04 | 7373.0ms/step | 8.1 samples/s
+Step 2630/50462 | Loss: 3.1277 | LR: 4.98e-04 | 7367.1ms/step | 8.1 samples/s
+Step 2640/50462 | Loss: 3.1604 | LR: 4.98e-04 | 7375.0ms/step | 8.1 samples/s
+Step 2650/50462 | Loss: 3.1427 | LR: 4.98e-04 | 7388.8ms/step | 8.1 samples/s
+Step 2660/50462 | Loss: 3.1738 | LR: 4.98e-04 | 7347.5ms/step | 8.2 samples/s
+Step 2670/50462 | Loss: 3.1324 | LR: 4.98e-04 | 7392.1ms/step | 8.1 samples/s
+Step 2680/50462 | Loss: 3.1455 | LR: 4.98e-04 | 7374.6ms/step | 8.1 samples/s
+Step 2690/50462 | Loss: 3.1413 | LR: 4.98e-04 | 7373.6ms/step | 8.1 samples/s
+Step 2700/50462 | Loss: 3.1465 | LR: 4.98e-04 | 7378.4ms/step | 8.1 samples/s
+Step 2710/50462 | Loss: 3.1472 | LR: 4.98e-04 | 7352.7ms/step | 8.2 samples/s
+Step 2720/50462 | Loss: 3.1333 | LR: 4.98e-04 | 7373.7ms/step | 8.1 samples/s
+Step 2730/50462 | Loss: 3.1362 | LR: 4.98e-04 | 7356.3ms/step | 8.2 samples/s
+Step 2740/50462 | Loss: 3.1527 | LR: 4.98e-04 | 7366.7ms/step | 8.1 samples/s
+Step 2750/50462 | Loss: 3.1580 | LR: 4.98e-04 | 7362.4ms/step | 8.1 samples/s
+Validation at step 2750: Loss = 3.1435, Perplexity = 23.18
+Step 2760/50462 | Loss: 3.1579 | LR: 4.98e-04 | 11246.1ms/step | 5.3 samples/s
+Step 2770/50462 | Loss: 3.1259 | LR: 4.98e-04 | 7389.8ms/step | 8.1 samples/s
+Step 2780/50462 | Loss: 3.1441 | LR: 4.98e-04 | 7386.7ms/step | 8.1 samples/s
+Step 2790/50462 | Loss: 3.1499 | LR: 4.98e-04 | 7361.5ms/step | 8.2 samples/s
+Step 2800/50462 | Loss: 3.1231 | LR: 4.98e-04 | 7362.2ms/step | 8.1 samples/s
+Step 2810/50462 | Loss: 3.1289 | LR: 4.98e-04 | 7354.5ms/step | 8.2 samples/s
+Step 2820/50462 | Loss: 3.1178 | LR: 4.98e-04 | 7373.0ms/step | 8.1 samples/s
+Step 2830/50462 | Loss: 3.0993 | LR: 4.98e-04 | 7355.6ms/step | 8.2 samples/s
+Step 2840/50462 | Loss: 3.1170 | LR: 4.98e-04 | 7344.4ms/step | 8.2 samples/s
+Step 2850/50462 | Loss: 3.1312 | LR: 4.98e-04 | 7373.1ms/step | 8.1 samples/s
+Step 2860/50462 | Loss: 3.1184 | LR: 4.98e-04 | 7376.8ms/step | 8.1 samples/s
+Step 2870/50462 | Loss: 3.1153 | LR: 4.98e-04 | 7374.4ms/step | 8.1 samples/s
+Step 2880/50462 | Loss: 3.1072 | LR: 4.98e-04 | 7364.4ms/step | 8.1 samples/s
+Step 2890/50462 | Loss: 3.1181 | LR: 4.98e-04 | 7380.4ms/step | 8.1 samples/s
+Step 2900/50462 | Loss: 3.1301 | LR: 4.98e-04 | 7360.8ms/step | 8.2 samples/s
+Step 2910/50462 | Loss: 3.0896 | LR: 4.98e-04 | 7358.5ms/step | 8.2 samples/s
+Step 2920/50462 | Loss: 3.1073 | LR: 4.98e-04 | 7381.8ms/step | 8.1 samples/s
+Step 2930/50462 | Loss: 3.1197 | LR: 4.98e-04 | 7370.4ms/step | 8.1 samples/s
+Step 2940/50462 | Loss: 3.0829 | LR: 4.98e-04 | 7377.3ms/step | 8.1 samples/s
+Step 2950/50462 | Loss: 3.1117 | LR: 4.98e-04 | 7384.8ms/step | 8.1 samples/s
+Step 2960/50462 | Loss: 3.1098 | LR: 4.98e-04 | 7358.4ms/step | 8.2 samples/s
+Step 2970/50462 | Loss: 3.1180 | LR: 4.98e-04 | 7372.1ms/step | 8.1 samples/s
+Step 2980/50462 | Loss: 3.1047 | LR: 4.98e-04 | 7366.9ms/step | 8.1 samples/s
+Step 2990/50462 | Loss: 3.1069 | LR: 4.97e-04 | 7360.7ms/step | 8.2 samples/s
+Step 3000/50462 | Loss: 3.0927 | LR: 4.97e-04 | 7360.1ms/step | 8.2 samples/s
+Validation at step 3000: Loss = 3.1122, Perplexity = 22.47
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-3000
+Step 3010/50462 | Loss: 3.0770 | LR: 4.97e-04 | 12146.7ms/step | 4.9 samples/s
+Step 3020/50462 | Loss: 3.0881 | LR: 4.97e-04 | 7369.0ms/step | 8.1 samples/s
+Step 3030/50462 | Loss: 3.1111 | LR: 4.97e-04 | 7362.5ms/step | 8.1 samples/s
+Step 3040/50462 | Loss: 3.1004 | LR: 4.97e-04 | 7355.5ms/step | 8.2 samples/s
+Step 3050/50462 | Loss: 3.0841 | LR: 4.97e-04 | 7369.9ms/step | 8.1 samples/s
+Step 3060/50462 | Loss: 3.1183 | LR: 4.97e-04 | 7351.9ms/step | 8.2 samples/s
+Step 3070/50462 | Loss: 3.1183 | LR: 4.97e-04 | 7372.4ms/step | 8.1 samples/s
+Step 3080/50462 | Loss: 3.0967 | LR: 4.97e-04 | 7378.5ms/step | 8.1 samples/s
+Step 3090/50462 | Loss: 3.1079 | LR: 4.97e-04 | 7395.7ms/step | 8.1 samples/s
+Step 3100/50462 | Loss: 3.1020 | LR: 4.97e-04 | 7380.6ms/step | 8.1 samples/s
+Step 3110/50462 | Loss: 3.0934 | LR: 4.97e-04 | 7358.4ms/step | 8.2 samples/s
+Step 3120/50462 | Loss: 3.0919 | LR: 4.97e-04 | 7363.8ms/step | 8.1 samples/s
+Step 3130/50462 | Loss: 3.0860 | LR: 4.97e-04 | 7368.5ms/step | 8.1 samples/s
+Step 3140/50462 | Loss: 3.1037 | LR: 4.97e-04 | 7366.7ms/step | 8.1 samples/s
+Step 3150/50462 | Loss: 3.0955 | LR: 4.97e-04 | 7369.2ms/step | 8.1 samples/s
+Step 3160/50462 | Loss: 3.0900 | LR: 4.97e-04 | 7368.1ms/step | 8.1 samples/s
+Step 3170/50462 | Loss: 3.0800 | LR: 4.97e-04 | 7368.7ms/step | 8.1 samples/s
+Step 3180/50462 | Loss: 3.0779 | LR: 4.97e-04 | 7364.3ms/step | 8.1 samples/s
+Step 3190/50462 | Loss: 3.0759 | LR: 4.97e-04 | 7358.5ms/step | 8.2 samples/s
+Step 3200/50462 | Loss: 3.0620 | LR: 4.97e-04 | 7349.9ms/step | 8.2 samples/s
+Step 3210/50462 | Loss: 3.0839 | LR: 4.97e-04 | 7366.5ms/step | 8.1 samples/s
+Step 3220/50462 | Loss: 3.0646 | LR: 4.97e-04 | 7389.4ms/step | 8.1 samples/s
+Step 3230/50462 | Loss: 3.0610 | LR: 4.97e-04 | 7372.2ms/step | 8.1 samples/s
+Step 3240/50462 | Loss: 3.0832 | LR: 4.97e-04 | 7349.6ms/step | 8.2 samples/s
+Step 3250/50462 | Loss: 3.0886 | LR: 4.97e-04 | 7361.0ms/step | 8.2 samples/s
+Validation at step 3250: Loss = 3.0823, Perplexity = 21.81
+Step 3260/50462 | Loss: 3.0558 | LR: 4.97e-04 | 11233.7ms/step | 5.3 samples/s
+Step 3270/50462 | Loss: 3.0566 | LR: 4.97e-04 | 7398.0ms/step | 8.1 samples/s
+Step 3280/50462 | Loss: 3.0632 | LR: 4.97e-04 | 7358.3ms/step | 8.2 samples/s
+Step 3290/50462 | Loss: 3.0667 | LR: 4.97e-04 | 7353.2ms/step | 8.2 samples/s
+Step 3300/50462 | Loss: 3.0615 | LR: 4.97e-04 | 7352.4ms/step | 8.2 samples/s
+Step 3310/50462 | Loss: 3.0545 | LR: 4.97e-04 | 7351.0ms/step | 8.2 samples/s
+Step 3320/50462 | Loss: 3.0483 | LR: 4.97e-04 | 7390.7ms/step | 8.1 samples/s

epfl_final/SparseAtt/wandb/run-20260822_210348-i8ewyy95/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260822_210348-i8ewyy95/files/wandb-metadata.json

@@ -0,0 +1,51 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-08-22T21:03:48.952755Z",
+  "args":  [
+    "--config",
+    "configs/p2_1p2b_mixed_seed70_2gpu.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "5637886f0cc760009989b14e68c6ae06d4827418"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-workspace-2-gpu-0-9",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  2,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "6420774977536"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-af79466d-4300-4171-b5bd-7697eca27264"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-0460dc7d-d83e-fd0f-7548-a3079803b79e"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "clb2ll0v1m1hv818bzb391b0hn7ensmv"
+}

epfl_final/SparseAtt/wandb/run-20260822_210348-i8ewyy95/logs/debug.log

@@ -0,0 +1,19 @@
+2026-08-22 21:03:49,027 INFO    MainThread:4690 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-08-22 21:03:49,027 INFO    MainThread:4690 [wandb_setup.py:_flush():81] Configure stats pid to 4690
+2026-08-22 21:03:49,027 INFO    MainThread:4690 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-22 21:03:49,027 INFO    MainThread:4690 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_210348-i8ewyy95/logs/debug.log
+2026-08-22 21:03:49,027 INFO    MainThread:4690 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_210348-i8ewyy95/logs/debug-internal.log
+2026-08-22 21:03:49,027 INFO    MainThread:4690 [wandb_init.py:init():850] calling init triggers
+2026-08-22 21:03:49,027 INFO    MainThread:4690 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107', 'resume_from_checkpoint': None, 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 15, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 70, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b mixed 8k seed70', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'mixed', 'seed70'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed', 'sparse_mask_seed': 70, 'sparse_mixed_nrb_set': [63, 47, 31, 21, 11], 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-08-22 21:03:49,027 INFO    MainThread:4690 [wandb_init.py:init():898] starting backend
+2026-08-22 21:03:49,306 INFO    MainThread:4690 [wandb_init.py:init():913] sending inform_init request
+2026-08-22 21:03:49,615 INFO    MainThread:4690 [wandb_init.py:init():918] backend started and connected
+2026-08-22 21:03:49,617 INFO    MainThread:4690 [wandb_init.py:init():988] updated telemetry
+2026-08-22 21:03:49,638 INFO    MainThread:4690 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-08-22 21:03:50,169 INFO    MainThread:4690 [wandb_init.py:init():1056] starting run threads in backend
+2026-08-22 21:03:50,405 INFO    MainThread:4690 [wandb_run.py:_console_start():2554] atexit reg
+2026-08-22 21:03:50,405 INFO    MainThread:4690 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-08-22 21:03:50,405 INFO    MainThread:4690 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-08-22 21:03:50,406 INFO    MainThread:4690 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-08-22 21:03:50,411 INFO    MainThread:4690 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/run-20260822_210348-i8ewyy95/run-i8ewyy95.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5faf62872164f62ff7093cf1309689deb3ffb1a7a7bcc372fce2efca315ad318
+size 5079040

epfl_final/SparseAtt/wandb/run-20260823_040101-y8clx6eu/files/output.log

@@ -0,0 +1,2 @@
+Starting training...
+Switching attention mode for step 1: dense (dense step interval)

epfl_final/SparseAtt/wandb/run-20260823_040101-y8clx6eu/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260823_040101-y8clx6eu/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-08-23T04:01:01.681514Z",
+  "args":  [
+    "--config",
+    "configs/p2_1p2b_dense_seed70_4gpu.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "5637886f0cc760009989b14e68c6ae06d4827418"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-workspace-4-gpu-3-0-9",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "9371709865984",
+      "used":  "6561452351488"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-9a903f43-9348-6fa6-bcf7-e12bd01803a7"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-d0294574-a180-e984-6ca3-0573e9724f2e"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-24b8d952-dc08-33b9-d5df-54ce871fd288"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-b87e8111-63b6-b364-28b9-c4ce747aec08"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "5xn9247xpp7bkytaub0n7rw61p5n7iwz"
+}

epfl_final/SparseAtt/wandb/run-20260823_040101-y8clx6eu/logs/debug-internal.log

@@ -0,0 +1,13 @@
+{"time":"2026-08-23T04:01:02.004539028Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-08-23T04:01:02.005119234Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-08-23T04:01:02.334253575Z","level":"INFO","msg":"stream: created new stream","id":"y8clx6eu"}
+{"time":"2026-08-23T04:01:02.334373184Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-08-23T04:01:02.336722983Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-08-23T04:01:02.336795701Z","level":"INFO","msg":"writer: started","stream_id":"y8clx6eu"}
+{"time":"2026-08-23T04:01:02.336846818Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-08-23T04:01:02.892387047Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-08-23T04:01:03.150807079Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-23T04:01:17.892692579Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":1,"console_offset":1,"console_lines":1,"uploaded_len":2}
+{"time":"2026-08-23T04:01:18.129623503Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-23T04:01:32.89303166Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":1,"events_lines":2}
+{"time":"2026-08-23T04:01:33.117742527Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

epfl_final/SparseAtt/wandb/run-20260823_040101-y8clx6eu/logs/debug.log

@@ -0,0 +1,19 @@
+2026-08-23 04:01:01,711 INFO    MainThread:4320 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-08-23 04:01:01,711 INFO    MainThread:4320 [wandb_setup.py:_flush():81] Configure stats pid to 4320
+2026-08-23 04:01:01,712 INFO    MainThread:4320 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-23 04:01:01,712 INFO    MainThread:4320 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260823_040101-y8clx6eu/logs/debug.log
+2026-08-23 04:01:01,712 INFO    MainThread:4320 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260823_040101-y8clx6eu/logs/debug-internal.log
+2026-08-23 04:01:01,712 INFO    MainThread:4320 [wandb_init.py:init():850] calling init triggers
+2026-08-23 04:01:01,712 INFO    MainThread:4320 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_040030', 'resume_from_checkpoint': None, 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 15, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 70, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b dense 8k seed70', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'dense', 'seed70'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
+2026-08-23 04:01:01,712 INFO    MainThread:4320 [wandb_init.py:init():898] starting backend
+2026-08-23 04:01:01,989 INFO    MainThread:4320 [wandb_init.py:init():913] sending inform_init request
+2026-08-23 04:01:02,337 INFO    MainThread:4320 [wandb_init.py:init():918] backend started and connected
+2026-08-23 04:01:02,342 INFO    MainThread:4320 [wandb_init.py:init():988] updated telemetry
+2026-08-23 04:01:02,359 INFO    MainThread:4320 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-08-23 04:01:02,695 INFO    MainThread:4320 [wandb_init.py:init():1056] starting run threads in backend
+2026-08-23 04:01:02,879 INFO    MainThread:4320 [wandb_run.py:_console_start():2554] atexit reg
+2026-08-23 04:01:02,879 INFO    MainThread:4320 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-08-23 04:01:02,879 INFO    MainThread:4320 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-08-23 04:01:02,879 INFO    MainThread:4320 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-08-23 04:01:02,883 INFO    MainThread:4320 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/run-20260823_041148-i8ewyy95/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260823_041148-i8ewyy95/files/wandb-metadata.json

@@ -0,0 +1,79 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-08-23T04:11:48.465653Z",
+  "args":  [
+    "--config",
+    "configs/p2_1p2b_mixed_seed70_6gpu.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "5637886f0cc760009989b14e68c6ae06d4827418"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-train-6-gpu-0-31",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  6,
+  "disk":  {
+    "/":  {
+      "total":  "9371709865984",
+      "used":  "6560433696768"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-98029aa4-c589-b65b-3ab7-6495256f31ed"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-37a1c6c2-d3df-3064-44df-33fa82304bd0"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-9a903f43-9348-6fa6-bcf7-e12bd01803a7"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-d0294574-a180-e984-6ca3-0573e9724f2e"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-24b8d952-dc08-33b9-d5df-54ce871fd288"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-b87e8111-63b6-b364-28b9-c4ce747aec08"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "xh8q4c30086d52gf48fc191wcdie0cdj"
+}

epfl_final/SparseAtt/wandb/run-20260823_041148-i8ewyy95/logs/debug.log

@@ -0,0 +1,20 @@
+2026-08-23 04:11:48,490 INFO    MainThread:217 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-08-23 04:11:48,490 INFO    MainThread:217 [wandb_setup.py:_flush():81] Configure stats pid to 217
+2026-08-23 04:11:48,490 INFO    MainThread:217 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-23 04:11:48,490 INFO    MainThread:217 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260823_041148-i8ewyy95/logs/debug.log
+2026-08-23 04:11:48,490 INFO    MainThread:217 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260823_041148-i8ewyy95/logs/debug-internal.log
+2026-08-23 04:11:48,490 INFO    MainThread:217 [wandb_init.py:init():850] calling init triggers
+2026-08-23 04:11:48,490 INFO    MainThread:217 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-3000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 70, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b mixed 8k seed70 6gpu resume', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'mixed', 'seed70'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed', 'sparse_mask_seed': 70, 'sparse_mixed_nrb_set': [63, 47, 31, 21, 11], 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-08-23 04:11:48,490 INFO    MainThread:217 [wandb_init.py:init():898] starting backend
+2026-08-23 04:11:49,135 INFO    MainThread:217 [wandb_init.py:init():913] sending inform_init request
+2026-08-23 04:11:49,695 INFO    MainThread:217 [wandb_init.py:init():918] backend started and connected
+2026-08-23 04:11:49,700 INFO    MainThread:217 [wandb_init.py:init():988] updated telemetry
+2026-08-23 04:11:49,737 INFO    MainThread:217 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-08-23 04:11:50,209 INFO    MainThread:217 [wandb_init.py:init():1051] run resumed
+2026-08-23 04:11:50,212 INFO    MainThread:217 [wandb_init.py:init():1056] starting run threads in backend
+2026-08-23 04:11:50,469 INFO    MainThread:217 [wandb_run.py:_console_start():2554] atexit reg
+2026-08-23 04:11:50,469 INFO    MainThread:217 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-08-23 04:11:50,469 INFO    MainThread:217 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-08-23 04:11:50,469 INFO    MainThread:217 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-08-23 04:11:50,477 INFO    MainThread:217 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/run-20260823_041148-i8ewyy95/run-i8ewyy95.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c560dc13b2b59868404a464767df09e0e8d758e3eaf372b8cc66bf4bf70352bd
+size 17661952

epfl_final/SparseAtt/wandb/run-20260823_043841-rgmkfpnl/files/output.log

@@ -0,0 +1,526 @@
+Starting training...
+Switching attention mode for step 1: dense (dense step interval)
+Step 10/50462 | Loss: 12.2299 | LR: 6.67e-06 | 13397.9ms/step | 4.5 samples/s
+Step 20/50462 | Loss: 11.9156 | LR: 1.33e-05 | 6965.1ms/step | 8.6 samples/s
+Step 30/50462 | Loss: 11.0080 | LR: 2.00e-05 | 6966.3ms/step | 8.6 samples/s
+Step 40/50462 | Loss: 10.1412 | LR: 2.67e-05 | 6953.0ms/step | 8.6 samples/s
+Step 50/50462 | Loss: 9.7365 | LR: 3.33e-05 | 6952.0ms/step | 8.6 samples/s
+Step 60/50462 | Loss: 9.2596 | LR: 4.00e-05 | 6956.4ms/step | 8.6 samples/s
+Step 70/50462 | Loss: 8.7069 | LR: 4.67e-05 | 6960.8ms/step | 8.6 samples/s
+Step 80/50462 | Loss: 8.2501 | LR: 5.33e-05 | 6957.5ms/step | 8.6 samples/s
+Step 90/50462 | Loss: 7.8702 | LR: 6.00e-05 | 6961.8ms/step | 8.6 samples/s
+Step 100/50462 | Loss: 7.4724 | LR: 6.67e-05 | 6967.9ms/step | 8.6 samples/s
+Step 110/50462 | Loss: 7.2534 | LR: 7.33e-05 | 6966.4ms/step | 8.6 samples/s
+Step 120/50462 | Loss: 7.0820 | LR: 8.00e-05 | 6971.5ms/step | 8.6 samples/s
+Step 130/50462 | Loss: 6.9967 | LR: 8.67e-05 | 6974.3ms/step | 8.6 samples/s
+Step 140/50462 | Loss: 6.8992 | LR: 9.33e-05 | 6976.3ms/step | 8.6 samples/s
+Step 150/50462 | Loss: 6.7704 | LR: 1.00e-04 | 6973.5ms/step | 8.6 samples/s
+Step 160/50462 | Loss: 6.6579 | LR: 1.07e-04 | 6975.6ms/step | 8.6 samples/s
+Step 170/50462 | Loss: 6.5643 | LR: 1.13e-04 | 6977.5ms/step | 8.6 samples/s
+Step 180/50462 | Loss: 6.4492 | LR: 1.20e-04 | 6976.6ms/step | 8.6 samples/s
+Step 190/50462 | Loss: 6.3763 | LR: 1.27e-04 | 6977.4ms/step | 8.6 samples/s
+Step 200/50462 | Loss: 6.3030 | LR: 1.33e-04 | 6977.5ms/step | 8.6 samples/s
+Step 210/50462 | Loss: 6.2139 | LR: 1.40e-04 | 6977.2ms/step | 8.6 samples/s
+Step 220/50462 | Loss: 6.2008 | LR: 1.47e-04 | 6976.3ms/step | 8.6 samples/s
+Step 230/50462 | Loss: 6.1205 | LR: 1.53e-04 | 6977.5ms/step | 8.6 samples/s
+Step 240/50462 | Loss: 6.0458 | LR: 1.60e-04 | 6976.5ms/step | 8.6 samples/s
+Step 250/50462 | Loss: 6.0234 | LR: 1.67e-04 | 6978.1ms/step | 8.6 samples/s
+Validation at step 250: Loss = 5.9605, Perplexity = 387.82
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-250
+Step 260/50462 | Loss: 5.9211 | LR: 1.73e-04 | 14290.4ms/step | 4.2 samples/s
+Step 270/50462 | Loss: 5.8880 | LR: 1.80e-04 | 6979.7ms/step | 8.6 samples/s
+Step 280/50462 | Loss: 5.8289 | LR: 1.87e-04 | 6978.4ms/step | 8.6 samples/s
+Step 290/50462 | Loss: 5.7598 | LR: 1.93e-04 | 6983.4ms/step | 8.6 samples/s
+Step 300/50462 | Loss: 5.7123 | LR: 2.00e-04 | 6978.2ms/step | 8.6 samples/s
+Step 310/50462 | Loss: 5.6371 | LR: 2.07e-04 | 6974.8ms/step | 8.6 samples/s
+Step 320/50462 | Loss: 5.6010 | LR: 2.13e-04 | 6974.8ms/step | 8.6 samples/s
+Step 330/50462 | Loss: 5.5710 | LR: 2.20e-04 | 6978.6ms/step | 8.6 samples/s
+Step 340/50462 | Loss: 5.5359 | LR: 2.27e-04 | 6974.4ms/step | 8.6 samples/s
+Step 350/50462 | Loss: 5.4892 | LR: 2.33e-04 | 6973.2ms/step | 8.6 samples/s
+Step 360/50462 | Loss: 5.4533 | LR: 2.40e-04 | 6972.2ms/step | 8.6 samples/s
+Step 370/50462 | Loss: 5.3670 | LR: 2.47e-04 | 6974.5ms/step | 8.6 samples/s
+Step 380/50462 | Loss: 5.3213 | LR: 2.53e-04 | 6973.7ms/step | 8.6 samples/s
+Step 390/50462 | Loss: 5.2728 | LR: 2.60e-04 | 6976.0ms/step | 8.6 samples/s
+Step 400/50462 | Loss: 5.2346 | LR: 2.67e-04 | 6992.1ms/step | 8.6 samples/s
+Step 410/50462 | Loss: 5.2008 | LR: 2.73e-04 | 7002.5ms/step | 8.6 samples/s
+Step 420/50462 | Loss: 5.1574 | LR: 2.80e-04 | 7002.8ms/step | 8.6 samples/s
+Step 430/50462 | Loss: 5.1082 | LR: 2.87e-04 | 7004.1ms/step | 8.6 samples/s
+Step 440/50462 | Loss: 5.0639 | LR: 2.93e-04 | 7001.7ms/step | 8.6 samples/s
+Step 450/50462 | Loss: 5.0605 | LR: 3.00e-04 | 6992.6ms/step | 8.6 samples/s
+Step 460/50462 | Loss: 5.0267 | LR: 3.07e-04 | 7001.6ms/step | 8.6 samples/s
+Step 470/50462 | Loss: 4.9638 | LR: 3.13e-04 | 6999.0ms/step | 8.6 samples/s
+Step 480/50462 | Loss: 4.9458 | LR: 3.20e-04 | 6985.4ms/step | 8.6 samples/s
+Step 490/50462 | Loss: 4.8491 | LR: 3.27e-04 | 6980.0ms/step | 8.6 samples/s
+Step 500/50462 | Loss: 4.8597 | LR: 3.33e-04 | 6981.3ms/step | 8.6 samples/s
+Validation at step 500: Loss = 4.8557, Perplexity = 128.48
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-500
+Step 510/50462 | Loss: 4.8445 | LR: 3.40e-04 | 10743.5ms/step | 5.6 samples/s
+Step 520/50462 | Loss: 4.7613 | LR: 3.47e-04 | 6994.9ms/step | 8.6 samples/s
+Step 530/50462 | Loss: 4.7293 | LR: 3.53e-04 | 6997.9ms/step | 8.6 samples/s
+Step 540/50462 | Loss: 4.7942 | LR: 3.60e-04 | 6991.0ms/step | 8.6 samples/s
+Step 550/50462 | Loss: 4.7975 | LR: 3.67e-04 | 6995.8ms/step | 8.6 samples/s
+Step 560/50462 | Loss: 4.7114 | LR: 3.73e-04 | 6991.5ms/step | 8.6 samples/s
+Step 570/50462 | Loss: 4.6380 | LR: 3.80e-04 | 6992.5ms/step | 8.6 samples/s
+Step 580/50462 | Loss: 4.6088 | LR: 3.87e-04 | 6995.6ms/step | 8.6 samples/s
+Step 590/50462 | Loss: 4.6519 | LR: 3.93e-04 | 6993.0ms/step | 8.6 samples/s
+Step 600/50462 | Loss: 4.5644 | LR: 4.00e-04 | 6992.5ms/step | 8.6 samples/s
+Step 610/50462 | Loss: 4.5056 | LR: 4.07e-04 | 6989.8ms/step | 8.6 samples/s
+Step 620/50462 | Loss: 4.5163 | LR: 4.13e-04 | 6991.3ms/step | 8.6 samples/s
+Step 630/50462 | Loss: 4.5185 | LR: 4.20e-04 | 6993.7ms/step | 8.6 samples/s
+Step 640/50462 | Loss: 4.4487 | LR: 4.27e-04 | 6991.2ms/step | 8.6 samples/s
+Step 650/50462 | Loss: 4.3603 | LR: 4.33e-04 | 6993.4ms/step | 8.6 samples/s
+Step 660/50462 | Loss: 4.3409 | LR: 4.40e-04 | 6994.4ms/step | 8.6 samples/s
+Step 670/50462 | Loss: 4.4596 | LR: 4.47e-04 | 6992.8ms/step | 8.6 samples/s
+Step 680/50462 | Loss: 4.3716 | LR: 4.53e-04 | 6994.4ms/step | 8.6 samples/s
+Step 690/50462 | Loss: 4.2619 | LR: 4.60e-04 | 6997.6ms/step | 8.6 samples/s
+Step 700/50462 | Loss: 4.2409 | LR: 4.67e-04 | 6991.7ms/step | 8.6 samples/s
+Step 710/50462 | Loss: 4.2600 | LR: 4.73e-04 | 6990.2ms/step | 8.6 samples/s
+Step 720/50462 | Loss: 4.2935 | LR: 4.80e-04 | 6996.9ms/step | 8.6 samples/s
+Step 730/50462 | Loss: 4.2164 | LR: 4.87e-04 | 6997.2ms/step | 8.6 samples/s
+Step 740/50462 | Loss: 4.1447 | LR: 4.93e-04 | 6999.5ms/step | 8.6 samples/s
+Step 750/50462 | Loss: 4.1294 | LR: 5.00e-04 | 6998.5ms/step | 8.6 samples/s
+Validation at step 750: Loss = 4.1136, Perplexity = 61.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-750
+Step 760/50462 | Loss: 4.0937 | LR: 5.00e-04 | 10758.6ms/step | 5.6 samples/s
+Step 770/50462 | Loss: 4.1263 | LR: 5.00e-04 | 6986.5ms/step | 8.6 samples/s
+Step 780/50462 | Loss: 4.0728 | LR: 5.00e-04 | 6989.5ms/step | 8.6 samples/s
+Step 790/50462 | Loss: 4.0538 | LR: 5.00e-04 | 6994.4ms/step | 8.6 samples/s
+Step 800/50462 | Loss: 4.0512 | LR: 5.00e-04 | 6995.7ms/step | 8.6 samples/s
+Step 810/50462 | Loss: 3.9448 | LR: 5.00e-04 | 6991.6ms/step | 8.6 samples/s
+Step 820/50462 | Loss: 4.0090 | LR: 5.00e-04 | 6996.2ms/step | 8.6 samples/s
+Step 830/50462 | Loss: 3.9644 | LR: 5.00e-04 | 6984.1ms/step | 8.6 samples/s
+Step 840/50462 | Loss: 3.9260 | LR: 5.00e-04 | 6986.2ms/step | 8.6 samples/s
+Step 850/50462 | Loss: 3.9136 | LR: 5.00e-04 | 6989.1ms/step | 8.6 samples/s
+Step 860/50462 | Loss: 3.8627 | LR: 5.00e-04 | 6981.9ms/step | 8.6 samples/s
+Step 870/50462 | Loss: 3.8631 | LR: 5.00e-04 | 6974.7ms/step | 8.6 samples/s
+Step 880/50462 | Loss: 3.8533 | LR: 5.00e-04 | 6977.1ms/step | 8.6 samples/s
+Step 890/50462 | Loss: 3.8499 | LR: 5.00e-04 | 6977.7ms/step | 8.6 samples/s
+Step 900/50462 | Loss: 3.8223 | LR: 5.00e-04 | 6974.0ms/step | 8.6 samples/s
+Step 910/50462 | Loss: 3.8381 | LR: 5.00e-04 | 6973.0ms/step | 8.6 samples/s
+Step 920/50462 | Loss: 3.8187 | LR: 5.00e-04 | 6972.2ms/step | 8.6 samples/s
+Step 930/50462 | Loss: 3.7817 | LR: 5.00e-04 | 6973.8ms/step | 8.6 samples/s
+Step 940/50462 | Loss: 3.8183 | LR: 5.00e-04 | 6976.2ms/step | 8.6 samples/s
+Step 950/50462 | Loss: 3.7734 | LR: 5.00e-04 | 6972.4ms/step | 8.6 samples/s
+Step 960/50462 | Loss: 3.7546 | LR: 5.00e-04 | 6980.5ms/step | 8.6 samples/s
+Step 970/50462 | Loss: 3.7664 | LR: 5.00e-04 | 6997.3ms/step | 8.6 samples/s
+Step 980/50462 | Loss: 3.7394 | LR: 5.00e-04 | 7001.2ms/step | 8.6 samples/s
+Step 990/50462 | Loss: 3.7606 | LR: 5.00e-04 | 7003.0ms/step | 8.6 samples/s
+Step 1000/50462 | Loss: 3.7216 | LR: 5.00e-04 | 7003.4ms/step | 8.6 samples/s
+Validation at step 1000: Loss = 3.6926, Perplexity = 40.15
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-1000
+Step 1010/50462 | Loss: 3.6989 | LR: 5.00e-04 | 10784.3ms/step | 5.6 samples/s
+Step 1020/50462 | Loss: 3.6964 | LR: 5.00e-04 | 7000.7ms/step | 8.6 samples/s
+Step 1030/50462 | Loss: 3.6747 | LR: 5.00e-04 | 6995.7ms/step | 8.6 samples/s
+Step 1040/50462 | Loss: 3.6733 | LR: 5.00e-04 | 6981.7ms/step | 8.6 samples/s
+Step 1050/50462 | Loss: 3.6548 | LR: 5.00e-04 | 6976.3ms/step | 8.6 samples/s
+Step 1060/50462 | Loss: 3.6781 | LR: 5.00e-04 | 6978.6ms/step | 8.6 samples/s
+Step 1070/50462 | Loss: 3.6458 | LR: 5.00e-04 | 6984.2ms/step | 8.6 samples/s
+Step 1080/50462 | Loss: 3.6464 | LR: 5.00e-04 | 6995.5ms/step | 8.6 samples/s
+Step 1090/50462 | Loss: 3.6192 | LR: 5.00e-04 | 6999.7ms/step | 8.6 samples/s
+Step 1100/50462 | Loss: 3.6117 | LR: 5.00e-04 | 6995.7ms/step | 8.6 samples/s
+Step 1110/50462 | Loss: 3.6503 | LR: 5.00e-04 | 6990.1ms/step | 8.6 samples/s
+Step 1120/50462 | Loss: 3.5846 | LR: 5.00e-04 | 6988.8ms/step | 8.6 samples/s
+Step 1130/50462 | Loss: 3.5687 | LR: 5.00e-04 | 6988.1ms/step | 8.6 samples/s
+Step 1140/50462 | Loss: 3.5888 | LR: 5.00e-04 | 6989.8ms/step | 8.6 samples/s
+Step 1150/50462 | Loss: 3.5869 | LR: 5.00e-04 | 6985.9ms/step | 8.6 samples/s
+Step 1160/50462 | Loss: 3.5741 | LR: 5.00e-04 | 6986.5ms/step | 8.6 samples/s
+Step 1170/50462 | Loss: 3.5614 | LR: 5.00e-04 | 6986.4ms/step | 8.6 samples/s
+Step 1180/50462 | Loss: 3.5469 | LR: 5.00e-04 | 6987.6ms/step | 8.6 samples/s
+Step 1190/50462 | Loss: 3.5895 | LR: 5.00e-04 | 6986.5ms/step | 8.6 samples/s
+Step 1200/50462 | Loss: 3.4897 | LR: 5.00e-04 | 6987.6ms/step | 8.6 samples/s
+Step 1210/50462 | Loss: 3.5457 | LR: 5.00e-04 | 6985.7ms/step | 8.6 samples/s
+Step 1220/50462 | Loss: 3.5586 | LR: 5.00e-04 | 6985.5ms/step | 8.6 samples/s
+Step 1230/50462 | Loss: 3.5576 | LR: 5.00e-04 | 6987.6ms/step | 8.6 samples/s
+Step 1240/50462 | Loss: 3.5320 | LR: 5.00e-04 | 6987.8ms/step | 8.6 samples/s
+Step 1250/50462 | Loss: 3.5228 | LR: 5.00e-04 | 6986.3ms/step | 8.6 samples/s
+Validation at step 1250: Loss = 3.4897, Perplexity = 32.78
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-1250
+Step 1260/50462 | Loss: 3.4950 | LR: 5.00e-04 | 10757.8ms/step | 5.6 samples/s
+Step 1270/50462 | Loss: 3.4530 | LR: 5.00e-04 | 6984.5ms/step | 8.6 samples/s
+Step 1280/50462 | Loss: 3.5241 | LR: 5.00e-04 | 6988.9ms/step | 8.6 samples/s
+Step 1290/50462 | Loss: 3.4532 | LR: 5.00e-04 | 6986.1ms/step | 8.6 samples/s
+Step 1300/50462 | Loss: 3.4954 | LR: 5.00e-04 | 6988.0ms/step | 8.6 samples/s
+Step 1310/50462 | Loss: 3.5101 | LR: 5.00e-04 | 6986.1ms/step | 8.6 samples/s
+Step 1320/50462 | Loss: 3.4786 | LR: 5.00e-04 | 6989.8ms/step | 8.6 samples/s
+Step 1330/50462 | Loss: 3.4448 | LR: 5.00e-04 | 6984.2ms/step | 8.6 samples/s
+Step 1340/50462 | Loss: 3.4296 | LR: 5.00e-04 | 6989.5ms/step | 8.6 samples/s
+Step 1350/50462 | Loss: 3.4514 | LR: 5.00e-04 | 6988.0ms/step | 8.6 samples/s
+Step 1360/50462 | Loss: 3.4206 | LR: 5.00e-04 | 6985.8ms/step | 8.6 samples/s
+Step 1370/50462 | Loss: 3.4638 | LR: 5.00e-04 | 6990.2ms/step | 8.6 samples/s
+Step 1380/50462 | Loss: 3.4280 | LR: 5.00e-04 | 6986.4ms/step | 8.6 samples/s
+Step 1390/50462 | Loss: 3.3971 | LR: 5.00e-04 | 6987.5ms/step | 8.6 samples/s
+Step 1400/50462 | Loss: 3.4146 | LR: 5.00e-04 | 6987.4ms/step | 8.6 samples/s
+Step 1410/50462 | Loss: 3.4468 | LR: 5.00e-04 | 6980.2ms/step | 8.6 samples/s
+Step 1420/50462 | Loss: 3.4424 | LR: 5.00e-04 | 6979.3ms/step | 8.6 samples/s
+Step 1430/50462 | Loss: 3.4211 | LR: 5.00e-04 | 6980.1ms/step | 8.6 samples/s
+Step 1440/50462 | Loss: 3.4276 | LR: 5.00e-04 | 6978.7ms/step | 8.6 samples/s
+Step 1450/50462 | Loss: 3.3767 | LR: 5.00e-04 | 6978.1ms/step | 8.6 samples/s
+Step 1460/50462 | Loss: 3.3895 | LR: 5.00e-04 | 6975.8ms/step | 8.6 samples/s
+Step 1470/50462 | Loss: 3.4021 | LR: 5.00e-04 | 6977.1ms/step | 8.6 samples/s
+Step 1480/50462 | Loss: 3.3726 | LR: 5.00e-04 | 6975.9ms/step | 8.6 samples/s
+Step 1490/50462 | Loss: 3.4028 | LR: 5.00e-04 | 6975.7ms/step | 8.6 samples/s
+Step 1500/50462 | Loss: 3.3659 | LR: 5.00e-04 | 6976.2ms/step | 8.6 samples/s
+Validation at step 1500: Loss = 3.3674, Perplexity = 29.00
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-1500
+Step 1510/50462 | Loss: 3.3810 | LR: 5.00e-04 | 10759.7ms/step | 5.6 samples/s
+Step 1520/50462 | Loss: 3.3442 | LR: 5.00e-04 | 7000.4ms/step | 8.6 samples/s
+Step 1530/50462 | Loss: 3.3526 | LR: 5.00e-04 | 7016.7ms/step | 8.6 samples/s
+Step 1540/50462 | Loss: 3.3672 | LR: 5.00e-04 | 7020.0ms/step | 8.5 samples/s
+Step 1550/50462 | Loss: 3.3788 | LR: 5.00e-04 | 7018.8ms/step | 8.5 samples/s
+Step 1560/50462 | Loss: 3.3520 | LR: 5.00e-04 | 7002.4ms/step | 8.6 samples/s
+Step 1570/50462 | Loss: 3.3618 | LR: 5.00e-04 | 7009.3ms/step | 8.6 samples/s
+Step 1580/50462 | Loss: 3.3222 | LR: 5.00e-04 | 7015.6ms/step | 8.6 samples/s
+Step 1590/50462 | Loss: 3.3415 | LR: 5.00e-04 | 6992.0ms/step | 8.6 samples/s
+Step 1600/50462 | Loss: 3.3620 | LR: 5.00e-04 | 6991.1ms/step | 8.6 samples/s
+Step 1610/50462 | Loss: 3.3402 | LR: 5.00e-04 | 6984.7ms/step | 8.6 samples/s
+Step 1620/50462 | Loss: 3.3494 | LR: 5.00e-04 | 6991.9ms/step | 8.6 samples/s
+Step 1630/50462 | Loss: 3.3329 | LR: 5.00e-04 | 7008.2ms/step | 8.6 samples/s
+Step 1640/50462 | Loss: 3.3089 | LR: 5.00e-04 | 7019.5ms/step | 8.5 samples/s
+Step 1650/50462 | Loss: 3.3211 | LR: 5.00e-04 | 7009.9ms/step | 8.6 samples/s
+Step 1660/50462 | Loss: 3.3309 | LR: 5.00e-04 | 7007.8ms/step | 8.6 samples/s
+Step 1670/50462 | Loss: 3.2594 | LR: 5.00e-04 | 7007.3ms/step | 8.6 samples/s
+Step 1680/50462 | Loss: 3.3242 | LR: 5.00e-04 | 7004.2ms/step | 8.6 samples/s
+Step 1690/50462 | Loss: 3.3271 | LR: 5.00e-04 | 6994.7ms/step | 8.6 samples/s
+Step 1700/50462 | Loss: 3.2683 | LR: 5.00e-04 | 6996.2ms/step | 8.6 samples/s
+Step 1710/50462 | Loss: 3.2889 | LR: 5.00e-04 | 6994.8ms/step | 8.6 samples/s
+Step 1720/50462 | Loss: 3.2799 | LR: 5.00e-04 | 6991.3ms/step | 8.6 samples/s
+Step 1730/50462 | Loss: 3.2937 | LR: 5.00e-04 | 6992.7ms/step | 8.6 samples/s
+Step 1740/50462 | Loss: 3.2602 | LR: 5.00e-04 | 6992.2ms/step | 8.6 samples/s
+Step 1750/50462 | Loss: 3.2789 | LR: 5.00e-04 | 6999.4ms/step | 8.6 samples/s
+Validation at step 1750: Loss = 3.2792, Perplexity = 26.55
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-1750
+Step 1760/50462 | Loss: 3.2848 | LR: 4.99e-04 | 10795.4ms/step | 5.6 samples/s
+Step 1770/50462 | Loss: 3.2934 | LR: 4.99e-04 | 6996.0ms/step | 8.6 samples/s
+Step 1780/50462 | Loss: 3.2656 | LR: 4.99e-04 | 7001.8ms/step | 8.6 samples/s
+Step 1790/50462 | Loss: 3.2855 | LR: 4.99e-04 | 7008.4ms/step | 8.6 samples/s
+Step 1800/50462 | Loss: 3.2852 | LR: 4.99e-04 | 7007.4ms/step | 8.6 samples/s
+Step 1810/50462 | Loss: 3.3031 | LR: 4.99e-04 | 7002.4ms/step | 8.6 samples/s
+Step 1820/50462 | Loss: 3.2804 | LR: 4.99e-04 | 7004.9ms/step | 8.6 samples/s
+Step 1830/50462 | Loss: 3.2724 | LR: 4.99e-04 | 7004.9ms/step | 8.6 samples/s
+Step 1840/50462 | Loss: 3.2435 | LR: 4.99e-04 | 6999.7ms/step | 8.6 samples/s
+Step 1850/50462 | Loss: 3.2467 | LR: 4.99e-04 | 6999.1ms/step | 8.6 samples/s
+Step 1860/50462 | Loss: 3.2667 | LR: 4.99e-04 | 7000.4ms/step | 8.6 samples/s
+Step 1870/50462 | Loss: 3.2725 | LR: 4.99e-04 | 7002.4ms/step | 8.6 samples/s
+Step 1880/50462 | Loss: 3.2516 | LR: 4.99e-04 | 7005.4ms/step | 8.6 samples/s
+Step 1890/50462 | Loss: 3.2262 | LR: 4.99e-04 | 7006.3ms/step | 8.6 samples/s
+Step 1900/50462 | Loss: 3.2665 | LR: 4.99e-04 | 7008.0ms/step | 8.6 samples/s
+Step 1910/50462 | Loss: 3.2557 | LR: 4.99e-04 | 6999.6ms/step | 8.6 samples/s
+Step 1920/50462 | Loss: 3.2344 | LR: 4.99e-04 | 7007.4ms/step | 8.6 samples/s
+Step 1930/50462 | Loss: 3.2331 | LR: 4.99e-04 | 7006.3ms/step | 8.6 samples/s
+Step 1940/50462 | Loss: 3.2540 | LR: 4.99e-04 | 7005.4ms/step | 8.6 samples/s
+Step 1950/50462 | Loss: 3.2642 | LR: 4.99e-04 | 7003.3ms/step | 8.6 samples/s
+Step 1960/50462 | Loss: 3.2328 | LR: 4.99e-04 | 7001.2ms/step | 8.6 samples/s
+Step 1970/50462 | Loss: 3.2330 | LR: 4.99e-04 | 6995.3ms/step | 8.6 samples/s
+Step 1980/50462 | Loss: 3.2341 | LR: 4.99e-04 | 6988.7ms/step | 8.6 samples/s
+Step 1990/50462 | Loss: 3.2301 | LR: 4.99e-04 | 6986.6ms/step | 8.6 samples/s
+Step 2000/50462 | Loss: 3.2461 | LR: 4.99e-04 | 6986.6ms/step | 8.6 samples/s
+Validation at step 2000: Loss = 3.2130, Perplexity = 24.85
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-2000
+Step 2010/50462 | Loss: 3.1880 | LR: 4.99e-04 | 10783.7ms/step | 5.6 samples/s
+Step 2020/50462 | Loss: 3.2372 | LR: 4.99e-04 | 6986.3ms/step | 8.6 samples/s
+Step 2030/50462 | Loss: 3.2083 | LR: 4.99e-04 | 6990.2ms/step | 8.6 samples/s
+Step 2040/50462 | Loss: 3.2101 | LR: 4.99e-04 | 6988.6ms/step | 8.6 samples/s
+Step 2050/50462 | Loss: 3.1618 | LR: 4.99e-04 | 6982.2ms/step | 8.6 samples/s
+Step 2060/50462 | Loss: 3.2344 | LR: 4.99e-04 | 6984.4ms/step | 8.6 samples/s
+Step 2070/50462 | Loss: 3.2079 | LR: 4.99e-04 | 7000.0ms/step | 8.6 samples/s
+Step 2080/50462 | Loss: 3.2033 | LR: 4.99e-04 | 7013.4ms/step | 8.6 samples/s
+Step 2090/50462 | Loss: 3.2283 | LR: 4.99e-04 | 7008.2ms/step | 8.6 samples/s
+Step 2100/50462 | Loss: 3.2070 | LR: 4.99e-04 | 7010.9ms/step | 8.6 samples/s
+Step 2110/50462 | Loss: 3.2140 | LR: 4.99e-04 | 7016.3ms/step | 8.6 samples/s
+Step 2120/50462 | Loss: 3.2027 | LR: 4.99e-04 | 7006.6ms/step | 8.6 samples/s
+Step 2130/50462 | Loss: 3.2023 | LR: 4.99e-04 | 7015.1ms/step | 8.6 samples/s
+Step 2140/50462 | Loss: 3.2078 | LR: 4.99e-04 | 7003.2ms/step | 8.6 samples/s
+Step 2150/50462 | Loss: 3.1806 | LR: 4.99e-04 | 6993.1ms/step | 8.6 samples/s
+Step 2160/50462 | Loss: 3.2147 | LR: 4.99e-04 | 6990.6ms/step | 8.6 samples/s
+Step 2170/50462 | Loss: 3.1855 | LR: 4.99e-04 | 6987.0ms/step | 8.6 samples/s
+Step 2180/50462 | Loss: 3.2051 | LR: 4.99e-04 | 6994.4ms/step | 8.6 samples/s
+Step 2190/50462 | Loss: 3.1860 | LR: 4.99e-04 | 7002.5ms/step | 8.6 samples/s
+Step 2200/50462 | Loss: 3.1900 | LR: 4.99e-04 | 7005.0ms/step | 8.6 samples/s
+Step 2210/50462 | Loss: 3.1696 | LR: 4.99e-04 | 6998.2ms/step | 8.6 samples/s
+Step 2220/50462 | Loss: 3.2034 | LR: 4.99e-04 | 6996.9ms/step | 8.6 samples/s
+Step 2230/50462 | Loss: 3.1856 | LR: 4.99e-04 | 6994.2ms/step | 8.6 samples/s
+Step 2240/50462 | Loss: 3.1857 | LR: 4.99e-04 | 6994.2ms/step | 8.6 samples/s
+Step 2250/50462 | Loss: 3.1901 | LR: 4.99e-04 | 6991.9ms/step | 8.6 samples/s
+Validation at step 2250: Loss = 3.1624, Perplexity = 23.63
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-2250
+Step 2260/50462 | Loss: 3.1717 | LR: 4.99e-04 | 10763.1ms/step | 5.6 samples/s
+Step 2270/50462 | Loss: 3.1833 | LR: 4.99e-04 | 6996.3ms/step | 8.6 samples/s
+Step 2280/50462 | Loss: 3.1842 | LR: 4.99e-04 | 6993.1ms/step | 8.6 samples/s
+Step 2290/50462 | Loss: 3.1703 | LR: 4.99e-04 | 6994.1ms/step | 8.6 samples/s
+Step 2300/50462 | Loss: 3.1720 | LR: 4.99e-04 | 6991.5ms/step | 8.6 samples/s
+Step 2310/50462 | Loss: 3.1915 | LR: 4.99e-04 | 6993.6ms/step | 8.6 samples/s
+Step 2320/50462 | Loss: 3.1855 | LR: 4.99e-04 | 6995.1ms/step | 8.6 samples/s
+Step 2330/50462 | Loss: 3.1540 | LR: 4.99e-04 | 7005.4ms/step | 8.6 samples/s
+Step 2340/50462 | Loss: 3.1747 | LR: 4.99e-04 | 7005.2ms/step | 8.6 samples/s
+Step 2350/50462 | Loss: 3.1328 | LR: 4.99e-04 | 7004.4ms/step | 8.6 samples/s
+Step 2360/50462 | Loss: 3.1299 | LR: 4.99e-04 | 7000.5ms/step | 8.6 samples/s
+Step 2370/50462 | Loss: 3.1529 | LR: 4.99e-04 | 6998.4ms/step | 8.6 samples/s
+Step 2380/50462 | Loss: 3.1293 | LR: 4.99e-04 | 6999.3ms/step | 8.6 samples/s
+Step 2390/50462 | Loss: 3.1752 | LR: 4.99e-04 | 6999.6ms/step | 8.6 samples/s
+Step 2400/50462 | Loss: 3.1625 | LR: 4.99e-04 | 6996.9ms/step | 8.6 samples/s
+Step 2410/50462 | Loss: 3.1279 | LR: 4.99e-04 | 7000.3ms/step | 8.6 samples/s
+Step 2420/50462 | Loss: 3.1246 | LR: 4.99e-04 | 7003.3ms/step | 8.6 samples/s
+Step 2430/50462 | Loss: 3.1375 | LR: 4.99e-04 | 7000.2ms/step | 8.6 samples/s
+Step 2440/50462 | Loss: 3.1167 | LR: 4.99e-04 | 7009.0ms/step | 8.6 samples/s
+Step 2450/50462 | Loss: 3.1839 | LR: 4.99e-04 | 7001.4ms/step | 8.6 samples/s
+Step 2460/50462 | Loss: 3.1271 | LR: 4.99e-04 | 6998.0ms/step | 8.6 samples/s
+Step 2470/50462 | Loss: 3.1414 | LR: 4.99e-04 | 7006.0ms/step | 8.6 samples/s
+Step 2480/50462 | Loss: 3.1352 | LR: 4.99e-04 | 7005.4ms/step | 8.6 samples/s
+Step 2490/50462 | Loss: 3.1253 | LR: 4.98e-04 | 7004.1ms/step | 8.6 samples/s
+Step 2500/50462 | Loss: 3.1194 | LR: 4.98e-04 | 7005.5ms/step | 8.6 samples/s
+Validation at step 2500: Loss = 3.1202, Perplexity = 22.65
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-2500
+Step 2510/50462 | Loss: 3.1033 | LR: 4.98e-04 | 10794.6ms/step | 5.6 samples/s
+Step 2520/50462 | Loss: 3.1173 | LR: 4.98e-04 | 6997.5ms/step | 8.6 samples/s
+Step 2530/50462 | Loss: 3.1184 | LR: 4.98e-04 | 6997.1ms/step | 8.6 samples/s
+Step 2540/50462 | Loss: 3.1084 | LR: 4.98e-04 | 6996.1ms/step | 8.6 samples/s
+Step 2550/50462 | Loss: 3.1290 | LR: 4.98e-04 | 6995.3ms/step | 8.6 samples/s
+Step 2560/50462 | Loss: 3.1058 | LR: 4.98e-04 | 6989.3ms/step | 8.6 samples/s
+Step 2570/50462 | Loss: 3.1115 | LR: 4.98e-04 | 6985.1ms/step | 8.6 samples/s
+Step 2580/50462 | Loss: 3.0768 | LR: 4.98e-04 | 6987.8ms/step | 8.6 samples/s
+Step 2590/50462 | Loss: 3.0940 | LR: 4.98e-04 | 6995.2ms/step | 8.6 samples/s
+Step 2600/50462 | Loss: 3.1402 | LR: 4.98e-04 | 6995.5ms/step | 8.6 samples/s
+Step 2610/50462 | Loss: 3.1374 | LR: 4.98e-04 | 6993.3ms/step | 8.6 samples/s
+Step 2620/50462 | Loss: 3.1452 | LR: 4.98e-04 | 6999.2ms/step | 8.6 samples/s
+Step 2630/50462 | Loss: 3.0915 | LR: 4.98e-04 | 6996.9ms/step | 8.6 samples/s
+Step 2640/50462 | Loss: 3.1067 | LR: 4.98e-04 | 6995.4ms/step | 8.6 samples/s
+Step 2650/50462 | Loss: 3.1091 | LR: 4.98e-04 | 7000.9ms/step | 8.6 samples/s
+Step 2660/50462 | Loss: 3.1264 | LR: 4.98e-04 | 7014.9ms/step | 8.6 samples/s
+Step 2670/50462 | Loss: 3.1062 | LR: 4.98e-04 | 7016.7ms/step | 8.6 samples/s
+Step 2680/50462 | Loss: 3.1093 | LR: 4.98e-04 | 7029.1ms/step | 8.5 samples/s
+Step 2690/50462 | Loss: 3.0895 | LR: 4.98e-04 | 7018.9ms/step | 8.5 samples/s
+Step 2700/50462 | Loss: 3.1212 | LR: 4.98e-04 | 7019.5ms/step | 8.5 samples/s
+Step 2710/50462 | Loss: 3.1282 | LR: 4.98e-04 | 7016.4ms/step | 8.6 samples/s
+Step 2720/50462 | Loss: 3.1085 | LR: 4.98e-04 | 7016.2ms/step | 8.6 samples/s
+Step 2730/50462 | Loss: 3.1022 | LR: 4.98e-04 | 7005.8ms/step | 8.6 samples/s
+Step 2740/50462 | Loss: 3.1090 | LR: 4.98e-04 | 6988.8ms/step | 8.6 samples/s
+Step 2750/50462 | Loss: 3.1129 | LR: 4.98e-04 | 6995.2ms/step | 8.6 samples/s
+Validation at step 2750: Loss = 3.0843, Perplexity = 21.85
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-2750
+Step 2760/50462 | Loss: 3.1199 | LR: 4.98e-04 | 10776.6ms/step | 5.6 samples/s
+Step 2770/50462 | Loss: 3.1243 | LR: 4.98e-04 | 7010.9ms/step | 8.6 samples/s
+Step 2780/50462 | Loss: 3.1189 | LR: 4.98e-04 | 7020.1ms/step | 8.5 samples/s
+Step 2790/50462 | Loss: 3.1252 | LR: 4.98e-04 | 7017.2ms/step | 8.6 samples/s
+Step 2800/50462 | Loss: 3.0913 | LR: 4.98e-04 | 7014.1ms/step | 8.6 samples/s
+Step 2810/50462 | Loss: 3.1074 | LR: 4.98e-04 | 7006.9ms/step | 8.6 samples/s
+Step 2820/50462 | Loss: 3.0836 | LR: 4.98e-04 | 7007.0ms/step | 8.6 samples/s
+Step 2830/50462 | Loss: 3.0726 | LR: 4.98e-04 | 7006.4ms/step | 8.6 samples/s
+Step 2840/50462 | Loss: 3.0717 | LR: 4.98e-04 | 7008.0ms/step | 8.6 samples/s
+Step 2850/50462 | Loss: 3.0960 | LR: 4.98e-04 | 7006.4ms/step | 8.6 samples/s
+Step 2860/50462 | Loss: 3.0877 | LR: 4.98e-04 | 7006.9ms/step | 8.6 samples/s
+Step 2870/50462 | Loss: 3.0788 | LR: 4.98e-04 | 7007.4ms/step | 8.6 samples/s
+Step 2880/50462 | Loss: 3.0712 | LR: 4.98e-04 | 7008.6ms/step | 8.6 samples/s
+Step 2890/50462 | Loss: 3.0741 | LR: 4.98e-04 | 7005.6ms/step | 8.6 samples/s
+Step 2900/50462 | Loss: 3.1151 | LR: 4.98e-04 | 7003.6ms/step | 8.6 samples/s
+Step 2910/50462 | Loss: 3.0641 | LR: 4.98e-04 | 7006.8ms/step | 8.6 samples/s
+Step 2920/50462 | Loss: 3.0694 | LR: 4.98e-04 | 7006.8ms/step | 8.6 samples/s
+Step 2930/50462 | Loss: 3.0819 | LR: 4.98e-04 | 7009.3ms/step | 8.6 samples/s
+Step 2940/50462 | Loss: 3.0491 | LR: 4.98e-04 | 7006.2ms/step | 8.6 samples/s
+Step 2950/50462 | Loss: 3.0619 | LR: 4.98e-04 | 7012.4ms/step | 8.6 samples/s
+Step 2960/50462 | Loss: 3.0773 | LR: 4.98e-04 | 7011.6ms/step | 8.6 samples/s
+Step 2970/50462 | Loss: 3.0798 | LR: 4.98e-04 | 7007.0ms/step | 8.6 samples/s
+Step 2980/50462 | Loss: 3.0732 | LR: 4.98e-04 | 7011.1ms/step | 8.6 samples/s
+Step 2990/50462 | Loss: 3.0639 | LR: 4.97e-04 | 7007.3ms/step | 8.6 samples/s
+Step 3000/50462 | Loss: 3.0502 | LR: 4.97e-04 | 7009.6ms/step | 8.6 samples/s
+Validation at step 3000: Loss = 3.0547, Perplexity = 21.22
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-3000
+Step 3010/50462 | Loss: 3.0558 | LR: 4.97e-04 | 10794.4ms/step | 5.6 samples/s
+Step 3020/50462 | Loss: 3.0383 | LR: 4.97e-04 | 7004.8ms/step | 8.6 samples/s
+Step 3030/50462 | Loss: 3.0735 | LR: 4.97e-04 | 7006.4ms/step | 8.6 samples/s
+Step 3040/50462 | Loss: 3.0494 | LR: 4.97e-04 | 7010.6ms/step | 8.6 samples/s
+Step 3050/50462 | Loss: 3.0584 | LR: 4.97e-04 | 7009.5ms/step | 8.6 samples/s
+Step 3060/50462 | Loss: 3.0662 | LR: 4.97e-04 | 7012.8ms/step | 8.6 samples/s
+Step 3070/50462 | Loss: 3.0913 | LR: 4.97e-04 | 7011.4ms/step | 8.6 samples/s
+Step 3080/50462 | Loss: 3.0764 | LR: 4.97e-04 | 7006.5ms/step | 8.6 samples/s
+Step 3090/50462 | Loss: 3.0916 | LR: 4.97e-04 | 7009.7ms/step | 8.6 samples/s
+Step 3100/50462 | Loss: 3.0698 | LR: 4.97e-04 | 7011.2ms/step | 8.6 samples/s
+Step 3110/50462 | Loss: 3.0645 | LR: 4.97e-04 | 7010.5ms/step | 8.6 samples/s
+Step 3120/50462 | Loss: 3.0299 | LR: 4.97e-04 | 7008.8ms/step | 8.6 samples/s
+Step 3130/50462 | Loss: 3.0276 | LR: 4.97e-04 | 7005.5ms/step | 8.6 samples/s
+Step 3140/50462 | Loss: 3.0592 | LR: 4.97e-04 | 7006.3ms/step | 8.6 samples/s
+Step 3150/50462 | Loss: 3.0562 | LR: 4.97e-04 | 7005.0ms/step | 8.6 samples/s
+Step 3160/50462 | Loss: 3.0228 | LR: 4.97e-04 | 7003.1ms/step | 8.6 samples/s
+Step 3170/50462 | Loss: 3.0266 | LR: 4.97e-04 | 7005.8ms/step | 8.6 samples/s
+Step 3180/50462 | Loss: 3.0595 | LR: 4.97e-04 | 7001.5ms/step | 8.6 samples/s
+Step 3190/50462 | Loss: 3.0480 | LR: 4.97e-04 | 6990.3ms/step | 8.6 samples/s
+Step 3200/50462 | Loss: 3.0290 | LR: 4.97e-04 | 6992.9ms/step | 8.6 samples/s
+Step 3210/50462 | Loss: 3.0576 | LR: 4.97e-04 | 6991.4ms/step | 8.6 samples/s
+Step 3220/50462 | Loss: 3.0312 | LR: 4.97e-04 | 6986.6ms/step | 8.6 samples/s
+Step 3230/50462 | Loss: 3.0388 | LR: 4.97e-04 | 6985.9ms/step | 8.6 samples/s
+Step 3240/50462 | Loss: 3.0631 | LR: 4.97e-04 | 6986.2ms/step | 8.6 samples/s
+Step 3250/50462 | Loss: 3.0602 | LR: 4.97e-04 | 6995.1ms/step | 8.6 samples/s
+Validation at step 3250: Loss = 3.0245, Perplexity = 20.58
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-3250
+Step 3260/50462 | Loss: 3.0145 | LR: 4.97e-04 | 10809.9ms/step | 5.6 samples/s
+Step 3270/50462 | Loss: 3.0456 | LR: 4.97e-04 | 7006.2ms/step | 8.6 samples/s
+Step 3280/50462 | Loss: 3.0094 | LR: 4.97e-04 | 7012.6ms/step | 8.6 samples/s
+Step 3290/50462 | Loss: 3.0381 | LR: 4.97e-04 | 7013.9ms/step | 8.6 samples/s
+Step 3300/50462 | Loss: 3.0133 | LR: 4.97e-04 | 7018.6ms/step | 8.5 samples/s
+Step 3310/50462 | Loss: 3.0206 | LR: 4.97e-04 | 7014.5ms/step | 8.6 samples/s
+Step 3320/50462 | Loss: 3.0257 | LR: 4.97e-04 | 7006.3ms/step | 8.6 samples/s
+Step 3330/50462 | Loss: 3.0168 | LR: 4.97e-04 | 7018.8ms/step | 8.5 samples/s
+Step 3340/50462 | Loss: 3.0034 | LR: 4.97e-04 | 7012.0ms/step | 8.6 samples/s
+Step 3350/50462 | Loss: 3.0155 | LR: 4.97e-04 | 6993.5ms/step | 8.6 samples/s
+Step 3360/50462 | Loss: 3.0274 | LR: 4.97e-04 | 6995.4ms/step | 8.6 samples/s
+Step 3370/50462 | Loss: 3.0142 | LR: 4.97e-04 | 6987.6ms/step | 8.6 samples/s
+Step 3380/50462 | Loss: 3.0444 | LR: 4.97e-04 | 6988.8ms/step | 8.6 samples/s
+Step 3390/50462 | Loss: 3.0144 | LR: 4.97e-04 | 7003.5ms/step | 8.6 samples/s
+Step 3400/50462 | Loss: 3.0029 | LR: 4.97e-04 | 7014.1ms/step | 8.6 samples/s
+Step 3410/50462 | Loss: 3.0591 | LR: 4.96e-04 | 7007.0ms/step | 8.6 samples/s
+Step 3420/50462 | Loss: 3.0262 | LR: 4.96e-04 | 7005.3ms/step | 8.6 samples/s
+Step 3430/50462 | Loss: 3.0395 | LR: 4.96e-04 | 7007.2ms/step | 8.6 samples/s
+Step 3440/50462 | Loss: 3.0176 | LR: 4.96e-04 | 7003.7ms/step | 8.6 samples/s
+Step 3450/50462 | Loss: 2.9979 | LR: 4.96e-04 | 7002.4ms/step | 8.6 samples/s
+Step 3460/50462 | Loss: 3.0453 | LR: 4.96e-04 | 7002.0ms/step | 8.6 samples/s
+Step 3470/50462 | Loss: 2.9988 | LR: 4.96e-04 | 7003.9ms/step | 8.6 samples/s
+Step 3480/50462 | Loss: 3.0340 | LR: 4.96e-04 | 6999.6ms/step | 8.6 samples/s
+Step 3490/50462 | Loss: 3.0024 | LR: 4.96e-04 | 7003.7ms/step | 8.6 samples/s
+Step 3500/50462 | Loss: 3.0125 | LR: 4.96e-04 | 7004.1ms/step | 8.6 samples/s
+Validation at step 3500: Loss = 2.9992, Perplexity = 20.07
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-3500
+Step 3510/50462 | Loss: 3.0054 | LR: 4.96e-04 | 10800.0ms/step | 5.6 samples/s
+Step 3520/50462 | Loss: 3.0265 | LR: 4.96e-04 | 6996.9ms/step | 8.6 samples/s
+Step 3530/50462 | Loss: 2.9760 | LR: 4.96e-04 | 6996.8ms/step | 8.6 samples/s
+Step 3540/50462 | Loss: 3.0349 | LR: 4.96e-04 | 6994.7ms/step | 8.6 samples/s
+Step 3550/50462 | Loss: 3.0334 | LR: 4.96e-04 | 6995.0ms/step | 8.6 samples/s
+Step 3560/50462 | Loss: 3.0091 | LR: 4.96e-04 | 6994.6ms/step | 8.6 samples/s
+Step 3570/50462 | Loss: 3.0159 | LR: 4.96e-04 | 6995.8ms/step | 8.6 samples/s
+Step 3580/50462 | Loss: 2.9684 | LR: 4.96e-04 | 6997.7ms/step | 8.6 samples/s
+Step 3590/50462 | Loss: 2.9992 | LR: 4.96e-04 | 6993.4ms/step | 8.6 samples/s
+Step 3600/50462 | Loss: 2.9951 | LR: 4.96e-04 | 6996.2ms/step | 8.6 samples/s
+Step 3610/50462 | Loss: 3.0092 | LR: 4.96e-04 | 6998.2ms/step | 8.6 samples/s
+Step 3620/50462 | Loss: 2.9983 | LR: 4.96e-04 | 6993.0ms/step | 8.6 samples/s
+Step 3630/50462 | Loss: 3.0109 | LR: 4.96e-04 | 6995.3ms/step | 8.6 samples/s
+Step 3640/50462 | Loss: 3.0164 | LR: 4.96e-04 | 6995.2ms/step | 8.6 samples/s
+Step 3650/50462 | Loss: 2.9972 | LR: 4.96e-04 | 6996.5ms/step | 8.6 samples/s
+Step 3660/50462 | Loss: 2.9938 | LR: 4.96e-04 | 6996.9ms/step | 8.6 samples/s
+Step 3670/50462 | Loss: 3.0070 | LR: 4.96e-04 | 6995.0ms/step | 8.6 samples/s
+Step 3680/50462 | Loss: 2.9592 | LR: 4.96e-04 | 6998.3ms/step | 8.6 samples/s
+Step 3690/50462 | Loss: 2.9704 | LR: 4.96e-04 | 6994.9ms/step | 8.6 samples/s
+Step 3700/50462 | Loss: 2.9944 | LR: 4.96e-04 | 6992.0ms/step | 8.6 samples/s
+Step 3710/50462 | Loss: 2.9934 | LR: 4.96e-04 | 6995.7ms/step | 8.6 samples/s
+Step 3720/50462 | Loss: 2.9773 | LR: 4.96e-04 | 6999.1ms/step | 8.6 samples/s
+Step 3730/50462 | Loss: 2.9717 | LR: 4.96e-04 | 6990.4ms/step | 8.6 samples/s
+Step 3740/50462 | Loss: 2.9750 | LR: 4.96e-04 | 6995.3ms/step | 8.6 samples/s
+Step 3750/50462 | Loss: 2.9603 | LR: 4.96e-04 | 6990.2ms/step | 8.6 samples/s
+Validation at step 3750: Loss = 2.9805, Perplexity = 19.70
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-3750
+Step 3760/50462 | Loss: 2.9941 | LR: 4.95e-04 | 10820.4ms/step | 5.5 samples/s
+Step 3770/50462 | Loss: 2.9756 | LR: 4.95e-04 | 6985.8ms/step | 8.6 samples/s
+Step 3780/50462 | Loss: 3.0174 | LR: 4.95e-04 | 6988.8ms/step | 8.6 samples/s
+Step 3790/50462 | Loss: 3.0043 | LR: 4.95e-04 | 6985.6ms/step | 8.6 samples/s
+Step 3800/50462 | Loss: 2.9699 | LR: 4.95e-04 | 6985.1ms/step | 8.6 samples/s
+Step 3810/50462 | Loss: 2.9569 | LR: 4.95e-04 | 6990.3ms/step | 8.6 samples/s
+Step 3820/50462 | Loss: 2.9567 | LR: 4.95e-04 | 6987.1ms/step | 8.6 samples/s
+Step 3830/50462 | Loss: 3.0193 | LR: 4.95e-04 | 6989.8ms/step | 8.6 samples/s
+Step 3840/50462 | Loss: 2.9586 | LR: 4.95e-04 | 6985.7ms/step | 8.6 samples/s
+Step 3850/50462 | Loss: 3.0179 | LR: 4.95e-04 | 6983.8ms/step | 8.6 samples/s
+Step 3860/50462 | Loss: 2.9404 | LR: 4.95e-04 | 6985.8ms/step | 8.6 samples/s
+Step 3870/50462 | Loss: 2.9509 | LR: 4.95e-04 | 6988.0ms/step | 8.6 samples/s
+Step 3880/50462 | Loss: 2.9673 | LR: 4.95e-04 | 6990.4ms/step | 8.6 samples/s
+Step 3890/50462 | Loss: 2.9607 | LR: 4.95e-04 | 6985.9ms/step | 8.6 samples/s
+Step 3900/50462 | Loss: 3.0096 | LR: 4.95e-04 | 6987.0ms/step | 8.6 samples/s
+Step 3910/50462 | Loss: 2.9598 | LR: 4.95e-04 | 6984.0ms/step | 8.6 samples/s
+Step 3920/50462 | Loss: 2.9733 | LR: 4.95e-04 | 6986.5ms/step | 8.6 samples/s
+Step 3930/50462 | Loss: 2.9840 | LR: 4.95e-04 | 6988.0ms/step | 8.6 samples/s
+Step 3940/50462 | Loss: 2.9849 | LR: 4.95e-04 | 6991.4ms/step | 8.6 samples/s
+Step 3950/50462 | Loss: 2.9811 | LR: 4.95e-04 | 6990.0ms/step | 8.6 samples/s
+Step 3960/50462 | Loss: 2.9710 | LR: 4.95e-04 | 6990.3ms/step | 8.6 samples/s
+Step 3970/50462 | Loss: 3.0009 | LR: 4.95e-04 | 6989.9ms/step | 8.6 samples/s
+Step 3980/50462 | Loss: 2.9627 | LR: 4.95e-04 | 6993.3ms/step | 8.6 samples/s
+Step 3990/50462 | Loss: 2.9685 | LR: 4.95e-04 | 6991.1ms/step | 8.6 samples/s
+Step 4000/50462 | Loss: 2.9596 | LR: 4.95e-04 | 6990.9ms/step | 8.6 samples/s
+Validation at step 4000: Loss = 2.9601, Perplexity = 19.30
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-4000
+Step 4010/50462 | Loss: 2.9981 | LR: 4.95e-04 | 10780.6ms/step | 5.6 samples/s
+Step 4020/50462 | Loss: 2.9723 | LR: 4.95e-04 | 6995.1ms/step | 8.6 samples/s
+Step 4030/50462 | Loss: 2.9815 | LR: 4.95e-04 | 6993.8ms/step | 8.6 samples/s
+Step 4040/50462 | Loss: 2.9473 | LR: 4.95e-04 | 6996.5ms/step | 8.6 samples/s
+Step 4050/50462 | Loss: 2.9688 | LR: 4.95e-04 | 6997.4ms/step | 8.6 samples/s
+Step 4060/50462 | Loss: 2.9749 | LR: 4.95e-04 | 6997.4ms/step | 8.6 samples/s
+Step 4070/50462 | Loss: 2.9614 | LR: 4.95e-04 | 7004.9ms/step | 8.6 samples/s
+Step 4080/50462 | Loss: 2.9888 | LR: 4.94e-04 | 6995.8ms/step | 8.6 samples/s
+Step 4090/50462 | Loss: 2.9840 | LR: 4.94e-04 | 6994.3ms/step | 8.6 samples/s
+Step 4100/50462 | Loss: 2.9587 | LR: 4.94e-04 | 6995.2ms/step | 8.6 samples/s
+Step 4110/50462 | Loss: 2.9552 | LR: 4.94e-04 | 6994.8ms/step | 8.6 samples/s
+Step 4120/50462 | Loss: 2.9543 | LR: 4.94e-04 | 6997.1ms/step | 8.6 samples/s
+Step 4130/50462 | Loss: 2.9524 | LR: 4.94e-04 | 6997.7ms/step | 8.6 samples/s
+Step 4140/50462 | Loss: 2.9478 | LR: 4.94e-04 | 6997.7ms/step | 8.6 samples/s
+Step 4150/50462 | Loss: 2.9574 | LR: 4.94e-04 | 6996.8ms/step | 8.6 samples/s
+Step 4160/50462 | Loss: 2.9604 | LR: 4.94e-04 | 6996.1ms/step | 8.6 samples/s
+Step 4170/50462 | Loss: 2.9167 | LR: 4.94e-04 | 6997.5ms/step | 8.6 samples/s
+Step 4180/50462 | Loss: 2.9211 | LR: 4.94e-04 | 6996.1ms/step | 8.6 samples/s
+Step 4190/50462 | Loss: 2.9405 | LR: 4.94e-04 | 6998.5ms/step | 8.6 samples/s
+Step 4200/50462 | Loss: 2.9799 | LR: 4.94e-04 | 6997.3ms/step | 8.6 samples/s
+Step 4210/50462 | Loss: 2.9519 | LR: 4.94e-04 | 6995.0ms/step | 8.6 samples/s
+Step 4220/50462 | Loss: 2.9385 | LR: 4.94e-04 | 6997.7ms/step | 8.6 samples/s
+Step 4230/50462 | Loss: 2.9527 | LR: 4.94e-04 | 6997.7ms/step | 8.6 samples/s
+Step 4240/50462 | Loss: 2.9633 | LR: 4.94e-04 | 6997.3ms/step | 8.6 samples/s
+Step 4250/50462 | Loss: 2.9604 | LR: 4.94e-04 | 6996.6ms/step | 8.6 samples/s
+Validation at step 4250: Loss = 2.9433, Perplexity = 18.98
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-4250
+Step 4260/50462 | Loss: 2.9435 | LR: 4.94e-04 | 10779.6ms/step | 5.6 samples/s
+Step 4270/50462 | Loss: 2.9536 | LR: 4.94e-04 | 6984.5ms/step | 8.6 samples/s
+Step 4280/50462 | Loss: 2.9440 | LR: 4.94e-04 | 6987.6ms/step | 8.6 samples/s
+Step 4290/50462 | Loss: 2.9702 | LR: 4.94e-04 | 6994.7ms/step | 8.6 samples/s
+Step 4300/50462 | Loss: 2.9500 | LR: 4.94e-04 | 6992.7ms/step | 8.6 samples/s
+Step 4310/50462 | Loss: 2.9406 | LR: 4.94e-04 | 6990.2ms/step | 8.6 samples/s
+Step 4320/50462 | Loss: 2.9522 | LR: 4.94e-04 | 6991.9ms/step | 8.6 samples/s
+Step 4330/50462 | Loss: 2.9094 | LR: 4.94e-04 | 6992.1ms/step | 8.6 samples/s
+Step 4340/50462 | Loss: 2.9586 | LR: 4.94e-04 | 6989.2ms/step | 8.6 samples/s
+Step 4350/50462 | Loss: 2.9547 | LR: 4.94e-04 | 6992.2ms/step | 8.6 samples/s
+Step 4360/50462 | Loss: 2.9547 | LR: 4.94e-04 | 6997.4ms/step | 8.6 samples/s
+Step 4370/50462 | Loss: 2.9486 | LR: 4.93e-04 | 7023.0ms/step | 8.5 samples/s
+Step 4380/50462 | Loss: 2.9227 | LR: 4.93e-04 | 7033.4ms/step | 8.5 samples/s
+Step 4390/50462 | Loss: 2.9447 | LR: 4.93e-04 | 7029.2ms/step | 8.5 samples/s
+Step 4400/50462 | Loss: 2.9726 | LR: 4.93e-04 | 7028.9ms/step | 8.5 samples/s
+Step 4410/50462 | Loss: 2.9471 | LR: 4.93e-04 | 7015.8ms/step | 8.6 samples/s
+Step 4420/50462 | Loss: 2.9220 | LR: 4.93e-04 | 7021.1ms/step | 8.5 samples/s
+Step 4430/50462 | Loss: 2.9621 | LR: 4.93e-04 | 7015.4ms/step | 8.6 samples/s
+Step 4440/50462 | Loss: 2.8961 | LR: 4.93e-04 | 6995.7ms/step | 8.6 samples/s
+Step 4450/50462 | Loss: 2.9816 | LR: 4.93e-04 | 6988.4ms/step | 8.6 samples/s
+Step 4460/50462 | Loss: 2.9553 | LR: 4.93e-04 | 6989.8ms/step | 8.6 samples/s
+Step 4470/50462 | Loss: 2.9377 | LR: 4.93e-04 | 6990.9ms/step | 8.6 samples/s
+Step 4480/50462 | Loss: 2.9308 | LR: 4.93e-04 | 7017.4ms/step | 8.6 samples/s
+Step 4490/50462 | Loss: 2.9559 | LR: 4.93e-04 | 7020.0ms/step | 8.5 samples/s
+Step 4500/50462 | Loss: 2.9532 | LR: 4.93e-04 | 7015.9ms/step | 8.6 samples/s
+Validation at step 4500: Loss = 2.9257, Perplexity = 18.65
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-4500
+Step 4510/50462 | Loss: 2.9406 | LR: 4.93e-04 | 10835.1ms/step | 5.5 samples/s
+Step 4520/50462 | Loss: 2.9427 | LR: 4.93e-04 | 6984.9ms/step | 8.6 samples/s
+Step 4530/50462 | Loss: 2.9319 | LR: 4.93e-04 | 6986.4ms/step | 8.6 samples/s
+Step 4540/50462 | Loss: 2.9591 | LR: 4.93e-04 | 6988.8ms/step | 8.6 samples/s
+Step 4550/50462 | Loss: 2.8968 | LR: 4.93e-04 | 6994.0ms/step | 8.6 samples/s
+Step 4560/50462 | Loss: 2.9353 | LR: 4.93e-04 | 6996.5ms/step | 8.6 samples/s
+Step 4570/50462 | Loss: 2.9200 | LR: 4.93e-04 | 6998.4ms/step | 8.6 samples/s
+Step 4580/50462 | Loss: 2.9691 | LR: 4.93e-04 | 7001.0ms/step | 8.6 samples/s
+Step 4590/50462 | Loss: 2.9395 | LR: 4.93e-04 | 7001.3ms/step | 8.6 samples/s
+Step 4600/50462 | Loss: 2.9255 | LR: 4.93e-04 | 7000.8ms/step | 8.6 samples/s
+Step 4610/50462 | Loss: 2.9342 | LR: 4.93e-04 | 6999.3ms/step | 8.6 samples/s
+Step 4620/50462 | Loss: 2.9498 | LR: 4.93e-04 | 7002.2ms/step | 8.6 samples/s
+Step 4630/50462 | Loss: 2.9489 | LR: 4.93e-04 | 7002.7ms/step | 8.6 samples/s
+Step 4640/50462 | Loss: 2.8864 | LR: 4.92e-04 | 7004.4ms/step | 8.6 samples/s
+Step 4650/50462 | Loss: 2.9097 | LR: 4.92e-04 | 7003.4ms/step | 8.6 samples/s
+Step 4660/50462 | Loss: 2.9420 | LR: 4.92e-04 | 7003.1ms/step | 8.6 samples/s
+Step 4670/50462 | Loss: 2.9389 | LR: 4.92e-04 | 7002.2ms/step | 8.6 samples/s
+Step 4680/50462 | Loss: 2.9490 | LR: 4.92e-04 | 6998.3ms/step | 8.6 samples/s
+Step 4690/50462 | Loss: 2.9286 | LR: 4.92e-04 | 6999.6ms/step | 8.6 samples/s
+Step 4700/50462 | Loss: 2.9467 | LR: 4.92e-04 | 6999.5ms/step | 8.6 samples/s
+Step 4710/50462 | Loss: 2.9263 | LR: 4.92e-04 | 6996.3ms/step | 8.6 samples/s
+Step 4720/50462 | Loss: 2.9237 | LR: 4.92e-04 | 6999.5ms/step | 8.6 samples/s
+Step 4730/50462 | Loss: 2.9466 | LR: 4.92e-04 | 7001.0ms/step | 8.6 samples/s
+Step 4740/50462 | Loss: 2.9121 | LR: 4.92e-04 | 6999.4ms/step | 8.6 samples/s
+Step 4750/50462 | Loss: 2.9019 | LR: 4.92e-04 | 6992.7ms/step | 8.6 samples/s
+Validation at step 4750: Loss = 2.9103, Perplexity = 18.36
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-4750
+Step 4760/50462 | Loss: 2.9249 | LR: 4.92e-04 | 10842.3ms/step | 5.5 samples/s
+Step 4770/50462 | Loss: 2.9126 | LR: 4.92e-04 | 7000.6ms/step | 8.6 samples/s
+Step 4780/50462 | Loss: 2.9187 | LR: 4.92e-04 | 7000.4ms/step | 8.6 samples/s
+Step 4790/50462 | Loss: 2.9223 | LR: 4.92e-04 | 6999.1ms/step | 8.6 samples/s
+Step 4800/50462 | Loss: 2.9357 | LR: 4.92e-04 | 7009.3ms/step | 8.6 samples/s
+Step 4810/50462 | Loss: 2.9072 | LR: 4.92e-04 | 6997.4ms/step | 8.6 samples/s
+Step 4820/50462 | Loss: 2.9050 | LR: 4.92e-04 | 6997.2ms/step | 8.6 samples/s
+Step 4830/50462 | Loss: 2.9059 | LR: 4.92e-04 | 6995.3ms/step | 8.6 samples/s
+Step 4840/50462 | Loss: 2.9244 | LR: 4.92e-04 | 6998.3ms/step | 8.6 samples/s
+Step 4850/50462 | Loss: 2.9042 | LR: 4.92e-04 | 6996.2ms/step | 8.6 samples/s
+Step 4860/50462 | Loss: 2.9137 | LR: 4.92e-04 | 6993.9ms/step | 8.6 samples/s

epfl_final/SparseAtt/wandb/run-20260823_043841-rgmkfpnl/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260823_043841-rgmkfpnl/files/wandb-metadata.json

@@ -0,0 +1,72 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-08-23T04:38:41.848448Z",
+  "args":  [
+    "--config",
+    "configs/p2_1p2b_dense_seed70_4gpu.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "5637886f0cc760009989b14e68c6ae06d4827418"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "a100-train-5-gpu-0-2",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python",
+  "cpu_count":  64,
+  "cpu_count_logical":  128,
+  "gpu":  "NVIDIA A100-SXM4-80GB",
+  "gpu_count":  5,
+  "disk":  {
+    "/":  {
+      "total":  "7679362727936",
+      "used":  "4492280492032"
+    }
+  },
+  "memory":  {
+    "total":  "1081884971008"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-da403873-77fd-0400-8874-ee85122e86ce"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-ac7c2b06-0e54-6c0e-5c5e-bef297b937e2"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-2f2cf903-6615-a5f8-353b-076dccd64125"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-75d03f5a-9fc2-0173-5629-3ef21c2f450c"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-1d4bb5b9-c680-19af-9659-b21aa13b3c4d"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "bsb4p1tr8xxk4uc4w45xux1wm5szyk5p"
+}

epfl_final/SparseAtt/wandb/run-20260823_043841-rgmkfpnl/logs/debug.log

@@ -0,0 +1,19 @@
+2026-08-23 04:38:41,885 INFO    MainThread:19202 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-08-23 04:38:41,885 INFO    MainThread:19202 [wandb_setup.py:_flush():81] Configure stats pid to 19202
+2026-08-23 04:38:41,885 INFO    MainThread:19202 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-23 04:38:41,886 INFO    MainThread:19202 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260823_043841-rgmkfpnl/logs/debug.log
+2026-08-23 04:38:41,886 INFO    MainThread:19202 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260823_043841-rgmkfpnl/logs/debug-internal.log
+2026-08-23 04:38:41,886 INFO    MainThread:19202 [wandb_init.py:init():850] calling init triggers
+2026-08-23 04:38:41,886 INFO    MainThread:19202 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810', 'resume_from_checkpoint': None, 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 15, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 70, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b dense 8k seed70', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'dense', 'seed70'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
+2026-08-23 04:38:41,886 INFO    MainThread:19202 [wandb_init.py:init():898] starting backend
+2026-08-23 04:38:42,416 INFO    MainThread:19202 [wandb_init.py:init():913] sending inform_init request
+2026-08-23 04:38:42,938 INFO    MainThread:19202 [wandb_init.py:init():918] backend started and connected
+2026-08-23 04:38:42,945 INFO    MainThread:19202 [wandb_init.py:init():988] updated telemetry
+2026-08-23 04:38:42,974 INFO    MainThread:19202 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-08-23 04:38:43,420 INFO    MainThread:19202 [wandb_init.py:init():1056] starting run threads in backend
+2026-08-23 04:38:43,596 INFO    MainThread:19202 [wandb_run.py:_console_start():2554] atexit reg
+2026-08-23 04:38:43,596 INFO    MainThread:19202 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-08-23 04:38:43,596 INFO    MainThread:19202 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-08-23 04:38:43,596 INFO    MainThread:19202 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-08-23 04:38:43,603 INFO    MainThread:19202 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/run-20260823_043841-rgmkfpnl/run-rgmkfpnl.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f64a10c8a14fe30c917bab29afc12077d78d48524e30fd9403cff0186c66262e
+size 6651904

epfl_final/SparseAtt/wandb/run-20260823_140341-qctui00g/files/wandb-metadata.json

@@ -0,0 +1,79 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-08-23T14:03:41.679954Z",
+  "args":  [
+    "--config",
+    "configs/p2_1p2b_dense_seed71_6gpu.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "5637886f0cc760009989b14e68c6ae06d4827418"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "a100-train-6-gpu-0-0",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python",
+  "cpu_count":  64,
+  "cpu_count_logical":  128,
+  "gpu":  "NVIDIA A100-SXM4-80GB",
+  "gpu_count":  6,
+  "disk":  {
+    "/":  {
+      "total":  "7679362727936",
+      "used":  "5253326848000"
+    }
+  },
+  "memory":  {
+    "total":  "1081884884992"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-d89fb348-1c38-9080-1e32-49dcf38bb677"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-8a962b76-efe8-388f-24a2-7f80426e9ebb"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-184e67df-fbb6-0afd-f911-a3467fac75a5"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-df0fae65-9451-78e9-7e2d-965a8a4f3427"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-da84a93e-cacd-38a5-d0c2-c6f7e46f0285"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-330b94fa-0daf-9a72-5a17-135faae05e20"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "lmk1a2m3lyyjdah2zmapfcvju4md0aif"
+}