EPFL final checkpoints and W&B history 851-900 of 1463

.gitattributes

@@ -111,3 +111,10 @@ epfl_final/SparseAtt/wandb/run-20260821_043849-zrelwtt2/run-zrelwtt2.wandb filte
 epfl_final/SparseAtt/wandb/run-20260821_164441-ino9n2jx/run-ino9n2jx.wandb filter=lfs diff=lfs merge=lfs -text
 epfl_final/SparseAtt/wandb/run-20260821_164441-bsc91ke5/run-bsc91ke5.wandb filter=lfs diff=lfs merge=lfs -text
 epfl_final/SparseAtt/wandb/run-20260821_164610-ryta2atg/run-ryta2atg.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260821_164610-n56uphgo/run-n56uphgo.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260821_180856-na2e5dc1/run-na2e5dc1.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260821_181107-27rw10jv/run-27rw10jv.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260822_042228-bsc91ke5/run-bsc91ke5.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260822_042228-ino9n2jx/run-ino9n2jx.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260822_042436-ryta2atg/run-ryta2atg.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260822_042436-n56uphgo/run-n56uphgo.wandb filter=lfs diff=lfs merge=lfs -text

epfl_final/SparseAtt/wandb/run-20260821_164610-n56uphgo/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260821_164610-n56uphgo/run-n56uphgo.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c125d1ddf1bd471b0756a327bc3317daced65e8a66036e961b3b5f5fd13c740d
+size 10354688

epfl_final/SparseAtt/wandb/run-20260821_180856-na2e5dc1/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260821_180856-na2e5dc1/files/wandb-metadata.json

@@ -0,0 +1,51 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-08-21T18:08:56.122670Z",
+  "args":  [
+    "--config",
+    "configs/p1_controls_nrb35_seed44_47/l8k365m_static_nrb35_seed44.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "5637886f0cc760009989b14e68c6ae06d4827418"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-workspace-2-gpu-0-7",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  2,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "6413804277760"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-af79466d-4300-4171-b5bd-7697eca27264"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-0460dc7d-d83e-fd0f-7548-a3079803b79e"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "zy9lggcsxfwz8lbaihu2c7gudadl9cdd"
+}

epfl_final/SparseAtt/wandb/run-20260821_180856-na2e5dc1/logs/debug.log

@@ -0,0 +1,19 @@
+2026-08-21 18:08:56,145 INFO    MainThread:657 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-08-21 18:08:56,145 INFO    MainThread:657 [wandb_setup.py:_flush():81] Configure stats pid to 657
+2026-08-21 18:08:56,145 INFO    MainThread:657 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-21 18:08:56,145 INFO    MainThread:657 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260821_180856-na2e5dc1/logs/debug.log
+2026-08-21 18:08:56,145 INFO    MainThread:657 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260821_180856-na2e5dc1/logs/debug-internal.log
+2026-08-21 18:08:56,145 INFO    MainThread:657 [wandb_init.py:init():850] calling init triggers
+2026-08-21 18:08:56,145 INFO    MainThread:657 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed44/run_20260821_180828', 'resume_from_checkpoint': None, 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 44, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'P1 365m static 8k nrb35 seed44', 'wandb_tags': ['p1', 'structured-control', 'qwen3', '365m', '8k', 'static', 'nrb35', 'seed44'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_static_nrb35_s1', 'sparse_mask_seed': 1, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-08-21 18:08:56,145 INFO    MainThread:657 [wandb_init.py:init():898] starting backend
+2026-08-21 18:08:57,103 INFO    MainThread:657 [wandb_init.py:init():913] sending inform_init request
+2026-08-21 18:08:57,691 INFO    MainThread:657 [wandb_init.py:init():918] backend started and connected
+2026-08-21 18:08:57,694 INFO    MainThread:657 [wandb_init.py:init():988] updated telemetry
+2026-08-21 18:08:57,785 INFO    MainThread:657 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-08-21 18:08:58,184 INFO    MainThread:657 [wandb_init.py:init():1056] starting run threads in backend
+2026-08-21 18:08:58,410 INFO    MainThread:657 [wandb_run.py:_console_start():2554] atexit reg
+2026-08-21 18:08:58,410 INFO    MainThread:657 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-08-21 18:08:58,410 INFO    MainThread:657 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-08-21 18:08:58,410 INFO    MainThread:657 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-08-21 18:08:58,438 INFO    MainThread:657 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/run-20260821_180856-na2e5dc1/run-na2e5dc1.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8c76743c2cc4c64c89e20fb111ac8b1a7bf1e788a648bdc9e5f6e7baf01eebb
+size 9371648

epfl_final/SparseAtt/wandb/run-20260821_181107-27rw10jv/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260821_181107-27rw10jv/files/wandb-metadata.json

@@ -0,0 +1,51 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-08-21T18:11:07.377810Z",
+  "args":  [
+    "--config",
+    "configs/p1_controls_nrb35_seed44_47/l8k365m_resampled_nrb35_seed44.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "5637886f0cc760009989b14e68c6ae06d4827418"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-workspace-2-gpu-0-7",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  2,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "6413878943744"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-af79466d-4300-4171-b5bd-7697eca27264"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-0460dc7d-d83e-fd0f-7548-a3079803b79e"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "avq79k8jgi4xry3pjja4gap809zl046n"
+}

epfl_final/SparseAtt/wandb/run-20260821_181107-27rw10jv/logs/debug.log

@@ -0,0 +1,19 @@
+2026-08-21 18:11:07,412 INFO    MainThread:659 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-08-21 18:11:07,412 INFO    MainThread:659 [wandb_setup.py:_flush():81] Configure stats pid to 659
+2026-08-21 18:11:07,412 INFO    MainThread:659 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-21 18:11:07,412 INFO    MainThread:659 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260821_181107-27rw10jv/logs/debug.log
+2026-08-21 18:11:07,412 INFO    MainThread:659 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260821_181107-27rw10jv/logs/debug-internal.log
+2026-08-21 18:11:07,412 INFO    MainThread:659 [wandb_init.py:init():850] calling init triggers
+2026-08-21 18:11:07,412 INFO    MainThread:659 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed44/run_20260821_180828', 'resume_from_checkpoint': None, 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 44, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'P1 365m resampled 8k nrb35 seed44', 'wandb_tags': ['p1', 'structured-control', 'qwen3', '365m', '8k', 'resampled', 'nrb35', 'seed44'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_resampled_nrb35_s0', 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-08-21 18:11:07,412 INFO    MainThread:659 [wandb_init.py:init():898] starting backend
+2026-08-21 18:11:07,698 INFO    MainThread:659 [wandb_init.py:init():913] sending inform_init request
+2026-08-21 18:11:08,019 INFO    MainThread:659 [wandb_init.py:init():918] backend started and connected
+2026-08-21 18:11:08,021 INFO    MainThread:659 [wandb_init.py:init():988] updated telemetry
+2026-08-21 18:11:08,039 INFO    MainThread:659 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-08-21 18:11:08,882 INFO    MainThread:659 [wandb_init.py:init():1056] starting run threads in backend
+2026-08-21 18:11:09,146 INFO    MainThread:659 [wandb_run.py:_console_start():2554] atexit reg
+2026-08-21 18:11:09,146 INFO    MainThread:659 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-08-21 18:11:09,146 INFO    MainThread:659 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-08-21 18:11:09,146 INFO    MainThread:659 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-08-21 18:11:09,167 INFO    MainThread:659 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/run-20260821_181107-27rw10jv/run-27rw10jv.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45035654347153dcad6fcef5dbd2a5f133b7d27322349f61325dbaa779bc03a3
+size 9338880

epfl_final/SparseAtt/wandb/run-20260822_042228-bsc91ke5/files/config.yaml

@@ -0,0 +1,200 @@
+_wandb:
+    value:
+        cli_version: 0.26.1
+        e:
+            2965ctjxhzcf4kq80c3tij3pry7vnir9:
+                args:
+                    - --config
+                    - configs/p1_resume_h1004_20260822/l8k365m_static_nrb35_seed42.json
+                codePath: train.py
+                codePathLocal: train.py
+                cpu_count: 96
+                cpu_count_logical: 192
+                cudaVersion: "13.0"
+                disk:
+                    /:
+                        total: "9371709865984"
+                        used: "6534294126592"
+                email: ruben.navasardyan@mail.utoronto.ca
+                executable: /mloscratch/homes/navasard/conda/bin/python
+                git:
+                    commit: 5637886f0cc760009989b14e68c6ae06d4827418
+                    remote: https://github.com/rubentium/TokenPrune.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 4
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-37a1c6c2-d3df-3064-44df-33fa82304bd0
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-9a903f43-9348-6fa6-bcf7-e12bd01803a7
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-24b8d952-dc08-33b9-d5df-54ce871fd288
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-b87e8111-63b6-b364-28b9-c4ce747aec08
+                host: h100-workspace-4-gpu-3-0-8
+                memory:
+                    total: "1623005732864"
+                os: Linux-6.8.0-124-generic-x86_64-with-glibc2.35
+                program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
+                python: CPython 3.13.13
+                root: /mloscratch/homes/navasard/sparse/SparseAtt
+                startedAt: "2026-08-22T04:22:28.765078Z"
+                writerId: 2965ctjxhzcf4kq80c3tij3pry7vnir9
+        m: []
+        python_version: 3.13.13
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 5
+                - 13
+                - 14
+                - 15
+                - 16
+                - 61
+                - 62
+            "4": 3.13.13
+            "5": 0.26.1
+            "6": 5.7.0
+            "12": 0.26.1
+            "13": linux-x86_64
+attention_dropout:
+    value: null
+attention_type:
+    value: sparse_stochastic
+checkpoint_activations:
+    value: false
+dense_mode:
+    value: false
+dense_step_intervals:
+    value: null
+eval_batches:
+    value: 50
+eval_steps:
+    value: 250
+force_new_run:
+    value: false
+gradient_accumulation_steps:
+    value: 10
+head_dim:
+    value: null
+hidden_size:
+    value: 768
+init_from_scratch:
+    value: true
+intermediate_size:
+    value: 3072
+learning_rate:
+    value: 0.0005
+logging_steps:
+    value: 10
+lr_scheduler:
+    value: cosine
+max_grad_norm:
+    value: 1
+max_seq_length:
+    value: 8192
+max_steps:
+    value: 44000
+mixed_precision:
+    value: bf16
+model_dtype:
+    value: bfloat16
+model_path:
+    value: /mloscratch/homes/navasard/qwen3_06b_base
+num_attention_heads:
+    value: 6
+num_epochs:
+    value: 1
+num_hidden_layers:
+    value: 28
+num_key_value_heads:
+    value: 2
+num_workers:
+    value: 8
+output_dir:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416
+per_device_batch_size:
+    value: 2
+resume_from_checkpoint:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-20000
+rope_train_position_scale_set:
+    value: null
+save_steps:
+    value: 1000
+seed:
+    value: 42
+sparse_block_size:
+    value: 128
+sparse_layer_indices:
+    value: null
+sparse_mask_cache_dir:
+    value: /mloscratch/homes/navasard/sparse/mask_cache_8k_static_nrb35_s1
+sparse_mask_pool_size:
+    value: 1
+sparse_mask_seed:
+    value: 1
+sparse_mixed_nrb_set:
+    value: null
+sparse_num_random_blocks:
+    value: 35
+sparse_val_dense_compare:
+    value: false
+strategy:
+    value: ddp
+train_path:
+    value: /mloscratch/homes/navasard/data/fineweb_large/tokenized
+val_path:
+    value: null
+verbose:
+    value: true
+wandb_enabled:
+    value: true
+wandb_name:
+    value: P1 resume 365m static 8k nrb35 seed42
+wandb_project:
+    value: stochastic_sparsity
+wandb_tags:
+    value:
+        - p1
+        - structured-control
+        - qwen3
+        - 365m
+        - 8k
+        - static
+        - nrb35
+        - seed42
+        - resume
+        - h1004
+        - step20000
+warmup_steps:
+    value: 750
+weight_decay:
+    value: 0.01

epfl_final/SparseAtt/wandb/run-20260822_042228-bsc91ke5/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260822_042228-bsc91ke5/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-08-22T04:22:28.765078Z",
+  "args":  [
+    "--config",
+    "configs/p1_resume_h1004_20260822/l8k365m_static_nrb35_seed42.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "5637886f0cc760009989b14e68c6ae06d4827418"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-workspace-4-gpu-3-0-8",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "9371709865984",
+      "used":  "6534294126592"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-37a1c6c2-d3df-3064-44df-33fa82304bd0"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-9a903f43-9348-6fa6-bcf7-e12bd01803a7"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-24b8d952-dc08-33b9-d5df-54ce871fd288"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-b87e8111-63b6-b364-28b9-c4ce747aec08"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "2965ctjxhzcf4kq80c3tij3pry7vnir9"
+}

epfl_final/SparseAtt/wandb/run-20260822_042228-bsc91ke5/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb.runtime":28629,"train/epoch":0.2906439083679023,"train/global_step":44000,"eval/loss":2.9291802616119385,"eval/dense_interval_active":0,"speed/ms_per_step":1304.6709775924683,"_step":44000,"eval/dense_interval_end":-1,"_runtime":62625.923406705,"speed/steps_per_sec":0.766476772439069,"eval/uses_sparse_attention":1,"_timestamp":1.78740654066076e+09,"speed/samples_per_sec":15.32953544878138,"_wandb":{"runtime":62625},"train/learning_rate":0,"eval/dense_interval_start":-1,"eval/uses_dense_attention":0,"eval/perplexity":18.712285029530918,"train/loss":2.960497133731842}

epfl_final/SparseAtt/wandb/run-20260822_042228-bsc91ke5/logs/debug.log

@@ -0,0 +1,25 @@
+2026-08-22 04:22:28,786 INFO    MainThread:245 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-08-22 04:22:28,786 INFO    MainThread:245 [wandb_setup.py:_flush():81] Configure stats pid to 245
+2026-08-22 04:22:28,787 INFO    MainThread:245 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-22 04:22:28,787 INFO    MainThread:245 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_042228-bsc91ke5/logs/debug.log
+2026-08-22 04:22:28,787 INFO    MainThread:245 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_042228-bsc91ke5/logs/debug-internal.log
+2026-08-22 04:22:28,787 INFO    MainThread:245 [wandb_init.py:init():850] calling init triggers
+2026-08-22 04:22:28,787 INFO    MainThread:245 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed42/run_20260821_164416/checkpoint-20000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'P1 resume 365m static 8k nrb35 seed42', 'wandb_tags': ['p1', 'structured-control', 'qwen3', '365m', '8k', 'static', 'nrb35', 'seed42', 'resume', 'h1004', 'step20000'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_static_nrb35_s1', 'sparse_mask_seed': 1, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-08-22 04:22:28,787 INFO    MainThread:245 [wandb_init.py:init():898] starting backend
+2026-08-22 04:22:29,644 INFO    MainThread:245 [wandb_init.py:init():913] sending inform_init request
+2026-08-22 04:22:30,098 INFO    MainThread:245 [wandb_init.py:init():918] backend started and connected
+2026-08-22 04:22:30,101 INFO    MainThread:245 [wandb_init.py:init():988] updated telemetry
+2026-08-22 04:22:30,132 INFO    MainThread:245 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-08-22 04:22:30,616 INFO    MainThread:245 [wandb_init.py:init():1051] run resumed
+2026-08-22 04:22:30,619 INFO    MainThread:245 [wandb_init.py:init():1056] starting run threads in backend
+2026-08-22 04:22:31,038 INFO    MainThread:245 [wandb_run.py:_console_start():2554] atexit reg
+2026-08-22 04:22:31,038 INFO    MainThread:245 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-08-22 04:22:31,038 INFO    MainThread:245 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-08-22 04:22:31,038 INFO    MainThread:245 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-08-22 04:22:31,044 INFO    MainThread:245 [wandb_init.py:init():1094] run started, returning control to user process
+2026-08-22 13:49:07,541 INFO    MainThread:245 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/bsc91ke5
+2026-08-22 13:49:07,542 INFO    MainThread:245 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
+2026-08-22 13:49:07,543 INFO    MainThread:245 [wandb_run.py:_restore():2502] restore
+2026-08-22 13:49:07,543 INFO    MainThread:245 [wandb_run.py:_restore():2508] restore done
+2026-08-22 13:49:09,241 INFO    MainThread:245 [wandb_run.py:_footer_sync_info():3889] logging synced files

epfl_final/SparseAtt/wandb/run-20260822_042228-bsc91ke5/run-bsc91ke5.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b0e4414f711817e233d82d7cb5a3e03ed766adfc520a9ca1fb89d8833a4bee9
+size 12539287

epfl_final/SparseAtt/wandb/run-20260822_042228-ino9n2jx/files/config.yaml

@@ -0,0 +1,200 @@
+_wandb:
+    value:
+        cli_version: 0.26.1
+        e:
+            qw8e8hdt66njwfipf4is9wc0gyoth8ew:
+                args:
+                    - --config
+                    - configs/p1_resume_h1004_20260822/l8k365m_static_nrb35_seed43.json
+                codePath: train.py
+                codePathLocal: train.py
+                cpu_count: 96
+                cpu_count_logical: 192
+                cudaVersion: "13.0"
+                disk:
+                    /:
+                        total: "9371709865984"
+                        used: "6534294126592"
+                email: ruben.navasardyan@mail.utoronto.ca
+                executable: /mloscratch/homes/navasard/conda/bin/python
+                git:
+                    commit: 5637886f0cc760009989b14e68c6ae06d4827418
+                    remote: https://github.com/rubentium/TokenPrune.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 4
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-37a1c6c2-d3df-3064-44df-33fa82304bd0
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-9a903f43-9348-6fa6-bcf7-e12bd01803a7
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-24b8d952-dc08-33b9-d5df-54ce871fd288
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-b87e8111-63b6-b364-28b9-c4ce747aec08
+                host: h100-workspace-4-gpu-3-0-8
+                memory:
+                    total: "1623005732864"
+                os: Linux-6.8.0-124-generic-x86_64-with-glibc2.35
+                program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
+                python: CPython 3.13.13
+                root: /mloscratch/homes/navasard/sparse/SparseAtt
+                startedAt: "2026-08-22T04:22:28.900203Z"
+                writerId: qw8e8hdt66njwfipf4is9wc0gyoth8ew
+        m: []
+        python_version: 3.13.13
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 5
+                - 13
+                - 14
+                - 15
+                - 16
+                - 61
+                - 62
+            "4": 3.13.13
+            "5": 0.26.1
+            "6": 5.7.0
+            "12": 0.26.1
+            "13": linux-x86_64
+attention_dropout:
+    value: null
+attention_type:
+    value: sparse_stochastic
+checkpoint_activations:
+    value: false
+dense_mode:
+    value: false
+dense_step_intervals:
+    value: null
+eval_batches:
+    value: 50
+eval_steps:
+    value: 250
+force_new_run:
+    value: false
+gradient_accumulation_steps:
+    value: 10
+head_dim:
+    value: null
+hidden_size:
+    value: 768
+init_from_scratch:
+    value: true
+intermediate_size:
+    value: 3072
+learning_rate:
+    value: 0.0005
+logging_steps:
+    value: 10
+lr_scheduler:
+    value: cosine
+max_grad_norm:
+    value: 1
+max_seq_length:
+    value: 8192
+max_steps:
+    value: 44000
+mixed_precision:
+    value: bf16
+model_dtype:
+    value: bfloat16
+model_path:
+    value: /mloscratch/homes/navasard/qwen3_06b_base
+num_attention_heads:
+    value: 6
+num_epochs:
+    value: 1
+num_hidden_layers:
+    value: 28
+num_key_value_heads:
+    value: 2
+num_workers:
+    value: 8
+output_dir:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417
+per_device_batch_size:
+    value: 2
+resume_from_checkpoint:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-20000
+rope_train_position_scale_set:
+    value: null
+save_steps:
+    value: 1000
+seed:
+    value: 43
+sparse_block_size:
+    value: 128
+sparse_layer_indices:
+    value: null
+sparse_mask_cache_dir:
+    value: /mloscratch/homes/navasard/sparse/mask_cache_8k_static_nrb35_s1
+sparse_mask_pool_size:
+    value: 1
+sparse_mask_seed:
+    value: 1
+sparse_mixed_nrb_set:
+    value: null
+sparse_num_random_blocks:
+    value: 35
+sparse_val_dense_compare:
+    value: false
+strategy:
+    value: ddp
+train_path:
+    value: /mloscratch/homes/navasard/data/fineweb_large/tokenized
+val_path:
+    value: null
+verbose:
+    value: true
+wandb_enabled:
+    value: true
+wandb_name:
+    value: P1 resume 365m static 8k nrb35 seed43
+wandb_project:
+    value: stochastic_sparsity
+wandb_tags:
+    value:
+        - p1
+        - structured-control
+        - qwen3
+        - 365m
+        - 8k
+        - static
+        - nrb35
+        - seed43
+        - resume
+        - h1004
+        - step20000
+warmup_steps:
+    value: 750
+weight_decay:
+    value: 0.01

epfl_final/SparseAtt/wandb/run-20260822_042228-ino9n2jx/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260822_042228-ino9n2jx/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-08-22T04:22:28.900203Z",
+  "args":  [
+    "--config",
+    "configs/p1_resume_h1004_20260822/l8k365m_static_nrb35_seed43.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "5637886f0cc760009989b14e68c6ae06d4827418"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-workspace-4-gpu-3-0-8",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "9371709865984",
+      "used":  "6534294126592"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-37a1c6c2-d3df-3064-44df-33fa82304bd0"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-9a903f43-9348-6fa6-bcf7-e12bd01803a7"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-24b8d952-dc08-33b9-d5df-54ce871fd288"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-b87e8111-63b6-b364-28b9-c4ce747aec08"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "qw8e8hdt66njwfipf4is9wc0gyoth8ew"
+}

epfl_final/SparseAtt/wandb/run-20260822_042228-ino9n2jx/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_runtime":63428.3206671,"speed/ms_per_step":1331.3699960708618,"train/learning_rate":0,"train/global_step":44000,"eval/perplexity":18.262208775344025,"train/loss":2.918302698135376,"speed/steps_per_sec":0.7511060058069502,"_wandb":{"runtime":63428},"speed/samples_per_sec":15.022120116139003,"eval/dense_interval_start":-1,"_step":44000,"train/epoch":0.2906439083679023,"eval/dense_interval_active":0,"eval/loss":2.9048338303565977,"_wandb.runtime":28631,"eval/dense_interval_end":-1,"eval/uses_dense_attention":0,"eval/uses_sparse_attention":1,"_timestamp":1.7874073412727184e+09}

epfl_final/SparseAtt/wandb/run-20260822_042228-ino9n2jx/logs/debug.log

@@ -0,0 +1,25 @@
+2026-08-22 04:22:28,919 INFO    MainThread:249 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-08-22 04:22:28,919 INFO    MainThread:249 [wandb_setup.py:_flush():81] Configure stats pid to 249
+2026-08-22 04:22:28,919 INFO    MainThread:249 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-22 04:22:28,919 INFO    MainThread:249 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_042228-ino9n2jx/logs/debug.log
+2026-08-22 04:22:28,919 INFO    MainThread:249 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_042228-ino9n2jx/logs/debug-internal.log
+2026-08-22 04:22:28,919 INFO    MainThread:249 [wandb_init.py:init():850] calling init triggers
+2026-08-22 04:22:28,920 INFO    MainThread:249 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_static_nrb35_seed43/run_20260821_164417/checkpoint-20000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 43, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'P1 resume 365m static 8k nrb35 seed43', 'wandb_tags': ['p1', 'structured-control', 'qwen3', '365m', '8k', 'static', 'nrb35', 'seed43', 'resume', 'h1004', 'step20000'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_static_nrb35_s1', 'sparse_mask_seed': 1, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-08-22 04:22:28,920 INFO    MainThread:249 [wandb_init.py:init():898] starting backend
+2026-08-22 04:22:29,647 INFO    MainThread:249 [wandb_init.py:init():913] sending inform_init request
+2026-08-22 04:22:30,098 INFO    MainThread:249 [wandb_init.py:init():918] backend started and connected
+2026-08-22 04:22:30,104 INFO    MainThread:249 [wandb_init.py:init():988] updated telemetry
+2026-08-22 04:22:30,132 INFO    MainThread:249 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-08-22 04:22:30,768 INFO    MainThread:249 [wandb_init.py:init():1051] run resumed
+2026-08-22 04:22:30,771 INFO    MainThread:249 [wandb_init.py:init():1056] starting run threads in backend
+2026-08-22 04:22:31,036 INFO    MainThread:249 [wandb_run.py:_console_start():2554] atexit reg
+2026-08-22 04:22:31,036 INFO    MainThread:249 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-08-22 04:22:31,036 INFO    MainThread:249 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-08-22 04:22:31,037 INFO    MainThread:249 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-08-22 04:22:31,044 INFO    MainThread:249 [wandb_init.py:init():1094] run started, returning control to user process
+2026-08-22 14:02:28,089 INFO    MainThread:249 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/ino9n2jx
+2026-08-22 14:02:28,091 INFO    MainThread:249 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
+2026-08-22 14:02:28,092 INFO    MainThread:249 [wandb_run.py:_restore():2502] restore
+2026-08-22 14:02:28,092 INFO    MainThread:249 [wandb_run.py:_restore():2508] restore done
+2026-08-22 14:02:29,803 INFO    MainThread:249 [wandb_run.py:_footer_sync_info():3889] logging synced files

epfl_final/SparseAtt/wandb/run-20260822_042228-ino9n2jx/run-ino9n2jx.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eef9e00d234e99a61f970a66b47cc9143fa4d1b02321e7a0aa08c41dff6c5f7f
+size 12771237

epfl_final/SparseAtt/wandb/run-20260822_042436-n56uphgo/files/config.yaml

@@ -0,0 +1,200 @@
+_wandb:
+    value:
+        cli_version: 0.26.1
+        e:
+            kqq1bydckvzxqiydk1wxxneo4hogjbdw:
+                args:
+                    - --config
+                    - configs/p1_resume_h1004_20260822/l8k365m_resampled_nrb35_seed42.json
+                codePath: train.py
+                codePathLocal: train.py
+                cpu_count: 96
+                cpu_count_logical: 192
+                cudaVersion: "13.0"
+                disk:
+                    /:
+                        total: "9371709865984"
+                        used: "6534322081792"
+                email: ruben.navasardyan@mail.utoronto.ca
+                executable: /mloscratch/homes/navasard/conda/bin/python
+                git:
+                    commit: 5637886f0cc760009989b14e68c6ae06d4827418
+                    remote: https://github.com/rubentium/TokenPrune.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 4
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-37a1c6c2-d3df-3064-44df-33fa82304bd0
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-9a903f43-9348-6fa6-bcf7-e12bd01803a7
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-24b8d952-dc08-33b9-d5df-54ce871fd288
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-b87e8111-63b6-b364-28b9-c4ce747aec08
+                host: h100-workspace-4-gpu-3-0-8
+                memory:
+                    total: "1623005732864"
+                os: Linux-6.8.0-124-generic-x86_64-with-glibc2.35
+                program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
+                python: CPython 3.13.13
+                root: /mloscratch/homes/navasard/sparse/SparseAtt
+                startedAt: "2026-08-22T04:24:36.958542Z"
+                writerId: kqq1bydckvzxqiydk1wxxneo4hogjbdw
+        m: []
+        python_version: 3.13.13
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 5
+                - 13
+                - 14
+                - 15
+                - 16
+                - 61
+                - 62
+            "4": 3.13.13
+            "5": 0.26.1
+            "6": 5.7.0
+            "12": 0.26.1
+            "13": linux-x86_64
+attention_dropout:
+    value: null
+attention_type:
+    value: sparse_stochastic
+checkpoint_activations:
+    value: false
+dense_mode:
+    value: false
+dense_step_intervals:
+    value: null
+eval_batches:
+    value: 50
+eval_steps:
+    value: 250
+force_new_run:
+    value: false
+gradient_accumulation_steps:
+    value: 10
+head_dim:
+    value: null
+hidden_size:
+    value: 768
+init_from_scratch:
+    value: true
+intermediate_size:
+    value: 3072
+learning_rate:
+    value: 0.0005
+logging_steps:
+    value: 10
+lr_scheduler:
+    value: cosine
+max_grad_norm:
+    value: 1
+max_seq_length:
+    value: 8192
+max_steps:
+    value: 44000
+mixed_precision:
+    value: bf16
+model_dtype:
+    value: bfloat16
+model_path:
+    value: /mloscratch/homes/navasard/qwen3_06b_base
+num_attention_heads:
+    value: 6
+num_epochs:
+    value: 1
+num_hidden_layers:
+    value: 28
+num_key_value_heads:
+    value: 2
+num_workers:
+    value: 8
+output_dir:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416
+per_device_batch_size:
+    value: 2
+resume_from_checkpoint:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-19000
+rope_train_position_scale_set:
+    value: null
+save_steps:
+    value: 1000
+seed:
+    value: 42
+sparse_block_size:
+    value: 128
+sparse_layer_indices:
+    value: null
+sparse_mask_cache_dir:
+    value: /mloscratch/homes/navasard/sparse/mask_cache_8k_resampled_nrb35_s0
+sparse_mask_pool_size:
+    value: 10000
+sparse_mask_seed:
+    value: 0
+sparse_mixed_nrb_set:
+    value: null
+sparse_num_random_blocks:
+    value: 35
+sparse_val_dense_compare:
+    value: false
+strategy:
+    value: ddp
+train_path:
+    value: /mloscratch/homes/navasard/data/fineweb_large/tokenized
+val_path:
+    value: null
+verbose:
+    value: true
+wandb_enabled:
+    value: true
+wandb_name:
+    value: P1 resume 365m resampled 8k nrb35 seed42
+wandb_project:
+    value: stochastic_sparsity
+wandb_tags:
+    value:
+        - p1
+        - structured-control
+        - qwen3
+        - 365m
+        - 8k
+        - resampled
+        - nrb35
+        - seed42
+        - resume
+        - h1004
+        - step19000
+warmup_steps:
+    value: 750
+weight_decay:
+    value: 0.01

epfl_final/SparseAtt/wandb/run-20260822_042436-n56uphgo/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260822_042436-n56uphgo/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-08-22T04:24:36.958542Z",
+  "args":  [
+    "--config",
+    "configs/p1_resume_h1004_20260822/l8k365m_resampled_nrb35_seed42.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "5637886f0cc760009989b14e68c6ae06d4827418"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-workspace-4-gpu-3-0-8",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "9371709865984",
+      "used":  "6534322081792"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-37a1c6c2-d3df-3064-44df-33fa82304bd0"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-9a903f43-9348-6fa6-bcf7-e12bd01803a7"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-24b8d952-dc08-33b9-d5df-54ce871fd288"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-b87e8111-63b6-b364-28b9-c4ce747aec08"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "kqq1bydckvzxqiydk1wxxneo4hogjbdw"
+}

epfl_final/SparseAtt/wandb/run-20260822_042436-n56uphgo/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_timestamp":1.7874091238222768e+09,"speed/ms_per_step":1349.8376369476318,"train/global_step":44000,"eval/loss":2.931060739517212,"eval/uses_dense_attention":0,"train/epoch":0.2906439083679023,"eval/perplexity":18.747506173999042,"eval/dense_interval_active":0,"_step":44000,"eval/uses_sparse_attention":1,"_wandb":{"runtime":64995},"eval/dense_interval_end":-1,"_runtime":64995.650841625,"train/loss":2.96248096704483,"speed/samples_per_sec":14.81659679102274,"_wandb.runtime":28544,"eval/dense_interval_start":-1,"train/learning_rate":0,"speed/steps_per_sec":0.740829839551137}

epfl_final/SparseAtt/wandb/run-20260822_042436-n56uphgo/logs/debug.log

@@ -0,0 +1,25 @@
+2026-08-22 04:24:36,998 INFO    MainThread:247 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-08-22 04:24:36,998 INFO    MainThread:247 [wandb_setup.py:_flush():81] Configure stats pid to 247
+2026-08-22 04:24:36,998 INFO    MainThread:247 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-22 04:24:36,998 INFO    MainThread:247 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_042436-n56uphgo/logs/debug.log
+2026-08-22 04:24:36,998 INFO    MainThread:247 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_042436-n56uphgo/logs/debug-internal.log
+2026-08-22 04:24:36,998 INFO    MainThread:247 [wandb_init.py:init():850] calling init triggers
+2026-08-22 04:24:36,998 INFO    MainThread:247 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed42/run_20260821_164416/checkpoint-19000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'P1 resume 365m resampled 8k nrb35 seed42', 'wandb_tags': ['p1', 'structured-control', 'qwen3', '365m', '8k', 'resampled', 'nrb35', 'seed42', 'resume', 'h1004', 'step19000'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_resampled_nrb35_s0', 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-08-22 04:24:36,998 INFO    MainThread:247 [wandb_init.py:init():898] starting backend
+2026-08-22 04:24:37,258 INFO    MainThread:247 [wandb_init.py:init():913] sending inform_init request
+2026-08-22 04:24:37,598 INFO    MainThread:247 [wandb_init.py:init():918] backend started and connected
+2026-08-22 04:24:37,601 INFO    MainThread:247 [wandb_init.py:init():988] updated telemetry
+2026-08-22 04:24:37,614 INFO    MainThread:247 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-08-22 04:24:38,172 INFO    MainThread:247 [wandb_init.py:init():1051] run resumed
+2026-08-22 04:24:38,175 INFO    MainThread:247 [wandb_init.py:init():1056] starting run threads in backend
+2026-08-22 04:24:38,368 INFO    MainThread:247 [wandb_run.py:_console_start():2554] atexit reg
+2026-08-22 04:24:38,368 INFO    MainThread:247 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-08-22 04:24:38,368 INFO    MainThread:247 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-08-22 04:24:38,368 INFO    MainThread:247 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-08-22 04:24:38,371 INFO    MainThread:247 [wandb_init.py:init():1094] run started, returning control to user process
+2026-08-22 14:32:09,825 INFO    MainThread:247 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/n56uphgo
+2026-08-22 14:32:09,826 INFO    MainThread:247 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
+2026-08-22 14:32:09,826 INFO    MainThread:247 [wandb_run.py:_restore():2502] restore
+2026-08-22 14:32:09,826 INFO    MainThread:247 [wandb_run.py:_restore():2508] restore done
+2026-08-22 14:32:11,657 INFO    MainThread:247 [wandb_run.py:_footer_sync_info():3889] logging synced files

epfl_final/SparseAtt/wandb/run-20260822_042436-n56uphgo/run-n56uphgo.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2db508558e5d1ea3a9ad0e3e8d1f8f925bd4db91554ffd0608ed6fb7e47b6a36
+size 13282203

epfl_final/SparseAtt/wandb/run-20260822_042436-ryta2atg/files/config.yaml

@@ -0,0 +1,200 @@
+_wandb:
+    value:
+        cli_version: 0.26.1
+        e:
+            jsj85vkalk9ofby14v7td25d3qnlnvss:
+                args:
+                    - --config
+                    - configs/p1_resume_h1004_20260822/l8k365m_resampled_nrb35_seed43.json
+                codePath: train.py
+                codePathLocal: train.py
+                cpu_count: 96
+                cpu_count_logical: 192
+                cudaVersion: "13.0"
+                disk:
+                    /:
+                        total: "9371709865984"
+                        used: "6534322081792"
+                email: ruben.navasardyan@mail.utoronto.ca
+                executable: /mloscratch/homes/navasard/conda/bin/python
+                git:
+                    commit: 5637886f0cc760009989b14e68c6ae06d4827418
+                    remote: https://github.com/rubentium/TokenPrune.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 4
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-37a1c6c2-d3df-3064-44df-33fa82304bd0
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-9a903f43-9348-6fa6-bcf7-e12bd01803a7
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-24b8d952-dc08-33b9-d5df-54ce871fd288
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-b87e8111-63b6-b364-28b9-c4ce747aec08
+                host: h100-workspace-4-gpu-3-0-8
+                memory:
+                    total: "1623005732864"
+                os: Linux-6.8.0-124-generic-x86_64-with-glibc2.35
+                program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
+                python: CPython 3.13.13
+                root: /mloscratch/homes/navasard/sparse/SparseAtt
+                startedAt: "2026-08-22T04:24:36.957073Z"
+                writerId: jsj85vkalk9ofby14v7td25d3qnlnvss
+        m: []
+        python_version: 3.13.13
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 5
+                - 13
+                - 14
+                - 15
+                - 16
+                - 61
+                - 62
+            "4": 3.13.13
+            "5": 0.26.1
+            "6": 5.7.0
+            "12": 0.26.1
+            "13": linux-x86_64
+attention_dropout:
+    value: null
+attention_type:
+    value: sparse_stochastic
+checkpoint_activations:
+    value: false
+dense_mode:
+    value: false
+dense_step_intervals:
+    value: null
+eval_batches:
+    value: 50
+eval_steps:
+    value: 250
+force_new_run:
+    value: false
+gradient_accumulation_steps:
+    value: 10
+head_dim:
+    value: null
+hidden_size:
+    value: 768
+init_from_scratch:
+    value: true
+intermediate_size:
+    value: 3072
+learning_rate:
+    value: 0.0005
+logging_steps:
+    value: 10
+lr_scheduler:
+    value: cosine
+max_grad_norm:
+    value: 1
+max_seq_length:
+    value: 8192
+max_steps:
+    value: 44000
+mixed_precision:
+    value: bf16
+model_dtype:
+    value: bfloat16
+model_path:
+    value: /mloscratch/homes/navasard/qwen3_06b_base
+num_attention_heads:
+    value: 6
+num_epochs:
+    value: 1
+num_hidden_layers:
+    value: 28
+num_key_value_heads:
+    value: 2
+num_workers:
+    value: 8
+output_dir:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416
+per_device_batch_size:
+    value: 2
+resume_from_checkpoint:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-20000
+rope_train_position_scale_set:
+    value: null
+save_steps:
+    value: 1000
+seed:
+    value: 43
+sparse_block_size:
+    value: 128
+sparse_layer_indices:
+    value: null
+sparse_mask_cache_dir:
+    value: /mloscratch/homes/navasard/sparse/mask_cache_8k_resampled_nrb35_s0
+sparse_mask_pool_size:
+    value: 10000
+sparse_mask_seed:
+    value: 0
+sparse_mixed_nrb_set:
+    value: null
+sparse_num_random_blocks:
+    value: 35
+sparse_val_dense_compare:
+    value: false
+strategy:
+    value: ddp
+train_path:
+    value: /mloscratch/homes/navasard/data/fineweb_large/tokenized
+val_path:
+    value: null
+verbose:
+    value: true
+wandb_enabled:
+    value: true
+wandb_name:
+    value: P1 resume 365m resampled 8k nrb35 seed43
+wandb_project:
+    value: stochastic_sparsity
+wandb_tags:
+    value:
+        - p1
+        - structured-control
+        - qwen3
+        - 365m
+        - 8k
+        - resampled
+        - nrb35
+        - seed43
+        - resume
+        - h1004
+        - step20000
+warmup_steps:
+    value: 750
+weight_decay:
+    value: 0.01

epfl_final/SparseAtt/wandb/run-20260822_042436-ryta2atg/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260822_042436-ryta2atg/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-08-22T04:24:36.957073Z",
+  "args":  [
+    "--config",
+    "configs/p1_resume_h1004_20260822/l8k365m_resampled_nrb35_seed43.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "5637886f0cc760009989b14e68c6ae06d4827418"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-workspace-4-gpu-3-0-8",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "9371709865984",
+      "used":  "6534322081792"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-37a1c6c2-d3df-3064-44df-33fa82304bd0"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-9a903f43-9348-6fa6-bcf7-e12bd01803a7"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-24b8d952-dc08-33b9-d5df-54ce871fd288"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-b87e8111-63b6-b364-28b9-c4ce747aec08"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "jsj85vkalk9ofby14v7td25d3qnlnvss"
+}

epfl_final/SparseAtt/wandb/run-20260822_042436-ryta2atg/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_timestamp":1.7874068678075686e+09,"speed/steps_per_sec":0.7616804511029692,"speed/ms_per_step":1312.8865242004395,"_step":44000,"train/epoch":0.2906439083679023,"train/global_step":44000,"speed/samples_per_sec":15.233609022059383,"eval/dense_interval_end":-1,"_wandb":{"runtime":62739},"eval/dense_interval_active":0,"train/loss":2.9203021836280825,"eval/dense_interval_start":-1,"eval/uses_sparse_attention":1,"eval/loss":2.9069794735908507,"eval/uses_dense_attention":0,"train/learning_rate":0,"eval/perplexity":18.301435027767816,"_wandb.runtime":28543,"_runtime":62739.043285146}

epfl_final/SparseAtt/wandb/run-20260822_042436-ryta2atg/logs/debug.log

@@ -0,0 +1,25 @@
+2026-08-22 04:24:36,993 INFO    MainThread:251 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-08-22 04:24:36,993 INFO    MainThread:251 [wandb_setup.py:_flush():81] Configure stats pid to 251
+2026-08-22 04:24:36,993 INFO    MainThread:251 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-22 04:24:36,993 INFO    MainThread:251 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_042436-ryta2atg/logs/debug.log
+2026-08-22 04:24:36,994 INFO    MainThread:251 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_042436-ryta2atg/logs/debug-internal.log
+2026-08-22 04:24:36,994 INFO    MainThread:251 [wandb_init.py:init():850] calling init triggers
+2026-08-22 04:24:36,994 INFO    MainThread:251 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_resampled_nrb35_seed43/run_20260821_164416/checkpoint-20000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 43, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'P1 resume 365m resampled 8k nrb35 seed43', 'wandb_tags': ['p1', 'structured-control', 'qwen3', '365m', '8k', 'resampled', 'nrb35', 'seed43', 'resume', 'h1004', 'step20000'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_resampled_nrb35_s0', 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-08-22 04:24:36,994 INFO    MainThread:251 [wandb_init.py:init():898] starting backend
+2026-08-22 04:24:37,258 INFO    MainThread:251 [wandb_init.py:init():913] sending inform_init request
+2026-08-22 04:24:37,594 INFO    MainThread:251 [wandb_init.py:init():918] backend started and connected
+2026-08-22 04:24:37,596 INFO    MainThread:251 [wandb_init.py:init():988] updated telemetry
+2026-08-22 04:24:37,614 INFO    MainThread:251 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-08-22 04:24:38,088 INFO    MainThread:251 [wandb_init.py:init():1051] run resumed
+2026-08-22 04:24:38,090 INFO    MainThread:251 [wandb_init.py:init():1056] starting run threads in backend
+2026-08-22 04:24:38,331 INFO    MainThread:251 [wandb_run.py:_console_start():2554] atexit reg
+2026-08-22 04:24:38,331 INFO    MainThread:251 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-08-22 04:24:38,331 INFO    MainThread:251 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-08-22 04:24:38,331 INFO    MainThread:251 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-08-22 04:24:38,336 INFO    MainThread:251 [wandb_init.py:init():1094] run started, returning control to user process
+2026-08-22 13:54:34,133 INFO    MainThread:251 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/ryta2atg
+2026-08-22 13:54:34,134 INFO    MainThread:251 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
+2026-08-22 13:54:34,134 INFO    MainThread:251 [wandb_run.py:_restore():2502] restore
+2026-08-22 13:54:34,134 INFO    MainThread:251 [wandb_run.py:_restore():2508] restore done
+2026-08-22 13:54:35,552 INFO    MainThread:251 [wandb_run.py:_footer_sync_info():3889] logging synced files

epfl_final/SparseAtt/wandb/run-20260822_042436-ryta2atg/run-ryta2atg.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:290d5f2fb9238a303b3dac4a5d3802e1a52c8e732b67c2f083d97ef2f021a6df
+size 12578629

epfl_final/SparseAtt/wandb/run-20260822_043357-na2e5dc1/files/wandb-metadata.json

@@ -0,0 +1,51 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-08-22T04:33:57.496973Z",
+  "args":  [
+    "--config",
+    "configs/p1_resume_h1002_20260822/l8k365m_static_nrb35_seed44.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "5637886f0cc760009989b14e68c6ae06d4827418"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-workspace-2-gpu-0-8",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  2,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "6420876230656"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-af79466d-4300-4171-b5bd-7697eca27264"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-0460dc7d-d83e-fd0f-7548-a3079803b79e"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "jpbvtfmuhlvm7j9d3zd6cbp2ruimxi5m"
+}