EPFL final checkpoints and W&B history 1401-1450 of 1463

.gitattributes

@@ -189,3 +189,15 @@ epfl_final/SparseAtt/wandb/offline-run-20260921_131249-xc6c08ps/run-xc6c08ps.wan
 epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3uan03m3/run-3uan03m3.wandb filter=lfs diff=lfs merge=lfs -text
 epfl_final/SparseAtt/wandb/offline-run-20260921_143024-kbn66izc/run-kbn66izc.wandb filter=lfs diff=lfs merge=lfs -text
 epfl_final/SparseAtt/wandb/offline-run-20260921_143024-zgrsvkbi/run-zgrsvkbi.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/offline-run-20260921_143024-ghfbqjah/run-ghfbqjah.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/offline-run-20260921_143024-6ijtsdl5/run-6ijtsdl5.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/offline-run-20260921_143024-wk3rt7ae/run-wk3rt7ae.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/offline-run-20260921_143024-nihe6o4e/run-nihe6o4e.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/offline-run-20260921_154845-ghfbqjah/run-ghfbqjah.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/offline-run-20260921_154845-zgrsvkbi/run-zgrsvkbi.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/offline-run-20260921_154845-nihe6o4e/run-nihe6o4e.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/offline-run-20260921_154845-wk3rt7ae/run-wk3rt7ae.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/offline-run-20260921_154846-kbn66izc/run-kbn66izc.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/offline-run-20260921_163219-nihe6o4e/run-nihe6o4e.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/offline-run-20260921_164533-kbn66izc/run-kbn66izc.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/offline-run-20260921_164537-zgrsvkbi/run-zgrsvkbi.wandb filter=lfs diff=lfs merge=lfs -text

epfl_final/SparseAtt/wandb/offline-run-20260921_143024-6ijtsdl5/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_143024-6ijtsdl5/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-09-21T14:30:25.205688293Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T14:30:25.205700601Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T14:30:25.544942881Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T14:30:25.545041428Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T14:30:25.545090001Z","level":"INFO","msg":"stream: created new stream","id":"6ijtsdl5"}
+{"time":"2026-09-21T14:30:25.545224552Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T14:30:25.547588391Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T14:30:25.548874862Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T14:30:25.548875933Z","level":"INFO","msg":"writer: started","stream_id":"6ijtsdl5"}
+{"time":"2026-09-21T14:30:25.588698359Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T14:30:25.58873973Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

epfl_final/SparseAtt/wandb/offline-run-20260921_143024-6ijtsdl5/logs/debug.log

@@ -0,0 +1,19 @@
+2026-09-21 14:30:24,663 INFO    MainThread:72 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-21 14:30:24,663 INFO    MainThread:72 [wandb_setup.py:_flush():81] Configure stats pid to 72
+2026-09-21 14:30:24,663 INFO    MainThread:72 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-21 14:30:24,663 INFO    MainThread:72 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_143024-6ijtsdl5/logs/debug.log
+2026-09-21 14:30:24,663 INFO    MainThread:72 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_143024-6ijtsdl5/logs/debug-internal.log
+2026-09-21 14:30:24,663 INFO    MainThread:72 [wandb_init.py:init():850] calling init triggers
+2026-09-21 14:30:24,664 INFO    MainThread:72 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951', 'resume_from_checkpoint': None, 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 47, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s47', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed47'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-21 14:30:24,664 INFO    MainThread:72 [wandb_init.py:init():898] starting backend
+2026-09-21 14:30:25,192 INFO    MainThread:72 [wandb_init.py:init():913] sending inform_init request
+2026-09-21 14:30:25,548 INFO    MainThread:72 [wandb_init.py:init():918] backend started and connected
+2026-09-21 14:30:25,552 INFO    MainThread:72 [wandb_init.py:init():988] updated telemetry
+2026-09-21 14:30:25,585 INFO    MainThread:72 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-21 14:30:25,591 INFO    MainThread:72 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-21 14:30:25,795 INFO    MainThread:72 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-21 14:30:25,795 INFO    MainThread:72 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-21 14:30:25,795 INFO    MainThread:72 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-21 14:30:25,796 INFO    MainThread:72 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-21 14:30:25,797 INFO    MainThread:72 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/offline-run-20260921_143024-6ijtsdl5/run-6ijtsdl5.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3cc5d982f52825586bc97397d7069a333bd09a3ba50406a9b0266cfef88e8421
+size 1376256

epfl_final/SparseAtt/wandb/offline-run-20260921_143024-ghfbqjah/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_143024-ghfbqjah/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-09-21T14:30:25.207855399Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T14:30:25.207885784Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T14:30:25.565662376Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T14:30:25.565698019Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T14:30:25.565724288Z","level":"INFO","msg":"stream: created new stream","id":"ghfbqjah"}
+{"time":"2026-09-21T14:30:25.565760121Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T14:30:25.567608171Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T14:30:25.567633909Z","level":"INFO","msg":"writer: started","stream_id":"ghfbqjah"}
+{"time":"2026-09-21T14:30:25.567661911Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T14:30:25.588444309Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T14:30:25.588477479Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

epfl_final/SparseAtt/wandb/offline-run-20260921_143024-ghfbqjah/run-ghfbqjah.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85bd743fed8b7029fefeb99cd0ad6c4ae0e10464be9097780d5aaef5bf31dc7f
+size 1376256

epfl_final/SparseAtt/wandb/offline-run-20260921_143024-nihe6o4e/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_143024-nihe6o4e/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-09-21T14:30:25.207122284Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T14:30:25.207148273Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T14:30:25.563932403Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T14:30:25.563969739Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T14:30:25.564003799Z","level":"INFO","msg":"stream: created new stream","id":"nihe6o4e"}
+{"time":"2026-09-21T14:30:25.564132572Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T14:30:25.566124576Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T14:30:25.566219718Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T14:30:25.566224274Z","level":"INFO","msg":"writer: started","stream_id":"nihe6o4e"}
+{"time":"2026-09-21T14:30:25.598552731Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T14:30:25.598580863Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

epfl_final/SparseAtt/wandb/offline-run-20260921_143024-nihe6o4e/logs/debug.log

@@ -0,0 +1,19 @@
+2026-09-21 14:30:24,666 INFO    MainThread:67 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-21 14:30:24,666 INFO    MainThread:67 [wandb_setup.py:_flush():81] Configure stats pid to 67
+2026-09-21 14:30:24,666 INFO    MainThread:67 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-21 14:30:24,666 INFO    MainThread:67 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_143024-nihe6o4e/logs/debug.log
+2026-09-21 14:30:24,667 INFO    MainThread:67 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_143024-nihe6o4e/logs/debug-internal.log
+2026-09-21 14:30:24,667 INFO    MainThread:67 [wandb_init.py:init():850] calling init triggers
+2026-09-21 14:30:24,667 INFO    MainThread:67 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s42/run_20260921_142951', 'resume_from_checkpoint': None, 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s42', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed42'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-21 14:30:24,667 INFO    MainThread:67 [wandb_init.py:init():898] starting backend
+2026-09-21 14:30:25,192 INFO    MainThread:67 [wandb_init.py:init():913] sending inform_init request
+2026-09-21 14:30:25,567 INFO    MainThread:67 [wandb_init.py:init():918] backend started and connected
+2026-09-21 14:30:25,571 INFO    MainThread:67 [wandb_init.py:init():988] updated telemetry
+2026-09-21 14:30:25,595 INFO    MainThread:67 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-21 14:30:25,601 INFO    MainThread:67 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-21 14:30:25,802 INFO    MainThread:67 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-21 14:30:25,802 INFO    MainThread:67 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-21 14:30:25,802 INFO    MainThread:67 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-21 14:30:25,802 INFO    MainThread:67 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-21 14:30:25,804 INFO    MainThread:67 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/offline-run-20260921_143024-nihe6o4e/run-nihe6o4e.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96a013ff2d1155a019f60391503db1a304dae2d90dfd1ded3ca4ff9e1ecb270e
+size 1376256

epfl_final/SparseAtt/wandb/offline-run-20260921_143024-wk3rt7ae/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_143024-wk3rt7ae/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-09-21T14:30:25.207082344Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T14:30:25.207109204Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T14:30:25.554324889Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T14:30:25.554389605Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T14:30:25.554438769Z","level":"INFO","msg":"stream: created new stream","id":"wk3rt7ae"}
+{"time":"2026-09-21T14:30:25.554579739Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T14:30:25.556817811Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T14:30:25.556913373Z","level":"INFO","msg":"writer: started","stream_id":"wk3rt7ae"}
+{"time":"2026-09-21T14:30:25.556961245Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T14:30:25.58822386Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T14:30:25.588239193Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

epfl_final/SparseAtt/wandb/offline-run-20260921_143024-wk3rt7ae/logs/debug.log

@@ -0,0 +1,19 @@
+2026-09-21 14:30:24,664 INFO    MainThread:71 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-21 14:30:24,664 INFO    MainThread:71 [wandb_setup.py:_flush():81] Configure stats pid to 71
+2026-09-21 14:30:24,664 INFO    MainThread:71 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-21 14:30:24,664 INFO    MainThread:71 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_143024-wk3rt7ae/logs/debug.log
+2026-09-21 14:30:24,664 INFO    MainThread:71 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_143024-wk3rt7ae/logs/debug-internal.log
+2026-09-21 14:30:24,665 INFO    MainThread:71 [wandb_init.py:init():850] calling init triggers
+2026-09-21 14:30:24,665 INFO    MainThread:71 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s46/run_20260921_142951', 'resume_from_checkpoint': None, 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 46, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s46', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed46'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-21 14:30:24,665 INFO    MainThread:71 [wandb_init.py:init():898] starting backend
+2026-09-21 14:30:25,192 INFO    MainThread:71 [wandb_init.py:init():913] sending inform_init request
+2026-09-21 14:30:25,557 INFO    MainThread:71 [wandb_init.py:init():918] backend started and connected
+2026-09-21 14:30:25,561 INFO    MainThread:71 [wandb_init.py:init():988] updated telemetry
+2026-09-21 14:30:25,585 INFO    MainThread:71 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-21 14:30:25,590 INFO    MainThread:71 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-21 14:30:25,799 INFO    MainThread:71 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-21 14:30:25,799 INFO    MainThread:71 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-21 14:30:25,799 INFO    MainThread:71 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-21 14:30:25,799 INFO    MainThread:71 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-21 14:30:25,801 INFO    MainThread:71 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/offline-run-20260921_143024-wk3rt7ae/run-wk3rt7ae.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eaa8febec31ee92d4896cc02cd635856a829e30a54c8794baf3d7b91821413ac
+size 1376256

epfl_final/SparseAtt/wandb/offline-run-20260921_154845-ghfbqjah/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_154845-ghfbqjah/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-09-21T15:48:45.97315144Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T15:48:45.973175436Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T15:48:46.317654694Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T15:48:46.317697138Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T15:48:46.317754213Z","level":"INFO","msg":"stream: created new stream","id":"ghfbqjah"}
+{"time":"2026-09-21T15:48:46.317790477Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T15:48:46.319985701Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T15:48:46.32000528Z","level":"INFO","msg":"writer: started","stream_id":"ghfbqjah"}
+{"time":"2026-09-21T15:48:46.320027153Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T15:48:46.353007627Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T15:48:46.353039114Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

epfl_final/SparseAtt/wandb/offline-run-20260921_154845-ghfbqjah/logs/debug.log

@@ -0,0 +1,19 @@
+2026-09-21 15:48:45,417 INFO    MainThread:189 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-21 15:48:45,417 INFO    MainThread:189 [wandb_setup.py:_flush():81] Configure stats pid to 189
+2026-09-21 15:48:45,418 INFO    MainThread:189 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-21 15:48:45,418 INFO    MainThread:189 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_154845-ghfbqjah/logs/debug.log
+2026-09-21 15:48:45,418 INFO    MainThread:189 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_154845-ghfbqjah/logs/debug-internal.log
+2026-09-21 15:48:45,418 INFO    MainThread:189 [wandb_init.py:init():850] calling init triggers
+2026-09-21 15:48:45,418 INFO    MainThread:189 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s45/run_20260921_142951', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s45/run_20260921_142951/checkpoint-2000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 45, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s45', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed45'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-21 15:48:45,418 INFO    MainThread:189 [wandb_init.py:init():898] starting backend
+2026-09-21 15:48:45,958 INFO    MainThread:189 [wandb_init.py:init():913] sending inform_init request
+2026-09-21 15:48:46,320 INFO    MainThread:189 [wandb_init.py:init():918] backend started and connected
+2026-09-21 15:48:46,324 INFO    MainThread:189 [wandb_init.py:init():988] updated telemetry
+2026-09-21 15:48:46,350 INFO    MainThread:189 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-21 15:48:46,355 INFO    MainThread:189 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-21 15:48:46,601 INFO    MainThread:189 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-21 15:48:46,601 INFO    MainThread:189 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-21 15:48:46,601 INFO    MainThread:189 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-21 15:48:46,601 INFO    MainThread:189 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-21 15:48:46,603 INFO    MainThread:189 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/offline-run-20260921_154845-ghfbqjah/run-ghfbqjah.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86e0ecf363da86a6737dc4e5bad2001c7af3f31850e9439be4b2d951331841e3
+size 557056

epfl_final/SparseAtt/wandb/offline-run-20260921_154845-nihe6o4e/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_154845-nihe6o4e/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-09-21T15:48:45.973155927Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T15:48:45.97317795Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T15:48:46.224512758Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T15:48:46.224589834Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T15:48:46.224627911Z","level":"INFO","msg":"stream: created new stream","id":"nihe6o4e"}
+{"time":"2026-09-21T15:48:46.224692076Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T15:48:46.226575624Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T15:48:46.226611888Z","level":"INFO","msg":"writer: started","stream_id":"nihe6o4e"}
+{"time":"2026-09-21T15:48:46.22665338Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T15:48:46.290800955Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T15:48:46.290818902Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

epfl_final/SparseAtt/wandb/offline-run-20260921_154845-nihe6o4e/logs/debug.log

@@ -0,0 +1,19 @@
+2026-09-21 15:48:45,723 INFO    MainThread:174 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-21 15:48:45,723 INFO    MainThread:174 [wandb_setup.py:_flush():81] Configure stats pid to 174
+2026-09-21 15:48:45,723 INFO    MainThread:174 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-21 15:48:45,723 INFO    MainThread:174 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_154845-nihe6o4e/logs/debug.log
+2026-09-21 15:48:45,723 INFO    MainThread:174 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_154845-nihe6o4e/logs/debug-internal.log
+2026-09-21 15:48:45,723 INFO    MainThread:174 [wandb_init.py:init():850] calling init triggers
+2026-09-21 15:48:45,723 INFO    MainThread:174 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s42/run_20260921_142951', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s42/run_20260921_142951/checkpoint-2000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s42', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed42'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-21 15:48:45,723 INFO    MainThread:174 [wandb_init.py:init():898] starting backend
+2026-09-21 15:48:45,955 INFO    MainThread:174 [wandb_init.py:init():913] sending inform_init request
+2026-09-21 15:48:46,227 INFO    MainThread:174 [wandb_init.py:init():918] backend started and connected
+2026-09-21 15:48:46,230 INFO    MainThread:174 [wandb_init.py:init():988] updated telemetry
+2026-09-21 15:48:46,285 INFO    MainThread:174 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-21 15:48:46,291 INFO    MainThread:174 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-21 15:48:46,522 INFO    MainThread:174 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-21 15:48:46,522 INFO    MainThread:174 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-21 15:48:46,522 INFO    MainThread:174 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-21 15:48:46,523 INFO    MainThread:174 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-21 15:48:46,529 INFO    MainThread:174 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/offline-run-20260921_154845-nihe6o4e/run-nihe6o4e.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca6c43495cd064bdb2d9b5967a5b6e3949b7d97bcb3d1dc8781de46e4089ea91
+size 557056

epfl_final/SparseAtt/wandb/offline-run-20260921_154845-wk3rt7ae/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_154845-wk3rt7ae/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-09-21T15:48:45.973165451Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T15:48:45.973200123Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T15:48:46.220742849Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T15:48:46.220815608Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T15:48:46.220868327Z","level":"INFO","msg":"stream: created new stream","id":"wk3rt7ae"}
+{"time":"2026-09-21T15:48:46.220971972Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T15:48:46.222811433Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T15:48:46.223799912Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T15:48:46.223838379Z","level":"INFO","msg":"writer: started","stream_id":"wk3rt7ae"}
+{"time":"2026-09-21T15:48:46.291016127Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T15:48:46.291045461Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

epfl_final/SparseAtt/wandb/offline-run-20260921_154845-wk3rt7ae/logs/debug.log

@@ -0,0 +1,19 @@
+2026-09-21 15:48:45,731 INFO    MainThread:194 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-21 15:48:45,731 INFO    MainThread:194 [wandb_setup.py:_flush():81] Configure stats pid to 194
+2026-09-21 15:48:45,731 INFO    MainThread:194 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-21 15:48:45,731 INFO    MainThread:194 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_154845-wk3rt7ae/logs/debug.log
+2026-09-21 15:48:45,731 INFO    MainThread:194 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_154845-wk3rt7ae/logs/debug-internal.log
+2026-09-21 15:48:45,731 INFO    MainThread:194 [wandb_init.py:init():850] calling init triggers
+2026-09-21 15:48:45,732 INFO    MainThread:194 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s46/run_20260921_142951', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s46/run_20260921_142951/checkpoint-2000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 46, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s46', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed46'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-21 15:48:45,732 INFO    MainThread:194 [wandb_init.py:init():898] starting backend
+2026-09-21 15:48:45,961 INFO    MainThread:194 [wandb_init.py:init():913] sending inform_init request
+2026-09-21 15:48:46,223 INFO    MainThread:194 [wandb_init.py:init():918] backend started and connected
+2026-09-21 15:48:46,228 INFO    MainThread:194 [wandb_init.py:init():988] updated telemetry
+2026-09-21 15:48:46,285 INFO    MainThread:194 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-21 15:48:46,293 INFO    MainThread:194 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-21 15:48:46,546 INFO    MainThread:194 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-21 15:48:46,546 INFO    MainThread:194 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-21 15:48:46,546 INFO    MainThread:194 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-21 15:48:46,546 INFO    MainThread:194 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-21 15:48:46,548 INFO    MainThread:194 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/offline-run-20260921_154845-wk3rt7ae/run-wk3rt7ae.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:977d8eb231aae9cd8adfc08e4b2a0c9d75d7151ebde1202a3b518e3f51c1fc2b
+size 557056

epfl_final/SparseAtt/wandb/offline-run-20260921_154845-zgrsvkbi/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_154845-zgrsvkbi/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-09-21T15:48:45.976437516Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T15:48:45.976465287Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T15:48:46.229963081Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T15:48:46.230010913Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T15:48:46.230057112Z","level":"INFO","msg":"stream: created new stream","id":"zgrsvkbi"}
+{"time":"2026-09-21T15:48:46.230134417Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T15:48:46.2381986Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T15:48:46.238231209Z","level":"INFO","msg":"writer: started","stream_id":"zgrsvkbi"}
+{"time":"2026-09-21T15:48:46.238269336Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T15:48:46.291039061Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T15:48:46.291076247Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

epfl_final/SparseAtt/wandb/offline-run-20260921_154845-zgrsvkbi/logs/debug.log

@@ -0,0 +1,19 @@
+2026-09-21 15:48:45,718 INFO    MainThread:179 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-21 15:48:45,718 INFO    MainThread:179 [wandb_setup.py:_flush():81] Configure stats pid to 179
+2026-09-21 15:48:45,718 INFO    MainThread:179 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-21 15:48:45,719 INFO    MainThread:179 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_154845-zgrsvkbi/logs/debug.log
+2026-09-21 15:48:45,719 INFO    MainThread:179 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_154845-zgrsvkbi/logs/debug-internal.log
+2026-09-21 15:48:45,719 INFO    MainThread:179 [wandb_init.py:init():850] calling init triggers
+2026-09-21 15:48:45,719 INFO    MainThread:179 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s43/run_20260921_142951', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s43/run_20260921_142951/checkpoint-2000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 43, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s43', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed43'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-21 15:48:45,719 INFO    MainThread:179 [wandb_init.py:init():898] starting backend
+2026-09-21 15:48:45,965 INFO    MainThread:179 [wandb_init.py:init():913] sending inform_init request
+2026-09-21 15:48:46,239 INFO    MainThread:179 [wandb_init.py:init():918] backend started and connected
+2026-09-21 15:48:46,243 INFO    MainThread:179 [wandb_init.py:init():988] updated telemetry
+2026-09-21 15:48:46,285 INFO    MainThread:179 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-21 15:48:46,292 INFO    MainThread:179 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-21 15:48:46,527 INFO    MainThread:179 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-21 15:48:46,527 INFO    MainThread:179 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-21 15:48:46,527 INFO    MainThread:179 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-21 15:48:46,527 INFO    MainThread:179 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-21 15:48:46,529 INFO    MainThread:179 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/offline-run-20260921_154845-zgrsvkbi/run-zgrsvkbi.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7211ab74941b2c2fbc03f25eda120604bf257819452998b7a19429fc7814243
+size 557056

epfl_final/SparseAtt/wandb/offline-run-20260921_154846-kbn66izc/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_154846-kbn66izc/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-09-21T15:48:46.438986648Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T15:48:46.43901461Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T15:48:46.574888667Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T15:48:46.574951741Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T15:48:46.574980875Z","level":"INFO","msg":"stream: created new stream","id":"kbn66izc"}
+{"time":"2026-09-21T15:48:46.5750312Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T15:48:46.577122839Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T15:48:46.57714365Z","level":"INFO","msg":"writer: started","stream_id":"kbn66izc"}
+{"time":"2026-09-21T15:48:46.577159904Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T15:48:46.596521556Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T15:48:46.596543008Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

epfl_final/SparseAtt/wandb/offline-run-20260921_154846-kbn66izc/logs/debug.log

@@ -0,0 +1,19 @@
+2026-09-21 15:48:46,197 INFO    MainThread:184 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-21 15:48:46,197 INFO    MainThread:184 [wandb_setup.py:_flush():81] Configure stats pid to 184
+2026-09-21 15:48:46,197 INFO    MainThread:184 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-21 15:48:46,197 INFO    MainThread:184 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_154846-kbn66izc/logs/debug.log
+2026-09-21 15:48:46,197 INFO    MainThread:184 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_154846-kbn66izc/logs/debug-internal.log
+2026-09-21 15:48:46,197 INFO    MainThread:184 [wandb_init.py:init():850] calling init triggers
+2026-09-21 15:48:46,197 INFO    MainThread:184 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s44/run_20260921_142951', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s44/run_20260921_142951/checkpoint-2000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 44, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s44', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed44'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-21 15:48:46,197 INFO    MainThread:184 [wandb_init.py:init():898] starting backend
+2026-09-21 15:48:46,427 INFO    MainThread:184 [wandb_init.py:init():913] sending inform_init request
+2026-09-21 15:48:46,577 INFO    MainThread:184 [wandb_init.py:init():918] backend started and connected
+2026-09-21 15:48:46,580 INFO    MainThread:184 [wandb_init.py:init():988] updated telemetry
+2026-09-21 15:48:46,594 INFO    MainThread:184 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-21 15:48:46,597 INFO    MainThread:184 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-21 15:48:46,811 INFO    MainThread:184 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-21 15:48:46,811 INFO    MainThread:184 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-21 15:48:46,811 INFO    MainThread:184 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-21 15:48:46,811 INFO    MainThread:184 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-21 15:48:46,813 INFO    MainThread:184 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/offline-run-20260921_154846-kbn66izc/run-kbn66izc.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aec2d054f22f5949d7d9d256bc1cfbad70df2d5f04db8d4745b8d208897ca7d2
+size 557056

epfl_final/SparseAtt/wandb/offline-run-20260921_163219-nihe6o4e/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_163219-nihe6o4e/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-09-21T16:32:19.893591207Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T16:32:19.893619479Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T16:32:20.046817338Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T16:32:20.046876216Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T16:32:20.046930778Z","level":"INFO","msg":"stream: created new stream","id":"nihe6o4e"}
+{"time":"2026-09-21T16:32:20.047043416Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T16:32:20.049160733Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T16:32:20.04919915Z","level":"INFO","msg":"writer: started","stream_id":"nihe6o4e"}
+{"time":"2026-09-21T16:32:20.049243957Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T16:32:20.089660068Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T16:32:20.089689632Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

epfl_final/SparseAtt/wandb/offline-run-20260921_163219-nihe6o4e/logs/debug.log

@@ -0,0 +1,19 @@
+2026-09-21 16:32:19,612 INFO    MainThread:268 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-21 16:32:19,612 INFO    MainThread:268 [wandb_setup.py:_flush():81] Configure stats pid to 268
+2026-09-21 16:32:19,612 INFO    MainThread:268 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-21 16:32:19,612 INFO    MainThread:268 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_163219-nihe6o4e/logs/debug.log
+2026-09-21 16:32:19,613 INFO    MainThread:268 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_163219-nihe6o4e/logs/debug-internal.log
+2026-09-21 16:32:19,613 INFO    MainThread:268 [wandb_init.py:init():850] calling init triggers
+2026-09-21 16:32:19,613 INFO    MainThread:268 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s42/run_20260921_142951', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s42/run_20260921_142951/checkpoint-2000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 5, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': 10000, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s42', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed42'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-21 16:32:19,613 INFO    MainThread:268 [wandb_init.py:init():898] starting backend
+2026-09-21 16:32:19,880 INFO    MainThread:268 [wandb_init.py:init():913] sending inform_init request
+2026-09-21 16:32:20,050 INFO    MainThread:268 [wandb_init.py:init():918] backend started and connected
+2026-09-21 16:32:20,055 INFO    MainThread:268 [wandb_init.py:init():988] updated telemetry
+2026-09-21 16:32:20,086 INFO    MainThread:268 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-21 16:32:20,100 INFO    MainThread:268 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-21 16:32:20,398 INFO    MainThread:268 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-21 16:32:20,398 INFO    MainThread:268 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-21 16:32:20,398 INFO    MainThread:268 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-21 16:32:20,399 INFO    MainThread:268 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-21 16:32:20,402 INFO    MainThread:268 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/offline-run-20260921_163219-nihe6o4e/run-nihe6o4e.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d55a048cb5bc4a9201d9644da58d274285914becd036f1bff4e4470fe88abf43
+size 2392064

epfl_final/SparseAtt/wandb/offline-run-20260921_164533-kbn66izc/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_164533-kbn66izc/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2026-09-21T16:45:34.951715029Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T16:45:34.951729165Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T16:45:35.478503057Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T16:45:35.478524787Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T16:45:35.478555224Z","level":"INFO","msg":"stream: created new stream","id":"kbn66izc"}
+{"time":"2026-09-21T16:45:35.478593225Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T16:45:35.501714375Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T16:45:35.505389607Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T16:45:35.505394967Z","level":"INFO","msg":"writer: started","stream_id":"kbn66izc"}
+{"time":"2026-09-21T16:45:35.549255731Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T16:45:35.549268585Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}
+{"time":"2026-09-22T03:50:08.983849971Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-09-22T03:50:08.990497594Z","level":"INFO","msg":"stream: finishing up"}
+{"time":"2026-09-22T03:50:08.99051679Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-09-22T03:50:08.991565127Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-09-22T03:50:08.99157224Z","level":"INFO","msg":"stream: all finished"}

epfl_final/SparseAtt/wandb/offline-run-20260921_164533-kbn66izc/logs/debug.log

@@ -0,0 +1,23 @@
+2026-09-21 16:45:33,941 INFO    MainThread:238 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-21 16:45:33,942 INFO    MainThread:238 [wandb_setup.py:_flush():81] Configure stats pid to 238
+2026-09-21 16:45:33,942 INFO    MainThread:238 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-21 16:45:33,942 INFO    MainThread:238 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_164533-kbn66izc/logs/debug.log
+2026-09-21 16:45:33,942 INFO    MainThread:238 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_164533-kbn66izc/logs/debug-internal.log
+2026-09-21 16:45:33,942 INFO    MainThread:238 [wandb_init.py:init():850] calling init triggers
+2026-09-21 16:45:33,942 INFO    MainThread:238 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s44/run_20260921_142951', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s44/run_20260921_142951/checkpoint-2000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 4, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 44, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': 8000, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s44', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed44'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-21 16:45:33,942 INFO    MainThread:238 [wandb_init.py:init():898] starting backend
+2026-09-21 16:45:34,935 INFO    MainThread:238 [wandb_init.py:init():913] sending inform_init request
+2026-09-21 16:45:35,502 INFO    MainThread:238 [wandb_init.py:init():918] backend started and connected
+2026-09-21 16:45:35,506 INFO    MainThread:238 [wandb_init.py:init():988] updated telemetry
+2026-09-21 16:45:35,546 INFO    MainThread:238 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-21 16:45:35,550 INFO    MainThread:238 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-21 16:45:35,784 INFO    MainThread:238 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-21 16:45:35,784 INFO    MainThread:238 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-21 16:45:35,784 INFO    MainThread:238 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-21 16:45:35,784 INFO    MainThread:238 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-21 16:45:35,799 INFO    MainThread:238 [wandb_init.py:init():1094] run started, returning control to user process
+2026-09-22 03:50:08,977 INFO    MainThread:238 [wandb_run.py:_finish():2321] finishing run stochastic_sparsity/kbn66izc
+2026-09-22 03:50:08,978 INFO    MainThread:238 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
+2026-09-22 03:50:08,980 INFO    MainThread:238 [wandb_run.py:_restore():2502] restore
+2026-09-22 03:50:08,980 INFO    MainThread:238 [wandb_run.py:_restore():2508] restore done

epfl_final/SparseAtt/wandb/offline-run-20260921_164533-kbn66izc/run-kbn66izc.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d917d1f5ecf537741b1509bda27a6cf13f48cb81298670cc7a75e03721986608
+size 7641216

epfl_final/SparseAtt/wandb/offline-run-20260921_164537-zgrsvkbi/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_164537-zgrsvkbi/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2026-09-21T16:45:37.803741511Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T16:45:37.803757971Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T16:45:38.139478759Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T16:45:38.139514896Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T16:45:38.139546384Z","level":"INFO","msg":"stream: created new stream","id":"zgrsvkbi"}
+{"time":"2026-09-21T16:45:38.139586388Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T16:45:38.141813423Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T16:45:38.142565807Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T16:45:38.142566037Z","level":"INFO","msg":"writer: started","stream_id":"zgrsvkbi"}
+{"time":"2026-09-21T16:45:38.193714108Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T16:45:38.193729426Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}
+{"time":"2026-09-22T04:08:06.511878708Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-09-22T04:08:06.522226378Z","level":"INFO","msg":"stream: finishing up"}
+{"time":"2026-09-22T04:08:06.52224334Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-09-22T04:08:06.523497333Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-09-22T04:08:06.523505869Z","level":"INFO","msg":"stream: all finished"}

epfl_final/SparseAtt/wandb/offline-run-20260921_164537-zgrsvkbi/logs/debug.log

@@ -0,0 +1,23 @@
+2026-09-21 16:45:37,291 INFO    MainThread:235 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-21 16:45:37,291 INFO    MainThread:235 [wandb_setup.py:_flush():81] Configure stats pid to 235
+2026-09-21 16:45:37,291 INFO    MainThread:235 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-21 16:45:37,291 INFO    MainThread:235 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_164537-zgrsvkbi/logs/debug.log
+2026-09-21 16:45:37,291 INFO    MainThread:235 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_164537-zgrsvkbi/logs/debug-internal.log
+2026-09-21 16:45:37,291 INFO    MainThread:235 [wandb_init.py:init():850] calling init triggers
+2026-09-21 16:45:37,291 INFO    MainThread:235 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s43/run_20260921_142951', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s43/run_20260921_142951/checkpoint-2000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 4, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 43, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': 8000, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s43', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed43'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-21 16:45:37,291 INFO    MainThread:235 [wandb_init.py:init():898] starting backend
+2026-09-21 16:45:37,788 INFO    MainThread:235 [wandb_init.py:init():913] sending inform_init request
+2026-09-21 16:45:38,142 INFO    MainThread:235 [wandb_init.py:init():918] backend started and connected
+2026-09-21 16:45:38,147 INFO    MainThread:235 [wandb_init.py:init():988] updated telemetry
+2026-09-21 16:45:38,190 INFO    MainThread:235 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-21 16:45:38,195 INFO    MainThread:235 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-21 16:45:38,433 INFO    MainThread:235 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-21 16:45:38,433 INFO    MainThread:235 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-21 16:45:38,433 INFO    MainThread:235 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-21 16:45:38,433 INFO    MainThread:235 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-21 16:45:38,439 INFO    MainThread:235 [wandb_init.py:init():1094] run started, returning control to user process
+2026-09-22 04:08:06,508 INFO    MainThread:235 [wandb_run.py:_finish():2321] finishing run stochastic_sparsity/zgrsvkbi
+2026-09-22 04:08:06,509 INFO    MainThread:235 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
+2026-09-22 04:08:06,510 INFO    MainThread:235 [wandb_run.py:_restore():2502] restore
+2026-09-22 04:08:06,510 INFO    MainThread:235 [wandb_run.py:_restore():2508] restore done

epfl_final/SparseAtt/wandb/offline-run-20260921_164537-zgrsvkbi/run-zgrsvkbi.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4000d30c7617d887accba66cddef90a2450a99e75aa1cd604be87994ec3a8f6
+size 7726453

epfl_final/SparseAtt/wandb/offline-run-20260921_195241-nihe6o4e/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_195241-nihe6o4e/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2026-09-21T19:52:42.319490993Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T19:52:42.319520437Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T19:52:42.58442705Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T19:52:42.584504607Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T19:52:42.584554181Z","level":"INFO","msg":"stream: created new stream","id":"nihe6o4e"}
+{"time":"2026-09-21T19:52:42.584644176Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T19:52:42.586982479Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T19:52:42.588333569Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T19:52:42.588362913Z","level":"INFO","msg":"writer: started","stream_id":"nihe6o4e"}
+{"time":"2026-09-21T19:52:42.666823239Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T19:52:42.666853084Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}
+{"time":"2026-09-21T23:17:02.832069883Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-09-21T23:17:02.836913023Z","level":"INFO","msg":"stream: finishing up"}
+{"time":"2026-09-21T23:17:02.836921235Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-09-21T23:17:02.838136842Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-09-21T23:17:02.838143012Z","level":"INFO","msg":"stream: all finished"}