EPFL final checkpoints and W&B history 1351-1400 of 1463

.gitattributes

@@ -176,3 +176,16 @@ epfl_final/SparseAtt/wandb/run-20260903_142704-7tfq6nwm/run-7tfq6nwm.wandb filte
 epfl_final/SparseAtt/wandb/run-20260903_191410-7tfq6nwm/run-7tfq6nwm.wandb filter=lfs diff=lfs merge=lfs -text
 epfl_final/SparseAtt/wandb/run-20260904_042057-7tfq6nwm/run-7tfq6nwm.wandb filter=lfs diff=lfs merge=lfs -text
 epfl_final/SparseAtt/wandb/offline-run-20260921_030515-1savcbqu/run-1savcbqu.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ao5edbbv/run-ao5edbbv.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ut49ok16/run-ut49ok16.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/offline-run-20260921_030515-6k91ldtc/run-6k91ldtc.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/offline-run-20260921_030515-hmdx78bp/run-hmdx78bp.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/offline-run-20260921_030515-8km2sc23/run-8km2sc23.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/offline-run-20260921_131249-ifgmkpzt/run-ifgmkpzt.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/offline-run-20260921_131249-meicy3md/run-meicy3md.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/offline-run-20260921_131249-n6s56yfr/run-n6s56yfr.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3la36nzy/run-3la36nzy.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/offline-run-20260921_131249-xc6c08ps/run-xc6c08ps.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3uan03m3/run-3uan03m3.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/offline-run-20260921_143024-kbn66izc/run-kbn66izc.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/offline-run-20260921_143024-zgrsvkbi/run-zgrsvkbi.wandb filter=lfs diff=lfs merge=lfs -text

epfl_final/SparseAtt/wandb/offline-run-20260921_030515-6k91ldtc/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_030515-6k91ldtc/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-09-21T03:05:15.627748328Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T03:05:15.627774557Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T03:05:15.977507774Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T03:05:15.977569276Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T03:05:15.977607514Z","level":"INFO","msg":"stream: created new stream","id":"6k91ldtc"}
+{"time":"2026-09-21T03:05:15.977692221Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T03:05:15.986110077Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T03:05:15.986165781Z","level":"INFO","msg":"writer: started","stream_id":"6k91ldtc"}
+{"time":"2026-09-21T03:05:15.986204038Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T03:05:16.013061227Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T03:05:16.0130764Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

epfl_final/SparseAtt/wandb/offline-run-20260921_030515-6k91ldtc/logs/debug.log

@@ -0,0 +1,19 @@
+2026-09-21 03:05:15,374 INFO    MainThread:4776 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-21 03:05:15,374 INFO    MainThread:4776 [wandb_setup.py:_flush():81] Configure stats pid to 4776
+2026-09-21 03:05:15,374 INFO    MainThread:4776 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-21 03:05:15,374 INFO    MainThread:4776 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_030515-6k91ldtc/logs/debug.log
+2026-09-21 03:05:15,374 INFO    MainThread:4776 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_030515-6k91ldtc/logs/debug-internal.log
+2026-09-21 03:05:15,374 INFO    MainThread:4776 [wandb_init.py:init():850] calling init triggers
+2026-09-21 03:05:15,374 INFO    MainThread:4776 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s44/run_20260921_030452', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 22000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 44, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s44', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed44'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-21 03:05:15,374 INFO    MainThread:4776 [wandb_init.py:init():898] starting backend
+2026-09-21 03:05:15,616 INFO    MainThread:4776 [wandb_init.py:init():913] sending inform_init request
+2026-09-21 03:05:15,987 INFO    MainThread:4776 [wandb_init.py:init():918] backend started and connected
+2026-09-21 03:05:15,988 INFO    MainThread:4776 [wandb_init.py:init():988] updated telemetry
+2026-09-21 03:05:16,009 INFO    MainThread:4776 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-21 03:05:16,014 INFO    MainThread:4776 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-21 03:05:16,229 INFO    MainThread:4776 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-21 03:05:16,229 INFO    MainThread:4776 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-21 03:05:16,229 INFO    MainThread:4776 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-21 03:05:16,230 INFO    MainThread:4776 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-21 03:05:16,235 INFO    MainThread:4776 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/offline-run-20260921_030515-6k91ldtc/run-6k91ldtc.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9fa5a8893b0d64471e1060bb2634231d6cfeb06f87b5f0bdf963159327fa592
+size 9502720

epfl_final/SparseAtt/wandb/offline-run-20260921_030515-8km2sc23/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_030515-8km2sc23/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-09-21T03:05:15.647686351Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T03:05:15.647714233Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T03:05:16.001225716Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T03:05:16.001272596Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T03:05:16.001306587Z","level":"INFO","msg":"stream: created new stream","id":"8km2sc23"}
+{"time":"2026-09-21T03:05:16.001376873Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T03:05:16.007121864Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T03:05:16.007230587Z","level":"INFO","msg":"writer: started","stream_id":"8km2sc23"}
+{"time":"2026-09-21T03:05:16.007241514Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T03:05:16.028513461Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T03:05:16.028534102Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

epfl_final/SparseAtt/wandb/offline-run-20260921_030515-8km2sc23/logs/debug.log

@@ -0,0 +1,19 @@
+2026-09-21 03:05:15,380 INFO    MainThread:4779 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-21 03:05:15,380 INFO    MainThread:4779 [wandb_setup.py:_flush():81] Configure stats pid to 4779
+2026-09-21 03:05:15,380 INFO    MainThread:4779 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-21 03:05:15,380 INFO    MainThread:4779 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_030515-8km2sc23/logs/debug.log
+2026-09-21 03:05:15,380 INFO    MainThread:4779 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_030515-8km2sc23/logs/debug-internal.log
+2026-09-21 03:05:15,380 INFO    MainThread:4779 [wandb_init.py:init():850] calling init triggers
+2026-09-21 03:05:15,381 INFO    MainThread:4779 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s47/run_20260921_030452', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 22000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 47, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s47', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed47'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-21 03:05:15,381 INFO    MainThread:4779 [wandb_init.py:init():898] starting backend
+2026-09-21 03:05:15,636 INFO    MainThread:4779 [wandb_init.py:init():913] sending inform_init request
+2026-09-21 03:05:16,008 INFO    MainThread:4779 [wandb_init.py:init():918] backend started and connected
+2026-09-21 03:05:16,010 INFO    MainThread:4779 [wandb_init.py:init():988] updated telemetry
+2026-09-21 03:05:16,025 INFO    MainThread:4779 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-21 03:05:16,030 INFO    MainThread:4779 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-21 03:05:16,260 INFO    MainThread:4779 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-21 03:05:16,260 INFO    MainThread:4779 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-21 03:05:16,260 INFO    MainThread:4779 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-21 03:05:16,260 INFO    MainThread:4779 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-21 03:05:16,262 INFO    MainThread:4779 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/offline-run-20260921_030515-8km2sc23/run-8km2sc23.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f20ea78ff25351a9482d7556532b12dfc04bb9feccf50b508a5d996379de376
+size 9502720

epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ao5edbbv/run-ao5edbbv.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a47ccd47018e8925ebe18a6bcc6a903833a70cb17c477052d08acb8df1e90288
+size 9502720

epfl_final/SparseAtt/wandb/offline-run-20260921_030515-hmdx78bp/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_030515-hmdx78bp/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-09-21T03:05:15.644133914Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T03:05:15.644161065Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T03:05:15.995507455Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T03:05:15.995566224Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T03:05:15.995608958Z","level":"INFO","msg":"stream: created new stream","id":"hmdx78bp"}
+{"time":"2026-09-21T03:05:15.995691291Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T03:05:15.997829013Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T03:05:15.997901802Z","level":"INFO","msg":"writer: started","stream_id":"hmdx78bp"}
+{"time":"2026-09-21T03:05:15.997918447Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T03:05:16.02085118Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T03:05:16.020873704Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

epfl_final/SparseAtt/wandb/offline-run-20260921_030515-hmdx78bp/logs/debug.log

@@ -0,0 +1,19 @@
+2026-09-21 03:05:15,378 INFO    MainThread:4777 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-21 03:05:15,378 INFO    MainThread:4777 [wandb_setup.py:_flush():81] Configure stats pid to 4777
+2026-09-21 03:05:15,378 INFO    MainThread:4777 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-21 03:05:15,378 INFO    MainThread:4777 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_030515-hmdx78bp/logs/debug.log
+2026-09-21 03:05:15,378 INFO    MainThread:4777 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_030515-hmdx78bp/logs/debug-internal.log
+2026-09-21 03:05:15,378 INFO    MainThread:4777 [wandb_init.py:init():850] calling init triggers
+2026-09-21 03:05:15,378 INFO    MainThread:4777 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s45/run_20260921_030452', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 22000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 45, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s45', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed45'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-21 03:05:15,378 INFO    MainThread:4777 [wandb_init.py:init():898] starting backend
+2026-09-21 03:05:15,632 INFO    MainThread:4777 [wandb_init.py:init():913] sending inform_init request
+2026-09-21 03:05:15,998 INFO    MainThread:4777 [wandb_init.py:init():918] backend started and connected
+2026-09-21 03:05:16,001 INFO    MainThread:4777 [wandb_init.py:init():988] updated telemetry
+2026-09-21 03:05:16,018 INFO    MainThread:4777 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-21 03:05:16,023 INFO    MainThread:4777 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-21 03:05:16,249 INFO    MainThread:4777 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-21 03:05:16,249 INFO    MainThread:4777 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-21 03:05:16,249 INFO    MainThread:4777 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-21 03:05:16,249 INFO    MainThread:4777 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-21 03:05:16,251 INFO    MainThread:4777 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/offline-run-20260921_030515-hmdx78bp/run-hmdx78bp.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:529c1489903aaab749fa0277af1d0a0149c7354fdf132a44620412bbaa1cc1d3
+size 9502720

epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ut49ok16/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ut49ok16/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-09-21T03:05:15.626653773Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T03:05:15.626680303Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T03:05:15.981530157Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T03:05:15.981590878Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T03:05:15.981626161Z","level":"INFO","msg":"stream: created new stream","id":"ut49ok16"}
+{"time":"2026-09-21T03:05:15.981707033Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T03:05:15.99192309Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T03:05:15.991947527Z","level":"INFO","msg":"writer: started","stream_id":"ut49ok16"}
+{"time":"2026-09-21T03:05:15.991976481Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T03:05:16.024492801Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T03:05:16.0245186Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ut49ok16/logs/debug.log

@@ -0,0 +1,19 @@
+2026-09-21 03:05:15,152 INFO    MainThread:4775 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-21 03:05:15,153 INFO    MainThread:4775 [wandb_setup.py:_flush():81] Configure stats pid to 4775
+2026-09-21 03:05:15,153 INFO    MainThread:4775 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-21 03:05:15,153 INFO    MainThread:4775 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_030515-ut49ok16/logs/debug.log
+2026-09-21 03:05:15,153 INFO    MainThread:4775 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_030515-ut49ok16/logs/debug-internal.log
+2026-09-21 03:05:15,153 INFO    MainThread:4775 [wandb_init.py:init():850] calling init triggers
+2026-09-21 03:05:15,153 INFO    MainThread:4775 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s43/run_20260921_030452', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 22000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 43, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s43', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed43'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-21 03:05:15,153 INFO    MainThread:4775 [wandb_init.py:init():898] starting backend
+2026-09-21 03:05:15,613 INFO    MainThread:4775 [wandb_init.py:init():913] sending inform_init request
+2026-09-21 03:05:15,992 INFO    MainThread:4775 [wandb_init.py:init():918] backend started and connected
+2026-09-21 03:05:15,998 INFO    MainThread:4775 [wandb_init.py:init():988] updated telemetry
+2026-09-21 03:05:16,020 INFO    MainThread:4775 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-21 03:05:16,028 INFO    MainThread:4775 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-21 03:05:16,251 INFO    MainThread:4775 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-21 03:05:16,251 INFO    MainThread:4775 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-21 03:05:16,251 INFO    MainThread:4775 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-21 03:05:16,251 INFO    MainThread:4775 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-21 03:05:16,253 INFO    MainThread:4775 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ut49ok16/run-ut49ok16.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:652398a1a92d2ce4c8f034a80d6119d7e6e8370b537dc3bfb78449df8b3d97dc
+size 9502720

epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3la36nzy/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3la36nzy/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-09-21T13:12:50.483986586Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T13:12:50.484020637Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T13:12:50.838023892Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T13:12:50.838063321Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T13:12:50.838094006Z","level":"INFO","msg":"stream: created new stream","id":"3la36nzy"}
+{"time":"2026-09-21T13:12:50.838213515Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T13:12:50.841655341Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T13:12:50.841683162Z","level":"INFO","msg":"writer: started","stream_id":"3la36nzy"}
+{"time":"2026-09-21T13:12:50.841732275Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T13:12:50.870894371Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T13:12:50.87092038Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3la36nzy/logs/debug.log

@@ -0,0 +1,19 @@
+2026-09-21 13:12:49,924 INFO    MainThread:356 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-21 13:12:49,924 INFO    MainThread:356 [wandb_setup.py:_flush():81] Configure stats pid to 356
+2026-09-21 13:12:49,924 INFO    MainThread:356 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-21 13:12:49,924 INFO    MainThread:356 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_131249-3la36nzy/logs/debug.log
+2026-09-21 13:12:49,925 INFO    MainThread:356 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_131249-3la36nzy/logs/debug-internal.log
+2026-09-21 13:12:49,925 INFO    MainThread:356 [wandb_init.py:init():850] calling init triggers
+2026-09-21 13:12:49,925 INFO    MainThread:356 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s45/run_20260921_131213', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 45, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s45', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed45'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-21 13:12:49,925 INFO    MainThread:356 [wandb_init.py:init():898] starting backend
+2026-09-21 13:12:50,469 INFO    MainThread:356 [wandb_init.py:init():913] sending inform_init request
+2026-09-21 13:12:50,842 INFO    MainThread:356 [wandb_init.py:init():918] backend started and connected
+2026-09-21 13:12:50,847 INFO    MainThread:356 [wandb_init.py:init():988] updated telemetry
+2026-09-21 13:12:50,867 INFO    MainThread:356 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-21 13:12:50,873 INFO    MainThread:356 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-21 13:12:51,121 INFO    MainThread:356 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-21 13:12:51,121 INFO    MainThread:356 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-21 13:12:51,121 INFO    MainThread:356 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-21 13:12:51,121 INFO    MainThread:356 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-21 13:12:51,127 INFO    MainThread:356 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3la36nzy/run-3la36nzy.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50d40438b665e32cf8402c79131dd92f63553951050e3e7266dc0c35d6e0b96c
+size 622592

epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3uan03m3/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3uan03m3/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-09-21T13:12:50.483874479Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T13:12:50.483901889Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T13:12:50.841088435Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T13:12:50.841137017Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T13:12:50.841184688Z","level":"INFO","msg":"stream: created new stream","id":"3uan03m3"}
+{"time":"2026-09-21T13:12:50.841265799Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T13:12:50.84313623Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T13:12:50.843167918Z","level":"INFO","msg":"writer: started","stream_id":"3uan03m3"}
+{"time":"2026-09-21T13:12:50.84320998Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T13:12:50.870785509Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T13:12:50.870814782Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3uan03m3/logs/debug.log

@@ -0,0 +1,19 @@
+2026-09-21 13:12:49,929 INFO    MainThread:353 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-21 13:12:49,929 INFO    MainThread:353 [wandb_setup.py:_flush():81] Configure stats pid to 353
+2026-09-21 13:12:49,929 INFO    MainThread:353 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-21 13:12:49,929 INFO    MainThread:353 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_131249-3uan03m3/logs/debug.log
+2026-09-21 13:12:49,930 INFO    MainThread:353 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_131249-3uan03m3/logs/debug-internal.log
+2026-09-21 13:12:49,930 INFO    MainThread:353 [wandb_init.py:init():850] calling init triggers
+2026-09-21 13:12:49,930 INFO    MainThread:353 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s42/run_20260921_131213', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s42', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed42'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-21 13:12:49,930 INFO    MainThread:353 [wandb_init.py:init():898] starting backend
+2026-09-21 13:12:50,469 INFO    MainThread:353 [wandb_init.py:init():913] sending inform_init request
+2026-09-21 13:12:50,844 INFO    MainThread:353 [wandb_init.py:init():918] backend started and connected
+2026-09-21 13:12:50,847 INFO    MainThread:353 [wandb_init.py:init():988] updated telemetry
+2026-09-21 13:12:50,867 INFO    MainThread:353 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-21 13:12:50,873 INFO    MainThread:353 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-21 13:12:51,108 INFO    MainThread:353 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-21 13:12:51,108 INFO    MainThread:353 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-21 13:12:51,108 INFO    MainThread:353 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-21 13:12:51,108 INFO    MainThread:353 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-21 13:12:51,115 INFO    MainThread:353 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3uan03m3/run-3uan03m3.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8131887d0f9a50fc1d08591b27b346f6ac4586d89dfbf1fd0cf022321587af0f
+size 622592

epfl_final/SparseAtt/wandb/offline-run-20260921_131249-ifgmkpzt/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_131249-ifgmkpzt/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-09-21T13:12:50.483216577Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T13:12:50.483239Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T13:12:50.836243114Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T13:12:50.836287921Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T13:12:50.836332528Z","level":"INFO","msg":"stream: created new stream","id":"ifgmkpzt"}
+{"time":"2026-09-21T13:12:50.836402232Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T13:12:50.839011786Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T13:12:50.839030384Z","level":"INFO","msg":"writer: started","stream_id":"ifgmkpzt"}
+{"time":"2026-09-21T13:12:50.839085496Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T13:12:50.870954671Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T13:12:50.870977545Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

epfl_final/SparseAtt/wandb/offline-run-20260921_131249-ifgmkpzt/logs/debug.log

@@ -0,0 +1,19 @@
+2026-09-21 13:12:49,902 INFO    MainThread:355 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-21 13:12:49,902 INFO    MainThread:355 [wandb_setup.py:_flush():81] Configure stats pid to 355
+2026-09-21 13:12:49,902 INFO    MainThread:355 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-21 13:12:49,902 INFO    MainThread:355 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_131249-ifgmkpzt/logs/debug.log
+2026-09-21 13:12:49,902 INFO    MainThread:355 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_131249-ifgmkpzt/logs/debug-internal.log
+2026-09-21 13:12:49,902 INFO    MainThread:355 [wandb_init.py:init():850] calling init triggers
+2026-09-21 13:12:49,902 INFO    MainThread:355 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s44/run_20260921_131213', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 44, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s44', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed44'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-21 13:12:49,902 INFO    MainThread:355 [wandb_init.py:init():898] starting backend
+2026-09-21 13:12:50,469 INFO    MainThread:355 [wandb_init.py:init():913] sending inform_init request
+2026-09-21 13:12:50,839 INFO    MainThread:355 [wandb_init.py:init():918] backend started and connected
+2026-09-21 13:12:50,843 INFO    MainThread:355 [wandb_init.py:init():988] updated telemetry
+2026-09-21 13:12:50,867 INFO    MainThread:355 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-21 13:12:50,873 INFO    MainThread:355 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-21 13:12:51,130 INFO    MainThread:355 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-21 13:12:51,130 INFO    MainThread:355 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-21 13:12:51,130 INFO    MainThread:355 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-21 13:12:51,131 INFO    MainThread:355 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-21 13:12:51,133 INFO    MainThread:355 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/offline-run-20260921_131249-ifgmkpzt/run-ifgmkpzt.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b50e684727aea5e38ea564a5e13082ab3e25791c61b34c009b307cc1a7eb8bbd
+size 622592

epfl_final/SparseAtt/wandb/offline-run-20260921_131249-meicy3md/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_131249-meicy3md/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-09-21T13:12:50.483553329Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T13:12:50.48358054Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T13:12:50.839693533Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T13:12:50.839788525Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T13:12:50.839831339Z","level":"INFO","msg":"stream: created new stream","id":"meicy3md"}
+{"time":"2026-09-21T13:12:50.83995932Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T13:12:50.843120577Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T13:12:50.843209209Z","level":"INFO","msg":"writer: started","stream_id":"meicy3md"}
+{"time":"2026-09-21T13:12:50.843275598Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T13:12:50.87087387Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T13:12:50.870909333Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

epfl_final/SparseAtt/wandb/offline-run-20260921_131249-meicy3md/logs/debug.log

@@ -0,0 +1,19 @@
+2026-09-21 13:12:49,918 INFO    MainThread:357 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-21 13:12:49,918 INFO    MainThread:357 [wandb_setup.py:_flush():81] Configure stats pid to 357
+2026-09-21 13:12:49,918 INFO    MainThread:357 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-21 13:12:49,919 INFO    MainThread:357 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_131249-meicy3md/logs/debug.log
+2026-09-21 13:12:49,919 INFO    MainThread:357 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_131249-meicy3md/logs/debug-internal.log
+2026-09-21 13:12:49,919 INFO    MainThread:357 [wandb_init.py:init():850] calling init triggers
+2026-09-21 13:12:49,919 INFO    MainThread:357 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s46/run_20260921_131213', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 46, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s46', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed46'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-21 13:12:49,919 INFO    MainThread:357 [wandb_init.py:init():898] starting backend
+2026-09-21 13:12:50,470 INFO    MainThread:357 [wandb_init.py:init():913] sending inform_init request
+2026-09-21 13:12:50,844 INFO    MainThread:357 [wandb_init.py:init():918] backend started and connected
+2026-09-21 13:12:50,849 INFO    MainThread:357 [wandb_init.py:init():988] updated telemetry
+2026-09-21 13:12:50,867 INFO    MainThread:357 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-21 13:12:50,873 INFO    MainThread:357 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-21 13:12:51,111 INFO    MainThread:357 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-21 13:12:51,111 INFO    MainThread:357 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-21 13:12:51,111 INFO    MainThread:357 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-21 13:12:51,111 INFO    MainThread:357 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-21 13:12:51,115 INFO    MainThread:357 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/offline-run-20260921_131249-meicy3md/run-meicy3md.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78693de0a766c541b0b8c83847e80ef9f3fb9a094f3b14113b5310df63b79b4c
+size 622592

epfl_final/SparseAtt/wandb/offline-run-20260921_131249-n6s56yfr/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_131249-n6s56yfr/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-09-21T13:12:50.484598128Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T13:12:50.484624147Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T13:12:50.831760545Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T13:12:50.831852012Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T13:12:50.831921986Z","level":"INFO","msg":"stream: created new stream","id":"n6s56yfr"}
+{"time":"2026-09-21T13:12:50.832074734Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T13:12:50.834780502Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T13:12:50.83493327Z","level":"INFO","msg":"writer: started","stream_id":"n6s56yfr"}
+{"time":"2026-09-21T13:12:50.834970566Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T13:12:50.870640873Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T13:12:50.870664909Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

epfl_final/SparseAtt/wandb/offline-run-20260921_131249-n6s56yfr/logs/debug.log

@@ -0,0 +1,19 @@
+2026-09-21 13:12:49,926 INFO    MainThread:354 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-21 13:12:49,927 INFO    MainThread:354 [wandb_setup.py:_flush():81] Configure stats pid to 354
+2026-09-21 13:12:49,927 INFO    MainThread:354 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-21 13:12:49,927 INFO    MainThread:354 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_131249-n6s56yfr/logs/debug.log
+2026-09-21 13:12:49,927 INFO    MainThread:354 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_131249-n6s56yfr/logs/debug-internal.log
+2026-09-21 13:12:49,927 INFO    MainThread:354 [wandb_init.py:init():850] calling init triggers
+2026-09-21 13:12:49,927 INFO    MainThread:354 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s43/run_20260921_131213', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 43, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s43', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed43'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-21 13:12:49,927 INFO    MainThread:354 [wandb_init.py:init():898] starting backend
+2026-09-21 13:12:50,469 INFO    MainThread:354 [wandb_init.py:init():913] sending inform_init request
+2026-09-21 13:12:50,836 INFO    MainThread:354 [wandb_init.py:init():918] backend started and connected
+2026-09-21 13:12:50,839 INFO    MainThread:354 [wandb_init.py:init():988] updated telemetry
+2026-09-21 13:12:50,867 INFO    MainThread:354 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-21 13:12:50,873 INFO    MainThread:354 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-21 13:12:51,107 INFO    MainThread:354 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-21 13:12:51,107 INFO    MainThread:354 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-21 13:12:51,107 INFO    MainThread:354 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-21 13:12:51,107 INFO    MainThread:354 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-21 13:12:51,115 INFO    MainThread:354 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/offline-run-20260921_131249-n6s56yfr/run-n6s56yfr.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c2e5252108c00a233fb04e5c68db422bc976a5a19e24447d297f1d4ab141507
+size 622592

epfl_final/SparseAtt/wandb/offline-run-20260921_131249-xc6c08ps/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_131249-xc6c08ps/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-09-21T13:12:50.484437969Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T13:12:50.484464449Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T13:12:50.838624218Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T13:12:50.838691207Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T13:12:50.838730746Z","level":"INFO","msg":"stream: created new stream","id":"xc6c08ps"}
+{"time":"2026-09-21T13:12:50.838795473Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T13:12:50.840941876Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T13:12:50.840979752Z","level":"INFO","msg":"writer: started","stream_id":"xc6c08ps"}
+{"time":"2026-09-21T13:12:50.841010919Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T13:12:50.870744397Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T13:12:50.870771448Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

epfl_final/SparseAtt/wandb/offline-run-20260921_131249-xc6c08ps/logs/debug.log

@@ -0,0 +1,19 @@
+2026-09-21 13:12:49,928 INFO    MainThread:358 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-21 13:12:49,928 INFO    MainThread:358 [wandb_setup.py:_flush():81] Configure stats pid to 358
+2026-09-21 13:12:49,928 INFO    MainThread:358 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-21 13:12:49,928 INFO    MainThread:358 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_131249-xc6c08ps/logs/debug.log
+2026-09-21 13:12:49,928 INFO    MainThread:358 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_131249-xc6c08ps/logs/debug-internal.log
+2026-09-21 13:12:49,928 INFO    MainThread:358 [wandb_init.py:init():850] calling init triggers
+2026-09-21 13:12:49,928 INFO    MainThread:358 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s47/run_20260921_131213', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 47, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s47', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed47'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-21 13:12:49,928 INFO    MainThread:358 [wandb_init.py:init():898] starting backend
+2026-09-21 13:12:50,469 INFO    MainThread:358 [wandb_init.py:init():913] sending inform_init request
+2026-09-21 13:12:50,841 INFO    MainThread:358 [wandb_init.py:init():918] backend started and connected
+2026-09-21 13:12:50,846 INFO    MainThread:358 [wandb_init.py:init():988] updated telemetry
+2026-09-21 13:12:50,867 INFO    MainThread:358 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-21 13:12:50,873 INFO    MainThread:358 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-21 13:12:51,127 INFO    MainThread:358 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-21 13:12:51,127 INFO    MainThread:358 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-21 13:12:51,127 INFO    MainThread:358 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-21 13:12:51,127 INFO    MainThread:358 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-21 13:12:51,129 INFO    MainThread:358 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/offline-run-20260921_131249-xc6c08ps/run-xc6c08ps.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9483abf7a5154cc7067a0597415e245fd9c190499f29c23aa5f133ed608d76c
+size 622592

epfl_final/SparseAtt/wandb/offline-run-20260921_143024-ghfbqjah/logs/debug.log

@@ -0,0 +1,19 @@
+2026-09-21 14:30:24,657 INFO    MainThread:70 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-21 14:30:24,657 INFO    MainThread:70 [wandb_setup.py:_flush():81] Configure stats pid to 70
+2026-09-21 14:30:24,658 INFO    MainThread:70 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-21 14:30:24,658 INFO    MainThread:70 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_143024-ghfbqjah/logs/debug.log
+2026-09-21 14:30:24,658 INFO    MainThread:70 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_143024-ghfbqjah/logs/debug-internal.log
+2026-09-21 14:30:24,658 INFO    MainThread:70 [wandb_init.py:init():850] calling init triggers
+2026-09-21 14:30:24,658 INFO    MainThread:70 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s45/run_20260921_142951', 'resume_from_checkpoint': None, 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 45, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s45', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed45'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-21 14:30:24,658 INFO    MainThread:70 [wandb_init.py:init():898] starting backend
+2026-09-21 14:30:25,191 INFO    MainThread:70 [wandb_init.py:init():913] sending inform_init request
+2026-09-21 14:30:25,568 INFO    MainThread:70 [wandb_init.py:init():918] backend started and connected
+2026-09-21 14:30:25,570 INFO    MainThread:70 [wandb_init.py:init():988] updated telemetry
+2026-09-21 14:30:25,584 INFO    MainThread:70 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-21 14:30:25,589 INFO    MainThread:70 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-21 14:30:25,756 INFO    MainThread:70 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-21 14:30:25,756 INFO    MainThread:70 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-21 14:30:25,757 INFO    MainThread:70 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-21 14:30:25,757 INFO    MainThread:70 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-21 14:30:25,763 INFO    MainThread:70 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/offline-run-20260921_143024-kbn66izc/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_143024-kbn66izc/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-09-21T14:30:25.206945921Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T14:30:25.206970507Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T14:30:25.559404152Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T14:30:25.559465233Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T14:30:25.559507276Z","level":"INFO","msg":"stream: created new stream","id":"kbn66izc"}
+{"time":"2026-09-21T14:30:25.559588297Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T14:30:25.561924124Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T14:30:25.561956583Z","level":"INFO","msg":"writer: started","stream_id":"kbn66izc"}
+{"time":"2026-09-21T14:30:25.562002852Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T14:30:25.588409287Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T14:30:25.588434474Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

epfl_final/SparseAtt/wandb/offline-run-20260921_143024-kbn66izc/logs/debug.log

@@ -0,0 +1,19 @@
+2026-09-21 14:30:24,644 INFO    MainThread:69 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-21 14:30:24,644 INFO    MainThread:69 [wandb_setup.py:_flush():81] Configure stats pid to 69
+2026-09-21 14:30:24,644 INFO    MainThread:69 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-21 14:30:24,644 INFO    MainThread:69 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_143024-kbn66izc/logs/debug.log
+2026-09-21 14:30:24,644 INFO    MainThread:69 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_143024-kbn66izc/logs/debug-internal.log
+2026-09-21 14:30:24,644 INFO    MainThread:69 [wandb_init.py:init():850] calling init triggers
+2026-09-21 14:30:24,644 INFO    MainThread:69 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s44/run_20260921_142951', 'resume_from_checkpoint': None, 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 44, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s44', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed44'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-21 14:30:24,644 INFO    MainThread:69 [wandb_init.py:init():898] starting backend
+2026-09-21 14:30:25,191 INFO    MainThread:69 [wandb_init.py:init():913] sending inform_init request
+2026-09-21 14:30:25,562 INFO    MainThread:69 [wandb_init.py:init():918] backend started and connected
+2026-09-21 14:30:25,565 INFO    MainThread:69 [wandb_init.py:init():988] updated telemetry
+2026-09-21 14:30:25,585 INFO    MainThread:69 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-21 14:30:25,591 INFO    MainThread:69 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-21 14:30:25,790 INFO    MainThread:69 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-21 14:30:25,790 INFO    MainThread:69 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-21 14:30:25,790 INFO    MainThread:69 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-21 14:30:25,791 INFO    MainThread:69 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-21 14:30:25,792 INFO    MainThread:69 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/offline-run-20260921_143024-kbn66izc/run-kbn66izc.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f8d7767727a9fb0c9c652f571b990bbae7a5074ddfc36a5ebeed9b64e493455
+size 1376256

epfl_final/SparseAtt/wandb/offline-run-20260921_143024-zgrsvkbi/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_143024-zgrsvkbi/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-09-21T14:30:25.206626644Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T14:30:25.20665084Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T14:30:25.554203978Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T14:30:25.554264238Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T14:30:25.554298309Z","level":"INFO","msg":"stream: created new stream","id":"zgrsvkbi"}
+{"time":"2026-09-21T14:30:25.554400832Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T14:30:25.556590301Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T14:30:25.556631613Z","level":"INFO","msg":"writer: started","stream_id":"zgrsvkbi"}
+{"time":"2026-09-21T14:30:25.556676189Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T14:30:25.588291171Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T14:30:25.588316949Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

epfl_final/SparseAtt/wandb/offline-run-20260921_143024-zgrsvkbi/logs/debug.log

@@ -0,0 +1,19 @@
+2026-09-21 14:30:24,658 INFO    MainThread:68 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-21 14:30:24,659 INFO    MainThread:68 [wandb_setup.py:_flush():81] Configure stats pid to 68
+2026-09-21 14:30:24,659 INFO    MainThread:68 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-21 14:30:24,659 INFO    MainThread:68 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_143024-zgrsvkbi/logs/debug.log
+2026-09-21 14:30:24,659 INFO    MainThread:68 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_143024-zgrsvkbi/logs/debug-internal.log
+2026-09-21 14:30:24,659 INFO    MainThread:68 [wandb_init.py:init():850] calling init triggers
+2026-09-21 14:30:24,659 INFO    MainThread:68 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s43/run_20260921_142951', 'resume_from_checkpoint': None, 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 43, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s43', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed43'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-21 14:30:24,659 INFO    MainThread:68 [wandb_init.py:init():898] starting backend
+2026-09-21 14:30:25,191 INFO    MainThread:68 [wandb_init.py:init():913] sending inform_init request
+2026-09-21 14:30:25,557 INFO    MainThread:68 [wandb_init.py:init():918] backend started and connected
+2026-09-21 14:30:25,558 INFO    MainThread:68 [wandb_init.py:init():988] updated telemetry
+2026-09-21 14:30:25,585 INFO    MainThread:68 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-21 14:30:25,590 INFO    MainThread:68 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-21 14:30:25,804 INFO    MainThread:68 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-21 14:30:25,805 INFO    MainThread:68 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-21 14:30:25,805 INFO    MainThread:68 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-21 14:30:25,805 INFO    MainThread:68 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-21 14:30:25,806 INFO    MainThread:68 [wandb_init.py:init():1094] run started, returning control to user process