EPFL final checkpoints and W&B history 1301-1350 of 1463

.gitattributes

@@ -168,3 +168,11 @@ epfl_final/SparseAtt/wandb/run-20260828_133629-8ynn1we6/run-8ynn1we6.wandb filte
 epfl_final/SparseAtt/wandb/run-20260828_214524-z1i60278/run-z1i60278.wandb filter=lfs diff=lfs merge=lfs -text
 epfl_final/SparseAtt/wandb/run-20260828_214632-8ynn1we6/run-8ynn1we6.wandb filter=lfs diff=lfs merge=lfs -text
 epfl_final/SparseAtt/wandb/run-20260828_215102-8ynn1we6/run-8ynn1we6.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260829_040843-z1i60278/run-z1i60278.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260829_174918-8ynn1we6/run-8ynn1we6.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260830_002911-8ynn1we6/run-8ynn1we6.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260903_094250-7tfq6nwm/run-7tfq6nwm.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260903_142704-7tfq6nwm/run-7tfq6nwm.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260903_191410-7tfq6nwm/run-7tfq6nwm.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260904_042057-7tfq6nwm/run-7tfq6nwm.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/offline-run-20260921_030515-1savcbqu/run-1savcbqu.wandb filter=lfs diff=lfs merge=lfs -text

epfl_final/SparseAtt/wandb/offline-run-20260921_030515-1savcbqu/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_030515-1savcbqu/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-09-21T03:05:15.624884555Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T03:05:15.624895883Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T03:05:15.974865003Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T03:05:15.974924302Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T03:05:15.974971473Z","level":"INFO","msg":"stream: created new stream","id":"1savcbqu"}
+{"time":"2026-09-21T03:05:15.97504917Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T03:05:15.977570007Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T03:05:15.977610859Z","level":"INFO","msg":"writer: started","stream_id":"1savcbqu"}
+{"time":"2026-09-21T03:05:15.97763849Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T03:05:16.019056183Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T03:05:16.019081682Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

epfl_final/SparseAtt/wandb/offline-run-20260921_030515-1savcbqu/logs/debug.log

@@ -0,0 +1,19 @@
+2026-09-21 03:05:15,135 INFO    MainThread:4778 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-21 03:05:15,136 INFO    MainThread:4778 [wandb_setup.py:_flush():81] Configure stats pid to 4778
+2026-09-21 03:05:15,136 INFO    MainThread:4778 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-21 03:05:15,136 INFO    MainThread:4778 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_030515-1savcbqu/logs/debug.log
+2026-09-21 03:05:15,136 INFO    MainThread:4778 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_030515-1savcbqu/logs/debug-internal.log
+2026-09-21 03:05:15,136 INFO    MainThread:4778 [wandb_init.py:init():850] calling init triggers
+2026-09-21 03:05:15,136 INFO    MainThread:4778 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s46/run_20260921_030452', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 22000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 46, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s46', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed46'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-21 03:05:15,136 INFO    MainThread:4778 [wandb_init.py:init():898] starting backend
+2026-09-21 03:05:15,608 INFO    MainThread:4778 [wandb_init.py:init():913] sending inform_init request
+2026-09-21 03:05:15,978 INFO    MainThread:4778 [wandb_init.py:init():918] backend started and connected
+2026-09-21 03:05:15,980 INFO    MainThread:4778 [wandb_init.py:init():988] updated telemetry
+2026-09-21 03:05:16,015 INFO    MainThread:4778 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-21 03:05:16,021 INFO    MainThread:4778 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-21 03:05:16,229 INFO    MainThread:4778 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-21 03:05:16,229 INFO    MainThread:4778 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-21 03:05:16,229 INFO    MainThread:4778 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-21 03:05:16,229 INFO    MainThread:4778 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-21 03:05:16,235 INFO    MainThread:4778 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/offline-run-20260921_030515-1savcbqu/run-1savcbqu.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69e921d2ffaef99e5c7ae6f07570c98135bbe8ae2cd39ed0766b16547745421d
+size 9502720

epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ao5edbbv/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ao5edbbv/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-09-21T03:05:15.626167763Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-21T03:05:15.626193712Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-21T03:05:15.980676033Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T03:05:15.9807137Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T03:05:15.980739319Z","level":"INFO","msg":"stream: created new stream","id":"ao5edbbv"}
+{"time":"2026-09-21T03:05:15.980805388Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-21T03:05:15.98562607Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-21T03:05:15.98573274Z","level":"INFO","msg":"writer: started","stream_id":"ao5edbbv"}
+{"time":"2026-09-21T03:05:15.985776957Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-21T03:05:16.013000075Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-09-21T03:05:16.013026845Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ao5edbbv/logs/debug.log

@@ -0,0 +1,19 @@
+2026-09-21 03:05:15,138 INFO    MainThread:4774 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-21 03:05:15,138 INFO    MainThread:4774 [wandb_setup.py:_flush():81] Configure stats pid to 4774
+2026-09-21 03:05:15,138 INFO    MainThread:4774 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-21 03:05:15,138 INFO    MainThread:4774 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_030515-ao5edbbv/logs/debug.log
+2026-09-21 03:05:15,138 INFO    MainThread:4774 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_030515-ao5edbbv/logs/debug-internal.log
+2026-09-21 03:05:15,138 INFO    MainThread:4774 [wandb_init.py:init():850] calling init triggers
+2026-09-21 03:05:15,138 INFO    MainThread:4774 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s42/run_20260921_030452', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 22000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s42', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed42'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-21 03:05:15,138 INFO    MainThread:4774 [wandb_init.py:init():898] starting backend
+2026-09-21 03:05:15,608 INFO    MainThread:4774 [wandb_init.py:init():913] sending inform_init request
+2026-09-21 03:05:15,986 INFO    MainThread:4774 [wandb_init.py:init():918] backend started and connected
+2026-09-21 03:05:15,988 INFO    MainThread:4774 [wandb_init.py:init():988] updated telemetry
+2026-09-21 03:05:16,008 INFO    MainThread:4774 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-21 03:05:16,014 INFO    MainThread:4774 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-21 03:05:16,237 INFO    MainThread:4774 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-21 03:05:16,237 INFO    MainThread:4774 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-21 03:05:16,237 INFO    MainThread:4774 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-21 03:05:16,237 INFO    MainThread:4774 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-21 03:05:16,239 INFO    MainThread:4774 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/run-20260829_040843-z1i60278/run-z1i60278.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45ea7619b9bd6e7ffe6e8c6519db1b30ec33288e8bd80b556267ff3509b74fc4
+size 8355840

epfl_final/SparseAtt/wandb/run-20260829_135046-z1i60278/files/output.log

@@ -0,0 +1,2 @@
+Starting training...
+Validation at step 31000: Loss = 2.5770, Perplexity = 13.16

epfl_final/SparseAtt/wandb/run-20260829_135046-z1i60278/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260829_135046-z1i60278/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-08-29T13:50:46.363333Z",
+  "args":  [
+    "--config",
+    "configs/p2_1p2b_mixed_seed72_4gpu_autoresume_current.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/SparseAtt.git",
+    "commit":  "8f862f749eb4d5c17f688eb1562235e3fe5b2550"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-workspace-4-gpu-3-0-16",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "6081338675200"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-e3358200-f776-7214-8269-c2c219f03c2e"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-788fbe73-59b3-e845-80c4-5eff0b2f64b5"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-123c77ea-9377-5bde-70b6-99ef1c791ff4"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-b52a5d1a-9d9a-90fe-c3b3-708dcea4146f"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "df01z8rghww9jcht4muhh2otitsjz5if"
+}

epfl_final/SparseAtt/wandb/run-20260829_135046-z1i60278/logs/debug-internal.log

@@ -0,0 +1,63 @@
+{"time":"2026-08-29T13:50:47.981034307Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-08-29T13:50:47.986645154Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-08-29T13:50:48.521480585Z","level":"INFO","msg":"stream: created new stream","id":"z1i60278"}
+{"time":"2026-08-29T13:50:48.521600804Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-08-29T13:50:48.524277896Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-08-29T13:50:48.525126612Z","level":"INFO","msg":"writer: started","stream_id":"z1i60278"}
+{"time":"2026-08-29T13:50:48.52515896Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-08-29T13:50:49.099393271Z","level":"INFO","msg":"filestream: sending request","total_files":1}
+{"time":"2026-08-29T13:50:49.351363718Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:51:04.099843648Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1,"uploaded_len":2}
+{"time":"2026-08-29T13:51:04.342041061Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:51:19.099941939Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":0,"events_lines":2}
+{"time":"2026-08-29T13:51:19.315900857Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:51:34.099862765Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":2,"events_lines":2,"console_offset":1,"console_lines":1}
+{"time":"2026-08-29T13:51:34.292533968Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:51:49.100261314Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":4,"events_lines":2}
+{"time":"2026-08-29T13:51:49.348056495Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:52:04.099674674Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":6,"events_lines":2}
+{"time":"2026-08-29T13:52:04.340704336Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:52:19.09967664Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":8,"events_lines":2}
+{"time":"2026-08-29T13:52:19.29667841Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:52:34.099820269Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":10,"events_lines":2}
+{"time":"2026-08-29T13:52:34.331685916Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:52:49.099890629Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":12,"events_lines":2}
+{"time":"2026-08-29T13:52:49.320969642Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:53:04.099857354Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":14,"events_lines":2}
+{"time":"2026-08-29T13:53:04.313074423Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:53:19.099632491Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":16,"events_lines":2}
+{"time":"2026-08-29T13:53:19.317279421Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:53:34.099496406Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":18,"events_lines":2}
+{"time":"2026-08-29T13:53:34.315149954Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:53:49.100184961Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":20,"events_lines":2}
+{"time":"2026-08-29T13:53:49.328145118Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:54:04.099435602Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":22,"events_lines":2}
+{"time":"2026-08-29T13:54:04.317297864Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:54:19.099544272Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":24,"events_lines":2}
+{"time":"2026-08-29T13:54:19.310332687Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:54:34.099345714Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":26,"events_lines":2}
+{"time":"2026-08-29T13:54:34.302087511Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:54:49.099815392Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":28,"events_lines":2}
+{"time":"2026-08-29T13:54:49.295982187Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:55:04.099422394Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":30,"events_lines":2}
+{"time":"2026-08-29T13:55:04.3283946Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:55:19.099458876Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":32,"events_lines":2}
+{"time":"2026-08-29T13:55:19.298213189Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:55:34.099673494Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":34,"events_lines":2}
+{"time":"2026-08-29T13:55:34.321042394Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:55:49.099609217Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":36,"events_lines":2}
+{"time":"2026-08-29T13:55:49.419016637Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:56:04.099466662Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":38,"events_lines":2}
+{"time":"2026-08-29T13:56:04.31720589Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:56:19.100065854Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":40,"events_lines":2}
+{"time":"2026-08-29T13:56:19.351131139Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:56:34.100231238Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":42,"events_lines":2}
+{"time":"2026-08-29T13:56:34.300728479Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:56:49.100224706Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":44,"events_lines":2}
+{"time":"2026-08-29T13:56:49.329094149Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:57:04.099381287Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":46,"events_lines":2}
+{"time":"2026-08-29T13:57:04.302973963Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:57:19.099959677Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":48,"events_lines":2}
+{"time":"2026-08-29T13:57:19.314266287Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-29T13:57:34.100121715Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":50,"events_lines":2}
+{"time":"2026-08-29T13:57:34.308947667Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

epfl_final/SparseAtt/wandb/run-20260829_135046-z1i60278/logs/debug.log

@@ -0,0 +1,20 @@
+2026-08-29 13:50:46,385 INFO    MainThread:218 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-08-29 13:50:46,385 INFO    MainThread:218 [wandb_setup.py:_flush():81] Configure stats pid to 218
+2026-08-29 13:50:46,385 INFO    MainThread:218 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-29 13:50:46,385 INFO    MainThread:218 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260829_135046-z1i60278/logs/debug.log
+2026-08-29 13:50:46,385 INFO    MainThread:218 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260829_135046-z1i60278/logs/debug-internal.log
+2026-08-29 13:50:46,385 INFO    MainThread:218 [wandb_init.py:init():850] calling init triggers
+2026-08-29 13:50:46,385 INFO    MainThread:218 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-31000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 15, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 72, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b mixed 8k seed72 6gpu', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'mixed', 'seed72'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed', 'sparse_mask_seed': 72, 'sparse_mixed_nrb_set': [63, 47, 31, 21, 11], 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-08-29 13:50:46,385 INFO    MainThread:218 [wandb_init.py:init():898] starting backend
+2026-08-29 13:50:47,957 INFO    MainThread:218 [wandb_init.py:init():913] sending inform_init request
+2026-08-29 13:50:48,525 INFO    MainThread:218 [wandb_init.py:init():918] backend started and connected
+2026-08-29 13:50:48,528 INFO    MainThread:218 [wandb_init.py:init():988] updated telemetry
+2026-08-29 13:50:48,564 INFO    MainThread:218 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-08-29 13:50:49,095 INFO    MainThread:218 [wandb_init.py:init():1051] run resumed
+2026-08-29 13:50:49,097 INFO    MainThread:218 [wandb_init.py:init():1056] starting run threads in backend
+2026-08-29 13:50:49,347 INFO    MainThread:218 [wandb_run.py:_console_start():2554] atexit reg
+2026-08-29 13:50:49,347 INFO    MainThread:218 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-08-29 13:50:49,347 INFO    MainThread:218 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-08-29 13:50:49,347 INFO    MainThread:218 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-08-29 13:50:49,356 INFO    MainThread:218 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/run-20260829_174918-8ynn1we6/files/output.log

@@ -0,0 +1,314 @@
+Starting training...
+Validation at step 34250: Loss = 2.5253, Perplexity = 12.50
+wandb: WARNING Tried to log to step 34250 that is less than the current step 34331. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 34251: sparse_stochastic (configured sparse attention)
+Step 34260/50462 | Loss: 2.5295 | LR: 1.20e-04 | 79690.4ms/step | 0.8 samples/s
+wandb: WARNING Tried to log to step 34260 that is less than the current step 34331. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 34270/50462 | Loss: 2.5307 | LR: 1.20e-04 | 4005.0ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 34270 that is less than the current step 34331. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 34280/50462 | Loss: 2.5218 | LR: 1.20e-04 | 4008.1ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 34280 that is less than the current step 34331. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 34290/50462 | Loss: 2.5158 | LR: 1.20e-04 | 4011.6ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 34290 that is less than the current step 34331. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 34300/50462 | Loss: 2.5569 | LR: 1.19e-04 | 4015.8ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 34300 that is less than the current step 34331. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 34310/50462 | Loss: 2.5269 | LR: 1.19e-04 | 4009.6ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 34310 that is less than the current step 34331. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 34320/50462 | Loss: 2.5510 | LR: 1.19e-04 | 4011.0ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 34320 that is less than the current step 34331. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 34330/50462 | Loss: 2.5320 | LR: 1.19e-04 | 4008.1ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 34330 that is less than the current step 34331. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 34340/50462 | Loss: 2.5267 | LR: 1.19e-04 | 4009.5ms/step | 15.0 samples/s
+Step 34350/50462 | Loss: 2.5445 | LR: 1.19e-04 | 4018.6ms/step | 14.9 samples/s
+Step 34360/50462 | Loss: 2.5204 | LR: 1.19e-04 | 4005.3ms/step | 15.0 samples/s
+Step 34370/50462 | Loss: 2.5200 | LR: 1.19e-04 | 4036.4ms/step | 14.9 samples/s
+Step 34380/50462 | Loss: 2.5552 | LR: 1.18e-04 | 4008.2ms/step | 15.0 samples/s
+Step 34390/50462 | Loss: 2.5581 | LR: 1.18e-04 | 4026.4ms/step | 14.9 samples/s
+Step 34400/50462 | Loss: 2.5278 | LR: 1.18e-04 | 4016.6ms/step | 14.9 samples/s
+Step 34410/50462 | Loss: 2.5462 | LR: 1.18e-04 | 4013.9ms/step | 14.9 samples/s
+Step 34420/50462 | Loss: 2.5239 | LR: 1.18e-04 | 4023.7ms/step | 14.9 samples/s
+Step 34430/50462 | Loss: 2.5254 | LR: 1.18e-04 | 4012.9ms/step | 15.0 samples/s
+Step 34440/50462 | Loss: 2.5449 | LR: 1.18e-04 | 4011.3ms/step | 15.0 samples/s
+Step 34450/50462 | Loss: 2.5153 | LR: 1.17e-04 | 4025.0ms/step | 14.9 samples/s
+Step 34460/50462 | Loss: 2.5579 | LR: 1.17e-04 | 4021.4ms/step | 14.9 samples/s
+Step 34470/50462 | Loss: 2.5449 | LR: 1.17e-04 | 4022.0ms/step | 14.9 samples/s
+Step 34480/50462 | Loss: 2.5341 | LR: 1.17e-04 | 4025.6ms/step | 14.9 samples/s
+Step 34490/50462 | Loss: 2.5312 | LR: 1.17e-04 | 4019.8ms/step | 14.9 samples/s
+Step 34500/50462 | Loss: 2.5539 | LR: 1.17e-04 | 4017.8ms/step | 14.9 samples/s
+Validation at step 34500: Loss = 2.5244, Perplexity = 12.48
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-34500
+Step 34510/50462 | Loss: 2.5325 | LR: 1.17e-04 | 5973.8ms/step | 10.0 samples/s
+Step 34520/50462 | Loss: 2.5269 | LR: 1.16e-04 | 4023.4ms/step | 14.9 samples/s
+Step 34530/50462 | Loss: 2.5297 | LR: 1.16e-04 | 4017.7ms/step | 14.9 samples/s
+Step 34540/50462 | Loss: 2.5602 | LR: 1.16e-04 | 4029.1ms/step | 14.9 samples/s
+Step 34550/50462 | Loss: 2.5648 | LR: 1.16e-04 | 4024.2ms/step | 14.9 samples/s
+Step 34560/50462 | Loss: 2.5035 | LR: 1.16e-04 | 4014.5ms/step | 14.9 samples/s
+Step 34570/50462 | Loss: 2.5666 | LR: 1.16e-04 | 4020.8ms/step | 14.9 samples/s
+Step 34580/50462 | Loss: 2.5642 | LR: 1.16e-04 | 4015.5ms/step | 14.9 samples/s
+Step 34590/50462 | Loss: 2.5220 | LR: 1.16e-04 | 4019.1ms/step | 14.9 samples/s
+Step 34600/50462 | Loss: 2.5484 | LR: 1.15e-04 | 4019.9ms/step | 14.9 samples/s
+Step 34610/50462 | Loss: 2.5392 | LR: 1.15e-04 | 4014.7ms/step | 14.9 samples/s
+Step 34620/50462 | Loss: 2.5612 | LR: 1.15e-04 | 4018.6ms/step | 14.9 samples/s
+Step 34630/50462 | Loss: 2.5521 | LR: 1.15e-04 | 4016.2ms/step | 14.9 samples/s
+Step 34640/50462 | Loss: 2.5098 | LR: 1.15e-04 | 4003.9ms/step | 15.0 samples/s
+Step 34650/50462 | Loss: 2.5224 | LR: 1.15e-04 | 4010.4ms/step | 15.0 samples/s
+Step 34660/50462 | Loss: 2.5692 | LR: 1.15e-04 | 4015.3ms/step | 14.9 samples/s
+Step 34670/50462 | Loss: 2.5276 | LR: 1.15e-04 | 4020.9ms/step | 14.9 samples/s
+Step 34680/50462 | Loss: 2.5193 | LR: 1.14e-04 | 4010.0ms/step | 15.0 samples/s
+Step 34690/50462 | Loss: 2.5741 | LR: 1.14e-04 | 4007.4ms/step | 15.0 samples/s
+Step 34700/50462 | Loss: 2.5128 | LR: 1.14e-04 | 4032.4ms/step | 14.9 samples/s
+Step 34710/50462 | Loss: 2.5240 | LR: 1.14e-04 | 4024.5ms/step | 14.9 samples/s
+Step 34720/50462 | Loss: 2.5527 | LR: 1.14e-04 | 4009.3ms/step | 15.0 samples/s
+Step 34730/50462 | Loss: 2.5113 | LR: 1.14e-04 | 4017.6ms/step | 14.9 samples/s
+Step 34740/50462 | Loss: 2.5015 | LR: 1.14e-04 | 4002.1ms/step | 15.0 samples/s
+Step 34750/50462 | Loss: 2.5349 | LR: 1.13e-04 | 4008.5ms/step | 15.0 samples/s
+Validation at step 34750: Loss = 2.5230, Perplexity = 12.47
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-34750
+Step 34760/50462 | Loss: 2.5588 | LR: 1.13e-04 | 5979.6ms/step | 10.0 samples/s
+Step 34770/50462 | Loss: 2.5299 | LR: 1.13e-04 | 4005.8ms/step | 15.0 samples/s
+Step 34780/50462 | Loss: 2.5303 | LR: 1.13e-04 | 4011.3ms/step | 15.0 samples/s
+Step 34790/50462 | Loss: 2.5336 | LR: 1.13e-04 | 4007.3ms/step | 15.0 samples/s
+Step 34800/50462 | Loss: 2.5455 | LR: 1.13e-04 | 4017.7ms/step | 14.9 samples/s
+Step 34810/50462 | Loss: 2.5243 | LR: 1.13e-04 | 4009.1ms/step | 15.0 samples/s
+Step 34820/50462 | Loss: 2.5563 | LR: 1.13e-04 | 4033.8ms/step | 14.9 samples/s
+Step 34830/50462 | Loss: 2.5607 | LR: 1.12e-04 | 4016.8ms/step | 14.9 samples/s
+Step 34840/50462 | Loss: 2.5298 | LR: 1.12e-04 | 4020.1ms/step | 14.9 samples/s
+Step 34850/50462 | Loss: 2.5112 | LR: 1.12e-04 | 4023.5ms/step | 14.9 samples/s
+Step 34860/50462 | Loss: 2.5664 | LR: 1.12e-04 | 4009.2ms/step | 15.0 samples/s
+Step 34870/50462 | Loss: 2.5437 | LR: 1.12e-04 | 4017.0ms/step | 14.9 samples/s
+Step 34880/50462 | Loss: 2.5579 | LR: 1.12e-04 | 4009.8ms/step | 15.0 samples/s
+Step 34890/50462 | Loss: 2.5077 | LR: 1.12e-04 | 4021.0ms/step | 14.9 samples/s
+Step 34900/50462 | Loss: 2.5464 | LR: 1.11e-04 | 4020.6ms/step | 14.9 samples/s
+Step 34910/50462 | Loss: 2.5150 | LR: 1.11e-04 | 4034.4ms/step | 14.9 samples/s
+Step 34920/50462 | Loss: 2.5349 | LR: 1.11e-04 | 4028.4ms/step | 14.9 samples/s
+Step 34930/50462 | Loss: 2.5198 | LR: 1.11e-04 | 4024.4ms/step | 14.9 samples/s
+Step 34940/50462 | Loss: 2.5309 | LR: 1.11e-04 | 4021.5ms/step | 14.9 samples/s
+Step 34950/50462 | Loss: 2.4777 | LR: 1.11e-04 | 4019.0ms/step | 14.9 samples/s
+Step 34960/50462 | Loss: 2.5168 | LR: 1.11e-04 | 4018.5ms/step | 14.9 samples/s
+Step 34970/50462 | Loss: 2.5148 | LR: 1.11e-04 | 4004.0ms/step | 15.0 samples/s
+Step 34980/50462 | Loss: 2.5357 | LR: 1.10e-04 | 4014.6ms/step | 14.9 samples/s
+Step 34990/50462 | Loss: 2.5604 | LR: 1.10e-04 | 4025.5ms/step | 14.9 samples/s
+Step 35000/50462 | Loss: 2.5520 | LR: 1.10e-04 | 4026.4ms/step | 14.9 samples/s
+Validation at step 35000: Loss = 2.5225, Perplexity = 12.46
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-35000
+Step 35010/50462 | Loss: 2.5116 | LR: 1.10e-04 | 5924.9ms/step | 10.1 samples/s
+Step 35020/50462 | Loss: 2.5388 | LR: 1.10e-04 | 4016.1ms/step | 14.9 samples/s
+Step 35030/50462 | Loss: 2.5464 | LR: 1.10e-04 | 3998.3ms/step | 15.0 samples/s
+Step 35040/50462 | Loss: 2.5211 | LR: 1.10e-04 | 4003.2ms/step | 15.0 samples/s
+Step 35050/50462 | Loss: 2.5373 | LR: 1.09e-04 | 4010.7ms/step | 15.0 samples/s
+Step 35060/50462 | Loss: 2.5419 | LR: 1.09e-04 | 4022.3ms/step | 14.9 samples/s
+Step 35070/50462 | Loss: 2.5417 | LR: 1.09e-04 | 4004.0ms/step | 15.0 samples/s
+Step 35080/50462 | Loss: 2.5076 | LR: 1.09e-04 | 4020.1ms/step | 14.9 samples/s
+Step 35090/50462 | Loss: 2.5604 | LR: 1.09e-04 | 4004.6ms/step | 15.0 samples/s
+Step 35100/50462 | Loss: 2.5408 | LR: 1.09e-04 | 4009.0ms/step | 15.0 samples/s
+Step 35110/50462 | Loss: 2.5242 | LR: 1.09e-04 | 4012.1ms/step | 15.0 samples/s
+Step 35120/50462 | Loss: 2.5540 | LR: 1.09e-04 | 4012.6ms/step | 15.0 samples/s
+Step 35130/50462 | Loss: 2.5148 | LR: 1.08e-04 | 4006.1ms/step | 15.0 samples/s
+Step 35140/50462 | Loss: 2.5324 | LR: 1.08e-04 | 4014.9ms/step | 14.9 samples/s
+Step 35150/50462 | Loss: 2.5577 | LR: 1.08e-04 | 4012.7ms/step | 15.0 samples/s
+Step 35160/50462 | Loss: 2.5730 | LR: 1.08e-04 | 4008.1ms/step | 15.0 samples/s
+Step 35170/50462 | Loss: 2.5207 | LR: 1.08e-04 | 4011.7ms/step | 15.0 samples/s
+Step 35180/50462 | Loss: 2.5442 | LR: 1.08e-04 | 4012.0ms/step | 15.0 samples/s
+Step 35190/50462 | Loss: 2.4948 | LR: 1.08e-04 | 3999.1ms/step | 15.0 samples/s
+Step 35200/50462 | Loss: 2.5474 | LR: 1.08e-04 | 4018.2ms/step | 14.9 samples/s
+Step 35210/50462 | Loss: 2.5174 | LR: 1.07e-04 | 4021.2ms/step | 14.9 samples/s
+Step 35220/50462 | Loss: 2.5127 | LR: 1.07e-04 | 4011.0ms/step | 15.0 samples/s
+Step 35230/50462 | Loss: 2.5335 | LR: 1.07e-04 | 4018.6ms/step | 14.9 samples/s
+Step 35240/50462 | Loss: 2.5686 | LR: 1.07e-04 | 4014.5ms/step | 14.9 samples/s
+Step 35250/50462 | Loss: 2.5247 | LR: 1.07e-04 | 4008.8ms/step | 15.0 samples/s
+Validation at step 35250: Loss = 2.5219, Perplexity = 12.45
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-35250
+Step 35260/50462 | Loss: 2.5233 | LR: 1.07e-04 | 6050.4ms/step | 9.9 samples/s
+Step 35270/50462 | Loss: 2.5183 | LR: 1.07e-04 | 4014.7ms/step | 14.9 samples/s
+Step 35280/50462 | Loss: 2.4972 | LR: 1.07e-04 | 4626.5ms/step | 13.0 samples/s
+Step 35290/50462 | Loss: 2.5205 | LR: 1.06e-04 | 4008.7ms/step | 15.0 samples/s
+Step 35300/50462 | Loss: 2.5727 | LR: 1.06e-04 | 4009.5ms/step | 15.0 samples/s
+Step 35310/50462 | Loss: 2.5299 | LR: 1.06e-04 | 4016.0ms/step | 14.9 samples/s
+Step 35320/50462 | Loss: 2.5099 | LR: 1.06e-04 | 4020.5ms/step | 14.9 samples/s
+Step 35330/50462 | Loss: 2.5235 | LR: 1.06e-04 | 3998.7ms/step | 15.0 samples/s
+Step 35340/50462 | Loss: 2.5591 | LR: 1.06e-04 | 4009.1ms/step | 15.0 samples/s
+Step 35350/50462 | Loss: 2.5479 | LR: 1.06e-04 | 4006.2ms/step | 15.0 samples/s
+Step 35360/50462 | Loss: 2.5261 | LR: 1.05e-04 | 4010.2ms/step | 15.0 samples/s
+Step 35370/50462 | Loss: 2.5126 | LR: 1.05e-04 | 4000.7ms/step | 15.0 samples/s
+Step 35380/50462 | Loss: 2.5480 | LR: 1.05e-04 | 4008.4ms/step | 15.0 samples/s
+Step 35390/50462 | Loss: 2.5339 | LR: 1.05e-04 | 4009.0ms/step | 15.0 samples/s
+Step 35400/50462 | Loss: 2.5355 | LR: 1.05e-04 | 4016.5ms/step | 14.9 samples/s
+Step 35410/50462 | Loss: 2.5563 | LR: 1.05e-04 | 4030.3ms/step | 14.9 samples/s
+Step 35420/50462 | Loss: 2.5303 | LR: 1.05e-04 | 4011.8ms/step | 15.0 samples/s
+Step 35430/50462 | Loss: 2.5179 | LR: 1.05e-04 | 4017.6ms/step | 14.9 samples/s
+Step 35440/50462 | Loss: 2.5297 | LR: 1.04e-04 | 4020.5ms/step | 14.9 samples/s
+Step 35450/50462 | Loss: 2.5393 | LR: 1.04e-04 | 4010.1ms/step | 15.0 samples/s
+Step 35460/50462 | Loss: 2.5552 | LR: 1.04e-04 | 4014.1ms/step | 14.9 samples/s
+Step 35470/50462 | Loss: 2.5197 | LR: 1.04e-04 | 4014.7ms/step | 14.9 samples/s
+Step 35480/50462 | Loss: 2.5634 | LR: 1.04e-04 | 4012.9ms/step | 15.0 samples/s
+Step 35490/50462 | Loss: 2.5298 | LR: 1.04e-04 | 4013.7ms/step | 14.9 samples/s
+Step 35500/50462 | Loss: 2.4988 | LR: 1.04e-04 | 4026.9ms/step | 14.9 samples/s
+Validation at step 35500: Loss = 2.5208, Perplexity = 12.44
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-35500
+Step 35510/50462 | Loss: 2.5346 | LR: 1.04e-04 | 5962.3ms/step | 10.1 samples/s
+Step 35520/50462 | Loss: 2.5298 | LR: 1.03e-04 | 4011.1ms/step | 15.0 samples/s
+Step 35530/50462 | Loss: 2.4897 | LR: 1.03e-04 | 4026.2ms/step | 14.9 samples/s
+Step 35540/50462 | Loss: 2.5122 | LR: 1.03e-04 | 4018.1ms/step | 14.9 samples/s
+Step 35550/50462 | Loss: 2.5125 | LR: 1.03e-04 | 4016.7ms/step | 14.9 samples/s
+Step 35560/50462 | Loss: 2.5303 | LR: 1.03e-04 | 4020.5ms/step | 14.9 samples/s
+Step 35570/50462 | Loss: 2.5376 | LR: 1.03e-04 | 4012.6ms/step | 15.0 samples/s
+Step 35580/50462 | Loss: 2.5466 | LR: 1.03e-04 | 4013.3ms/step | 15.0 samples/s
+Step 35590/50462 | Loss: 2.5081 | LR: 1.03e-04 | 4009.0ms/step | 15.0 samples/s
+Step 35600/50462 | Loss: 2.5461 | LR: 1.02e-04 | 4015.9ms/step | 14.9 samples/s
+Step 35610/50462 | Loss: 2.5229 | LR: 1.02e-04 | 4013.6ms/step | 14.9 samples/s
+Step 35620/50462 | Loss: 2.5312 | LR: 1.02e-04 | 4001.3ms/step | 15.0 samples/s
+Step 35630/50462 | Loss: 2.5188 | LR: 1.02e-04 | 4015.8ms/step | 14.9 samples/s
+Step 35640/50462 | Loss: 2.5152 | LR: 1.02e-04 | 4020.4ms/step | 14.9 samples/s
+Step 35650/50462 | Loss: 2.5260 | LR: 1.02e-04 | 4012.2ms/step | 15.0 samples/s
+Step 35660/50462 | Loss: 2.5310 | LR: 1.02e-04 | 4009.3ms/step | 15.0 samples/s
+Step 35670/50462 | Loss: 2.5373 | LR: 1.02e-04 | 4014.3ms/step | 14.9 samples/s
+Step 35680/50462 | Loss: 2.5261 | LR: 1.01e-04 | 4029.4ms/step | 14.9 samples/s
+Step 35690/50462 | Loss: 2.5123 | LR: 1.01e-04 | 4026.2ms/step | 14.9 samples/s
+Step 35700/50462 | Loss: 2.5476 | LR: 1.01e-04 | 4028.5ms/step | 14.9 samples/s
+Step 35710/50462 | Loss: 2.4814 | LR: 1.01e-04 | 4019.7ms/step | 14.9 samples/s
+Step 35720/50462 | Loss: 2.5257 | LR: 1.01e-04 | 4000.6ms/step | 15.0 samples/s
+Step 35730/50462 | Loss: 2.5289 | LR: 1.01e-04 | 4005.9ms/step | 15.0 samples/s
+Step 35740/50462 | Loss: 2.5200 | LR: 1.01e-04 | 4012.5ms/step | 15.0 samples/s
+Step 35750/50462 | Loss: 2.5227 | LR: 1.00e-04 | 4005.6ms/step | 15.0 samples/s
+Validation at step 35750: Loss = 2.5209, Perplexity = 12.44
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-35750
+Step 35760/50462 | Loss: 2.5489 | LR: 1.00e-04 | 6039.7ms/step | 9.9 samples/s
+Step 35770/50462 | Loss: 2.5250 | LR: 1.00e-04 | 4005.4ms/step | 15.0 samples/s
+Step 35780/50462 | Loss: 2.5408 | LR: 1.00e-04 | 3999.1ms/step | 15.0 samples/s
+Step 35790/50462 | Loss: 2.4973 | LR: 1.00e-04 | 4005.4ms/step | 15.0 samples/s
+Step 35800/50462 | Loss: 2.5341 | LR: 9.99e-05 | 4017.2ms/step | 14.9 samples/s
+Step 35810/50462 | Loss: 2.5531 | LR: 9.97e-05 | 4003.5ms/step | 15.0 samples/s
+Step 35820/50462 | Loss: 2.5459 | LR: 9.96e-05 | 4008.9ms/step | 15.0 samples/s
+Step 35830/50462 | Loss: 2.5224 | LR: 9.95e-05 | 4020.4ms/step | 14.9 samples/s
+Step 35840/50462 | Loss: 2.5282 | LR: 9.94e-05 | 4017.6ms/step | 14.9 samples/s
+Step 35850/50462 | Loss: 2.5165 | LR: 9.92e-05 | 4011.7ms/step | 15.0 samples/s
+Step 35860/50462 | Loss: 2.5354 | LR: 9.91e-05 | 4020.8ms/step | 14.9 samples/s
+Step 35870/50462 | Loss: 2.5510 | LR: 9.90e-05 | 4022.1ms/step | 14.9 samples/s
+Step 35880/50462 | Loss: 2.5354 | LR: 9.88e-05 | 4017.0ms/step | 14.9 samples/s
+Step 35890/50462 | Loss: 2.5194 | LR: 9.87e-05 | 4027.3ms/step | 14.9 samples/s
+Step 35900/50462 | Loss: 2.5286 | LR: 9.86e-05 | 4002.3ms/step | 15.0 samples/s
+Step 35910/50462 | Loss: 2.5463 | LR: 9.85e-05 | 4010.4ms/step | 15.0 samples/s
+Step 35920/50462 | Loss: 2.5333 | LR: 9.83e-05 | 4001.0ms/step | 15.0 samples/s
+Step 35930/50462 | Loss: 2.5047 | LR: 9.82e-05 | 4788.9ms/step | 12.5 samples/s
+Step 35940/50462 | Loss: 2.5115 | LR: 9.81e-05 | 4010.6ms/step | 15.0 samples/s
+Step 35950/50462 | Loss: 2.5324 | LR: 9.80e-05 | 4000.4ms/step | 15.0 samples/s
+Step 35960/50462 | Loss: 2.5399 | LR: 9.78e-05 | 4578.2ms/step | 13.1 samples/s
+Step 35970/50462 | Loss: 2.5408 | LR: 9.77e-05 | 4017.4ms/step | 14.9 samples/s
+Step 35980/50462 | Loss: 2.5010 | LR: 9.76e-05 | 4012.7ms/step | 15.0 samples/s
+Step 35990/50462 | Loss: 2.4870 | LR: 9.75e-05 | 4009.5ms/step | 15.0 samples/s
+Step 36000/50462 | Loss: 2.5162 | LR: 9.73e-05 | 4016.4ms/step | 14.9 samples/s
+Validation at step 36000: Loss = 2.5207, Perplexity = 12.44
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-36000
+Step 36010/50462 | Loss: 2.5543 | LR: 9.72e-05 | 6012.2ms/step | 10.0 samples/s
+Step 36020/50462 | Loss: 2.5155 | LR: 9.71e-05 | 4010.3ms/step | 15.0 samples/s
+Step 36030/50462 | Loss: 2.5246 | LR: 9.70e-05 | 4592.2ms/step | 13.1 samples/s
+Step 36040/50462 | Loss: 2.5354 | LR: 9.68e-05 | 4011.1ms/step | 15.0 samples/s
+Step 36050/50462 | Loss: 2.5228 | LR: 9.67e-05 | 4013.1ms/step | 15.0 samples/s
+Step 36060/50462 | Loss: 2.5228 | LR: 9.66e-05 | 4009.2ms/step | 15.0 samples/s
+Step 36070/50462 | Loss: 2.5632 | LR: 9.65e-05 | 4026.1ms/step | 14.9 samples/s
+Step 36080/50462 | Loss: 2.5276 | LR: 9.63e-05 | 4011.1ms/step | 15.0 samples/s
+Step 36090/50462 | Loss: 2.5277 | LR: 9.62e-05 | 3999.8ms/step | 15.0 samples/s
+Step 36100/50462 | Loss: 2.5209 | LR: 9.61e-05 | 4007.4ms/step | 15.0 samples/s
+Step 36110/50462 | Loss: 2.5169 | LR: 9.60e-05 | 4015.4ms/step | 14.9 samples/s
+Step 36120/50462 | Loss: 2.5280 | LR: 9.58e-05 | 4009.9ms/step | 15.0 samples/s
+Step 36130/50462 | Loss: 2.5401 | LR: 9.57e-05 | 4006.8ms/step | 15.0 samples/s
+Step 36140/50462 | Loss: 2.5506 | LR: 9.56e-05 | 4006.0ms/step | 15.0 samples/s
+Step 36150/50462 | Loss: 2.5169 | LR: 9.55e-05 | 4002.5ms/step | 15.0 samples/s
+Step 36160/50462 | Loss: 2.5164 | LR: 9.53e-05 | 4020.7ms/step | 14.9 samples/s
+Step 36170/50462 | Loss: 2.5375 | LR: 9.52e-05 | 4011.4ms/step | 15.0 samples/s
+Step 36180/50462 | Loss: 2.5221 | LR: 9.51e-05 | 4007.1ms/step | 15.0 samples/s
+Step 36190/50462 | Loss: 2.5787 | LR: 9.50e-05 | 4024.1ms/step | 14.9 samples/s
+Step 36200/50462 | Loss: 2.5372 | LR: 9.49e-05 | 4000.8ms/step | 15.0 samples/s
+Step 36210/50462 | Loss: 2.5342 | LR: 9.47e-05 | 4002.6ms/step | 15.0 samples/s
+Step 36220/50462 | Loss: 2.5146 | LR: 9.46e-05 | 4004.9ms/step | 15.0 samples/s
+Step 36230/50462 | Loss: 2.5496 | LR: 9.45e-05 | 4001.2ms/step | 15.0 samples/s
+Step 36240/50462 | Loss: 2.5357 | LR: 9.44e-05 | 4008.1ms/step | 15.0 samples/s
+Step 36250/50462 | Loss: 2.5324 | LR: 9.42e-05 | 4004.5ms/step | 15.0 samples/s
+Validation at step 36250: Loss = 2.5197, Perplexity = 12.42
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-36250
+Step 36260/50462 | Loss: 2.4829 | LR: 9.41e-05 | 5979.5ms/step | 10.0 samples/s
+Step 36270/50462 | Loss: 2.5536 | LR: 9.40e-05 | 4015.5ms/step | 14.9 samples/s
+Step 36280/50462 | Loss: 2.5448 | LR: 9.39e-05 | 4012.5ms/step | 15.0 samples/s
+Step 36290/50462 | Loss: 2.5206 | LR: 9.37e-05 | 4005.7ms/step | 15.0 samples/s
+Step 36300/50462 | Loss: 2.5289 | LR: 9.36e-05 | 4007.7ms/step | 15.0 samples/s
+Step 36310/50462 | Loss: 2.5285 | LR: 9.35e-05 | 4007.8ms/step | 15.0 samples/s
+Step 36320/50462 | Loss: 2.5442 | LR: 9.34e-05 | 4015.3ms/step | 14.9 samples/s
+Step 36330/50462 | Loss: 2.5320 | LR: 9.32e-05 | 4002.4ms/step | 15.0 samples/s
+Step 36340/50462 | Loss: 2.5322 | LR: 9.31e-05 | 4014.7ms/step | 14.9 samples/s
+Step 36350/50462 | Loss: 2.5404 | LR: 9.30e-05 | 3998.5ms/step | 15.0 samples/s
+Step 36360/50462 | Loss: 2.5360 | LR: 9.29e-05 | 3997.9ms/step | 15.0 samples/s
+Step 36370/50462 | Loss: 2.5276 | LR: 9.28e-05 | 4009.5ms/step | 15.0 samples/s
+Step 36380/50462 | Loss: 2.5442 | LR: 9.26e-05 | 4009.0ms/step | 15.0 samples/s
+Step 36390/50462 | Loss: 2.5160 | LR: 9.25e-05 | 4011.2ms/step | 15.0 samples/s
+Step 36400/50462 | Loss: 2.5252 | LR: 9.24e-05 | 4011.2ms/step | 15.0 samples/s
+Step 36410/50462 | Loss: 2.5149 | LR: 9.23e-05 | 4014.4ms/step | 14.9 samples/s
+Step 36420/50462 | Loss: 2.5040 | LR: 9.21e-05 | 4013.6ms/step | 14.9 samples/s
+Step 36430/50462 | Loss: 2.5291 | LR: 9.20e-05 | 4012.1ms/step | 15.0 samples/s
+Step 36440/50462 | Loss: 2.5311 | LR: 9.19e-05 | 3999.6ms/step | 15.0 samples/s
+Step 36450/50462 | Loss: 2.5197 | LR: 9.18e-05 | 4000.0ms/step | 15.0 samples/s
+Step 36460/50462 | Loss: 2.5115 | LR: 9.17e-05 | 4010.0ms/step | 15.0 samples/s
+Step 36470/50462 | Loss: 2.5109 | LR: 9.15e-05 | 4019.0ms/step | 14.9 samples/s
+Step 36480/50462 | Loss: 2.5090 | LR: 9.14e-05 | 4012.0ms/step | 15.0 samples/s
+Step 36490/50462 | Loss: 2.5172 | LR: 9.13e-05 | 4016.6ms/step | 14.9 samples/s
+Step 36500/50462 | Loss: 2.5063 | LR: 9.12e-05 | 4013.1ms/step | 15.0 samples/s
+Validation at step 36500: Loss = 2.5191, Perplexity = 12.42
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-36500
+Step 36510/50462 | Loss: 2.5201 | LR: 9.10e-05 | 5965.9ms/step | 10.1 samples/s
+Step 36520/50462 | Loss: 2.5520 | LR: 9.09e-05 | 3998.6ms/step | 15.0 samples/s
+Step 36530/50462 | Loss: 2.5210 | LR: 9.08e-05 | 4003.7ms/step | 15.0 samples/s
+Step 36540/50462 | Loss: 2.5408 | LR: 9.07e-05 | 4011.9ms/step | 15.0 samples/s
+Step 36550/50462 | Loss: 2.5400 | LR: 9.06e-05 | 4009.7ms/step | 15.0 samples/s
+Step 36560/50462 | Loss: 2.5334 | LR: 9.04e-05 | 4017.7ms/step | 14.9 samples/s
+Step 36570/50462 | Loss: 2.4692 | LR: 9.03e-05 | 4017.9ms/step | 14.9 samples/s
+Step 36580/50462 | Loss: 2.5159 | LR: 9.02e-05 | 4011.0ms/step | 15.0 samples/s
+Step 36590/50462 | Loss: 2.5443 | LR: 9.01e-05 | 3991.4ms/step | 15.0 samples/s
+Step 36600/50462 | Loss: 2.5868 | LR: 8.99e-05 | 4000.3ms/step | 15.0 samples/s
+Step 36610/50462 | Loss: 2.5105 | LR: 8.98e-05 | 4004.4ms/step | 15.0 samples/s
+Step 36620/50462 | Loss: 2.5493 | LR: 8.97e-05 | 4018.8ms/step | 14.9 samples/s
+Step 36630/50462 | Loss: 2.5100 | LR: 8.96e-05 | 4014.5ms/step | 14.9 samples/s
+Step 36640/50462 | Loss: 2.5401 | LR: 8.95e-05 | 4000.1ms/step | 15.0 samples/s
+Step 36650/50462 | Loss: 2.5508 | LR: 8.93e-05 | 4010.1ms/step | 15.0 samples/s
+Step 36660/50462 | Loss: 2.5201 | LR: 8.92e-05 | 4006.3ms/step | 15.0 samples/s
+Step 36670/50462 | Loss: 2.5645 | LR: 8.91e-05 | 4009.9ms/step | 15.0 samples/s
+Step 36680/50462 | Loss: 2.5091 | LR: 8.90e-05 | 4006.8ms/step | 15.0 samples/s
+Step 36690/50462 | Loss: 2.5254 | LR: 8.89e-05 | 4001.2ms/step | 15.0 samples/s
+Step 36700/50462 | Loss: 2.5203 | LR: 8.87e-05 | 4005.7ms/step | 15.0 samples/s
+Step 36710/50462 | Loss: 2.5155 | LR: 8.86e-05 | 4017.2ms/step | 14.9 samples/s
+Step 36720/50462 | Loss: 2.5342 | LR: 8.85e-05 | 4005.9ms/step | 15.0 samples/s
+Step 36730/50462 | Loss: 2.5213 | LR: 8.84e-05 | 4007.6ms/step | 15.0 samples/s
+Step 36740/50462 | Loss: 2.5368 | LR: 8.83e-05 | 4014.3ms/step | 14.9 samples/s
+Step 36750/50462 | Loss: 2.4980 | LR: 8.81e-05 | 4011.1ms/step | 15.0 samples/s
+Validation at step 36750: Loss = 2.5186, Perplexity = 12.41
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-36750
+Step 36760/50462 | Loss: 2.5350 | LR: 8.80e-05 | 5960.2ms/step | 10.1 samples/s
+Step 36770/50462 | Loss: 2.5130 | LR: 8.79e-05 | 4000.3ms/step | 15.0 samples/s
+Step 36780/50462 | Loss: 2.5424 | LR: 8.78e-05 | 4004.6ms/step | 15.0 samples/s
+Step 36790/50462 | Loss: 2.5619 | LR: 8.77e-05 | 4008.2ms/step | 15.0 samples/s
+Step 36800/50462 | Loss: 2.5399 | LR: 8.75e-05 | 4007.4ms/step | 15.0 samples/s
+Step 36810/50462 | Loss: 2.4872 | LR: 8.74e-05 | 4010.8ms/step | 15.0 samples/s
+Step 36820/50462 | Loss: 2.5151 | LR: 8.73e-05 | 4011.5ms/step | 15.0 samples/s
+Step 36830/50462 | Loss: 2.5531 | LR: 8.72e-05 | 4007.2ms/step | 15.0 samples/s
+Step 36840/50462 | Loss: 2.5234 | LR: 8.71e-05 | 4015.4ms/step | 14.9 samples/s
+Step 36850/50462 | Loss: 2.5282 | LR: 8.69e-05 | 4014.1ms/step | 14.9 samples/s
+Step 36860/50462 | Loss: 2.5357 | LR: 8.68e-05 | 4013.8ms/step | 14.9 samples/s
+Step 36870/50462 | Loss: 2.4795 | LR: 8.67e-05 | 3994.2ms/step | 15.0 samples/s
+Step 36880/50462 | Loss: 2.5289 | LR: 8.66e-05 | 4006.1ms/step | 15.0 samples/s
+Step 36890/50462 | Loss: 2.5420 | LR: 8.65e-05 | 4012.2ms/step | 15.0 samples/s
+Step 36900/50462 | Loss: 2.5646 | LR: 8.63e-05 | 4012.3ms/step | 15.0 samples/s
+Step 36910/50462 | Loss: 2.5077 | LR: 8.62e-05 | 4006.7ms/step | 15.0 samples/s
+Step 36920/50462 | Loss: 2.5406 | LR: 8.61e-05 | 4006.6ms/step | 15.0 samples/s
+Step 36930/50462 | Loss: 2.5513 | LR: 8.60e-05 | 4009.5ms/step | 15.0 samples/s
+Step 36940/50462 | Loss: 2.5333 | LR: 8.59e-05 | 4000.8ms/step | 15.0 samples/s
+Step 36950/50462 | Loss: 2.5341 | LR: 8.57e-05 | 3996.6ms/step | 15.0 samples/s
+Step 36960/50462 | Loss: 2.5054 | LR: 8.56e-05 | 4020.8ms/step | 14.9 samples/s
+Step 36970/50462 | Loss: 2.4908 | LR: 8.55e-05 | 4015.0ms/step | 14.9 samples/s
+Step 36980/50462 | Loss: 2.5400 | LR: 8.54e-05 | 4011.5ms/step | 15.0 samples/s
+Step 36990/50462 | Loss: 2.5302 | LR: 8.53e-05 | 4016.5ms/step | 14.9 samples/s
+Step 37000/50462 | Loss: 2.5417 | LR: 8.51e-05 | 4009.3ms/step | 15.0 samples/s
+Validation at step 37000: Loss = 2.5185, Perplexity = 12.41
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-37000
+Step 37010/50462 | Loss: 2.5209 | LR: 8.50e-05 | 6067.3ms/step | 9.9 samples/s
+Step 37020/50462 | Loss: 2.4995 | LR: 8.49e-05 | 4021.3ms/step | 14.9 samples/s
+Step 37030/50462 | Loss: 2.5161 | LR: 8.48e-05 | 4025.2ms/step | 14.9 samples/s
+Step 37040/50462 | Loss: 2.5463 | LR: 8.47e-05 | 4000.8ms/step | 15.0 samples/s
+Step 37050/50462 | Loss: 2.5419 | LR: 8.46e-05 | 4011.3ms/step | 15.0 samples/s

epfl_final/SparseAtt/wandb/run-20260829_174918-8ynn1we6/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260829_174918-8ynn1we6/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-08-29T17:49:18.717996Z",
+  "args":  [
+    "--config",
+    "configs/p2_1p2b_mixed_seed71_4gpu_resume28500.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/SparseAtt.git",
+    "commit":  "8f862f749eb4d5c17f688eb1562235e3fe5b2550"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-train-4-gpu-0-31",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "6236834574336"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-c65885f7-52dc-6f23-a3dd-6addf8b518a2"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-3a3f0ac7-1596-1797-bf0d-31f3418789cd"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-410cba62-f098-3454-c960-ebc55bd081af"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-9e3de42d-518e-8887-4b3d-6c8316629c5e"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "ysjgvi3l9m5g6bnbldkpkj0tf9kxxbff"
+}

epfl_final/SparseAtt/wandb/run-20260829_174918-8ynn1we6/logs/debug.log

@@ -0,0 +1,20 @@
+2026-08-29 17:49:18,739 INFO    MainThread:196 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-08-29 17:49:18,740 INFO    MainThread:196 [wandb_setup.py:_flush():81] Configure stats pid to 196
+2026-08-29 17:49:18,740 INFO    MainThread:196 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-29 17:49:18,740 INFO    MainThread:196 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260829_174918-8ynn1we6/logs/debug.log
+2026-08-29 17:49:18,740 INFO    MainThread:196 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260829_174918-8ynn1we6/logs/debug-internal.log
+2026-08-29 17:49:18,740 INFO    MainThread:196 [wandb_init.py:init():850] calling init triggers
+2026-08-29 17:49:18,740 INFO    MainThread:196 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-34250', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 15, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 71, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b mixed 8k seed71 4gpu', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'mixed', 'seed71'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed', 'sparse_mask_seed': 71, 'sparse_mixed_nrb_set': [63, 47, 31, 21, 11], 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-08-29 17:49:18,740 INFO    MainThread:196 [wandb_init.py:init():898] starting backend
+2026-08-29 17:49:19,568 INFO    MainThread:196 [wandb_init.py:init():913] sending inform_init request
+2026-08-29 17:49:20,140 INFO    MainThread:196 [wandb_init.py:init():918] backend started and connected
+2026-08-29 17:49:20,142 INFO    MainThread:196 [wandb_init.py:init():988] updated telemetry
+2026-08-29 17:49:20,234 INFO    MainThread:196 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-08-29 17:49:20,772 INFO    MainThread:196 [wandb_init.py:init():1051] run resumed
+2026-08-29 17:49:20,775 INFO    MainThread:196 [wandb_init.py:init():1056] starting run threads in backend
+2026-08-29 17:49:21,122 INFO    MainThread:196 [wandb_run.py:_console_start():2554] atexit reg
+2026-08-29 17:49:21,122 INFO    MainThread:196 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-08-29 17:49:21,122 INFO    MainThread:196 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-08-29 17:49:21,122 INFO    MainThread:196 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-08-29 17:49:21,132 INFO    MainThread:196 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/run-20260829_174918-8ynn1we6/run-8ynn1we6.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41da6431de4f3cec9cdc7457c35b97463d4444b59427128ed2e1c7dd69c7b382
+size 4030464

epfl_final/SparseAtt/wandb/run-20260830_002911-8ynn1we6/files/output.log

@@ -0,0 +1,545 @@
+Starting training...
+Validation at step 37000: Loss = 2.5187, Perplexity = 12.41
+wandb: WARNING Tried to log to step 37000 that is less than the current step 37041. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 37001: sparse_stochastic (configured sparse attention)
+Step 37010/50462 | Loss: 2.5209 | LR: 8.50e-05 | 88560.5ms/step | 0.7 samples/s
+wandb: WARNING Tried to log to step 37010 that is less than the current step 37041. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 37020/50462 | Loss: 2.4994 | LR: 8.49e-05 | 4008.4ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 37020 that is less than the current step 37041. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 37030/50462 | Loss: 2.5164 | LR: 8.48e-05 | 4012.4ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 37030 that is less than the current step 37041. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 37040/50462 | Loss: 2.5460 | LR: 8.47e-05 | 4015.4ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 37040 that is less than the current step 37041. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 37050/50462 | Loss: 2.5402 | LR: 8.46e-05 | 4023.2ms/step | 14.9 samples/s
+Step 37060/50462 | Loss: 2.5259 | LR: 8.44e-05 | 4021.0ms/step | 14.9 samples/s
+Step 37070/50462 | Loss: 2.5400 | LR: 8.43e-05 | 4019.5ms/step | 14.9 samples/s
+Step 37080/50462 | Loss: 2.5154 | LR: 8.42e-05 | 4014.8ms/step | 14.9 samples/s
+Step 37090/50462 | Loss: 2.4991 | LR: 8.41e-05 | 4015.8ms/step | 14.9 samples/s
+Step 37100/50462 | Loss: 2.5339 | LR: 8.40e-05 | 4002.4ms/step | 15.0 samples/s
+Step 37110/50462 | Loss: 2.5480 | LR: 8.38e-05 | 4007.3ms/step | 15.0 samples/s
+Step 37120/50462 | Loss: 2.5428 | LR: 8.37e-05 | 4016.9ms/step | 14.9 samples/s
+Step 37130/50462 | Loss: 2.4922 | LR: 8.36e-05 | 4015.0ms/step | 14.9 samples/s
+Step 37140/50462 | Loss: 2.5174 | LR: 8.35e-05 | 4015.5ms/step | 14.9 samples/s
+Step 37150/50462 | Loss: 2.5209 | LR: 8.34e-05 | 4024.3ms/step | 14.9 samples/s
+Step 37160/50462 | Loss: 2.5147 | LR: 8.33e-05 | 4010.5ms/step | 15.0 samples/s
+Step 37170/50462 | Loss: 2.5313 | LR: 8.31e-05 | 4013.3ms/step | 15.0 samples/s
+Step 37180/50462 | Loss: 2.5433 | LR: 8.30e-05 | 4005.7ms/step | 15.0 samples/s
+Step 37190/50462 | Loss: 2.5234 | LR: 8.29e-05 | 4007.3ms/step | 15.0 samples/s
+Step 37200/50462 | Loss: 2.5430 | LR: 8.28e-05 | 4016.2ms/step | 14.9 samples/s
+Step 37210/50462 | Loss: 2.5253 | LR: 8.27e-05 | 4016.7ms/step | 14.9 samples/s
+Step 37220/50462 | Loss: 2.5080 | LR: 8.25e-05 | 4004.9ms/step | 15.0 samples/s
+Step 37230/50462 | Loss: 2.5245 | LR: 8.24e-05 | 4015.3ms/step | 14.9 samples/s
+Step 37240/50462 | Loss: 2.5479 | LR: 8.23e-05 | 4022.2ms/step | 14.9 samples/s
+Step 37250/50462 | Loss: 2.5347 | LR: 8.22e-05 | 3999.2ms/step | 15.0 samples/s
+Validation at step 37250: Loss = 2.5186, Perplexity = 12.41
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-37250
+Step 37260/50462 | Loss: 2.5185 | LR: 8.21e-05 | 5974.0ms/step | 10.0 samples/s
+Step 37270/50462 | Loss: 2.5397 | LR: 8.20e-05 | 4007.0ms/step | 15.0 samples/s
+Step 37280/50462 | Loss: 2.5277 | LR: 8.18e-05 | 4014.6ms/step | 14.9 samples/s
+Step 37290/50462 | Loss: 2.5529 | LR: 8.17e-05 | 4023.6ms/step | 14.9 samples/s
+Step 37300/50462 | Loss: 2.5347 | LR: 8.16e-05 | 4005.6ms/step | 15.0 samples/s
+Step 37310/50462 | Loss: 2.5321 | LR: 8.15e-05 | 4010.9ms/step | 15.0 samples/s
+Step 37320/50462 | Loss: 2.5161 | LR: 8.14e-05 | 4022.1ms/step | 14.9 samples/s
+Step 37330/50462 | Loss: 2.5420 | LR: 8.13e-05 | 4015.8ms/step | 14.9 samples/s
+Step 37340/50462 | Loss: 2.5075 | LR: 8.11e-05 | 4007.2ms/step | 15.0 samples/s
+Step 37350/50462 | Loss: 2.5490 | LR: 8.10e-05 | 4018.0ms/step | 14.9 samples/s
+Step 37360/50462 | Loss: 2.4989 | LR: 8.09e-05 | 4015.8ms/step | 14.9 samples/s
+Step 37370/50462 | Loss: 2.5496 | LR: 8.08e-05 | 4011.7ms/step | 15.0 samples/s
+Step 37380/50462 | Loss: 2.5320 | LR: 8.07e-05 | 4022.4ms/step | 14.9 samples/s
+Step 37390/50462 | Loss: 2.5320 | LR: 8.06e-05 | 4013.0ms/step | 15.0 samples/s
+Step 37400/50462 | Loss: 2.5390 | LR: 8.04e-05 | 3999.2ms/step | 15.0 samples/s
+Step 37410/50462 | Loss: 2.5283 | LR: 8.03e-05 | 4006.2ms/step | 15.0 samples/s
+Step 37420/50462 | Loss: 2.5621 | LR: 8.02e-05 | 4008.7ms/step | 15.0 samples/s
+Step 37430/50462 | Loss: 2.4928 | LR: 8.01e-05 | 4023.6ms/step | 14.9 samples/s
+Step 37440/50462 | Loss: 2.4970 | LR: 8.00e-05 | 4015.3ms/step | 14.9 samples/s
+Step 37450/50462 | Loss: 2.4874 | LR: 7.99e-05 | 4012.5ms/step | 15.0 samples/s
+Step 37460/50462 | Loss: 2.5284 | LR: 7.98e-05 | 4003.2ms/step | 15.0 samples/s
+Step 37470/50462 | Loss: 2.5782 | LR: 7.96e-05 | 4014.5ms/step | 14.9 samples/s
+Step 37480/50462 | Loss: 2.5119 | LR: 7.95e-05 | 4004.4ms/step | 15.0 samples/s
+Step 37490/50462 | Loss: 2.5167 | LR: 7.94e-05 | 4009.0ms/step | 15.0 samples/s
+Step 37500/50462 | Loss: 2.5421 | LR: 7.93e-05 | 4022.6ms/step | 14.9 samples/s
+Validation at step 37500: Loss = 2.5174, Perplexity = 12.40
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-37500
+Step 37510/50462 | Loss: 2.5200 | LR: 7.92e-05 | 5983.4ms/step | 10.0 samples/s
+Step 37520/50462 | Loss: 2.5330 | LR: 7.91e-05 | 4009.3ms/step | 15.0 samples/s
+Step 37530/50462 | Loss: 2.5239 | LR: 7.89e-05 | 4021.3ms/step | 14.9 samples/s
+Step 37540/50462 | Loss: 2.5474 | LR: 7.88e-05 | 4009.3ms/step | 15.0 samples/s
+Step 37550/50462 | Loss: 2.5640 | LR: 7.87e-05 | 4002.1ms/step | 15.0 samples/s
+Step 37560/50462 | Loss: 2.5490 | LR: 7.86e-05 | 4005.6ms/step | 15.0 samples/s
+Step 37570/50462 | Loss: 2.5234 | LR: 7.85e-05 | 4020.8ms/step | 14.9 samples/s
+Step 37580/50462 | Loss: 2.4794 | LR: 7.84e-05 | 4009.1ms/step | 15.0 samples/s
+Step 37590/50462 | Loss: 2.5330 | LR: 7.83e-05 | 4008.6ms/step | 15.0 samples/s
+Step 37600/50462 | Loss: 2.5372 | LR: 7.81e-05 | 4014.1ms/step | 14.9 samples/s
+Step 37610/50462 | Loss: 2.5169 | LR: 7.80e-05 | 3998.5ms/step | 15.0 samples/s
+Step 37620/50462 | Loss: 2.4926 | LR: 7.79e-05 | 4007.2ms/step | 15.0 samples/s
+Step 37630/50462 | Loss: 2.5177 | LR: 7.78e-05 | 4009.7ms/step | 15.0 samples/s
+Step 37640/50462 | Loss: 2.5319 | LR: 7.77e-05 | 4007.7ms/step | 15.0 samples/s
+Step 37650/50462 | Loss: 2.5355 | LR: 7.76e-05 | 4009.1ms/step | 15.0 samples/s
+Step 37660/50462 | Loss: 2.5330 | LR: 7.75e-05 | 4001.7ms/step | 15.0 samples/s
+Step 37670/50462 | Loss: 2.5413 | LR: 7.73e-05 | 4009.0ms/step | 15.0 samples/s
+Step 37680/50462 | Loss: 2.5254 | LR: 7.72e-05 | 4007.4ms/step | 15.0 samples/s
+Step 37690/50462 | Loss: 2.5601 | LR: 7.71e-05 | 4011.0ms/step | 15.0 samples/s
+Step 37700/50462 | Loss: 2.5274 | LR: 7.70e-05 | 4015.3ms/step | 14.9 samples/s
+Step 37710/50462 | Loss: 2.5339 | LR: 7.69e-05 | 4017.6ms/step | 14.9 samples/s
+Step 37720/50462 | Loss: 2.5433 | LR: 7.68e-05 | 4002.3ms/step | 15.0 samples/s
+Step 37730/50462 | Loss: 2.5434 | LR: 7.67e-05 | 4006.3ms/step | 15.0 samples/s
+Step 37740/50462 | Loss: 2.5303 | LR: 7.65e-05 | 4011.6ms/step | 15.0 samples/s
+Step 37750/50462 | Loss: 2.5163 | LR: 7.64e-05 | 4023.7ms/step | 14.9 samples/s
+Validation at step 37750: Loss = 2.5170, Perplexity = 12.39
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-37750
+Step 37760/50462 | Loss: 2.5241 | LR: 7.63e-05 | 5948.0ms/step | 10.1 samples/s
+Step 37770/50462 | Loss: 2.5658 | LR: 7.62e-05 | 4011.1ms/step | 15.0 samples/s
+Step 37780/50462 | Loss: 2.5006 | LR: 7.61e-05 | 4009.0ms/step | 15.0 samples/s
+Step 37790/50462 | Loss: 2.5109 | LR: 7.60e-05 | 4019.7ms/step | 14.9 samples/s
+Step 37800/50462 | Loss: 2.5383 | LR: 7.59e-05 | 4020.7ms/step | 14.9 samples/s
+Step 37810/50462 | Loss: 2.5317 | LR: 7.57e-05 | 4015.3ms/step | 14.9 samples/s
+Step 37820/50462 | Loss: 2.5175 | LR: 7.56e-05 | 4009.0ms/step | 15.0 samples/s
+Step 37830/50462 | Loss: 2.5493 | LR: 7.55e-05 | 4009.4ms/step | 15.0 samples/s
+Step 37840/50462 | Loss: 2.5404 | LR: 7.54e-05 | 4002.2ms/step | 15.0 samples/s
+Step 37850/50462 | Loss: 2.5368 | LR: 7.53e-05 | 4006.6ms/step | 15.0 samples/s
+Step 37860/50462 | Loss: 2.5151 | LR: 7.52e-05 | 4027.0ms/step | 14.9 samples/s
+Step 37870/50462 | Loss: 2.5120 | LR: 7.51e-05 | 4003.2ms/step | 15.0 samples/s
+Step 37880/50462 | Loss: 2.4851 | LR: 7.50e-05 | 4008.4ms/step | 15.0 samples/s
+Step 37890/50462 | Loss: 2.5321 | LR: 7.48e-05 | 4016.0ms/step | 14.9 samples/s
+Step 37900/50462 | Loss: 2.5021 | LR: 7.47e-05 | 4010.1ms/step | 15.0 samples/s
+Step 37910/50462 | Loss: 2.5287 | LR: 7.46e-05 | 4003.1ms/step | 15.0 samples/s
+Step 37920/50462 | Loss: 2.5196 | LR: 7.45e-05 | 4011.0ms/step | 15.0 samples/s
+Step 37930/50462 | Loss: 2.5343 | LR: 7.44e-05 | 4013.4ms/step | 14.9 samples/s
+Step 37940/50462 | Loss: 2.5294 | LR: 7.43e-05 | 4004.5ms/step | 15.0 samples/s
+Step 37950/50462 | Loss: 2.4690 | LR: 7.42e-05 | 4014.6ms/step | 14.9 samples/s
+Step 37960/50462 | Loss: 2.5227 | LR: 7.41e-05 | 4009.8ms/step | 15.0 samples/s
+Step 37970/50462 | Loss: 2.5276 | LR: 7.39e-05 | 4017.4ms/step | 14.9 samples/s
+Step 37980/50462 | Loss: 2.5424 | LR: 7.38e-05 | 4013.4ms/step | 14.9 samples/s
+Step 37990/50462 | Loss: 2.5276 | LR: 7.37e-05 | 3998.6ms/step | 15.0 samples/s
+Step 38000/50462 | Loss: 2.5303 | LR: 7.36e-05 | 4003.7ms/step | 15.0 samples/s
+Validation at step 38000: Loss = 2.5168, Perplexity = 12.39
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-38000
+Step 38010/50462 | Loss: 2.5436 | LR: 7.35e-05 | 5989.7ms/step | 10.0 samples/s
+Step 38020/50462 | Loss: 2.5508 | LR: 7.34e-05 | 4003.7ms/step | 15.0 samples/s
+Step 38030/50462 | Loss: 2.5237 | LR: 7.33e-05 | 4006.4ms/step | 15.0 samples/s
+Step 38040/50462 | Loss: 2.5327 | LR: 7.32e-05 | 4012.7ms/step | 15.0 samples/s
+Step 38050/50462 | Loss: 2.5654 | LR: 7.30e-05 | 4573.6ms/step | 13.1 samples/s
+Step 38060/50462 | Loss: 2.5513 | LR: 7.29e-05 | 4016.9ms/step | 14.9 samples/s
+Step 38070/50462 | Loss: 2.5554 | LR: 7.28e-05 | 4013.8ms/step | 14.9 samples/s
+Step 38080/50462 | Loss: 2.5710 | LR: 7.27e-05 | 4004.5ms/step | 15.0 samples/s
+Step 38090/50462 | Loss: 2.4927 | LR: 7.26e-05 | 4001.5ms/step | 15.0 samples/s
+Step 38100/50462 | Loss: 2.5545 | LR: 7.25e-05 | 4001.9ms/step | 15.0 samples/s
+Step 38110/50462 | Loss: 2.5445 | LR: 7.24e-05 | 3996.8ms/step | 15.0 samples/s
+Step 38120/50462 | Loss: 2.5296 | LR: 7.23e-05 | 4007.2ms/step | 15.0 samples/s
+Step 38130/50462 | Loss: 2.5132 | LR: 7.22e-05 | 4016.1ms/step | 14.9 samples/s
+Step 38140/50462 | Loss: 2.5305 | LR: 7.20e-05 | 4009.1ms/step | 15.0 samples/s
+Step 38150/50462 | Loss: 2.5286 | LR: 7.19e-05 | 4010.0ms/step | 15.0 samples/s
+Step 38160/50462 | Loss: 2.5233 | LR: 7.18e-05 | 4007.6ms/step | 15.0 samples/s
+Step 38170/50462 | Loss: 2.5372 | LR: 7.17e-05 | 4008.4ms/step | 15.0 samples/s
+Step 38180/50462 | Loss: 2.5496 | LR: 7.16e-05 | 4006.9ms/step | 15.0 samples/s
+Step 38190/50462 | Loss: 2.5129 | LR: 7.15e-05 | 4001.8ms/step | 15.0 samples/s
+Step 38200/50462 | Loss: 2.5115 | LR: 7.14e-05 | 4016.4ms/step | 14.9 samples/s
+Step 38210/50462 | Loss: 2.5442 | LR: 7.13e-05 | 4001.1ms/step | 15.0 samples/s
+Step 38220/50462 | Loss: 2.4845 | LR: 7.12e-05 | 4000.1ms/step | 15.0 samples/s
+Step 38230/50462 | Loss: 2.5413 | LR: 7.10e-05 | 4011.6ms/step | 15.0 samples/s
+Step 38240/50462 | Loss: 2.5150 | LR: 7.09e-05 | 4019.6ms/step | 14.9 samples/s
+Step 38250/50462 | Loss: 2.5721 | LR: 7.08e-05 | 4013.4ms/step | 15.0 samples/s
+Validation at step 38250: Loss = 2.5166, Perplexity = 12.39
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-38250
+Step 38260/50462 | Loss: 2.5265 | LR: 7.07e-05 | 6002.0ms/step | 10.0 samples/s
+Step 38270/50462 | Loss: 2.5039 | LR: 7.06e-05 | 3991.4ms/step | 15.0 samples/s
+Step 38280/50462 | Loss: 2.5187 | LR: 7.05e-05 | 4007.0ms/step | 15.0 samples/s
+Step 38290/50462 | Loss: 2.5397 | LR: 7.04e-05 | 4011.1ms/step | 15.0 samples/s
+Step 38300/50462 | Loss: 2.5738 | LR: 7.03e-05 | 4009.2ms/step | 15.0 samples/s
+Step 38310/50462 | Loss: 2.5500 | LR: 7.02e-05 | 3992.4ms/step | 15.0 samples/s
+Step 38320/50462 | Loss: 2.5473 | LR: 7.01e-05 | 4017.2ms/step | 14.9 samples/s
+Step 38330/50462 | Loss: 2.5522 | LR: 6.99e-05 | 3996.7ms/step | 15.0 samples/s
+Step 38340/50462 | Loss: 2.5050 | LR: 6.98e-05 | 3988.7ms/step | 15.0 samples/s
+Step 38350/50462 | Loss: 2.5426 | LR: 6.97e-05 | 4002.1ms/step | 15.0 samples/s
+Step 38360/50462 | Loss: 2.5391 | LR: 6.96e-05 | 3992.2ms/step | 15.0 samples/s
+Step 38370/50462 | Loss: 2.5113 | LR: 6.95e-05 | 4003.4ms/step | 15.0 samples/s
+Step 38380/50462 | Loss: 2.5209 | LR: 6.94e-05 | 4005.0ms/step | 15.0 samples/s
+Step 38390/50462 | Loss: 2.5366 | LR: 6.93e-05 | 4001.4ms/step | 15.0 samples/s
+Step 38400/50462 | Loss: 2.5408 | LR: 6.92e-05 | 4003.6ms/step | 15.0 samples/s
+Step 38410/50462 | Loss: 2.5018 | LR: 6.91e-05 | 4001.8ms/step | 15.0 samples/s
+Step 38420/50462 | Loss: 2.5188 | LR: 6.90e-05 | 4590.6ms/step | 13.1 samples/s
+Step 38430/50462 | Loss: 2.5445 | LR: 6.89e-05 | 4004.6ms/step | 15.0 samples/s
+Step 38440/50462 | Loss: 2.5409 | LR: 6.87e-05 | 3999.5ms/step | 15.0 samples/s
+Step 38450/50462 | Loss: 2.5168 | LR: 6.86e-05 | 4002.1ms/step | 15.0 samples/s
+Step 38460/50462 | Loss: 2.5426 | LR: 6.85e-05 | 4003.1ms/step | 15.0 samples/s
+Step 38470/50462 | Loss: 2.4927 | LR: 6.84e-05 | 4002.8ms/step | 15.0 samples/s
+Step 38480/50462 | Loss: 2.5251 | LR: 6.83e-05 | 4000.6ms/step | 15.0 samples/s
+Step 38490/50462 | Loss: 2.5102 | LR: 6.82e-05 | 4002.5ms/step | 15.0 samples/s
+Step 38500/50462 | Loss: 2.5297 | LR: 6.81e-05 | 4002.4ms/step | 15.0 samples/s
+Validation at step 38500: Loss = 2.5156, Perplexity = 12.37
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-38500
+Step 38510/50462 | Loss: 2.5386 | LR: 6.80e-05 | 5951.1ms/step | 10.1 samples/s
+Step 38520/50462 | Loss: 2.5286 | LR: 6.79e-05 | 4013.1ms/step | 15.0 samples/s
+Step 38530/50462 | Loss: 2.5254 | LR: 6.78e-05 | 4004.9ms/step | 15.0 samples/s
+Step 38540/50462 | Loss: 2.5558 | LR: 6.77e-05 | 4000.5ms/step | 15.0 samples/s
+Step 38550/50462 | Loss: 2.5272 | LR: 6.76e-05 | 4011.4ms/step | 15.0 samples/s
+Step 38560/50462 | Loss: 2.5052 | LR: 6.74e-05 | 4015.1ms/step | 14.9 samples/s
+Step 38570/50462 | Loss: 2.5224 | LR: 6.73e-05 | 3994.2ms/step | 15.0 samples/s
+Step 38580/50462 | Loss: 2.5193 | LR: 6.72e-05 | 4005.1ms/step | 15.0 samples/s
+Step 38590/50462 | Loss: 2.5426 | LR: 6.71e-05 | 4005.6ms/step | 15.0 samples/s
+Step 38600/50462 | Loss: 2.5274 | LR: 6.70e-05 | 4001.6ms/step | 15.0 samples/s
+Step 38610/50462 | Loss: 2.5188 | LR: 6.69e-05 | 4009.7ms/step | 15.0 samples/s
+Step 38620/50462 | Loss: 2.5197 | LR: 6.68e-05 | 4004.1ms/step | 15.0 samples/s
+Step 38630/50462 | Loss: 2.5226 | LR: 6.67e-05 | 4003.8ms/step | 15.0 samples/s
+Step 38640/50462 | Loss: 2.5265 | LR: 6.66e-05 | 3998.0ms/step | 15.0 samples/s
+Step 38650/50462 | Loss: 2.5219 | LR: 6.65e-05 | 3999.3ms/step | 15.0 samples/s
+Step 38660/50462 | Loss: 2.4942 | LR: 6.64e-05 | 4000.2ms/step | 15.0 samples/s
+Step 38670/50462 | Loss: 2.5160 | LR: 6.63e-05 | 3997.9ms/step | 15.0 samples/s
+Step 38680/50462 | Loss: 2.5272 | LR: 6.62e-05 | 4010.0ms/step | 15.0 samples/s
+Step 38690/50462 | Loss: 2.5409 | LR: 6.60e-05 | 3995.9ms/step | 15.0 samples/s
+Step 38700/50462 | Loss: 2.5198 | LR: 6.59e-05 | 4018.2ms/step | 14.9 samples/s
+Step 38710/50462 | Loss: 2.5478 | LR: 6.58e-05 | 4587.0ms/step | 13.1 samples/s
+Step 38720/50462 | Loss: 2.5239 | LR: 6.57e-05 | 4001.9ms/step | 15.0 samples/s
+Step 38730/50462 | Loss: 2.5330 | LR: 6.56e-05 | 4006.8ms/step | 15.0 samples/s
+Step 38740/50462 | Loss: 2.5448 | LR: 6.55e-05 | 4001.4ms/step | 15.0 samples/s
+Step 38750/50462 | Loss: 2.5414 | LR: 6.54e-05 | 4008.8ms/step | 15.0 samples/s
+Validation at step 38750: Loss = 2.5156, Perplexity = 12.37
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-38750
+Step 38760/50462 | Loss: 2.5087 | LR: 6.53e-05 | 5975.8ms/step | 10.0 samples/s
+Step 38770/50462 | Loss: 2.5608 | LR: 6.52e-05 | 4577.5ms/step | 13.1 samples/s
+Step 38780/50462 | Loss: 2.5204 | LR: 6.51e-05 | 4010.7ms/step | 15.0 samples/s
+Step 38790/50462 | Loss: 2.5456 | LR: 6.50e-05 | 4014.4ms/step | 14.9 samples/s
+Step 38800/50462 | Loss: 2.5340 | LR: 6.49e-05 | 4011.2ms/step | 15.0 samples/s
+Step 38810/50462 | Loss: 2.5234 | LR: 6.48e-05 | 4003.9ms/step | 15.0 samples/s
+Step 38820/50462 | Loss: 2.5060 | LR: 6.47e-05 | 4003.1ms/step | 15.0 samples/s
+Step 38830/50462 | Loss: 2.5149 | LR: 6.46e-05 | 4001.2ms/step | 15.0 samples/s
+Step 38840/50462 | Loss: 2.5306 | LR: 6.45e-05 | 4016.2ms/step | 14.9 samples/s
+Step 38850/50462 | Loss: 2.5183 | LR: 6.43e-05 | 4008.1ms/step | 15.0 samples/s
+Step 38860/50462 | Loss: 2.5169 | LR: 6.42e-05 | 4003.5ms/step | 15.0 samples/s
+Step 38870/50462 | Loss: 2.5164 | LR: 6.41e-05 | 3991.1ms/step | 15.0 samples/s
+Step 38880/50462 | Loss: 2.5213 | LR: 6.40e-05 | 4004.3ms/step | 15.0 samples/s
+Step 38890/50462 | Loss: 2.5466 | LR: 6.39e-05 | 3988.8ms/step | 15.0 samples/s
+Step 38900/50462 | Loss: 2.5454 | LR: 6.38e-05 | 4000.0ms/step | 15.0 samples/s
+Step 38910/50462 | Loss: 2.5072 | LR: 6.37e-05 | 4007.2ms/step | 15.0 samples/s
+Step 38920/50462 | Loss: 2.5254 | LR: 6.36e-05 | 4001.9ms/step | 15.0 samples/s
+Step 38930/50462 | Loss: 2.5420 | LR: 6.35e-05 | 4010.4ms/step | 15.0 samples/s
+Step 38940/50462 | Loss: 2.4852 | LR: 6.34e-05 | 4007.4ms/step | 15.0 samples/s
+Step 38950/50462 | Loss: 2.5488 | LR: 6.33e-05 | 4008.1ms/step | 15.0 samples/s
+Step 38960/50462 | Loss: 2.4928 | LR: 6.32e-05 | 4004.9ms/step | 15.0 samples/s
+Step 38970/50462 | Loss: 2.5180 | LR: 6.31e-05 | 4014.2ms/step | 14.9 samples/s
+Step 38980/50462 | Loss: 2.5604 | LR: 6.30e-05 | 3999.0ms/step | 15.0 samples/s
+Step 38990/50462 | Loss: 2.5140 | LR: 6.29e-05 | 4000.7ms/step | 15.0 samples/s
+Step 39000/50462 | Loss: 2.5415 | LR: 6.28e-05 | 4008.3ms/step | 15.0 samples/s
+Validation at step 39000: Loss = 2.5153, Perplexity = 12.37
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-39000
+Step 39010/50462 | Loss: 2.5268 | LR: 6.27e-05 | 5968.9ms/step | 10.1 samples/s
+Step 39020/50462 | Loss: 2.4951 | LR: 6.26e-05 | 4003.1ms/step | 15.0 samples/s
+Step 39030/50462 | Loss: 2.5821 | LR: 6.25e-05 | 3999.4ms/step | 15.0 samples/s
+Step 39040/50462 | Loss: 2.5043 | LR: 6.23e-05 | 4009.1ms/step | 15.0 samples/s
+Step 39050/50462 | Loss: 2.4923 | LR: 6.22e-05 | 4007.6ms/step | 15.0 samples/s
+Step 39060/50462 | Loss: 2.5380 | LR: 6.21e-05 | 4004.9ms/step | 15.0 samples/s
+Step 39070/50462 | Loss: 2.5371 | LR: 6.20e-05 | 3988.1ms/step | 15.0 samples/s
+Step 39080/50462 | Loss: 2.5212 | LR: 6.19e-05 | 4006.2ms/step | 15.0 samples/s
+Step 39090/50462 | Loss: 2.5256 | LR: 6.18e-05 | 4008.7ms/step | 15.0 samples/s
+Step 39100/50462 | Loss: 2.5145 | LR: 6.17e-05 | 4001.5ms/step | 15.0 samples/s
+Step 39110/50462 | Loss: 2.5222 | LR: 6.16e-05 | 4003.8ms/step | 15.0 samples/s
+Step 39120/50462 | Loss: 2.5528 | LR: 6.15e-05 | 4003.7ms/step | 15.0 samples/s
+Step 39130/50462 | Loss: 2.5391 | LR: 6.14e-05 | 4000.8ms/step | 15.0 samples/s
+Step 39140/50462 | Loss: 2.5232 | LR: 6.13e-05 | 4001.8ms/step | 15.0 samples/s
+Step 39150/50462 | Loss: 2.5392 | LR: 6.12e-05 | 4010.7ms/step | 15.0 samples/s
+Step 39160/50462 | Loss: 2.5816 | LR: 6.11e-05 | 4006.3ms/step | 15.0 samples/s
+Step 39170/50462 | Loss: 2.5041 | LR: 6.10e-05 | 4001.2ms/step | 15.0 samples/s
+Step 39180/50462 | Loss: 2.5259 | LR: 6.09e-05 | 4013.6ms/step | 14.9 samples/s
+Step 39190/50462 | Loss: 2.5302 | LR: 6.08e-05 | 4010.9ms/step | 15.0 samples/s
+Step 39200/50462 | Loss: 2.5225 | LR: 6.07e-05 | 4009.2ms/step | 15.0 samples/s
+Step 39210/50462 | Loss: 2.5333 | LR: 6.06e-05 | 4004.9ms/step | 15.0 samples/s
+Step 39220/50462 | Loss: 2.5238 | LR: 6.05e-05 | 4006.6ms/step | 15.0 samples/s
+Step 39230/50462 | Loss: 2.5580 | LR: 6.04e-05 | 4005.6ms/step | 15.0 samples/s
+Step 39240/50462 | Loss: 2.5330 | LR: 6.03e-05 | 4005.1ms/step | 15.0 samples/s
+Step 39250/50462 | Loss: 2.5298 | LR: 6.02e-05 | 4008.3ms/step | 15.0 samples/s
+Validation at step 39250: Loss = 2.5147, Perplexity = 12.36
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-39250
+Step 39260/50462 | Loss: 2.5355 | LR: 6.01e-05 | 5946.9ms/step | 10.1 samples/s
+Step 39270/50462 | Loss: 2.5374 | LR: 6.00e-05 | 4011.4ms/step | 15.0 samples/s
+Step 39280/50462 | Loss: 2.5800 | LR: 5.99e-05 | 4001.8ms/step | 15.0 samples/s
+Step 39290/50462 | Loss: 2.5258 | LR: 5.98e-05 | 4002.2ms/step | 15.0 samples/s
+Step 39300/50462 | Loss: 2.5512 | LR: 5.97e-05 | 4005.1ms/step | 15.0 samples/s
+Step 39310/50462 | Loss: 2.5680 | LR: 5.96e-05 | 4012.6ms/step | 15.0 samples/s
+Step 39320/50462 | Loss: 2.5396 | LR: 5.95e-05 | 4005.5ms/step | 15.0 samples/s
+Step 39330/50462 | Loss: 2.4976 | LR: 5.94e-05 | 4007.1ms/step | 15.0 samples/s
+Step 39340/50462 | Loss: 2.5320 | LR: 5.93e-05 | 4014.7ms/step | 14.9 samples/s
+Step 39350/50462 | Loss: 2.5123 | LR: 5.91e-05 | 4008.1ms/step | 15.0 samples/s
+Step 39360/50462 | Loss: 2.5155 | LR: 5.90e-05 | 4018.9ms/step | 14.9 samples/s
+Step 39370/50462 | Loss: 2.5245 | LR: 5.89e-05 | 4006.9ms/step | 15.0 samples/s
+Step 39380/50462 | Loss: 2.5099 | LR: 5.88e-05 | 4000.3ms/step | 15.0 samples/s
+Step 39390/50462 | Loss: 2.5313 | LR: 5.87e-05 | 4008.8ms/step | 15.0 samples/s
+Step 39400/50462 | Loss: 2.5152 | LR: 5.86e-05 | 4012.8ms/step | 15.0 samples/s
+Step 39410/50462 | Loss: 2.5459 | LR: 5.85e-05 | 4014.3ms/step | 14.9 samples/s
+Step 39420/50462 | Loss: 2.5242 | LR: 5.84e-05 | 3995.2ms/step | 15.0 samples/s
+Step 39430/50462 | Loss: 2.5140 | LR: 5.83e-05 | 4006.9ms/step | 15.0 samples/s
+Step 39440/50462 | Loss: 2.5428 | LR: 5.82e-05 | 4015.4ms/step | 14.9 samples/s
+Step 39450/50462 | Loss: 2.5944 | LR: 5.81e-05 | 4006.8ms/step | 15.0 samples/s
+Step 39460/50462 | Loss: 2.5178 | LR: 5.80e-05 | 4017.0ms/step | 14.9 samples/s
+Step 39470/50462 | Loss: 2.5450 | LR: 5.79e-05 | 3998.8ms/step | 15.0 samples/s
+Step 39480/50462 | Loss: 2.5051 | LR: 5.78e-05 | 4020.4ms/step | 14.9 samples/s
+Step 39490/50462 | Loss: 2.4963 | LR: 5.77e-05 | 4021.2ms/step | 14.9 samples/s
+Step 39500/50462 | Loss: 2.5452 | LR: 5.76e-05 | 4013.7ms/step | 14.9 samples/s
+Validation at step 39500: Loss = 2.5140, Perplexity = 12.35
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-39500
+Step 39510/50462 | Loss: 2.5403 | LR: 5.75e-05 | 5975.0ms/step | 10.0 samples/s
+Step 39520/50462 | Loss: 2.5472 | LR: 5.74e-05 | 4003.1ms/step | 15.0 samples/s
+Step 39530/50462 | Loss: 2.5334 | LR: 5.73e-05 | 4010.2ms/step | 15.0 samples/s
+Step 39540/50462 | Loss: 2.5224 | LR: 5.72e-05 | 4021.7ms/step | 14.9 samples/s
+Step 39550/50462 | Loss: 2.5452 | LR: 5.71e-05 | 4019.2ms/step | 14.9 samples/s
+Step 39560/50462 | Loss: 2.5217 | LR: 5.70e-05 | 3999.4ms/step | 15.0 samples/s
+Step 39570/50462 | Loss: 2.5778 | LR: 5.69e-05 | 4000.3ms/step | 15.0 samples/s
+Step 39580/50462 | Loss: 2.5182 | LR: 5.68e-05 | 4008.9ms/step | 15.0 samples/s
+Step 39590/50462 | Loss: 2.5446 | LR: 5.67e-05 | 4017.3ms/step | 14.9 samples/s
+Step 39600/50462 | Loss: 2.5355 | LR: 5.66e-05 | 4001.4ms/step | 15.0 samples/s
+Step 39610/50462 | Loss: 2.5099 | LR: 5.65e-05 | 4001.9ms/step | 15.0 samples/s
+Step 39620/50462 | Loss: 2.4746 | LR: 5.64e-05 | 4008.0ms/step | 15.0 samples/s
+Step 39630/50462 | Loss: 2.5512 | LR: 5.63e-05 | 4012.7ms/step | 15.0 samples/s
+Step 39640/50462 | Loss: 2.5394 | LR: 5.62e-05 | 4018.7ms/step | 14.9 samples/s
+Step 39650/50462 | Loss: 2.5413 | LR: 5.61e-05 | 4014.3ms/step | 14.9 samples/s
+Step 39660/50462 | Loss: 2.5323 | LR: 5.60e-05 | 4017.1ms/step | 14.9 samples/s
+Step 39670/50462 | Loss: 2.5298 | LR: 5.59e-05 | 4006.0ms/step | 15.0 samples/s
+Step 39680/50462 | Loss: 2.5260 | LR: 5.58e-05 | 4002.2ms/step | 15.0 samples/s
+Step 39690/50462 | Loss: 2.5366 | LR: 5.57e-05 | 3997.4ms/step | 15.0 samples/s
+Step 39700/50462 | Loss: 2.5185 | LR: 5.56e-05 | 4009.3ms/step | 15.0 samples/s
+Step 39710/50462 | Loss: 2.5375 | LR: 5.55e-05 | 4009.0ms/step | 15.0 samples/s
+Step 39720/50462 | Loss: 2.5514 | LR: 5.54e-05 | 4014.6ms/step | 14.9 samples/s
+Step 39730/50462 | Loss: 2.5145 | LR: 5.53e-05 | 4003.4ms/step | 15.0 samples/s
+Step 39740/50462 | Loss: 2.5117 | LR: 5.52e-05 | 4010.6ms/step | 15.0 samples/s
+Step 39750/50462 | Loss: 2.5405 | LR: 5.51e-05 | 4012.6ms/step | 15.0 samples/s
+Validation at step 39750: Loss = 2.5142, Perplexity = 12.36
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-39750
+Step 39760/50462 | Loss: 2.5424 | LR: 5.50e-05 | 6040.2ms/step | 9.9 samples/s
+Step 39770/50462 | Loss: 2.4993 | LR: 5.49e-05 | 4016.8ms/step | 14.9 samples/s
+Step 39780/50462 | Loss: 2.5170 | LR: 5.48e-05 | 4014.7ms/step | 14.9 samples/s
+Step 39790/50462 | Loss: 2.5228 | LR: 5.47e-05 | 4009.8ms/step | 15.0 samples/s
+Step 39800/50462 | Loss: 2.5443 | LR: 5.46e-05 | 4001.9ms/step | 15.0 samples/s
+Step 39810/50462 | Loss: 2.4825 | LR: 5.45e-05 | 4009.5ms/step | 15.0 samples/s
+Step 39820/50462 | Loss: 2.5189 | LR: 5.44e-05 | 4015.0ms/step | 14.9 samples/s
+Step 39830/50462 | Loss: 2.5114 | LR: 5.43e-05 | 4011.0ms/step | 15.0 samples/s
+Step 39840/50462 | Loss: 2.5643 | LR: 5.42e-05 | 4013.9ms/step | 14.9 samples/s
+Step 39850/50462 | Loss: 2.5269 | LR: 5.41e-05 | 4012.2ms/step | 15.0 samples/s
+Step 39860/50462 | Loss: 2.5235 | LR: 5.40e-05 | 4013.5ms/step | 14.9 samples/s
+Step 39870/50462 | Loss: 2.5342 | LR: 5.39e-05 | 4014.5ms/step | 14.9 samples/s
+Step 39880/50462 | Loss: 2.5516 | LR: 5.38e-05 | 4016.3ms/step | 14.9 samples/s
+Step 39890/50462 | Loss: 2.4943 | LR: 5.38e-05 | 3998.2ms/step | 15.0 samples/s
+Step 39900/50462 | Loss: 2.5347 | LR: 5.37e-05 | 4007.6ms/step | 15.0 samples/s
+Step 39910/50462 | Loss: 2.4953 | LR: 5.36e-05 | 4024.9ms/step | 14.9 samples/s
+Step 39920/50462 | Loss: 2.5496 | LR: 5.35e-05 | 4006.5ms/step | 15.0 samples/s
+Step 39930/50462 | Loss: 2.5228 | LR: 5.34e-05 | 4002.6ms/step | 15.0 samples/s
+Step 39940/50462 | Loss: 2.5518 | LR: 5.33e-05 | 4013.9ms/step | 14.9 samples/s
+Step 39950/50462 | Loss: 2.5025 | LR: 5.32e-05 | 4008.0ms/step | 15.0 samples/s
+Step 39960/50462 | Loss: 2.5485 | LR: 5.31e-05 | 4013.8ms/step | 14.9 samples/s
+Step 39970/50462 | Loss: 2.5016 | LR: 5.30e-05 | 3999.7ms/step | 15.0 samples/s
+Step 39980/50462 | Loss: 2.5409 | LR: 5.29e-05 | 4007.2ms/step | 15.0 samples/s
+Step 39990/50462 | Loss: 2.5106 | LR: 5.28e-05 | 3993.4ms/step | 15.0 samples/s
+Step 40000/50462 | Loss: 2.5219 | LR: 5.27e-05 | 4006.8ms/step | 15.0 samples/s
+Validation at step 40000: Loss = 2.5144, Perplexity = 12.36
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-40000
+Step 40010/50462 | Loss: 2.5225 | LR: 5.26e-05 | 5978.1ms/step | 10.0 samples/s
+Step 40020/50462 | Loss: 2.5309 | LR: 5.25e-05 | 4015.9ms/step | 14.9 samples/s
+Step 40030/50462 | Loss: 2.5096 | LR: 5.24e-05 | 4008.3ms/step | 15.0 samples/s
+Step 40040/50462 | Loss: 2.5430 | LR: 5.23e-05 | 4016.0ms/step | 14.9 samples/s
+Step 40050/50462 | Loss: 2.5232 | LR: 5.22e-05 | 4004.6ms/step | 15.0 samples/s
+Step 40060/50462 | Loss: 2.5315 | LR: 5.21e-05 | 4019.3ms/step | 14.9 samples/s
+Step 40070/50462 | Loss: 2.5196 | LR: 5.20e-05 | 3990.4ms/step | 15.0 samples/s
+Step 40080/50462 | Loss: 2.5333 | LR: 5.19e-05 | 4003.2ms/step | 15.0 samples/s
+Step 40090/50462 | Loss: 2.5457 | LR: 5.18e-05 | 4000.8ms/step | 15.0 samples/s
+Step 40100/50462 | Loss: 2.5135 | LR: 5.17e-05 | 3998.7ms/step | 15.0 samples/s
+Step 40110/50462 | Loss: 2.5633 | LR: 5.16e-05 | 4005.9ms/step | 15.0 samples/s
+Step 40120/50462 | Loss: 2.5690 | LR: 5.15e-05 | 4015.2ms/step | 14.9 samples/s
+Step 40130/50462 | Loss: 2.5397 | LR: 5.14e-05 | 4017.3ms/step | 14.9 samples/s
+Step 40140/50462 | Loss: 2.5211 | LR: 5.13e-05 | 4019.6ms/step | 14.9 samples/s
+Step 40150/50462 | Loss: 2.5124 | LR: 5.12e-05 | 4017.4ms/step | 14.9 samples/s
+Step 40160/50462 | Loss: 2.5518 | LR: 5.11e-05 | 4014.8ms/step | 14.9 samples/s
+Step 40170/50462 | Loss: 2.4963 | LR: 5.10e-05 | 4003.6ms/step | 15.0 samples/s
+Step 40180/50462 | Loss: 2.5508 | LR: 5.09e-05 | 4010.8ms/step | 15.0 samples/s
+Step 40190/50462 | Loss: 2.5099 | LR: 5.09e-05 | 4004.3ms/step | 15.0 samples/s
+Step 40200/50462 | Loss: 2.5201 | LR: 5.08e-05 | 4004.1ms/step | 15.0 samples/s
+Step 40210/50462 | Loss: 2.4976 | LR: 5.07e-05 | 4005.2ms/step | 15.0 samples/s
+Step 40220/50462 | Loss: 2.5117 | LR: 5.06e-05 | 4005.2ms/step | 15.0 samples/s
+Step 40230/50462 | Loss: 2.5380 | LR: 5.05e-05 | 4005.2ms/step | 15.0 samples/s
+Step 40240/50462 | Loss: 2.5342 | LR: 5.04e-05 | 4008.8ms/step | 15.0 samples/s
+Step 40250/50462 | Loss: 2.5363 | LR: 5.03e-05 | 4003.6ms/step | 15.0 samples/s
+Validation at step 40250: Loss = 2.5140, Perplexity = 12.35
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-40250
+Step 40260/50462 | Loss: 2.5399 | LR: 5.02e-05 | 6007.2ms/step | 10.0 samples/s
+Step 40270/50462 | Loss: 2.5376 | LR: 5.01e-05 | 4024.9ms/step | 14.9 samples/s
+Step 40280/50462 | Loss: 2.5174 | LR: 5.00e-05 | 4011.5ms/step | 15.0 samples/s
+Step 40290/50462 | Loss: 2.5202 | LR: 4.99e-05 | 4016.7ms/step | 14.9 samples/s
+Step 40300/50462 | Loss: 2.5231 | LR: 4.98e-05 | 4000.3ms/step | 15.0 samples/s
+Step 40310/50462 | Loss: 2.5439 | LR: 4.97e-05 | 4009.4ms/step | 15.0 samples/s
+Step 40320/50462 | Loss: 2.5283 | LR: 4.96e-05 | 4002.1ms/step | 15.0 samples/s
+Step 40330/50462 | Loss: 2.5249 | LR: 4.95e-05 | 4014.1ms/step | 14.9 samples/s
+Step 40340/50462 | Loss: 2.5171 | LR: 4.94e-05 | 4000.6ms/step | 15.0 samples/s
+Step 40350/50462 | Loss: 2.4994 | LR: 4.93e-05 | 4007.6ms/step | 15.0 samples/s
+Step 40360/50462 | Loss: 2.5192 | LR: 4.92e-05 | 4003.8ms/step | 15.0 samples/s
+Step 40370/50462 | Loss: 2.4739 | LR: 4.91e-05 | 4000.2ms/step | 15.0 samples/s
+Step 40380/50462 | Loss: 2.5332 | LR: 4.90e-05 | 4018.2ms/step | 14.9 samples/s
+Step 40390/50462 | Loss: 2.5144 | LR: 4.90e-05 | 4014.3ms/step | 14.9 samples/s
+Step 40400/50462 | Loss: 2.5392 | LR: 4.89e-05 | 4011.1ms/step | 15.0 samples/s
+Step 40410/50462 | Loss: 2.5479 | LR: 4.88e-05 | 4019.2ms/step | 14.9 samples/s
+Step 40420/50462 | Loss: 2.5107 | LR: 4.87e-05 | 4001.6ms/step | 15.0 samples/s
+Step 40430/50462 | Loss: 2.5407 | LR: 4.86e-05 | 4012.6ms/step | 15.0 samples/s
+Step 40440/50462 | Loss: 2.5588 | LR: 4.85e-05 | 4010.6ms/step | 15.0 samples/s
+Step 40450/50462 | Loss: 2.5069 | LR: 4.84e-05 | 4016.1ms/step | 14.9 samples/s
+Step 40460/50462 | Loss: 2.5123 | LR: 4.83e-05 | 4013.3ms/step | 15.0 samples/s
+Step 40470/50462 | Loss: 2.5297 | LR: 4.82e-05 | 4008.5ms/step | 15.0 samples/s
+Step 40480/50462 | Loss: 2.5401 | LR: 4.81e-05 | 4004.4ms/step | 15.0 samples/s
+Step 40490/50462 | Loss: 2.5168 | LR: 4.80e-05 | 4008.5ms/step | 15.0 samples/s
+Step 40500/50462 | Loss: 2.5467 | LR: 4.79e-05 | 4016.3ms/step | 14.9 samples/s
+Validation at step 40500: Loss = 2.5138, Perplexity = 12.35
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-40500
+Step 40510/50462 | Loss: 2.5449 | LR: 4.78e-05 | 6018.6ms/step | 10.0 samples/s
+Step 40520/50462 | Loss: 2.5054 | LR: 4.77e-05 | 4016.1ms/step | 14.9 samples/s
+Step 40530/50462 | Loss: 2.5126 | LR: 4.76e-05 | 4009.9ms/step | 15.0 samples/s
+Step 40540/50462 | Loss: 2.5323 | LR: 4.76e-05 | 4013.4ms/step | 15.0 samples/s
+Step 40550/50462 | Loss: 2.5121 | LR: 4.75e-05 | 4013.6ms/step | 14.9 samples/s
+Step 40560/50462 | Loss: 2.5098 | LR: 4.74e-05 | 4013.8ms/step | 14.9 samples/s
+Step 40570/50462 | Loss: 2.5412 | LR: 4.73e-05 | 4013.2ms/step | 15.0 samples/s
+Step 40580/50462 | Loss: 2.5172 | LR: 4.72e-05 | 4014.5ms/step | 14.9 samples/s
+Step 40590/50462 | Loss: 2.5357 | LR: 4.71e-05 | 4016.8ms/step | 14.9 samples/s
+Step 40600/50462 | Loss: 2.5051 | LR: 4.70e-05 | 4016.5ms/step | 14.9 samples/s
+Step 40610/50462 | Loss: 2.5517 | LR: 4.69e-05 | 3996.6ms/step | 15.0 samples/s
+Step 40620/50462 | Loss: 2.5123 | LR: 4.68e-05 | 4005.0ms/step | 15.0 samples/s
+Step 40630/50462 | Loss: 2.5527 | LR: 4.67e-05 | 4010.3ms/step | 15.0 samples/s
+Step 40640/50462 | Loss: 2.5468 | LR: 4.66e-05 | 4003.2ms/step | 15.0 samples/s
+Step 40650/50462 | Loss: 2.5537 | LR: 4.65e-05 | 4005.6ms/step | 15.0 samples/s
+Step 40660/50462 | Loss: 2.5046 | LR: 4.64e-05 | 4015.7ms/step | 14.9 samples/s
+Step 40670/50462 | Loss: 2.5486 | LR: 4.64e-05 | 4005.3ms/step | 15.0 samples/s
+Step 40680/50462 | Loss: 2.5265 | LR: 4.63e-05 | 4006.0ms/step | 15.0 samples/s
+Step 40690/50462 | Loss: 2.5485 | LR: 4.62e-05 | 4017.1ms/step | 14.9 samples/s
+Step 40700/50462 | Loss: 2.5038 | LR: 4.61e-05 | 4007.0ms/step | 15.0 samples/s
+Step 40710/50462 | Loss: 2.4995 | LR: 4.60e-05 | 4005.6ms/step | 15.0 samples/s
+Step 40720/50462 | Loss: 2.5343 | LR: 4.59e-05 | 4007.6ms/step | 15.0 samples/s
+Step 40730/50462 | Loss: 2.5376 | LR: 4.58e-05 | 4006.0ms/step | 15.0 samples/s
+Step 40740/50462 | Loss: 2.5358 | LR: 4.57e-05 | 4007.9ms/step | 15.0 samples/s
+Step 40750/50462 | Loss: 2.5441 | LR: 4.56e-05 | 4018.8ms/step | 14.9 samples/s
+Validation at step 40750: Loss = 2.5142, Perplexity = 12.36
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-40750
+Step 40760/50462 | Loss: 2.5616 | LR: 4.55e-05 | 6113.1ms/step | 9.8 samples/s
+Step 40770/50462 | Loss: 2.5284 | LR: 4.54e-05 | 4006.6ms/step | 15.0 samples/s
+Step 40780/50462 | Loss: 2.5351 | LR: 4.54e-05 | 4013.7ms/step | 14.9 samples/s
+Step 40790/50462 | Loss: 2.5405 | LR: 4.53e-05 | 4011.4ms/step | 15.0 samples/s
+Step 40800/50462 | Loss: 2.5533 | LR: 4.52e-05 | 4011.3ms/step | 15.0 samples/s
+Step 40810/50462 | Loss: 2.5353 | LR: 4.51e-05 | 3999.1ms/step | 15.0 samples/s
+Step 40820/50462 | Loss: 2.5097 | LR: 4.50e-05 | 4003.0ms/step | 15.0 samples/s
+Step 40830/50462 | Loss: 2.5362 | LR: 4.49e-05 | 4021.2ms/step | 14.9 samples/s
+Step 40840/50462 | Loss: 2.4932 | LR: 4.48e-05 | 3998.5ms/step | 15.0 samples/s
+Step 40850/50462 | Loss: 2.5236 | LR: 4.47e-05 | 4009.0ms/step | 15.0 samples/s
+Step 40860/50462 | Loss: 2.5116 | LR: 4.46e-05 | 4000.3ms/step | 15.0 samples/s
+Step 40870/50462 | Loss: 2.5206 | LR: 4.45e-05 | 4013.8ms/step | 14.9 samples/s
+Step 40880/50462 | Loss: 2.5283 | LR: 4.45e-05 | 4022.5ms/step | 14.9 samples/s
+Step 40890/50462 | Loss: 2.5058 | LR: 4.44e-05 | 4011.7ms/step | 15.0 samples/s
+Step 40900/50462 | Loss: 2.5314 | LR: 4.43e-05 | 4013.5ms/step | 14.9 samples/s
+Step 40910/50462 | Loss: 2.5079 | LR: 4.42e-05 | 4001.1ms/step | 15.0 samples/s
+Step 40920/50462 | Loss: 2.5369 | LR: 4.41e-05 | 4017.9ms/step | 14.9 samples/s
+Step 40930/50462 | Loss: 2.5192 | LR: 4.40e-05 | 4004.8ms/step | 15.0 samples/s
+Step 40940/50462 | Loss: 2.5180 | LR: 4.39e-05 | 4008.0ms/step | 15.0 samples/s
+Step 40950/50462 | Loss: 2.5770 | LR: 4.38e-05 | 4024.4ms/step | 14.9 samples/s
+Step 40960/50462 | Loss: 2.5337 | LR: 4.37e-05 | 4004.6ms/step | 15.0 samples/s
+Step 40970/50462 | Loss: 2.5261 | LR: 4.36e-05 | 4016.9ms/step | 14.9 samples/s
+Step 40980/50462 | Loss: 2.5395 | LR: 4.36e-05 | 4010.8ms/step | 15.0 samples/s
+Step 40990/50462 | Loss: 2.5225 | LR: 4.35e-05 | 4024.8ms/step | 14.9 samples/s
+Step 41000/50462 | Loss: 2.5120 | LR: 4.34e-05 | 4018.7ms/step | 14.9 samples/s
+Validation at step 41000: Loss = 2.5138, Perplexity = 12.35
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-41000
+Step 41010/50462 | Loss: 2.5142 | LR: 4.33e-05 | 6083.0ms/step | 9.9 samples/s
+Step 41020/50462 | Loss: 2.5342 | LR: 4.32e-05 | 4014.6ms/step | 14.9 samples/s
+Step 41030/50462 | Loss: 2.4986 | LR: 4.31e-05 | 4014.1ms/step | 14.9 samples/s
+Step 41040/50462 | Loss: 2.5457 | LR: 4.30e-05 | 4018.3ms/step | 14.9 samples/s
+Step 41050/50462 | Loss: 2.5463 | LR: 4.29e-05 | 4008.5ms/step | 15.0 samples/s
+Step 41060/50462 | Loss: 2.5428 | LR: 4.28e-05 | 4010.4ms/step | 15.0 samples/s
+Step 41070/50462 | Loss: 2.5159 | LR: 4.28e-05 | 3997.6ms/step | 15.0 samples/s
+Step 41080/50462 | Loss: 2.5320 | LR: 4.27e-05 | 4013.1ms/step | 15.0 samples/s
+Step 41090/50462 | Loss: 2.5029 | LR: 4.26e-05 | 4016.8ms/step | 14.9 samples/s
+Step 41100/50462 | Loss: 2.5280 | LR: 4.25e-05 | 4012.7ms/step | 15.0 samples/s
+Step 41110/50462 | Loss: 2.5120 | LR: 4.24e-05 | 4004.6ms/step | 15.0 samples/s
+Step 41120/50462 | Loss: 2.5210 | LR: 4.23e-05 | 4009.0ms/step | 15.0 samples/s
+Step 41130/50462 | Loss: 2.5365 | LR: 4.22e-05 | 3998.4ms/step | 15.0 samples/s
+Step 41140/50462 | Loss: 2.5316 | LR: 4.21e-05 | 4011.5ms/step | 15.0 samples/s
+Step 41150/50462 | Loss: 2.5744 | LR: 4.21e-05 | 4007.3ms/step | 15.0 samples/s
+Step 41160/50462 | Loss: 2.5142 | LR: 4.20e-05 | 4011.6ms/step | 15.0 samples/s
+Step 41170/50462 | Loss: 2.5081 | LR: 4.19e-05 | 4008.9ms/step | 15.0 samples/s
+Step 41180/50462 | Loss: 2.4817 | LR: 4.18e-05 | 4007.7ms/step | 15.0 samples/s
+Step 41190/50462 | Loss: 2.5155 | LR: 4.17e-05 | 4012.6ms/step | 15.0 samples/s
+Step 41200/50462 | Loss: 2.5207 | LR: 4.16e-05 | 4011.1ms/step | 15.0 samples/s
+Step 41210/50462 | Loss: 2.5278 | LR: 4.15e-05 | 4015.0ms/step | 14.9 samples/s
+Step 41220/50462 | Loss: 2.5038 | LR: 4.14e-05 | 4008.0ms/step | 15.0 samples/s
+Step 41230/50462 | Loss: 2.5054 | LR: 4.14e-05 | 4021.4ms/step | 14.9 samples/s
+Step 41240/50462 | Loss: 2.4993 | LR: 4.13e-05 | 4002.6ms/step | 15.0 samples/s
+Step 41250/50462 | Loss: 2.4888 | LR: 4.12e-05 | 4014.7ms/step | 14.9 samples/s
+Validation at step 41250: Loss = 2.5137, Perplexity = 12.35
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-41250
+Step 41260/50462 | Loss: 2.5110 | LR: 4.11e-05 | 6007.1ms/step | 10.0 samples/s
+Step 41270/50462 | Loss: 2.5246 | LR: 4.10e-05 | 4025.7ms/step | 14.9 samples/s
+Step 41280/50462 | Loss: 2.4996 | LR: 4.09e-05 | 4016.9ms/step | 14.9 samples/s
+Step 41290/50462 | Loss: 2.5258 | LR: 4.08e-05 | 4002.0ms/step | 15.0 samples/s
+Step 41300/50462 | Loss: 2.5121 | LR: 4.07e-05 | 4015.8ms/step | 14.9 samples/s
+Step 41310/50462 | Loss: 2.4976 | LR: 4.07e-05 | 4001.5ms/step | 15.0 samples/s
+Step 41320/50462 | Loss: 2.5293 | LR: 4.06e-05 | 4005.9ms/step | 15.0 samples/s
+Step 41330/50462 | Loss: 2.5348 | LR: 4.05e-05 | 4009.0ms/step | 15.0 samples/s
+Step 41340/50462 | Loss: 2.5139 | LR: 4.04e-05 | 4010.9ms/step | 15.0 samples/s
+Step 41350/50462 | Loss: 2.5209 | LR: 4.03e-05 | 4010.9ms/step | 15.0 samples/s
+Step 41360/50462 | Loss: 2.5241 | LR: 4.02e-05 | 3997.1ms/step | 15.0 samples/s
+Step 41370/50462 | Loss: 2.5372 | LR: 4.01e-05 | 4012.6ms/step | 15.0 samples/s
+Step 41380/50462 | Loss: 2.5285 | LR: 4.01e-05 | 4006.5ms/step | 15.0 samples/s
+Step 41390/50462 | Loss: 2.5382 | LR: 4.00e-05 | 4001.6ms/step | 15.0 samples/s
+Step 41400/50462 | Loss: 2.5235 | LR: 3.99e-05 | 4012.8ms/step | 15.0 samples/s
+Step 41410/50462 | Loss: 2.5132 | LR: 3.98e-05 | 4010.1ms/step | 15.0 samples/s
+Step 41420/50462 | Loss: 2.5363 | LR: 3.97e-05 | 4019.4ms/step | 14.9 samples/s
+Step 41430/50462 | Loss: 2.5527 | LR: 3.96e-05 | 4014.1ms/step | 14.9 samples/s
+Step 41440/50462 | Loss: 2.5370 | LR: 3.95e-05 | 4008.4ms/step | 15.0 samples/s
+Step 41450/50462 | Loss: 2.5460 | LR: 3.95e-05 | 4019.8ms/step | 14.9 samples/s
+Step 41460/50462 | Loss: 2.5152 | LR: 3.94e-05 | 4009.3ms/step | 15.0 samples/s
+Step 41470/50462 | Loss: 2.5329 | LR: 3.93e-05 | 4002.4ms/step | 15.0 samples/s
+Step 41480/50462 | Loss: 2.5015 | LR: 3.92e-05 | 4020.1ms/step | 14.9 samples/s
+Step 41490/50462 | Loss: 2.5214 | LR: 3.91e-05 | 4009.2ms/step | 15.0 samples/s
+Step 41500/50462 | Loss: 2.5022 | LR: 3.90e-05 | 4006.1ms/step | 15.0 samples/s
+Validation at step 41500: Loss = 2.5127, Perplexity = 12.34
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-41500
+Step 41510/50462 | Loss: 2.5158 | LR: 3.90e-05 | 6015.6ms/step | 10.0 samples/s
+Step 41520/50462 | Loss: 2.5272 | LR: 3.89e-05 | 4000.3ms/step | 15.0 samples/s
+Step 41530/50462 | Loss: 2.5400 | LR: 3.88e-05 | 3998.2ms/step | 15.0 samples/s
+Step 41540/50462 | Loss: 2.5086 | LR: 3.87e-05 | 4008.2ms/step | 15.0 samples/s
+Step 41550/50462 | Loss: 2.5544 | LR: 3.86e-05 | 4015.9ms/step | 14.9 samples/s
+Step 41560/50462 | Loss: 2.5107 | LR: 3.85e-05 | 4008.5ms/step | 15.0 samples/s
+Step 41570/50462 | Loss: 2.5127 | LR: 3.84e-05 | 4018.9ms/step | 14.9 samples/s
+Step 41580/50462 | Loss: 2.5236 | LR: 3.84e-05 | 4005.6ms/step | 15.0 samples/s
+Step 41590/50462 | Loss: 2.5112 | LR: 3.83e-05 | 4009.0ms/step | 15.0 samples/s
+Step 41600/50462 | Loss: 2.5354 | LR: 3.82e-05 | 3996.0ms/step | 15.0 samples/s
+Step 41610/50462 | Loss: 2.5519 | LR: 3.81e-05 | 4012.0ms/step | 15.0 samples/s
+Step 41620/50462 | Loss: 2.5336 | LR: 3.80e-05 | 4022.8ms/step | 14.9 samples/s
+Step 41630/50462 | Loss: 2.5274 | LR: 3.79e-05 | 4008.2ms/step | 15.0 samples/s
+Step 41640/50462 | Loss: 2.5526 | LR: 3.79e-05 | 4017.5ms/step | 14.9 samples/s
+Step 41650/50462 | Loss: 2.5316 | LR: 3.78e-05 | 4013.2ms/step | 15.0 samples/s
+Step 41660/50462 | Loss: 2.5129 | LR: 3.77e-05 | 4008.6ms/step | 15.0 samples/s
+Step 41670/50462 | Loss: 2.5274 | LR: 3.76e-05 | 4005.7ms/step | 15.0 samples/s
+Step 41680/50462 | Loss: 2.5415 | LR: 3.75e-05 | 4013.5ms/step | 14.9 samples/s
+Step 41690/50462 | Loss: 2.5204 | LR: 3.74e-05 | 4023.1ms/step | 14.9 samples/s
+Step 41700/50462 | Loss: 2.5329 | LR: 3.74e-05 | 4021.8ms/step | 14.9 samples/s
+Step 41710/50462 | Loss: 2.5468 | LR: 3.73e-05 | 4015.7ms/step | 14.9 samples/s
+Step 41720/50462 | Loss: 2.5391 | LR: 3.72e-05 | 4011.2ms/step | 15.0 samples/s
+Step 41730/50462 | Loss: 2.4665 | LR: 3.71e-05 | 4015.2ms/step | 14.9 samples/s
+Step 41740/50462 | Loss: 2.5286 | LR: 3.70e-05 | 4008.1ms/step | 15.0 samples/s
+Step 41750/50462 | Loss: 2.4952 | LR: 3.69e-05 | 4019.0ms/step | 14.9 samples/s
+Validation at step 41750: Loss = 2.5134, Perplexity = 12.35
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-41750
+Step 41760/50462 | Loss: 2.5262 | LR: 3.69e-05 | 5985.5ms/step | 10.0 samples/s
+Step 41770/50462 | Loss: 2.5425 | LR: 3.68e-05 | 4010.0ms/step | 15.0 samples/s
+Step 41780/50462 | Loss: 2.5098 | LR: 3.67e-05 | 4015.9ms/step | 14.9 samples/s
+Step 41790/50462 | Loss: 2.5031 | LR: 3.66e-05 | 4013.3ms/step | 15.0 samples/s
+Step 41800/50462 | Loss: 2.5011 | LR: 3.65e-05 | 4015.4ms/step | 14.9 samples/s
+Step 41810/50462 | Loss: 2.5073 | LR: 3.64e-05 | 4013.6ms/step | 14.9 samples/s
+Step 41820/50462 | Loss: 2.5483 | LR: 3.64e-05 | 4017.3ms/step | 14.9 samples/s
+Step 41830/50462 | Loss: 2.5514 | LR: 3.63e-05 | 4005.4ms/step | 15.0 samples/s
+Step 41840/50462 | Loss: 2.5439 | LR: 3.62e-05 | 4003.0ms/step | 15.0 samples/s
+Step 41850/50462 | Loss: 2.5347 | LR: 3.61e-05 | 4015.6ms/step | 14.9 samples/s
+Step 41860/50462 | Loss: 2.5376 | LR: 3.60e-05 | 4010.9ms/step | 15.0 samples/s
+Step 41870/50462 | Loss: 2.5340 | LR: 3.60e-05 | 4007.8ms/step | 15.0 samples/s
+Step 41880/50462 | Loss: 2.5312 | LR: 3.59e-05 | 4014.3ms/step | 14.9 samples/s
+Step 41890/50462 | Loss: 2.5281 | LR: 3.58e-05 | 4012.0ms/step | 15.0 samples/s
+Step 41900/50462 | Loss: 2.4941 | LR: 3.57e-05 | 4003.7ms/step | 15.0 samples/s
+Step 41910/50462 | Loss: 2.5519 | LR: 3.56e-05 | 4010.2ms/step | 15.0 samples/s
+Step 41920/50462 | Loss: 2.5021 | LR: 3.55e-05 | 4012.3ms/step | 15.0 samples/s
+Step 41930/50462 | Loss: 2.5074 | LR: 3.55e-05 | 4009.7ms/step | 15.0 samples/s
+Step 41940/50462 | Loss: 2.5100 | LR: 3.54e-05 | 4008.2ms/step | 15.0 samples/s
+Step 41950/50462 | Loss: 2.5295 | LR: 3.53e-05 | 4006.8ms/step | 15.0 samples/s
+Step 41960/50462 | Loss: 2.4884 | LR: 3.52e-05 | 4016.7ms/step | 14.9 samples/s
+Step 41970/50462 | Loss: 2.5388 | LR: 3.51e-05 | 3997.9ms/step | 15.0 samples/s
+Step 41980/50462 | Loss: 2.4969 | LR: 3.51e-05 | 4018.1ms/step | 14.9 samples/s
+Step 41990/50462 | Loss: 2.5505 | LR: 3.50e-05 | 4016.9ms/step | 14.9 samples/s

epfl_final/SparseAtt/wandb/run-20260830_002911-8ynn1we6/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260830_002911-8ynn1we6/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-08-30T00:29:11.512071Z",
+  "args":  [
+    "--config",
+    "configs/p2_1p2b_mixed_seed71_4gpu_resume28500.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/SparseAtt.git",
+    "commit":  "8f862f749eb4d5c17f688eb1562235e3fe5b2550"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-train-4-gpu-0-32",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "6099864145920"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-d22bd9b7-89b0-ad1d-3ee4-0f537734e4b6"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-12856a3f-ee83-808a-6c9c-72aea4c95839"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-a0a4877f-b532-7a97-dd64-6d98c46f4daa"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-0c16e256-87dd-d37d-f2d1-80989e45e199"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "lu97jr8qzpiuuvvbhvwtwdys879404q0"
+}

epfl_final/SparseAtt/wandb/run-20260830_002911-8ynn1we6/logs/debug.log

@@ -0,0 +1,20 @@
+2026-08-30 00:29:11,535 INFO    MainThread:211 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-08-30 00:29:11,535 INFO    MainThread:211 [wandb_setup.py:_flush():81] Configure stats pid to 211
+2026-08-30 00:29:11,535 INFO    MainThread:211 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-30 00:29:11,535 INFO    MainThread:211 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260830_002911-8ynn1we6/logs/debug.log
+2026-08-30 00:29:11,535 INFO    MainThread:211 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260830_002911-8ynn1we6/logs/debug-internal.log
+2026-08-30 00:29:11,535 INFO    MainThread:211 [wandb_init.py:init():850] calling init triggers
+2026-08-30 00:29:11,535 INFO    MainThread:211 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-37000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 15, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 71, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b mixed 8k seed71 4gpu', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'mixed', 'seed71'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed', 'sparse_mask_seed': 71, 'sparse_mixed_nrb_set': [63, 47, 31, 21, 11], 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-08-30 00:29:11,536 INFO    MainThread:211 [wandb_init.py:init():898] starting backend
+2026-08-30 00:29:12,322 INFO    MainThread:211 [wandb_init.py:init():913] sending inform_init request
+2026-08-30 00:29:12,934 INFO    MainThread:211 [wandb_init.py:init():918] backend started and connected
+2026-08-30 00:29:12,940 INFO    MainThread:211 [wandb_init.py:init():988] updated telemetry
+2026-08-30 00:29:12,977 INFO    MainThread:211 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-08-30 00:29:13,444 INFO    MainThread:211 [wandb_init.py:init():1051] run resumed
+2026-08-30 00:29:13,451 INFO    MainThread:211 [wandb_init.py:init():1056] starting run threads in backend
+2026-08-30 00:29:13,717 INFO    MainThread:211 [wandb_run.py:_console_start():2554] atexit reg
+2026-08-30 00:29:13,718 INFO    MainThread:211 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-08-30 00:29:13,718 INFO    MainThread:211 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-08-30 00:29:13,718 INFO    MainThread:211 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-08-30 00:29:13,727 INFO    MainThread:211 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/run-20260830_002911-8ynn1we6/run-8ynn1we6.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7a78ecf1457d5841fd259440d0969fc36ecd41eb6b727360b7a77d2965b6219
+size 7077888

epfl_final/SparseAtt/wandb/run-20260903_094250-7tfq6nwm/files/output.log

@@ -0,0 +1,318 @@
+Starting training...
+Switching attention mode for step 1: dense (dense step interval)
+Step 10/50462 | Loss: 12.2551 | LR: 6.67e-06 | 7102.4ms/step | 8.4 samples/s
+Step 20/50462 | Loss: 11.9598 | LR: 1.33e-05 | 2812.8ms/step | 21.3 samples/s
+Step 30/50462 | Loss: 11.0754 | LR: 2.00e-05 | 2815.7ms/step | 21.3 samples/s
+Step 40/50462 | Loss: 10.1714 | LR: 2.67e-05 | 2807.9ms/step | 21.4 samples/s
+Step 50/50462 | Loss: 9.7674 | LR: 3.33e-05 | 2810.0ms/step | 21.4 samples/s
+Step 60/50462 | Loss: 9.3025 | LR: 4.00e-05 | 2813.9ms/step | 21.3 samples/s
+Step 70/50462 | Loss: 8.7641 | LR: 4.67e-05 | 2814.0ms/step | 21.3 samples/s
+Step 80/50462 | Loss: 8.3041 | LR: 5.33e-05 | 2804.3ms/step | 21.4 samples/s
+Step 90/50462 | Loss: 7.9228 | LR: 6.00e-05 | 2804.0ms/step | 21.4 samples/s
+Step 100/50462 | Loss: 7.5357 | LR: 6.67e-05 | 2816.2ms/step | 21.3 samples/s
+Step 110/50462 | Loss: 7.2514 | LR: 7.33e-05 | 2812.8ms/step | 21.3 samples/s
+Step 120/50462 | Loss: 7.0840 | LR: 8.00e-05 | 2816.8ms/step | 21.3 samples/s
+Step 130/50462 | Loss: 6.9494 | LR: 8.67e-05 | 2813.0ms/step | 21.3 samples/s
+Step 140/50462 | Loss: 6.8809 | LR: 9.33e-05 | 2816.8ms/step | 21.3 samples/s
+Step 150/50462 | Loss: 6.7717 | LR: 1.00e-04 | 2820.0ms/step | 21.3 samples/s
+Step 160/50462 | Loss: 6.6834 | LR: 1.07e-04 | 2819.2ms/step | 21.3 samples/s
+Step 170/50462 | Loss: 6.5857 | LR: 1.13e-04 | 2820.4ms/step | 21.3 samples/s
+Step 180/50462 | Loss: 6.4664 | LR: 1.20e-04 | 2820.4ms/step | 21.3 samples/s
+Step 190/50462 | Loss: 6.3764 | LR: 1.27e-04 | 2821.6ms/step | 21.3 samples/s
+Step 200/50462 | Loss: 6.3100 | LR: 1.33e-04 | 2820.9ms/step | 21.3 samples/s
+Step 210/50462 | Loss: 6.2129 | LR: 1.40e-04 | 2820.0ms/step | 21.3 samples/s
+Step 220/50462 | Loss: 6.1440 | LR: 1.47e-04 | 2821.8ms/step | 21.3 samples/s
+Step 230/50462 | Loss: 6.1100 | LR: 1.53e-04 | 2824.6ms/step | 21.2 samples/s
+Step 240/50462 | Loss: 6.0559 | LR: 1.60e-04 | 2826.9ms/step | 21.2 samples/s
+Step 250/50462 | Loss: 6.0007 | LR: 1.67e-04 | 2827.9ms/step | 21.2 samples/s
+Validation at step 250: Loss = 5.9550, Perplexity = 385.66
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-250
+Step 260/50462 | Loss: 5.9754 | LR: 1.73e-04 | 7358.2ms/step | 8.2 samples/s
+Step 270/50462 | Loss: 5.8872 | LR: 1.80e-04 | 2818.4ms/step | 21.3 samples/s
+Step 280/50462 | Loss: 5.8567 | LR: 1.87e-04 | 2827.1ms/step | 21.2 samples/s
+Step 290/50462 | Loss: 5.7731 | LR: 1.93e-04 | 2828.9ms/step | 21.2 samples/s
+Step 300/50462 | Loss: 5.7089 | LR: 2.00e-04 | 2830.0ms/step | 21.2 samples/s
+Step 310/50462 | Loss: 5.6774 | LR: 2.07e-04 | 2831.4ms/step | 21.2 samples/s
+Step 320/50462 | Loss: 5.6239 | LR: 2.13e-04 | 2830.4ms/step | 21.2 samples/s
+Step 330/50462 | Loss: 5.5787 | LR: 2.20e-04 | 2827.7ms/step | 21.2 samples/s
+Step 340/50462 | Loss: 5.5174 | LR: 2.27e-04 | 2830.8ms/step | 21.2 samples/s
+Step 350/50462 | Loss: 5.4471 | LR: 2.33e-04 | 2831.1ms/step | 21.2 samples/s
+Step 360/50462 | Loss: 5.4239 | LR: 2.40e-04 | 2830.3ms/step | 21.2 samples/s
+Step 370/50462 | Loss: 5.3810 | LR: 2.47e-04 | 2833.3ms/step | 21.2 samples/s
+Step 380/50462 | Loss: 5.3305 | LR: 2.53e-04 | 2826.2ms/step | 21.2 samples/s
+Step 390/50462 | Loss: 5.2998 | LR: 2.60e-04 | 2831.7ms/step | 21.2 samples/s
+Step 400/50462 | Loss: 5.2540 | LR: 2.67e-04 | 2831.9ms/step | 21.2 samples/s
+Step 410/50462 | Loss: 5.2052 | LR: 2.73e-04 | 2831.6ms/step | 21.2 samples/s
+Step 420/50462 | Loss: 5.1640 | LR: 2.80e-04 | 2836.0ms/step | 21.2 samples/s
+Step 430/50462 | Loss: 5.1449 | LR: 2.87e-04 | 2830.6ms/step | 21.2 samples/s
+Step 440/50462 | Loss: 5.0836 | LR: 2.93e-04 | 2830.7ms/step | 21.2 samples/s
+Step 450/50462 | Loss: 5.0509 | LR: 3.00e-04 | 2835.3ms/step | 21.2 samples/s
+Step 460/50462 | Loss: 4.9936 | LR: 3.07e-04 | 2835.2ms/step | 21.2 samples/s
+Step 470/50462 | Loss: 4.9375 | LR: 3.13e-04 | 2834.5ms/step | 21.2 samples/s
+Step 480/50462 | Loss: 4.9336 | LR: 3.20e-04 | 2835.2ms/step | 21.2 samples/s
+Step 490/50462 | Loss: 4.9644 | LR: 3.27e-04 | 2834.1ms/step | 21.2 samples/s
+Step 500/50462 | Loss: 4.8714 | LR: 3.33e-04 | 2835.8ms/step | 21.2 samples/s
+Validation at step 500: Loss = 4.8220, Perplexity = 124.22
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-500
+Step 510/50462 | Loss: 4.7846 | LR: 3.40e-04 | 4784.4ms/step | 12.5 samples/s
+Step 520/50462 | Loss: 4.8335 | LR: 3.47e-04 | 2823.5ms/step | 21.3 samples/s
+Step 530/50462 | Loss: 4.8520 | LR: 3.53e-04 | 2830.0ms/step | 21.2 samples/s
+Step 540/50462 | Loss: 4.7551 | LR: 3.60e-04 | 2829.8ms/step | 21.2 samples/s
+Step 550/50462 | Loss: 4.7056 | LR: 3.67e-04 | 2828.8ms/step | 21.2 samples/s
+Step 560/50462 | Loss: 4.6716 | LR: 3.73e-04 | 2831.2ms/step | 21.2 samples/s
+Step 570/50462 | Loss: 4.7708 | LR: 3.80e-04 | 2835.8ms/step | 21.2 samples/s
+Step 580/50462 | Loss: 4.7270 | LR: 3.87e-04 | 2829.3ms/step | 21.2 samples/s
+Step 590/50462 | Loss: 4.6082 | LR: 3.93e-04 | 2830.9ms/step | 21.2 samples/s
+Step 600/50462 | Loss: 4.5363 | LR: 4.00e-04 | 2828.5ms/step | 21.2 samples/s
+Step 610/50462 | Loss: 4.5271 | LR: 4.07e-04 | 2828.2ms/step | 21.2 samples/s
+Step 620/50462 | Loss: 4.5835 | LR: 4.13e-04 | 2829.7ms/step | 21.2 samples/s
+Step 630/50462 | Loss: 4.5551 | LR: 4.20e-04 | 2826.9ms/step | 21.2 samples/s
+Step 640/50462 | Loss: 4.4513 | LR: 4.27e-04 | 2829.1ms/step | 21.2 samples/s
+Step 650/50462 | Loss: 4.3682 | LR: 4.33e-04 | 2828.8ms/step | 21.2 samples/s
+Step 660/50462 | Loss: 4.4221 | LR: 4.40e-04 | 2830.3ms/step | 21.2 samples/s
+Step 670/50462 | Loss: 4.4865 | LR: 4.47e-04 | 2826.9ms/step | 21.2 samples/s
+Step 680/50462 | Loss: 4.4094 | LR: 4.53e-04 | 2817.4ms/step | 21.3 samples/s
+Step 690/50462 | Loss: 4.2839 | LR: 4.60e-04 | 2828.3ms/step | 21.2 samples/s
+Step 700/50462 | Loss: 4.2525 | LR: 4.67e-04 | 2829.7ms/step | 21.2 samples/s
+Step 710/50462 | Loss: 4.2754 | LR: 4.73e-04 | 2830.2ms/step | 21.2 samples/s
+Step 720/50462 | Loss: 4.2294 | LR: 4.80e-04 | 2833.6ms/step | 21.2 samples/s
+Step 730/50462 | Loss: 4.1987 | LR: 4.87e-04 | 2832.7ms/step | 21.2 samples/s
+Step 740/50462 | Loss: 4.1587 | LR: 4.93e-04 | 2832.6ms/step | 21.2 samples/s
+Step 750/50462 | Loss: 4.1572 | LR: 5.00e-04 | 2832.4ms/step | 21.2 samples/s
+Validation at step 750: Loss = 4.1413, Perplexity = 62.89
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-750
+Step 760/50462 | Loss: 4.1212 | LR: 5.00e-04 | 4775.5ms/step | 12.6 samples/s
+Step 770/50462 | Loss: 4.0936 | LR: 5.00e-04 | 2820.0ms/step | 21.3 samples/s
+Step 780/50462 | Loss: 4.0432 | LR: 5.00e-04 | 2832.9ms/step | 21.2 samples/s
+Step 790/50462 | Loss: 4.0721 | LR: 5.00e-04 | 2831.8ms/step | 21.2 samples/s
+Step 800/50462 | Loss: 4.0250 | LR: 5.00e-04 | 2831.9ms/step | 21.2 samples/s
+Step 810/50462 | Loss: 3.9828 | LR: 5.00e-04 | 2832.6ms/step | 21.2 samples/s
+Step 820/50462 | Loss: 3.9879 | LR: 5.00e-04 | 2833.5ms/step | 21.2 samples/s
+Step 830/50462 | Loss: 3.9419 | LR: 5.00e-04 | 2834.4ms/step | 21.2 samples/s
+Step 840/50462 | Loss: 3.9689 | LR: 5.00e-04 | 2836.0ms/step | 21.2 samples/s
+Step 850/50462 | Loss: 3.9334 | LR: 5.00e-04 | 2832.6ms/step | 21.2 samples/s
+Step 860/50462 | Loss: 3.9061 | LR: 5.00e-04 | 2830.4ms/step | 21.2 samples/s
+Step 870/50462 | Loss: 3.8820 | LR: 5.00e-04 | 2833.4ms/step | 21.2 samples/s
+Step 880/50462 | Loss: 3.8880 | LR: 5.00e-04 | 2833.7ms/step | 21.2 samples/s
+Step 890/50462 | Loss: 3.8578 | LR: 5.00e-04 | 2834.7ms/step | 21.2 samples/s
+Step 900/50462 | Loss: 3.8720 | LR: 5.00e-04 | 2832.5ms/step | 21.2 samples/s
+Step 910/50462 | Loss: 3.7852 | LR: 5.00e-04 | 2833.9ms/step | 21.2 samples/s
+Step 920/50462 | Loss: 3.8342 | LR: 5.00e-04 | 2836.7ms/step | 21.2 samples/s
+Step 930/50462 | Loss: 3.8007 | LR: 5.00e-04 | 2837.0ms/step | 21.1 samples/s
+Step 940/50462 | Loss: 3.8378 | LR: 5.00e-04 | 2835.7ms/step | 21.2 samples/s
+Step 950/50462 | Loss: 3.7768 | LR: 5.00e-04 | 2836.2ms/step | 21.2 samples/s
+Step 960/50462 | Loss: 3.7685 | LR: 5.00e-04 | 2834.7ms/step | 21.2 samples/s
+Step 970/50462 | Loss: 3.7626 | LR: 5.00e-04 | 2835.7ms/step | 21.2 samples/s
+Step 980/50462 | Loss: 3.7532 | LR: 5.00e-04 | 2835.7ms/step | 21.2 samples/s
+Step 990/50462 | Loss: 3.7420 | LR: 5.00e-04 | 2833.1ms/step | 21.2 samples/s
+Step 1000/50462 | Loss: 3.6888 | LR: 5.00e-04 | 2835.1ms/step | 21.2 samples/s
+Validation at step 1000: Loss = 3.7120, Perplexity = 40.94
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-1000
+Step 1010/50462 | Loss: 3.7392 | LR: 5.00e-04 | 4769.8ms/step | 12.6 samples/s
+Step 1020/50462 | Loss: 3.6858 | LR: 5.00e-04 | 2823.0ms/step | 21.3 samples/s
+Step 1030/50462 | Loss: 3.6780 | LR: 5.00e-04 | 2821.9ms/step | 21.3 samples/s
+Step 1040/50462 | Loss: 3.6940 | LR: 5.00e-04 | 2831.0ms/step | 21.2 samples/s
+Step 1050/50462 | Loss: 3.6797 | LR: 5.00e-04 | 2832.5ms/step | 21.2 samples/s
+Step 1060/50462 | Loss: 3.6590 | LR: 5.00e-04 | 2833.9ms/step | 21.2 samples/s
+Step 1070/50462 | Loss: 3.6346 | LR: 5.00e-04 | 2834.3ms/step | 21.2 samples/s
+Step 1080/50462 | Loss: 3.6330 | LR: 5.00e-04 | 2831.1ms/step | 21.2 samples/s
+Step 1090/50462 | Loss: 3.6322 | LR: 5.00e-04 | 2830.9ms/step | 21.2 samples/s
+Step 1100/50462 | Loss: 3.6065 | LR: 5.00e-04 | 2832.0ms/step | 21.2 samples/s
+Step 1110/50462 | Loss: 3.6215 | LR: 5.00e-04 | 2835.8ms/step | 21.2 samples/s
+Step 1120/50462 | Loss: 3.6081 | LR: 5.00e-04 | 2834.7ms/step | 21.2 samples/s
+Step 1130/50462 | Loss: 3.5864 | LR: 5.00e-04 | 2836.9ms/step | 21.2 samples/s
+Step 1140/50462 | Loss: 3.5764 | LR: 5.00e-04 | 2833.3ms/step | 21.2 samples/s
+Step 1150/50462 | Loss: 3.5661 | LR: 5.00e-04 | 2834.6ms/step | 21.2 samples/s
+Step 1160/50462 | Loss: 3.5526 | LR: 5.00e-04 | 2837.5ms/step | 21.1 samples/s
+Step 1170/50462 | Loss: 3.5200 | LR: 5.00e-04 | 2838.2ms/step | 21.1 samples/s
+Step 1180/50462 | Loss: 3.5774 | LR: 5.00e-04 | 2835.7ms/step | 21.2 samples/s
+Step 1190/50462 | Loss: 3.5498 | LR: 5.00e-04 | 2832.9ms/step | 21.2 samples/s
+Step 1200/50462 | Loss: 3.5842 | LR: 5.00e-04 | 2835.5ms/step | 21.2 samples/s
+Step 1210/50462 | Loss: 3.5219 | LR: 5.00e-04 | 2834.7ms/step | 21.2 samples/s
+Step 1220/50462 | Loss: 3.5338 | LR: 5.00e-04 | 2834.9ms/step | 21.2 samples/s
+Step 1230/50462 | Loss: 3.5167 | LR: 5.00e-04 | 2830.6ms/step | 21.2 samples/s
+Step 1240/50462 | Loss: 3.5601 | LR: 5.00e-04 | 2830.1ms/step | 21.2 samples/s
+Step 1250/50462 | Loss: 3.5059 | LR: 5.00e-04 | 2833.6ms/step | 21.2 samples/s
+Validation at step 1250: Loss = 3.5163, Perplexity = 33.66
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-1250
+Step 1260/50462 | Loss: 3.4926 | LR: 5.00e-04 | 4778.8ms/step | 12.6 samples/s
+Step 1270/50462 | Loss: 3.4753 | LR: 5.00e-04 | 2822.1ms/step | 21.3 samples/s
+Step 1280/50462 | Loss: 3.5182 | LR: 5.00e-04 | 2831.1ms/step | 21.2 samples/s
+Step 1290/50462 | Loss: 3.4682 | LR: 5.00e-04 | 2833.9ms/step | 21.2 samples/s
+Step 1300/50462 | Loss: 3.4608 | LR: 5.00e-04 | 2831.3ms/step | 21.2 samples/s
+Step 1310/50462 | Loss: 3.4649 | LR: 5.00e-04 | 2833.8ms/step | 21.2 samples/s
+Step 1320/50462 | Loss: 3.4572 | LR: 5.00e-04 | 2836.6ms/step | 21.2 samples/s
+Step 1330/50462 | Loss: 3.4792 | LR: 5.00e-04 | 2836.4ms/step | 21.2 samples/s
+Step 1340/50462 | Loss: 3.4735 | LR: 5.00e-04 | 2833.5ms/step | 21.2 samples/s
+Step 1350/50462 | Loss: 3.4699 | LR: 5.00e-04 | 2832.1ms/step | 21.2 samples/s
+Step 1360/50462 | Loss: 3.4395 | LR: 5.00e-04 | 2835.7ms/step | 21.2 samples/s
+Step 1370/50462 | Loss: 3.4476 | LR: 5.00e-04 | 2833.9ms/step | 21.2 samples/s
+Step 1380/50462 | Loss: 3.4749 | LR: 5.00e-04 | 2836.8ms/step | 21.2 samples/s
+Step 1390/50462 | Loss: 3.4820 | LR: 5.00e-04 | 2832.5ms/step | 21.2 samples/s
+Step 1400/50462 | Loss: 3.4441 | LR: 5.00e-04 | 2832.5ms/step | 21.2 samples/s
+Step 1410/50462 | Loss: 3.4216 | LR: 5.00e-04 | 2834.7ms/step | 21.2 samples/s
+Step 1420/50462 | Loss: 3.3950 | LR: 5.00e-04 | 2833.8ms/step | 21.2 samples/s
+Step 1430/50462 | Loss: 3.3444 | LR: 5.00e-04 | 2831.1ms/step | 21.2 samples/s
+Step 1440/50462 | Loss: 3.4054 | LR: 5.00e-04 | 2837.5ms/step | 21.1 samples/s
+Step 1450/50462 | Loss: 3.3769 | LR: 5.00e-04 | 2834.2ms/step | 21.2 samples/s
+Step 1460/50462 | Loss: 3.3750 | LR: 5.00e-04 | 2834.5ms/step | 21.2 samples/s
+Step 1470/50462 | Loss: 3.4102 | LR: 5.00e-04 | 2835.8ms/step | 21.2 samples/s
+Step 1480/50462 | Loss: 3.3778 | LR: 5.00e-04 | 2836.2ms/step | 21.2 samples/s
+Step 1490/50462 | Loss: 3.3993 | LR: 5.00e-04 | 2833.3ms/step | 21.2 samples/s
+Step 1500/50462 | Loss: 3.3561 | LR: 5.00e-04 | 2832.9ms/step | 21.2 samples/s
+Validation at step 1500: Loss = 3.3873, Perplexity = 29.59
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-1500
+Step 1510/50462 | Loss: 3.3690 | LR: 5.00e-04 | 4772.0ms/step | 12.6 samples/s
+Step 1520/50462 | Loss: 3.3898 | LR: 5.00e-04 | 2826.9ms/step | 21.2 samples/s
+Step 1530/50462 | Loss: 3.3916 | LR: 5.00e-04 | 2831.4ms/step | 21.2 samples/s
+Step 1540/50462 | Loss: 3.3581 | LR: 5.00e-04 | 2832.5ms/step | 21.2 samples/s
+Step 1550/50462 | Loss: 3.3440 | LR: 5.00e-04 | 2833.3ms/step | 21.2 samples/s
+Step 1560/50462 | Loss: 3.3866 | LR: 5.00e-04 | 2834.3ms/step | 21.2 samples/s
+Step 1570/50462 | Loss: 3.3657 | LR: 5.00e-04 | 2834.3ms/step | 21.2 samples/s
+Step 1580/50462 | Loss: 3.3420 | LR: 5.00e-04 | 2832.6ms/step | 21.2 samples/s
+Step 1590/50462 | Loss: 3.3942 | LR: 5.00e-04 | 2832.5ms/step | 21.2 samples/s
+Step 1600/50462 | Loss: 3.3448 | LR: 5.00e-04 | 2830.8ms/step | 21.2 samples/s
+Step 1610/50462 | Loss: 3.3820 | LR: 5.00e-04 | 2831.1ms/step | 21.2 samples/s
+Step 1620/50462 | Loss: 3.3357 | LR: 5.00e-04 | 2832.2ms/step | 21.2 samples/s
+Step 1630/50462 | Loss: 3.3315 | LR: 5.00e-04 | 2832.8ms/step | 21.2 samples/s
+Step 1640/50462 | Loss: 3.3203 | LR: 5.00e-04 | 2833.5ms/step | 21.2 samples/s
+Step 1650/50462 | Loss: 3.3406 | LR: 5.00e-04 | 2835.3ms/step | 21.2 samples/s
+Step 1660/50462 | Loss: 3.3265 | LR: 5.00e-04 | 2832.0ms/step | 21.2 samples/s
+Step 1670/50462 | Loss: 3.3385 | LR: 5.00e-04 | 2833.9ms/step | 21.2 samples/s
+Step 1680/50462 | Loss: 3.3123 | LR: 5.00e-04 | 2831.5ms/step | 21.2 samples/s
+Step 1690/50462 | Loss: 3.2875 | LR: 5.00e-04 | 2833.7ms/step | 21.2 samples/s
+Step 1700/50462 | Loss: 3.3084 | LR: 5.00e-04 | 2834.1ms/step | 21.2 samples/s
+Step 1710/50462 | Loss: 3.2776 | LR: 5.00e-04 | 2835.5ms/step | 21.2 samples/s
+Step 1720/50462 | Loss: 3.3152 | LR: 5.00e-04 | 2832.9ms/step | 21.2 samples/s
+Step 1730/50462 | Loss: 3.3060 | LR: 5.00e-04 | 2832.0ms/step | 21.2 samples/s
+Step 1740/50462 | Loss: 3.2981 | LR: 5.00e-04 | 2832.6ms/step | 21.2 samples/s
+Step 1750/50462 | Loss: 3.3277 | LR: 5.00e-04 | 2833.0ms/step | 21.2 samples/s
+Validation at step 1750: Loss = 3.3060, Perplexity = 27.27
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-1750
+Step 1760/50462 | Loss: 3.3017 | LR: 4.99e-04 | 4767.8ms/step | 12.6 samples/s
+Step 1770/50462 | Loss: 3.2843 | LR: 4.99e-04 | 2821.2ms/step | 21.3 samples/s
+Step 1780/50462 | Loss: 3.2876 | LR: 4.99e-04 | 2825.9ms/step | 21.2 samples/s
+Step 1790/50462 | Loss: 3.2969 | LR: 4.99e-04 | 2827.5ms/step | 21.2 samples/s
+Step 1800/50462 | Loss: 3.3030 | LR: 4.99e-04 | 2826.7ms/step | 21.2 samples/s
+Step 1810/50462 | Loss: 3.2772 | LR: 4.99e-04 | 2828.0ms/step | 21.2 samples/s
+Step 1820/50462 | Loss: 3.2778 | LR: 4.99e-04 | 2831.3ms/step | 21.2 samples/s
+Step 1830/50462 | Loss: 3.3006 | LR: 4.99e-04 | 2827.9ms/step | 21.2 samples/s
+Step 1840/50462 | Loss: 3.3068 | LR: 4.99e-04 | 2831.2ms/step | 21.2 samples/s
+Step 1850/50462 | Loss: 3.2800 | LR: 4.99e-04 | 2831.1ms/step | 21.2 samples/s
+Step 1860/50462 | Loss: 3.2934 | LR: 4.99e-04 | 2828.0ms/step | 21.2 samples/s
+Step 1870/50462 | Loss: 3.3034 | LR: 4.99e-04 | 2831.1ms/step | 21.2 samples/s
+Step 1880/50462 | Loss: 3.2440 | LR: 4.99e-04 | 2836.0ms/step | 21.2 samples/s
+Step 1890/50462 | Loss: 3.2497 | LR: 4.99e-04 | 2835.8ms/step | 21.2 samples/s
+Step 1900/50462 | Loss: 3.2477 | LR: 4.99e-04 | 2832.5ms/step | 21.2 samples/s
+Step 1910/50462 | Loss: 3.2819 | LR: 4.99e-04 | 2831.3ms/step | 21.2 samples/s
+Step 1920/50462 | Loss: 3.2103 | LR: 4.99e-04 | 2831.0ms/step | 21.2 samples/s
+Step 1930/50462 | Loss: 3.2519 | LR: 4.99e-04 | 2827.6ms/step | 21.2 samples/s
+Step 1940/50462 | Loss: 3.2573 | LR: 4.99e-04 | 2829.5ms/step | 21.2 samples/s
+Step 1950/50462 | Loss: 3.2266 | LR: 4.99e-04 | 2829.8ms/step | 21.2 samples/s
+Step 1960/50462 | Loss: 3.2253 | LR: 4.99e-04 | 2827.9ms/step | 21.2 samples/s
+Step 1970/50462 | Loss: 3.2139 | LR: 4.99e-04 | 2822.1ms/step | 21.3 samples/s
+Step 1980/50462 | Loss: 3.2390 | LR: 4.99e-04 | 2824.2ms/step | 21.2 samples/s
+Step 1990/50462 | Loss: 3.2532 | LR: 4.99e-04 | 2826.8ms/step | 21.2 samples/s
+Step 2000/50462 | Loss: 3.2330 | LR: 4.99e-04 | 2827.6ms/step | 21.2 samples/s
+Validation at step 2000: Loss = 3.2365, Perplexity = 25.44
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-2000
+Step 2010/50462 | Loss: 3.2142 | LR: 4.99e-04 | 4786.8ms/step | 12.5 samples/s
+Step 2020/50462 | Loss: 3.2049 | LR: 4.99e-04 | 2827.6ms/step | 21.2 samples/s
+Step 2030/50462 | Loss: 3.2170 | LR: 4.99e-04 | 2829.6ms/step | 21.2 samples/s
+Step 2040/50462 | Loss: 3.2108 | LR: 4.99e-04 | 2832.1ms/step | 21.2 samples/s
+Step 2050/50462 | Loss: 3.2076 | LR: 4.99e-04 | 2830.7ms/step | 21.2 samples/s
+Step 2060/50462 | Loss: 3.2354 | LR: 4.99e-04 | 2830.8ms/step | 21.2 samples/s
+Step 2070/50462 | Loss: 3.1763 | LR: 4.99e-04 | 2830.0ms/step | 21.2 samples/s
+Step 2080/50462 | Loss: 3.1992 | LR: 4.99e-04 | 2832.8ms/step | 21.2 samples/s
+Step 2090/50462 | Loss: 3.1724 | LR: 4.99e-04 | 2832.2ms/step | 21.2 samples/s
+Step 2100/50462 | Loss: 3.1840 | LR: 4.99e-04 | 2833.0ms/step | 21.2 samples/s
+Step 2110/50462 | Loss: 3.1862 | LR: 4.99e-04 | 2829.5ms/step | 21.2 samples/s
+Step 2120/50462 | Loss: 3.1750 | LR: 4.99e-04 | 2832.5ms/step | 21.2 samples/s
+Step 2130/50462 | Loss: 3.2136 | LR: 4.99e-04 | 2828.8ms/step | 21.2 samples/s
+Step 2140/50462 | Loss: 3.2211 | LR: 4.99e-04 | 2828.8ms/step | 21.2 samples/s
+Step 2150/50462 | Loss: 3.1850 | LR: 4.99e-04 | 2830.5ms/step | 21.2 samples/s
+Step 2160/50462 | Loss: 3.2123 | LR: 4.99e-04 | 2833.0ms/step | 21.2 samples/s
+Step 2170/50462 | Loss: 3.1769 | LR: 4.99e-04 | 2827.5ms/step | 21.2 samples/s
+Step 2180/50462 | Loss: 3.1684 | LR: 4.99e-04 | 2833.1ms/step | 21.2 samples/s
+Step 2190/50462 | Loss: 3.2109 | LR: 4.99e-04 | 2832.6ms/step | 21.2 samples/s
+Step 2200/50462 | Loss: 3.2057 | LR: 4.99e-04 | 2828.3ms/step | 21.2 samples/s
+Step 2210/50462 | Loss: 3.1768 | LR: 4.99e-04 | 2829.8ms/step | 21.2 samples/s
+Step 2220/50462 | Loss: 3.1737 | LR: 4.99e-04 | 2831.3ms/step | 21.2 samples/s
+Step 2230/50462 | Loss: 3.2153 | LR: 4.99e-04 | 2829.1ms/step | 21.2 samples/s
+Step 2240/50462 | Loss: 3.1607 | LR: 4.99e-04 | 2834.0ms/step | 21.2 samples/s
+Step 2250/50462 | Loss: 3.1798 | LR: 4.99e-04 | 2835.3ms/step | 21.2 samples/s
+Validation at step 2250: Loss = 3.1867, Perplexity = 24.21
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-2250
+Step 2260/50462 | Loss: 3.1829 | LR: 4.99e-04 | 4783.3ms/step | 12.5 samples/s
+Step 2270/50462 | Loss: 3.1597 | LR: 4.99e-04 | 2823.9ms/step | 21.2 samples/s
+Step 2280/50462 | Loss: 3.1746 | LR: 4.99e-04 | 2819.4ms/step | 21.3 samples/s
+Step 2290/50462 | Loss: 3.1686 | LR: 4.99e-04 | 2821.6ms/step | 21.3 samples/s
+Step 2300/50462 | Loss: 3.1546 | LR: 4.99e-04 | 2827.4ms/step | 21.2 samples/s
+Step 2310/50462 | Loss: 3.1692 | LR: 4.99e-04 | 2831.3ms/step | 21.2 samples/s
+Step 2320/50462 | Loss: 3.1352 | LR: 4.99e-04 | 2831.3ms/step | 21.2 samples/s
+Step 2330/50462 | Loss: 3.1464 | LR: 4.99e-04 | 2832.4ms/step | 21.2 samples/s
+Step 2340/50462 | Loss: 3.1830 | LR: 4.99e-04 | 2832.4ms/step | 21.2 samples/s
+Step 2350/50462 | Loss: 3.1474 | LR: 4.99e-04 | 2832.7ms/step | 21.2 samples/s
+Step 2360/50462 | Loss: 3.1615 | LR: 4.99e-04 | 2834.3ms/step | 21.2 samples/s
+Step 2370/50462 | Loss: 3.1840 | LR: 4.99e-04 | 2830.4ms/step | 21.2 samples/s
+Step 2380/50462 | Loss: 3.1846 | LR: 4.99e-04 | 2832.5ms/step | 21.2 samples/s
+Step 2390/50462 | Loss: 3.1568 | LR: 4.99e-04 | 2834.6ms/step | 21.2 samples/s
+Step 2400/50462 | Loss: 3.1432 | LR: 4.99e-04 | 2831.5ms/step | 21.2 samples/s
+Step 2410/50462 | Loss: 3.1630 | LR: 4.99e-04 | 2830.8ms/step | 21.2 samples/s
+Step 2420/50462 | Loss: 3.1276 | LR: 4.99e-04 | 2835.0ms/step | 21.2 samples/s
+Step 2430/50462 | Loss: 3.1093 | LR: 4.99e-04 | 2830.0ms/step | 21.2 samples/s
+Step 2440/50462 | Loss: 3.1122 | LR: 4.99e-04 | 2827.3ms/step | 21.2 samples/s
+Step 2450/50462 | Loss: 3.1366 | LR: 4.99e-04 | 2829.7ms/step | 21.2 samples/s
+Step 2460/50462 | Loss: 3.1121 | LR: 4.99e-04 | 2829.3ms/step | 21.2 samples/s
+Step 2470/50462 | Loss: 3.1566 | LR: 4.99e-04 | 2829.9ms/step | 21.2 samples/s
+Step 2480/50462 | Loss: 3.1532 | LR: 4.99e-04 | 2832.4ms/step | 21.2 samples/s
+Step 2490/50462 | Loss: 3.1394 | LR: 4.98e-04 | 2828.7ms/step | 21.2 samples/s
+Step 2500/50462 | Loss: 3.1445 | LR: 4.98e-04 | 2832.1ms/step | 21.2 samples/s
+Validation at step 2500: Loss = 3.1455, Perplexity = 23.23
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-2500
+Step 2510/50462 | Loss: 3.1498 | LR: 4.98e-04 | 4811.5ms/step | 12.5 samples/s
+Step 2520/50462 | Loss: 3.1560 | LR: 4.98e-04 | 2825.7ms/step | 21.2 samples/s
+Step 2530/50462 | Loss: 3.1448 | LR: 4.98e-04 | 2828.1ms/step | 21.2 samples/s
+Step 2540/50462 | Loss: 3.1235 | LR: 4.98e-04 | 2831.4ms/step | 21.2 samples/s
+Step 2550/50462 | Loss: 3.1093 | LR: 4.98e-04 | 2830.7ms/step | 21.2 samples/s
+Step 2560/50462 | Loss: 3.1367 | LR: 4.98e-04 | 2828.9ms/step | 21.2 samples/s
+Step 2570/50462 | Loss: 3.1204 | LR: 4.98e-04 | 2830.7ms/step | 21.2 samples/s
+Step 2580/50462 | Loss: 3.1045 | LR: 4.98e-04 | 2830.2ms/step | 21.2 samples/s
+Step 2590/50462 | Loss: 3.0966 | LR: 4.98e-04 | 2832.2ms/step | 21.2 samples/s
+Step 2600/50462 | Loss: 3.1554 | LR: 4.98e-04 | 2828.6ms/step | 21.2 samples/s
+Step 2610/50462 | Loss: 3.1346 | LR: 4.98e-04 | 2823.5ms/step | 21.3 samples/s
+Step 2620/50462 | Loss: 3.1211 | LR: 4.98e-04 | 2829.3ms/step | 21.2 samples/s
+Step 2630/50462 | Loss: 3.1216 | LR: 4.98e-04 | 2831.2ms/step | 21.2 samples/s
+Step 2640/50462 | Loss: 3.0922 | LR: 4.98e-04 | 2831.7ms/step | 21.2 samples/s
+Step 2650/50462 | Loss: 3.1067 | LR: 4.98e-04 | 2834.0ms/step | 21.2 samples/s
+Step 2660/50462 | Loss: 3.0699 | LR: 4.98e-04 | 2832.7ms/step | 21.2 samples/s
+Step 2670/50462 | Loss: 3.0782 | LR: 4.98e-04 | 2831.8ms/step | 21.2 samples/s
+Step 2680/50462 | Loss: 3.1291 | LR: 4.98e-04 | 2832.4ms/step | 21.2 samples/s
+Step 2690/50462 | Loss: 3.1374 | LR: 4.98e-04 | 2824.9ms/step | 21.2 samples/s
+Step 2700/50462 | Loss: 3.1127 | LR: 4.98e-04 | 2831.0ms/step | 21.2 samples/s
+Step 2710/50462 | Loss: 3.0626 | LR: 4.98e-04 | 2832.2ms/step | 21.2 samples/s
+Step 2720/50462 | Loss: 3.0887 | LR: 4.98e-04 | 2834.6ms/step | 21.2 samples/s
+Step 2730/50462 | Loss: 3.0877 | LR: 4.98e-04 | 2832.9ms/step | 21.2 samples/s
+Step 2740/50462 | Loss: 3.1018 | LR: 4.98e-04 | 2832.1ms/step | 21.2 samples/s
+Step 2750/50462 | Loss: 3.0758 | LR: 4.98e-04 | 2834.3ms/step | 21.2 samples/s
+Validation at step 2750: Loss = 3.1075, Perplexity = 22.37
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-2750
+Step 2760/50462 | Loss: 3.1105 | LR: 4.98e-04 | 4764.3ms/step | 12.6 samples/s
+Step 2770/50462 | Loss: 3.0902 | LR: 4.98e-04 | 2827.1ms/step | 21.2 samples/s
+Step 2780/50462 | Loss: 3.1024 | LR: 4.98e-04 | 2831.5ms/step | 21.2 samples/s
+Step 2790/50462 | Loss: 3.1225 | LR: 4.98e-04 | 2834.2ms/step | 21.2 samples/s
+Step 2800/50462 | Loss: 3.0828 | LR: 4.98e-04 | 2834.6ms/step | 21.2 samples/s
+Step 2810/50462 | Loss: 3.0838 | LR: 4.98e-04 | 2833.9ms/step | 21.2 samples/s
+Step 2820/50462 | Loss: 3.0972 | LR: 4.98e-04 | 2831.6ms/step | 21.2 samples/s
+Step 2830/50462 | Loss: 3.0914 | LR: 4.98e-04 | 2831.6ms/step | 21.2 samples/s
+Step 2840/50462 | Loss: 3.0679 | LR: 4.98e-04 | 2835.1ms/step | 21.2 samples/s
+Step 2850/50462 | Loss: 3.0960 | LR: 4.98e-04 | 2832.3ms/step | 21.2 samples/s
+Step 2860/50462 | Loss: 3.0825 | LR: 4.98e-04 | 2830.1ms/step | 21.2 samples/s
+Step 2870/50462 | Loss: 3.0577 | LR: 4.98e-04 | 2831.8ms/step | 21.2 samples/s
+Step 2880/50462 | Loss: 3.0816 | LR: 4.98e-04 | 2830.6ms/step | 21.2 samples/s
+Step 2890/50462 | Loss: 3.0624 | LR: 4.98e-04 | 2832.9ms/step | 21.2 samples/s
+Step 2900/50462 | Loss: 3.0911 | LR: 4.98e-04 | 2828.0ms/step | 21.2 samples/s
+Step 2910/50462 | Loss: 3.0675 | LR: 4.98e-04 | 2827.4ms/step | 21.2 samples/s
+Step 2920/50462 | Loss: 3.0661 | LR: 4.98e-04 | 2825.4ms/step | 21.2 samples/s
+Step 2930/50462 | Loss: 3.0918 | LR: 4.98e-04 | 2818.3ms/step | 21.3 samples/s
+Step 2940/50462 | Loss: 3.0960 | LR: 4.98e-04 | 2825.7ms/step | 21.2 samples/s

epfl_final/SparseAtt/wandb/run-20260903_094250-7tfq6nwm/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260903_094250-7tfq6nwm/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-09-03T09:42:50.607930Z",
+  "args":  [
+    "--config",
+    "configs/p2_1p2b_dense_seed72_4gpu_chained.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/SparseAtt.git",
+    "commit":  "5b79c4bdda9b10dca72a01f318ead9d505227eb5"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-train-4-gpu-0-51",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "7216043995136"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-c9ff8a50-9103-3e73-0066-6e7c0568cad4"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-ac14b2f5-a0c5-12c4-9fa0-46c3aa9e75c9"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-b817d241-acce-e5dd-a749-3447239c5207"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-47444fc7-ea8c-cef6-aac8-ef408dc8047b"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "6onqv5wofo3cx216m294gwbpwvc0obr4"
+}

epfl_final/SparseAtt/wandb/run-20260903_094250-7tfq6nwm/logs/debug.log

@@ -0,0 +1,19 @@
+2026-09-03 09:42:50,629 INFO    MainThread:4816 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-03 09:42:50,629 INFO    MainThread:4816 [wandb_setup.py:_flush():81] Configure stats pid to 4816
+2026-09-03 09:42:50,629 INFO    MainThread:4816 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-03 09:42:50,629 INFO    MainThread:4816 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260903_094250-7tfq6nwm/logs/debug.log
+2026-09-03 09:42:50,629 INFO    MainThread:4816 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260903_094250-7tfq6nwm/logs/debug-internal.log
+2026-09-03 09:42:50,629 INFO    MainThread:4816 [wandb_init.py:init():850] calling init triggers
+2026-09-03 09:42:50,629 INFO    MainThread:4816 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216', 'resume_from_checkpoint': None, 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 15, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 72, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b dense 8k seed72 4gpu', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'dense', 'seed72'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-03 09:42:50,629 INFO    MainThread:4816 [wandb_init.py:init():898] starting backend
+2026-09-03 09:42:51,194 INFO    MainThread:4816 [wandb_init.py:init():913] sending inform_init request
+2026-09-03 09:42:51,653 INFO    MainThread:4816 [wandb_init.py:init():918] backend started and connected
+2026-09-03 09:42:51,656 INFO    MainThread:4816 [wandb_init.py:init():988] updated telemetry
+2026-09-03 09:42:51,684 INFO    MainThread:4816 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-03 09:42:52,095 INFO    MainThread:4816 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-03 09:42:52,302 INFO    MainThread:4816 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-03 09:42:52,302 INFO    MainThread:4816 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-03 09:42:52,302 INFO    MainThread:4816 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-03 09:42:52,302 INFO    MainThread:4816 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-03 09:42:52,309 INFO    MainThread:4816 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/run-20260903_094250-7tfq6nwm/run-7tfq6nwm.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1eea3ea8f45339cebb2098200dd77abceaf87af644c89eeabb00c754283d6d65
+size 2949120

epfl_final/SparseAtt/wandb/run-20260903_142704-7tfq6nwm/files/output.log

@@ -0,0 +1,396 @@
+Starting training...
+Validation at step 2750: Loss = 3.1075, Perplexity = 22.37
+wandb: WARNING Tried to log to step 2750 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 2751: dense (dense step interval)
+Step 2760/50462 | Loss: 3.1106 | LR: 4.98e-04 | 19044.7ms/step | 3.2 samples/s
+wandb: WARNING Tried to log to step 2760 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 2770/50462 | Loss: 3.0902 | LR: 4.98e-04 | 2807.4ms/step | 21.4 samples/s
+wandb: WARNING Tried to log to step 2770 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 2780/50462 | Loss: 3.1024 | LR: 4.98e-04 | 2812.9ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 2780 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 2790/50462 | Loss: 3.1225 | LR: 4.98e-04 | 2815.0ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 2790 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 2800/50462 | Loss: 3.0828 | LR: 4.98e-04 | 2817.2ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 2800 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 2810/50462 | Loss: 3.0838 | LR: 4.98e-04 | 2814.2ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 2810 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 2820/50462 | Loss: 3.0972 | LR: 4.98e-04 | 2818.8ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 2820 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 2830/50462 | Loss: 3.0915 | LR: 4.98e-04 | 2820.7ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 2830 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 2840/50462 | Loss: 3.0679 | LR: 4.98e-04 | 2816.4ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 2840 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 2850/50462 | Loss: 3.0960 | LR: 4.98e-04 | 2817.4ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 2850 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 2860/50462 | Loss: 3.0825 | LR: 4.98e-04 | 2816.8ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 2860 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 2870/50462 | Loss: 3.0577 | LR: 4.98e-04 | 2818.4ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 2870 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 2880/50462 | Loss: 3.0816 | LR: 4.98e-04 | 2822.5ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 2880 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 2890/50462 | Loss: 3.0625 | LR: 4.98e-04 | 2819.0ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 2890 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 2900/50462 | Loss: 3.0911 | LR: 4.98e-04 | 2820.2ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 2900 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 2910/50462 | Loss: 3.0675 | LR: 4.98e-04 | 2820.1ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 2910 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 2920/50462 | Loss: 3.0661 | LR: 4.98e-04 | 2816.4ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 2920 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 2930/50462 | Loss: 3.0918 | LR: 4.98e-04 | 2823.1ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 2930 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 2940/50462 | Loss: 3.0960 | LR: 4.98e-04 | 2820.2ms/step | 21.3 samples/s
+Step 2950/50462 | Loss: 3.0752 | LR: 4.98e-04 | 2819.6ms/step | 21.3 samples/s
+Step 2960/50462 | Loss: 3.0743 | LR: 4.98e-04 | 2816.4ms/step | 21.3 samples/s
+Step 2970/50462 | Loss: 3.0729 | LR: 4.98e-04 | 2819.9ms/step | 21.3 samples/s
+Step 2980/50462 | Loss: 3.0705 | LR: 4.98e-04 | 2817.0ms/step | 21.3 samples/s
+Step 2990/50462 | Loss: 3.0481 | LR: 4.97e-04 | 2821.6ms/step | 21.3 samples/s
+Step 3000/50462 | Loss: 3.0682 | LR: 4.97e-04 | 2819.9ms/step | 21.3 samples/s
+Validation at step 3000: Loss = 3.0768, Perplexity = 21.69
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-3000
+Step 3010/50462 | Loss: 3.1099 | LR: 4.97e-04 | 4716.5ms/step | 12.7 samples/s
+Step 3020/50462 | Loss: 3.0968 | LR: 4.97e-04 | 2819.9ms/step | 21.3 samples/s
+Step 3030/50462 | Loss: 3.0538 | LR: 4.97e-04 | 2824.1ms/step | 21.2 samples/s
+Step 3040/50462 | Loss: 3.0730 | LR: 4.97e-04 | 2823.9ms/step | 21.2 samples/s
+Step 3050/50462 | Loss: 3.0666 | LR: 4.97e-04 | 2827.8ms/step | 21.2 samples/s
+Step 3060/50462 | Loss: 3.0805 | LR: 4.97e-04 | 2828.1ms/step | 21.2 samples/s
+Step 3070/50462 | Loss: 3.0570 | LR: 4.97e-04 | 2823.8ms/step | 21.2 samples/s
+Step 3080/50462 | Loss: 3.0695 | LR: 4.97e-04 | 2828.4ms/step | 21.2 samples/s
+Step 3090/50462 | Loss: 3.0407 | LR: 4.97e-04 | 2821.7ms/step | 21.3 samples/s
+Step 3100/50462 | Loss: 3.0628 | LR: 4.97e-04 | 2825.4ms/step | 21.2 samples/s
+Step 3110/50462 | Loss: 3.0624 | LR: 4.97e-04 | 2823.5ms/step | 21.3 samples/s
+Step 3120/50462 | Loss: 3.0609 | LR: 4.97e-04 | 2827.7ms/step | 21.2 samples/s
+Step 3130/50462 | Loss: 3.0310 | LR: 4.97e-04 | 2824.6ms/step | 21.2 samples/s
+Step 3140/50462 | Loss: 3.0491 | LR: 4.97e-04 | 2822.7ms/step | 21.3 samples/s
+Step 3150/50462 | Loss: 3.0343 | LR: 4.97e-04 | 2825.1ms/step | 21.2 samples/s
+Step 3160/50462 | Loss: 3.0224 | LR: 4.97e-04 | 2826.8ms/step | 21.2 samples/s
+Step 3170/50462 | Loss: 3.0623 | LR: 4.97e-04 | 2828.3ms/step | 21.2 samples/s
+Step 3180/50462 | Loss: 3.0616 | LR: 4.97e-04 | 2826.7ms/step | 21.2 samples/s
+Step 3190/50462 | Loss: 3.0235 | LR: 4.97e-04 | 2828.1ms/step | 21.2 samples/s
+Step 3200/50462 | Loss: 3.0479 | LR: 4.97e-04 | 2823.7ms/step | 21.2 samples/s
+Step 3210/50462 | Loss: 3.0746 | LR: 4.97e-04 | 2829.2ms/step | 21.2 samples/s
+Step 3220/50462 | Loss: 3.0593 | LR: 4.97e-04 | 2829.8ms/step | 21.2 samples/s
+Step 3230/50462 | Loss: 3.0127 | LR: 4.97e-04 | 2825.7ms/step | 21.2 samples/s
+Step 3240/50462 | Loss: 3.0381 | LR: 4.97e-04 | 2823.8ms/step | 21.2 samples/s
+Step 3250/50462 | Loss: 3.0697 | LR: 4.97e-04 | 2818.8ms/step | 21.3 samples/s
+Validation at step 3250: Loss = 3.0495, Perplexity = 21.10
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-3250
+Step 3260/50462 | Loss: 3.0493 | LR: 4.97e-04 | 4697.0ms/step | 12.8 samples/s
+Step 3270/50462 | Loss: 3.0001 | LR: 4.97e-04 | 2820.6ms/step | 21.3 samples/s
+Step 3280/50462 | Loss: 3.0378 | LR: 4.97e-04 | 2829.6ms/step | 21.2 samples/s
+Step 3290/50462 | Loss: 3.0070 | LR: 4.97e-04 | 2827.4ms/step | 21.2 samples/s
+Step 3300/50462 | Loss: 3.0433 | LR: 4.97e-04 | 2827.4ms/step | 21.2 samples/s
+Step 3310/50462 | Loss: 3.0677 | LR: 4.97e-04 | 2829.2ms/step | 21.2 samples/s
+Step 3320/50462 | Loss: 3.0180 | LR: 4.97e-04 | 2831.2ms/step | 21.2 samples/s
+Step 3330/50462 | Loss: 2.9940 | LR: 4.97e-04 | 2830.3ms/step | 21.2 samples/s
+Step 3340/50462 | Loss: 3.0433 | LR: 4.97e-04 | 2823.6ms/step | 21.2 samples/s
+Step 3350/50462 | Loss: 3.0249 | LR: 4.97e-04 | 2826.9ms/step | 21.2 samples/s
+Step 3360/50462 | Loss: 3.0302 | LR: 4.97e-04 | 2829.3ms/step | 21.2 samples/s
+Step 3370/50462 | Loss: 3.0396 | LR: 4.97e-04 | 2828.0ms/step | 21.2 samples/s
+Step 3380/50462 | Loss: 3.0325 | LR: 4.97e-04 | 2826.0ms/step | 21.2 samples/s
+Step 3390/50462 | Loss: 3.0429 | LR: 4.97e-04 | 2819.1ms/step | 21.3 samples/s
+Step 3400/50462 | Loss: 3.0181 | LR: 4.97e-04 | 2826.6ms/step | 21.2 samples/s
+Step 3410/50462 | Loss: 2.9776 | LR: 4.96e-04 | 2827.2ms/step | 21.2 samples/s
+Step 3420/50462 | Loss: 2.9792 | LR: 4.96e-04 | 2822.3ms/step | 21.3 samples/s
+Step 3430/50462 | Loss: 3.0163 | LR: 4.96e-04 | 2818.7ms/step | 21.3 samples/s
+Step 3440/50462 | Loss: 2.9832 | LR: 4.96e-04 | 2825.3ms/step | 21.2 samples/s
+Step 3450/50462 | Loss: 3.0095 | LR: 4.96e-04 | 2826.8ms/step | 21.2 samples/s
+Step 3460/50462 | Loss: 2.9921 | LR: 4.96e-04 | 2828.3ms/step | 21.2 samples/s
+Step 3470/50462 | Loss: 3.0155 | LR: 4.96e-04 | 2825.9ms/step | 21.2 samples/s
+Step 3480/50462 | Loss: 3.0064 | LR: 4.96e-04 | 2829.7ms/step | 21.2 samples/s
+Step 3490/50462 | Loss: 3.0077 | LR: 4.96e-04 | 2827.7ms/step | 21.2 samples/s
+Step 3500/50462 | Loss: 2.9834 | LR: 4.96e-04 | 2824.3ms/step | 21.2 samples/s
+Validation at step 3500: Loss = 3.0263, Perplexity = 20.62
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-3500
+Step 3510/50462 | Loss: 3.0408 | LR: 4.96e-04 | 4701.2ms/step | 12.8 samples/s
+Step 3520/50462 | Loss: 3.0308 | LR: 4.96e-04 | 2825.3ms/step | 21.2 samples/s
+Step 3530/50462 | Loss: 2.9976 | LR: 4.96e-04 | 2825.0ms/step | 21.2 samples/s
+Step 3540/50462 | Loss: 3.0082 | LR: 4.96e-04 | 2825.4ms/step | 21.2 samples/s
+Step 3550/50462 | Loss: 2.9987 | LR: 4.96e-04 | 2824.7ms/step | 21.2 samples/s
+Step 3560/50462 | Loss: 3.0430 | LR: 4.96e-04 | 2826.1ms/step | 21.2 samples/s
+Step 3570/50462 | Loss: 3.0030 | LR: 4.96e-04 | 2827.4ms/step | 21.2 samples/s
+Step 3580/50462 | Loss: 2.9930 | LR: 4.96e-04 | 2829.3ms/step | 21.2 samples/s
+Step 3590/50462 | Loss: 2.9794 | LR: 4.96e-04 | 2825.5ms/step | 21.2 samples/s
+Step 3600/50462 | Loss: 2.9900 | LR: 4.96e-04 | 2823.8ms/step | 21.2 samples/s
+Step 3610/50462 | Loss: 3.0096 | LR: 4.96e-04 | 2827.5ms/step | 21.2 samples/s
+Step 3620/50462 | Loss: 3.0113 | LR: 4.96e-04 | 2827.2ms/step | 21.2 samples/s
+Step 3630/50462 | Loss: 2.9664 | LR: 4.96e-04 | 2830.4ms/step | 21.2 samples/s
+Step 3640/50462 | Loss: 3.0067 | LR: 4.96e-04 | 2829.6ms/step | 21.2 samples/s
+Step 3650/50462 | Loss: 2.9829 | LR: 4.96e-04 | 2826.1ms/step | 21.2 samples/s
+Step 3660/50462 | Loss: 3.0274 | LR: 4.96e-04 | 2824.3ms/step | 21.2 samples/s
+Step 3670/50462 | Loss: 3.0084 | LR: 4.96e-04 | 2825.4ms/step | 21.2 samples/s
+Step 3680/50462 | Loss: 3.0090 | LR: 4.96e-04 | 2823.5ms/step | 21.3 samples/s
+Step 3690/50462 | Loss: 3.0014 | LR: 4.96e-04 | 2830.7ms/step | 21.2 samples/s
+Step 3700/50462 | Loss: 3.0171 | LR: 4.96e-04 | 2832.3ms/step | 21.2 samples/s
+Step 3710/50462 | Loss: 2.9566 | LR: 4.96e-04 | 2827.9ms/step | 21.2 samples/s
+Step 3720/50462 | Loss: 2.9820 | LR: 4.96e-04 | 2828.1ms/step | 21.2 samples/s
+Step 3730/50462 | Loss: 2.9866 | LR: 4.96e-04 | 2827.0ms/step | 21.2 samples/s
+Step 3740/50462 | Loss: 2.9623 | LR: 4.96e-04 | 2829.6ms/step | 21.2 samples/s
+Step 3750/50462 | Loss: 3.0068 | LR: 4.96e-04 | 2825.0ms/step | 21.2 samples/s
+Validation at step 3750: Loss = 2.9998, Perplexity = 20.08
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-3750
+Step 3760/50462 | Loss: 3.0054 | LR: 4.95e-04 | 4701.9ms/step | 12.8 samples/s
+Step 3770/50462 | Loss: 3.0212 | LR: 4.95e-04 | 2824.0ms/step | 21.2 samples/s
+Step 3780/50462 | Loss: 3.0116 | LR: 4.95e-04 | 2824.1ms/step | 21.2 samples/s
+Step 3790/50462 | Loss: 3.0014 | LR: 4.95e-04 | 2819.7ms/step | 21.3 samples/s
+Step 3800/50462 | Loss: 2.9719 | LR: 4.95e-04 | 2824.0ms/step | 21.2 samples/s
+Step 3810/50462 | Loss: 3.0187 | LR: 4.95e-04 | 2827.6ms/step | 21.2 samples/s
+Step 3820/50462 | Loss: 2.9626 | LR: 4.95e-04 | 2827.9ms/step | 21.2 samples/s
+Step 3830/50462 | Loss: 3.0101 | LR: 4.95e-04 | 2825.0ms/step | 21.2 samples/s
+Step 3840/50462 | Loss: 2.9450 | LR: 4.95e-04 | 2819.9ms/step | 21.3 samples/s
+Step 3850/50462 | Loss: 2.9439 | LR: 4.95e-04 | 2822.3ms/step | 21.3 samples/s
+Step 3860/50462 | Loss: 2.9598 | LR: 4.95e-04 | 2828.4ms/step | 21.2 samples/s
+Step 3870/50462 | Loss: 2.9943 | LR: 4.95e-04 | 2828.8ms/step | 21.2 samples/s
+Step 3880/50462 | Loss: 2.9785 | LR: 4.95e-04 | 2829.3ms/step | 21.2 samples/s
+Step 3890/50462 | Loss: 2.9711 | LR: 4.95e-04 | 2829.3ms/step | 21.2 samples/s
+Step 3900/50462 | Loss: 3.0054 | LR: 4.95e-04 | 2830.9ms/step | 21.2 samples/s
+Step 3910/50462 | Loss: 3.0202 | LR: 4.95e-04 | 2831.5ms/step | 21.2 samples/s
+Step 3920/50462 | Loss: 3.0050 | LR: 4.95e-04 | 2829.1ms/step | 21.2 samples/s
+Step 3930/50462 | Loss: 2.9790 | LR: 4.95e-04 | 2834.8ms/step | 21.2 samples/s
+Step 3940/50462 | Loss: 2.9762 | LR: 4.95e-04 | 2835.1ms/step | 21.2 samples/s
+Step 3950/50462 | Loss: 2.9570 | LR: 4.95e-04 | 2830.0ms/step | 21.2 samples/s
+Step 3960/50462 | Loss: 2.9829 | LR: 4.95e-04 | 2826.5ms/step | 21.2 samples/s
+Step 3970/50462 | Loss: 2.9854 | LR: 4.95e-04 | 2829.6ms/step | 21.2 samples/s
+Step 3980/50462 | Loss: 2.9603 | LR: 4.95e-04 | 2827.0ms/step | 21.2 samples/s
+Step 3990/50462 | Loss: 2.9983 | LR: 4.95e-04 | 2825.7ms/step | 21.2 samples/s
+Step 4000/50462 | Loss: 2.9842 | LR: 4.95e-04 | 2825.1ms/step | 21.2 samples/s
+Validation at step 4000: Loss = 2.9835, Perplexity = 19.76
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-4000
+Step 4010/50462 | Loss: 2.9511 | LR: 4.95e-04 | 4804.3ms/step | 12.5 samples/s
+Step 4020/50462 | Loss: 2.9581 | LR: 4.95e-04 | 2825.5ms/step | 21.2 samples/s
+Step 4030/50462 | Loss: 2.9613 | LR: 4.95e-04 | 2825.8ms/step | 21.2 samples/s
+Step 4040/50462 | Loss: 2.9703 | LR: 4.95e-04 | 2824.9ms/step | 21.2 samples/s
+Step 4050/50462 | Loss: 2.9497 | LR: 4.95e-04 | 2826.4ms/step | 21.2 samples/s
+Step 4060/50462 | Loss: 2.9299 | LR: 4.95e-04 | 2821.6ms/step | 21.3 samples/s
+Step 4070/50462 | Loss: 2.9786 | LR: 4.95e-04 | 2830.0ms/step | 21.2 samples/s
+Step 4080/50462 | Loss: 2.9543 | LR: 4.94e-04 | 2830.3ms/step | 21.2 samples/s
+Step 4090/50462 | Loss: 2.9482 | LR: 4.94e-04 | 2828.1ms/step | 21.2 samples/s
+Step 4100/50462 | Loss: 2.9623 | LR: 4.94e-04 | 2828.1ms/step | 21.2 samples/s
+Step 4110/50462 | Loss: 2.9613 | LR: 4.94e-04 | 2829.5ms/step | 21.2 samples/s
+Step 4120/50462 | Loss: 2.9466 | LR: 4.94e-04 | 2832.9ms/step | 21.2 samples/s
+Step 4130/50462 | Loss: 2.9853 | LR: 4.94e-04 | 2830.2ms/step | 21.2 samples/s
+Step 4140/50462 | Loss: 2.9945 | LR: 4.94e-04 | 2830.0ms/step | 21.2 samples/s
+Step 4150/50462 | Loss: 2.9667 | LR: 4.94e-04 | 2834.5ms/step | 21.2 samples/s
+Step 4160/50462 | Loss: 2.9433 | LR: 4.94e-04 | 2831.8ms/step | 21.2 samples/s
+Step 4170/50462 | Loss: 2.9527 | LR: 4.94e-04 | 2831.8ms/step | 21.2 samples/s
+Step 4180/50462 | Loss: 2.9914 | LR: 4.94e-04 | 2834.8ms/step | 21.2 samples/s
+Step 4190/50462 | Loss: 2.9828 | LR: 4.94e-04 | 2833.7ms/step | 21.2 samples/s
+Step 4200/50462 | Loss: 2.9292 | LR: 4.94e-04 | 2834.3ms/step | 21.2 samples/s
+Step 4210/50462 | Loss: 2.9583 | LR: 4.94e-04 | 2831.8ms/step | 21.2 samples/s
+Step 4220/50462 | Loss: 2.9774 | LR: 4.94e-04 | 2831.2ms/step | 21.2 samples/s
+Step 4230/50462 | Loss: 2.9596 | LR: 4.94e-04 | 2831.4ms/step | 21.2 samples/s
+Step 4240/50462 | Loss: 2.9755 | LR: 4.94e-04 | 2827.6ms/step | 21.2 samples/s
+Step 4250/50462 | Loss: 2.9683 | LR: 4.94e-04 | 2822.5ms/step | 21.3 samples/s
+Validation at step 4250: Loss = 2.9662, Perplexity = 19.42
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-4250
+Step 4260/50462 | Loss: 2.9682 | LR: 4.94e-04 | 4772.9ms/step | 12.6 samples/s
+Step 4270/50462 | Loss: 2.9680 | LR: 4.94e-04 | 2824.6ms/step | 21.2 samples/s
+Step 4280/50462 | Loss: 2.9415 | LR: 4.94e-04 | 2829.0ms/step | 21.2 samples/s
+Step 4290/50462 | Loss: 2.9117 | LR: 4.94e-04 | 2831.9ms/step | 21.2 samples/s
+Step 4300/50462 | Loss: 2.9448 | LR: 4.94e-04 | 2827.4ms/step | 21.2 samples/s
+Step 4310/50462 | Loss: 2.9895 | LR: 4.94e-04 | 2826.8ms/step | 21.2 samples/s
+Step 4320/50462 | Loss: 2.9625 | LR: 4.94e-04 | 2824.8ms/step | 21.2 samples/s
+Step 4330/50462 | Loss: 2.9566 | LR: 4.94e-04 | 2827.1ms/step | 21.2 samples/s
+Step 4340/50462 | Loss: 2.9272 | LR: 4.94e-04 | 2826.5ms/step | 21.2 samples/s
+Step 4350/50462 | Loss: 2.9196 | LR: 4.94e-04 | 2824.7ms/step | 21.2 samples/s
+Step 4360/50462 | Loss: 2.9507 | LR: 4.94e-04 | 2827.7ms/step | 21.2 samples/s
+Step 4370/50462 | Loss: 2.9366 | LR: 4.93e-04 | 2828.7ms/step | 21.2 samples/s
+Step 4380/50462 | Loss: 2.9304 | LR: 4.93e-04 | 2827.5ms/step | 21.2 samples/s
+Step 4390/50462 | Loss: 2.9474 | LR: 4.93e-04 | 2823.3ms/step | 21.3 samples/s
+Step 4400/50462 | Loss: 2.9487 | LR: 4.93e-04 | 2824.9ms/step | 21.2 samples/s
+Step 4410/50462 | Loss: 2.9371 | LR: 4.93e-04 | 2826.6ms/step | 21.2 samples/s
+Step 4420/50462 | Loss: 2.9730 | LR: 4.93e-04 | 2831.6ms/step | 21.2 samples/s
+Step 4430/50462 | Loss: 2.9602 | LR: 4.93e-04 | 2830.6ms/step | 21.2 samples/s
+Step 4440/50462 | Loss: 2.9637 | LR: 4.93e-04 | 2830.0ms/step | 21.2 samples/s
+Step 4450/50462 | Loss: 2.9654 | LR: 4.93e-04 | 2830.3ms/step | 21.2 samples/s
+Step 4460/50462 | Loss: 2.9312 | LR: 4.93e-04 | 2828.5ms/step | 21.2 samples/s
+Step 4470/50462 | Loss: 2.9378 | LR: 4.93e-04 | 2830.7ms/step | 21.2 samples/s
+Step 4480/50462 | Loss: 2.9645 | LR: 4.93e-04 | 2827.0ms/step | 21.2 samples/s
+Step 4490/50462 | Loss: 2.9449 | LR: 4.93e-04 | 2827.0ms/step | 21.2 samples/s
+Step 4500/50462 | Loss: 2.9202 | LR: 4.93e-04 | 2828.6ms/step | 21.2 samples/s
+Validation at step 4500: Loss = 2.9516, Perplexity = 19.14
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-4500
+Step 4510/50462 | Loss: 2.9540 | LR: 4.93e-04 | 4772.5ms/step | 12.6 samples/s
+Step 4520/50462 | Loss: 2.9086 | LR: 4.93e-04 | 2821.9ms/step | 21.3 samples/s
+Step 4530/50462 | Loss: 2.9385 | LR: 4.93e-04 | 2824.3ms/step | 21.2 samples/s
+Step 4540/50462 | Loss: 2.9316 | LR: 4.93e-04 | 2826.6ms/step | 21.2 samples/s
+Step 4550/50462 | Loss: 2.9277 | LR: 4.93e-04 | 2827.8ms/step | 21.2 samples/s
+Step 4560/50462 | Loss: 2.9409 | LR: 4.93e-04 | 2821.7ms/step | 21.3 samples/s
+Step 4570/50462 | Loss: 2.9113 | LR: 4.93e-04 | 2827.5ms/step | 21.2 samples/s
+Step 4580/50462 | Loss: 2.9662 | LR: 4.93e-04 | 2831.0ms/step | 21.2 samples/s
+Step 4590/50462 | Loss: 2.9381 | LR: 4.93e-04 | 2828.3ms/step | 21.2 samples/s
+Step 4600/50462 | Loss: 2.9329 | LR: 4.93e-04 | 2828.5ms/step | 21.2 samples/s
+Step 4610/50462 | Loss: 2.9172 | LR: 4.93e-04 | 2827.7ms/step | 21.2 samples/s
+Step 4620/50462 | Loss: 2.9270 | LR: 4.93e-04 | 2827.1ms/step | 21.2 samples/s
+Step 4630/50462 | Loss: 2.9073 | LR: 4.93e-04 | 2826.0ms/step | 21.2 samples/s
+Step 4640/50462 | Loss: 2.9119 | LR: 4.92e-04 | 2823.2ms/step | 21.3 samples/s
+Step 4650/50462 | Loss: 2.9419 | LR: 4.92e-04 | 2823.0ms/step | 21.3 samples/s
+Step 4660/50462 | Loss: 2.9446 | LR: 4.92e-04 | 2824.9ms/step | 21.2 samples/s
+Step 4670/50462 | Loss: 2.9459 | LR: 4.92e-04 | 2823.2ms/step | 21.3 samples/s
+Step 4680/50462 | Loss: 2.9351 | LR: 4.92e-04 | 2825.8ms/step | 21.2 samples/s
+Step 4690/50462 | Loss: 2.9205 | LR: 4.92e-04 | 2823.9ms/step | 21.2 samples/s
+Step 4700/50462 | Loss: 2.9090 | LR: 4.92e-04 | 2819.4ms/step | 21.3 samples/s
+Step 4710/50462 | Loss: 2.9001 | LR: 4.92e-04 | 2822.0ms/step | 21.3 samples/s
+Step 4720/50462 | Loss: 2.9211 | LR: 4.92e-04 | 2824.6ms/step | 21.2 samples/s
+Step 4730/50462 | Loss: 2.9216 | LR: 4.92e-04 | 2820.6ms/step | 21.3 samples/s
+Step 4740/50462 | Loss: 2.9168 | LR: 4.92e-04 | 2824.8ms/step | 21.2 samples/s
+Step 4750/50462 | Loss: 2.9001 | LR: 4.92e-04 | 2823.3ms/step | 21.3 samples/s
+Validation at step 4750: Loss = 2.9330, Perplexity = 18.78
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-4750
+Step 4760/50462 | Loss: 2.9258 | LR: 4.92e-04 | 4762.7ms/step | 12.6 samples/s
+Step 4770/50462 | Loss: 2.9153 | LR: 4.92e-04 | 2818.6ms/step | 21.3 samples/s
+Step 4780/50462 | Loss: 2.9257 | LR: 4.92e-04 | 2825.6ms/step | 21.2 samples/s
+Step 4790/50462 | Loss: 2.9101 | LR: 4.92e-04 | 2826.3ms/step | 21.2 samples/s
+Step 4800/50462 | Loss: 2.9249 | LR: 4.92e-04 | 2824.2ms/step | 21.2 samples/s
+Step 4810/50462 | Loss: 2.9117 | LR: 4.92e-04 | 2819.0ms/step | 21.3 samples/s
+Step 4820/50462 | Loss: 2.8973 | LR: 4.92e-04 | 2824.3ms/step | 21.2 samples/s
+Step 4830/50462 | Loss: 2.8972 | LR: 4.92e-04 | 2825.3ms/step | 21.2 samples/s
+Step 4840/50462 | Loss: 2.9174 | LR: 4.92e-04 | 2825.8ms/step | 21.2 samples/s
+Step 4850/50462 | Loss: 2.9290 | LR: 4.92e-04 | 2825.2ms/step | 21.2 samples/s
+Step 4860/50462 | Loss: 2.9151 | LR: 4.92e-04 | 2821.8ms/step | 21.3 samples/s
+Step 4870/50462 | Loss: 2.9073 | LR: 4.92e-04 | 2820.1ms/step | 21.3 samples/s
+Step 4880/50462 | Loss: 2.9417 | LR: 4.92e-04 | 2824.9ms/step | 21.2 samples/s
+Step 4890/50462 | Loss: 2.9004 | LR: 4.91e-04 | 2821.7ms/step | 21.3 samples/s
+Step 4900/50462 | Loss: 2.9330 | LR: 4.91e-04 | 2824.4ms/step | 21.2 samples/s
+Step 4910/50462 | Loss: 2.9109 | LR: 4.91e-04 | 2826.3ms/step | 21.2 samples/s
+Step 4920/50462 | Loss: 2.9211 | LR: 4.91e-04 | 2821.8ms/step | 21.3 samples/s
+Step 4930/50462 | Loss: 2.9027 | LR: 4.91e-04 | 2822.0ms/step | 21.3 samples/s
+Step 4940/50462 | Loss: 2.9499 | LR: 4.91e-04 | 2824.6ms/step | 21.2 samples/s
+Step 4950/50462 | Loss: 2.9034 | LR: 4.91e-04 | 2824.7ms/step | 21.2 samples/s
+Step 4960/50462 | Loss: 2.8956 | LR: 4.91e-04 | 2823.4ms/step | 21.3 samples/s
+Step 4970/50462 | Loss: 2.8969 | LR: 4.91e-04 | 2824.9ms/step | 21.2 samples/s
+Step 4980/50462 | Loss: 2.9309 | LR: 4.91e-04 | 2821.2ms/step | 21.3 samples/s
+Step 4990/50462 | Loss: 2.9132 | LR: 4.91e-04 | 2824.5ms/step | 21.2 samples/s
+Step 5000/50462 | Loss: 2.8978 | LR: 4.91e-04 | 2822.2ms/step | 21.3 samples/s
+Validation at step 5000: Loss = 2.9191, Perplexity = 18.52
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-5000
+Step 5010/50462 | Loss: 2.9316 | LR: 4.91e-04 | 4750.4ms/step | 12.6 samples/s
+Step 5020/50462 | Loss: 2.8803 | LR: 4.91e-04 | 2817.5ms/step | 21.3 samples/s
+Step 5030/50462 | Loss: 2.9282 | LR: 4.91e-04 | 2824.3ms/step | 21.2 samples/s
+Step 5040/50462 | Loss: 2.9254 | LR: 4.91e-04 | 2828.1ms/step | 21.2 samples/s
+Step 5050/50462 | Loss: 2.8792 | LR: 4.91e-04 | 2829.3ms/step | 21.2 samples/s
+Step 5060/50462 | Loss: 2.8882 | LR: 4.91e-04 | 2827.4ms/step | 21.2 samples/s
+Step 5070/50462 | Loss: 2.8775 | LR: 4.91e-04 | 2829.3ms/step | 21.2 samples/s
+Step 5080/50462 | Loss: 2.9056 | LR: 4.91e-04 | 2828.5ms/step | 21.2 samples/s
+Step 5090/50462 | Loss: 2.8806 | LR: 4.91e-04 | 2830.7ms/step | 21.2 samples/s
+Step 5100/50462 | Loss: 2.9018 | LR: 4.91e-04 | 2836.4ms/step | 21.2 samples/s
+Step 5110/50462 | Loss: 2.9334 | LR: 4.91e-04 | 2832.6ms/step | 21.2 samples/s
+Step 5120/50462 | Loss: 2.9076 | LR: 4.91e-04 | 2822.5ms/step | 21.3 samples/s
+Step 5130/50462 | Loss: 2.8508 | LR: 4.90e-04 | 2825.5ms/step | 21.2 samples/s
+Step 5140/50462 | Loss: 2.8856 | LR: 4.90e-04 | 2823.4ms/step | 21.3 samples/s
+Step 5150/50462 | Loss: 2.8929 | LR: 4.90e-04 | 2818.7ms/step | 21.3 samples/s
+Step 5160/50462 | Loss: 2.9245 | LR: 4.90e-04 | 2823.5ms/step | 21.2 samples/s
+Step 5170/50462 | Loss: 2.8828 | LR: 4.90e-04 | 2824.4ms/step | 21.2 samples/s
+Step 5180/50462 | Loss: 2.9112 | LR: 4.90e-04 | 2823.9ms/step | 21.2 samples/s
+Step 5190/50462 | Loss: 2.8910 | LR: 4.90e-04 | 2824.0ms/step | 21.2 samples/s
+Step 5200/50462 | Loss: 2.9031 | LR: 4.90e-04 | 2823.3ms/step | 21.3 samples/s
+Step 5210/50462 | Loss: 2.8920 | LR: 4.90e-04 | 2820.8ms/step | 21.3 samples/s
+Step 5220/50462 | Loss: 2.8773 | LR: 4.90e-04 | 2821.7ms/step | 21.3 samples/s
+Step 5230/50462 | Loss: 2.9029 | LR: 4.90e-04 | 2821.0ms/step | 21.3 samples/s
+Step 5240/50462 | Loss: 2.8666 | LR: 4.90e-04 | 2823.9ms/step | 21.2 samples/s
+Step 5250/50462 | Loss: 2.8929 | LR: 4.90e-04 | 2827.5ms/step | 21.2 samples/s
+Validation at step 5250: Loss = 2.9067, Perplexity = 18.30
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-5250
+Step 5260/50462 | Loss: 2.8819 | LR: 4.90e-04 | 4767.7ms/step | 12.6 samples/s
+Step 5270/50462 | Loss: 2.8807 | LR: 4.90e-04 | 2824.3ms/step | 21.2 samples/s
+Step 5280/50462 | Loss: 2.8894 | LR: 4.90e-04 | 2831.5ms/step | 21.2 samples/s
+Step 5290/50462 | Loss: 2.9180 | LR: 4.90e-04 | 2828.8ms/step | 21.2 samples/s
+Step 5300/50462 | Loss: 2.8778 | LR: 4.90e-04 | 2829.2ms/step | 21.2 samples/s
+Step 5310/50462 | Loss: 2.9280 | LR: 4.90e-04 | 2827.9ms/step | 21.2 samples/s
+Step 5320/50462 | Loss: 2.9204 | LR: 4.90e-04 | 2831.6ms/step | 21.2 samples/s
+Step 5330/50462 | Loss: 2.8728 | LR: 4.90e-04 | 2829.1ms/step | 21.2 samples/s
+Step 5340/50462 | Loss: 2.8712 | LR: 4.90e-04 | 2830.1ms/step | 21.2 samples/s
+Step 5350/50462 | Loss: 2.9084 | LR: 4.90e-04 | 2828.1ms/step | 21.2 samples/s
+Step 5360/50462 | Loss: 2.8968 | LR: 4.89e-04 | 2829.3ms/step | 21.2 samples/s
+Step 5370/50462 | Loss: 2.8702 | LR: 4.89e-04 | 2826.8ms/step | 21.2 samples/s
+Step 5380/50462 | Loss: 2.9112 | LR: 4.89e-04 | 2832.5ms/step | 21.2 samples/s
+Step 5390/50462 | Loss: 2.9066 | LR: 4.89e-04 | 2830.3ms/step | 21.2 samples/s
+Step 5400/50462 | Loss: 2.8966 | LR: 4.89e-04 | 2827.6ms/step | 21.2 samples/s
+Step 5410/50462 | Loss: 2.8947 | LR: 4.89e-04 | 2826.6ms/step | 21.2 samples/s
+Step 5420/50462 | Loss: 2.8699 | LR: 4.89e-04 | 2827.3ms/step | 21.2 samples/s
+Step 5430/50462 | Loss: 2.8956 | LR: 4.89e-04 | 2828.5ms/step | 21.2 samples/s
+Step 5440/50462 | Loss: 2.8675 | LR: 4.89e-04 | 2830.4ms/step | 21.2 samples/s
+Step 5450/50462 | Loss: 2.8723 | LR: 4.89e-04 | 2831.2ms/step | 21.2 samples/s
+Step 5460/50462 | Loss: 2.8548 | LR: 4.89e-04 | 2826.4ms/step | 21.2 samples/s
+Step 5470/50462 | Loss: 2.8828 | LR: 4.89e-04 | 2828.1ms/step | 21.2 samples/s
+Step 5480/50462 | Loss: 2.8631 | LR: 4.89e-04 | 2828.7ms/step | 21.2 samples/s
+Step 5490/50462 | Loss: 2.8714 | LR: 4.89e-04 | 2828.4ms/step | 21.2 samples/s
+Step 5500/50462 | Loss: 2.8743 | LR: 4.89e-04 | 2828.7ms/step | 21.2 samples/s
+Validation at step 5500: Loss = 2.8967, Perplexity = 18.11
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-5500
+Step 5510/50462 | Loss: 2.8640 | LR: 4.89e-04 | 4749.9ms/step | 12.6 samples/s
+Step 5520/50462 | Loss: 2.8895 | LR: 4.89e-04 | 2826.1ms/step | 21.2 samples/s
+Step 5530/50462 | Loss: 2.8971 | LR: 4.89e-04 | 2825.5ms/step | 21.2 samples/s
+Step 5540/50462 | Loss: 2.8604 | LR: 4.89e-04 | 2826.6ms/step | 21.2 samples/s
+Step 5550/50462 | Loss: 2.8802 | LR: 4.89e-04 | 2827.0ms/step | 21.2 samples/s
+Step 5560/50462 | Loss: 2.8711 | LR: 4.89e-04 | 2827.9ms/step | 21.2 samples/s
+Step 5570/50462 | Loss: 2.8547 | LR: 4.88e-04 | 2827.7ms/step | 21.2 samples/s
+Step 5580/50462 | Loss: 2.8673 | LR: 4.88e-04 | 2824.9ms/step | 21.2 samples/s
+Step 5590/50462 | Loss: 2.9145 | LR: 4.88e-04 | 2825.2ms/step | 21.2 samples/s
+Step 5600/50462 | Loss: 2.8603 | LR: 4.88e-04 | 2829.6ms/step | 21.2 samples/s
+Step 5610/50462 | Loss: 2.8738 | LR: 4.88e-04 | 2831.7ms/step | 21.2 samples/s
+Step 5620/50462 | Loss: 2.8950 | LR: 4.88e-04 | 2824.6ms/step | 21.2 samples/s
+Step 5630/50462 | Loss: 2.8686 | LR: 4.88e-04 | 2824.8ms/step | 21.2 samples/s
+Step 5640/50462 | Loss: 2.8893 | LR: 4.88e-04 | 2825.5ms/step | 21.2 samples/s
+Step 5650/50462 | Loss: 2.8340 | LR: 4.88e-04 | 2832.0ms/step | 21.2 samples/s
+Step 5660/50462 | Loss: 2.8603 | LR: 4.88e-04 | 2829.7ms/step | 21.2 samples/s
+Step 5670/50462 | Loss: 2.8687 | LR: 4.88e-04 | 2829.1ms/step | 21.2 samples/s
+Step 5680/50462 | Loss: 2.8670 | LR: 4.88e-04 | 2825.5ms/step | 21.2 samples/s
+Step 5690/50462 | Loss: 2.8678 | LR: 4.88e-04 | 2828.6ms/step | 21.2 samples/s
+Step 5700/50462 | Loss: 2.8565 | LR: 4.88e-04 | 2826.6ms/step | 21.2 samples/s
+Step 5710/50462 | Loss: 2.8708 | LR: 4.88e-04 | 2825.7ms/step | 21.2 samples/s
+Step 5720/50462 | Loss: 2.8999 | LR: 4.88e-04 | 2826.7ms/step | 21.2 samples/s
+Step 5730/50462 | Loss: 2.8736 | LR: 4.88e-04 | 2831.8ms/step | 21.2 samples/s
+Step 5740/50462 | Loss: 2.8853 | LR: 4.88e-04 | 2832.3ms/step | 21.2 samples/s
+Step 5750/50462 | Loss: 2.8778 | LR: 4.88e-04 | 2826.5ms/step | 21.2 samples/s
+Validation at step 5750: Loss = 2.8842, Perplexity = 17.89
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-5750
+Step 5760/50462 | Loss: 2.8811 | LR: 4.88e-04 | 4756.4ms/step | 12.6 samples/s
+Step 5770/50462 | Loss: 2.8951 | LR: 4.88e-04 | 2822.1ms/step | 21.3 samples/s
+Step 5780/50462 | Loss: 2.8813 | LR: 4.87e-04 | 2826.0ms/step | 21.2 samples/s
+Step 5790/50462 | Loss: 2.8404 | LR: 4.87e-04 | 2826.3ms/step | 21.2 samples/s
+Step 5800/50462 | Loss: 2.8783 | LR: 4.87e-04 | 2823.0ms/step | 21.3 samples/s
+Step 5810/50462 | Loss: 2.8830 | LR: 4.87e-04 | 2824.6ms/step | 21.2 samples/s
+Step 5820/50462 | Loss: 2.8514 | LR: 4.87e-04 | 2825.9ms/step | 21.2 samples/s
+Step 5830/50462 | Loss: 2.8776 | LR: 4.87e-04 | 2824.4ms/step | 21.2 samples/s
+Step 5840/50462 | Loss: 2.8238 | LR: 4.87e-04 | 2829.3ms/step | 21.2 samples/s
+Step 5850/50462 | Loss: 2.8744 | LR: 4.87e-04 | 2826.8ms/step | 21.2 samples/s
+Step 5860/50462 | Loss: 2.8967 | LR: 4.87e-04 | 2828.7ms/step | 21.2 samples/s
+Step 5870/50462 | Loss: 2.8746 | LR: 4.87e-04 | 2824.4ms/step | 21.2 samples/s
+Step 5880/50462 | Loss: 2.8556 | LR: 4.87e-04 | 2825.6ms/step | 21.2 samples/s
+Step 5890/50462 | Loss: 2.8700 | LR: 4.87e-04 | 2827.4ms/step | 21.2 samples/s
+Step 5900/50462 | Loss: 2.8365 | LR: 4.87e-04 | 2828.1ms/step | 21.2 samples/s
+Step 5910/50462 | Loss: 2.8588 | LR: 4.87e-04 | 2831.0ms/step | 21.2 samples/s
+Step 5920/50462 | Loss: 2.8830 | LR: 4.87e-04 | 2832.3ms/step | 21.2 samples/s
+Step 5930/50462 | Loss: 2.8810 | LR: 4.87e-04 | 2835.2ms/step | 21.2 samples/s
+Step 5940/50462 | Loss: 2.8708 | LR: 4.87e-04 | 2833.6ms/step | 21.2 samples/s
+Step 5950/50462 | Loss: 2.8651 | LR: 4.87e-04 | 2832.6ms/step | 21.2 samples/s
+Step 5960/50462 | Loss: 2.8942 | LR: 4.87e-04 | 2832.2ms/step | 21.2 samples/s
+Step 5970/50462 | Loss: 2.8955 | LR: 4.87e-04 | 2829.8ms/step | 21.2 samples/s
+Step 5980/50462 | Loss: 2.8690 | LR: 4.86e-04 | 2824.9ms/step | 21.2 samples/s
+Step 5990/50462 | Loss: 2.8288 | LR: 4.86e-04 | 2826.1ms/step | 21.2 samples/s
+Step 6000/50462 | Loss: 2.8741 | LR: 4.86e-04 | 2829.3ms/step | 21.2 samples/s
+Validation at step 6000: Loss = 2.8723, Perplexity = 17.68
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-6000
+Step 6010/50462 | Loss: 2.8597 | LR: 4.86e-04 | 4744.6ms/step | 12.6 samples/s
+Step 6020/50462 | Loss: 2.8519 | LR: 4.86e-04 | 2824.0ms/step | 21.2 samples/s
+Step 6030/50462 | Loss: 2.8259 | LR: 4.86e-04 | 2826.1ms/step | 21.2 samples/s
+Step 6040/50462 | Loss: 2.8711 | LR: 4.86e-04 | 2830.3ms/step | 21.2 samples/s
+Step 6050/50462 | Loss: 2.8437 | LR: 4.86e-04 | 2824.0ms/step | 21.2 samples/s
+Step 6060/50462 | Loss: 2.8692 | LR: 4.86e-04 | 2825.0ms/step | 21.2 samples/s
+Step 6070/50462 | Loss: 2.8517 | LR: 4.86e-04 | 2828.4ms/step | 21.2 samples/s
+Step 6080/50462 | Loss: 2.8451 | LR: 4.86e-04 | 2829.4ms/step | 21.2 samples/s
+Step 6090/50462 | Loss: 2.8652 | LR: 4.86e-04 | 2825.3ms/step | 21.2 samples/s
+Step 6100/50462 | Loss: 2.8646 | LR: 4.86e-04 | 2833.5ms/step | 21.2 samples/s
+Step 6110/50462 | Loss: 2.8353 | LR: 4.86e-04 | 2830.6ms/step | 21.2 samples/s
+Step 6120/50462 | Loss: 2.8590 | LR: 4.86e-04 | 2829.5ms/step | 21.2 samples/s
+Step 6130/50462 | Loss: 2.8722 | LR: 4.86e-04 | 2826.8ms/step | 21.2 samples/s
+Step 6140/50462 | Loss: 2.8601 | LR: 4.86e-04 | 2830.5ms/step | 21.2 samples/s
+Step 6150/50462 | Loss: 2.8556 | LR: 4.86e-04 | 2830.8ms/step | 21.2 samples/s
+Step 6160/50462 | Loss: 2.8648 | LR: 4.86e-04 | 2827.6ms/step | 21.2 samples/s
+Step 6170/50462 | Loss: 2.8784 | LR: 4.85e-04 | 2830.2ms/step | 21.2 samples/s
+Step 6180/50462 | Loss: 2.8731 | LR: 4.85e-04 | 2826.2ms/step | 21.2 samples/s
+Step 6190/50462 | Loss: 2.8551 | LR: 4.85e-04 | 2827.7ms/step | 21.2 samples/s
+Step 6200/50462 | Loss: 2.8627 | LR: 4.85e-04 | 2824.1ms/step | 21.2 samples/s
+Step 6210/50462 | Loss: 2.8903 | LR: 4.85e-04 | 2828.8ms/step | 21.2 samples/s
+Step 6220/50462 | Loss: 2.8524 | LR: 4.85e-04 | 2831.8ms/step | 21.2 samples/s
+Step 6230/50462 | Loss: 2.8157 | LR: 4.85e-04 | 2830.2ms/step | 21.2 samples/s

epfl_final/SparseAtt/wandb/run-20260903_142704-7tfq6nwm/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260903_142704-7tfq6nwm/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-09-03T14:27:04.846539Z",
+  "args":  [
+    "--config",
+    "configs/p2_1p2b_dense_seed72_4gpu_resume2750_fixed.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/SparseAtt.git",
+    "commit":  "5b79c4bdda9b10dca72a01f318ead9d505227eb5"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-train-4-gpu-0-53",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "7226737491968"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-c9ff8a50-9103-3e73-0066-6e7c0568cad4"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-ac14b2f5-a0c5-12c4-9fa0-46c3aa9e75c9"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-b817d241-acce-e5dd-a749-3447239c5207"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-47444fc7-ea8c-cef6-aac8-ef408dc8047b"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "sn6mkwuu3x10mknvaidjrjhqujk66dlr"
+}

epfl_final/SparseAtt/wandb/run-20260903_142704-7tfq6nwm/logs/debug.log

@@ -0,0 +1,20 @@
+2026-09-03 14:27:04,872 INFO    MainThread:442 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-03 14:27:04,873 INFO    MainThread:442 [wandb_setup.py:_flush():81] Configure stats pid to 442
+2026-09-03 14:27:04,873 INFO    MainThread:442 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-03 14:27:04,873 INFO    MainThread:442 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260903_142704-7tfq6nwm/logs/debug.log
+2026-09-03 14:27:04,873 INFO    MainThread:442 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260903_142704-7tfq6nwm/logs/debug-internal.log
+2026-09-03 14:27:04,873 INFO    MainThread:442 [wandb_init.py:init():850] calling init triggers
+2026-09-03 14:27:04,873 INFO    MainThread:442 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-2750', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 15, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 72, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b dense 8k seed72 4gpu', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'dense', 'seed72'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-03 14:27:04,873 INFO    MainThread:442 [wandb_init.py:init():898] starting backend
+2026-09-03 14:27:06,027 INFO    MainThread:442 [wandb_init.py:init():913] sending inform_init request
+2026-09-03 14:27:06,590 INFO    MainThread:442 [wandb_init.py:init():918] backend started and connected
+2026-09-03 14:27:06,592 INFO    MainThread:442 [wandb_init.py:init():988] updated telemetry
+2026-09-03 14:27:06,642 INFO    MainThread:442 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-03 14:27:07,190 INFO    MainThread:442 [wandb_init.py:init():1051] run resumed
+2026-09-03 14:27:07,193 INFO    MainThread:442 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-03 14:27:07,541 INFO    MainThread:442 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-03 14:27:07,541 INFO    MainThread:442 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-03 14:27:07,541 INFO    MainThread:442 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-03 14:27:07,541 INFO    MainThread:442 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-03 14:27:07,550 INFO    MainThread:442 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/run-20260903_142704-7tfq6nwm/run-7tfq6nwm.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:086a0de4d0363dabcf09a4bb13f20d095fec8873f55ed7989fd1583d4f9d0dfb
+size 3473408

epfl_final/SparseAtt/wandb/run-20260903_191410-7tfq6nwm/files/output.log

@@ -0,0 +1,324 @@
+Starting training...
+Validation at step 6000: Loss = 2.8723, Perplexity = 17.68
+wandb: WARNING Tried to log to step 6000 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 6001: dense (dense step interval)
+Step 6010/50462 | Loss: 2.8597 | LR: 4.86e-04 | 21236.9ms/step | 2.8 samples/s
+wandb: WARNING Tried to log to step 6010 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 6020/50462 | Loss: 2.8519 | LR: 4.86e-04 | 2809.8ms/step | 21.4 samples/s
+wandb: WARNING Tried to log to step 6020 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 6030/50462 | Loss: 2.8259 | LR: 4.86e-04 | 2816.6ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 6030 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 6040/50462 | Loss: 2.8712 | LR: 4.86e-04 | 2816.1ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 6040 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 6050/50462 | Loss: 2.8437 | LR: 4.86e-04 | 2822.8ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 6050 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 6060/50462 | Loss: 2.8693 | LR: 4.86e-04 | 2818.7ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 6060 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 6070/50462 | Loss: 2.8517 | LR: 4.86e-04 | 2820.4ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 6070 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 6080/50462 | Loss: 2.8451 | LR: 4.86e-04 | 2817.7ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 6080 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 6090/50462 | Loss: 2.8651 | LR: 4.86e-04 | 2822.0ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 6090 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 6100/50462 | Loss: 2.8645 | LR: 4.86e-04 | 2818.9ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 6100 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 6110/50462 | Loss: 2.8353 | LR: 4.86e-04 | 2822.6ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 6110 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 6120/50462 | Loss: 2.8590 | LR: 4.86e-04 | 2822.6ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 6120 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 6130/50462 | Loss: 2.8722 | LR: 4.86e-04 | 2819.8ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 6130 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 6140/50462 | Loss: 2.8600 | LR: 4.86e-04 | 2818.8ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 6140 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 6150/50462 | Loss: 2.8555 | LR: 4.86e-04 | 2819.9ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 6150 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 6160/50462 | Loss: 2.8649 | LR: 4.86e-04 | 2818.8ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 6160 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 6170/50462 | Loss: 2.8784 | LR: 4.85e-04 | 2822.2ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 6170 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 6180/50462 | Loss: 2.8731 | LR: 4.85e-04 | 2821.1ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 6180 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 6190/50462 | Loss: 2.8551 | LR: 4.85e-04 | 2819.8ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 6190 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 6200/50462 | Loss: 2.8627 | LR: 4.85e-04 | 2817.7ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 6200 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 6210/50462 | Loss: 2.8903 | LR: 4.85e-04 | 2820.1ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 6210 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 6220/50462 | Loss: 2.8524 | LR: 4.85e-04 | 2819.5ms/step | 21.3 samples/s
+wandb: WARNING Tried to log to step 6220 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 6230/50462 | Loss: 2.8157 | LR: 4.85e-04 | 2821.0ms/step | 21.3 samples/s
+Step 6240/50462 | Loss: 2.8295 | LR: 4.85e-04 | 2822.1ms/step | 21.3 samples/s
+Step 6250/50462 | Loss: 2.8252 | LR: 4.85e-04 | 2819.6ms/step | 21.3 samples/s
+Validation at step 6250: Loss = 2.8645, Perplexity = 17.54
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-6250
+Step 6260/50462 | Loss: 2.8655 | LR: 4.85e-04 | 4744.2ms/step | 12.6 samples/s
+Step 6270/50462 | Loss: 2.8385 | LR: 4.85e-04 | 2825.2ms/step | 21.2 samples/s
+Step 6280/50462 | Loss: 2.8255 | LR: 4.85e-04 | 2823.4ms/step | 21.3 samples/s
+Step 6290/50462 | Loss: 2.8550 | LR: 4.85e-04 | 2824.9ms/step | 21.2 samples/s
+Step 6300/50462 | Loss: 2.8523 | LR: 4.85e-04 | 2830.5ms/step | 21.2 samples/s
+Step 6310/50462 | Loss: 2.8538 | LR: 4.85e-04 | 2821.9ms/step | 21.3 samples/s
+Step 6320/50462 | Loss: 2.8421 | LR: 4.85e-04 | 2827.6ms/step | 21.2 samples/s
+Step 6330/50462 | Loss: 2.8555 | LR: 4.85e-04 | 2826.2ms/step | 21.2 samples/s
+Step 6340/50462 | Loss: 2.8568 | LR: 4.85e-04 | 2827.0ms/step | 21.2 samples/s
+Step 6350/50462 | Loss: 2.8298 | LR: 4.85e-04 | 2824.4ms/step | 21.2 samples/s
+Step 6360/50462 | Loss: 2.8044 | LR: 4.84e-04 | 2826.0ms/step | 21.2 samples/s
+Step 6370/50462 | Loss: 2.8636 | LR: 4.84e-04 | 2825.6ms/step | 21.2 samples/s
+Step 6380/50462 | Loss: 2.8133 | LR: 4.84e-04 | 2824.3ms/step | 21.2 samples/s
+Step 6390/50462 | Loss: 2.8303 | LR: 4.84e-04 | 2826.0ms/step | 21.2 samples/s
+Step 6400/50462 | Loss: 2.7921 | LR: 4.84e-04 | 2819.7ms/step | 21.3 samples/s
+Step 6410/50462 | Loss: 2.8089 | LR: 4.84e-04 | 2820.7ms/step | 21.3 samples/s
+Step 6420/50462 | Loss: 2.8516 | LR: 4.84e-04 | 2825.8ms/step | 21.2 samples/s
+Step 6430/50462 | Loss: 2.8520 | LR: 4.84e-04 | 2829.3ms/step | 21.2 samples/s
+Step 6440/50462 | Loss: 2.8771 | LR: 4.84e-04 | 2825.5ms/step | 21.2 samples/s
+Step 6450/50462 | Loss: 2.8404 | LR: 4.84e-04 | 2826.3ms/step | 21.2 samples/s
+Step 6460/50462 | Loss: 2.8609 | LR: 4.84e-04 | 2825.8ms/step | 21.2 samples/s
+Step 6470/50462 | Loss: 2.8627 | LR: 4.84e-04 | 2827.6ms/step | 21.2 samples/s
+Step 6480/50462 | Loss: 2.8227 | LR: 4.84e-04 | 2828.5ms/step | 21.2 samples/s
+Step 6490/50462 | Loss: 2.8173 | LR: 4.84e-04 | 2832.5ms/step | 21.2 samples/s
+Step 6500/50462 | Loss: 2.8416 | LR: 4.84e-04 | 2830.2ms/step | 21.2 samples/s
+Validation at step 6500: Loss = 2.8544, Perplexity = 17.36
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-6500
+Step 6510/50462 | Loss: 2.8339 | LR: 4.84e-04 | 4724.2ms/step | 12.7 samples/s
+Step 6520/50462 | Loss: 2.8390 | LR: 4.84e-04 | 2821.9ms/step | 21.3 samples/s
+Step 6530/50462 | Loss: 2.8509 | LR: 4.84e-04 | 2821.2ms/step | 21.3 samples/s
+Step 6540/50462 | Loss: 2.8420 | LR: 4.83e-04 | 2823.8ms/step | 21.2 samples/s
+Step 6550/50462 | Loss: 2.8535 | LR: 4.83e-04 | 2827.1ms/step | 21.2 samples/s
+Step 6560/50462 | Loss: 2.8307 | LR: 4.83e-04 | 2826.6ms/step | 21.2 samples/s
+Step 6570/50462 | Loss: 2.8257 | LR: 4.83e-04 | 2828.4ms/step | 21.2 samples/s
+Step 6580/50462 | Loss: 2.8226 | LR: 4.83e-04 | 2827.2ms/step | 21.2 samples/s
+Step 6590/50462 | Loss: 2.8594 | LR: 4.83e-04 | 2827.3ms/step | 21.2 samples/s
+Step 6600/50462 | Loss: 2.8827 | LR: 4.83e-04 | 2825.0ms/step | 21.2 samples/s
+Step 6610/50462 | Loss: 2.8160 | LR: 4.83e-04 | 2826.7ms/step | 21.2 samples/s
+Step 6620/50462 | Loss: 2.8175 | LR: 4.83e-04 | 2825.4ms/step | 21.2 samples/s
+Step 6630/50462 | Loss: 2.8634 | LR: 4.83e-04 | 2828.3ms/step | 21.2 samples/s
+Step 6640/50462 | Loss: 2.8553 | LR: 4.83e-04 | 2830.2ms/step | 21.2 samples/s
+Step 6650/50462 | Loss: 2.8787 | LR: 4.83e-04 | 2831.6ms/step | 21.2 samples/s
+Step 6660/50462 | Loss: 2.8487 | LR: 4.83e-04 | 2832.4ms/step | 21.2 samples/s
+Step 6670/50462 | Loss: 2.8219 | LR: 4.83e-04 | 2833.4ms/step | 21.2 samples/s
+Step 6680/50462 | Loss: 2.8446 | LR: 4.83e-04 | 2835.7ms/step | 21.2 samples/s
+Step 6690/50462 | Loss: 2.8274 | LR: 4.83e-04 | 2837.0ms/step | 21.1 samples/s
+Step 6700/50462 | Loss: 2.8233 | LR: 4.83e-04 | 2822.0ms/step | 21.3 samples/s
+Step 6710/50462 | Loss: 2.7692 | LR: 4.82e-04 | 2826.6ms/step | 21.2 samples/s
+Step 6720/50462 | Loss: 2.8327 | LR: 4.82e-04 | 2829.6ms/step | 21.2 samples/s
+Step 6730/50462 | Loss: 2.8745 | LR: 4.82e-04 | 2827.9ms/step | 21.2 samples/s
+Step 6740/50462 | Loss: 2.8562 | LR: 4.82e-04 | 2828.4ms/step | 21.2 samples/s
+Step 6750/50462 | Loss: 2.8172 | LR: 4.82e-04 | 2830.9ms/step | 21.2 samples/s
+Validation at step 6750: Loss = 2.8447, Perplexity = 17.20
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-6750
+Step 6760/50462 | Loss: 2.7999 | LR: 4.82e-04 | 4742.2ms/step | 12.7 samples/s
+Step 6770/50462 | Loss: 2.8325 | LR: 4.82e-04 | 2826.8ms/step | 21.2 samples/s
+Step 6780/50462 | Loss: 2.8265 | LR: 4.82e-04 | 2827.0ms/step | 21.2 samples/s
+Step 6790/50462 | Loss: 2.8427 | LR: 4.82e-04 | 2826.8ms/step | 21.2 samples/s
+Step 6800/50462 | Loss: 2.8308 | LR: 4.82e-04 | 2831.5ms/step | 21.2 samples/s
+Step 6810/50462 | Loss: 2.8202 | LR: 4.82e-04 | 2825.0ms/step | 21.2 samples/s
+Step 6820/50462 | Loss: 2.8387 | LR: 4.82e-04 | 2829.4ms/step | 21.2 samples/s
+Step 6830/50462 | Loss: 2.8369 | LR: 4.82e-04 | 2829.2ms/step | 21.2 samples/s
+Step 6840/50462 | Loss: 2.8424 | LR: 4.82e-04 | 2827.2ms/step | 21.2 samples/s
+Step 6850/50462 | Loss: 2.8462 | LR: 4.82e-04 | 2827.3ms/step | 21.2 samples/s
+Step 6860/50462 | Loss: 2.8360 | LR: 4.82e-04 | 2827.5ms/step | 21.2 samples/s
+Step 6870/50462 | Loss: 2.8156 | LR: 4.82e-04 | 2823.5ms/step | 21.3 samples/s
+Step 6880/50462 | Loss: 2.8322 | LR: 4.81e-04 | 2827.1ms/step | 21.2 samples/s
+Step 6890/50462 | Loss: 2.8454 | LR: 4.81e-04 | 2824.9ms/step | 21.2 samples/s
+Step 6900/50462 | Loss: 2.8269 | LR: 4.81e-04 | 2824.2ms/step | 21.2 samples/s
+Step 6910/50462 | Loss: 2.8452 | LR: 4.81e-04 | 2829.2ms/step | 21.2 samples/s
+Step 6920/50462 | Loss: 2.8197 | LR: 4.81e-04 | 2826.6ms/step | 21.2 samples/s
+Step 6930/50462 | Loss: 2.8001 | LR: 4.81e-04 | 2827.0ms/step | 21.2 samples/s
+Step 6940/50462 | Loss: 2.8079 | LR: 4.81e-04 | 2827.8ms/step | 21.2 samples/s
+Step 6950/50462 | Loss: 2.8084 | LR: 4.81e-04 | 2828.2ms/step | 21.2 samples/s
+Step 6960/50462 | Loss: 2.7995 | LR: 4.81e-04 | 2827.1ms/step | 21.2 samples/s
+Step 6970/50462 | Loss: 2.8182 | LR: 4.81e-04 | 2828.7ms/step | 21.2 samples/s
+Step 6980/50462 | Loss: 2.8333 | LR: 4.81e-04 | 2825.2ms/step | 21.2 samples/s
+Step 6990/50462 | Loss: 2.8061 | LR: 4.81e-04 | 2823.9ms/step | 21.2 samples/s
+Step 7000/50462 | Loss: 2.8077 | LR: 4.81e-04 | 2826.0ms/step | 21.2 samples/s
+Validation at step 7000: Loss = 2.8372, Perplexity = 17.07
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-7000
+Step 7010/50462 | Loss: 2.8285 | LR: 4.81e-04 | 4713.6ms/step | 12.7 samples/s
+Step 7020/50462 | Loss: 2.8495 | LR: 4.81e-04 | 2824.9ms/step | 21.2 samples/s
+Step 7030/50462 | Loss: 2.8164 | LR: 4.81e-04 | 2828.3ms/step | 21.2 samples/s
+Step 7040/50462 | Loss: 2.8421 | LR: 4.81e-04 | 2831.6ms/step | 21.2 samples/s
+Step 7050/50462 | Loss: 2.8165 | LR: 4.80e-04 | 2831.3ms/step | 21.2 samples/s
+Step 7060/50462 | Loss: 2.8048 | LR: 4.80e-04 | 2831.8ms/step | 21.2 samples/s
+Step 7070/50462 | Loss: 2.8053 | LR: 4.80e-04 | 2830.2ms/step | 21.2 samples/s
+Step 7080/50462 | Loss: 2.8174 | LR: 4.80e-04 | 2829.2ms/step | 21.2 samples/s
+Step 7090/50462 | Loss: 2.8121 | LR: 4.80e-04 | 2829.8ms/step | 21.2 samples/s
+Step 7100/50462 | Loss: 2.8201 | LR: 4.80e-04 | 2831.6ms/step | 21.2 samples/s
+Step 7110/50462 | Loss: 2.7884 | LR: 4.80e-04 | 2828.6ms/step | 21.2 samples/s
+Step 7120/50462 | Loss: 2.8225 | LR: 4.80e-04 | 2827.6ms/step | 21.2 samples/s
+Step 7130/50462 | Loss: 2.8209 | LR: 4.80e-04 | 2829.3ms/step | 21.2 samples/s
+Step 7140/50462 | Loss: 2.8234 | LR: 4.80e-04 | 2837.5ms/step | 21.1 samples/s
+Step 7150/50462 | Loss: 2.8532 | LR: 4.80e-04 | 2838.6ms/step | 21.1 samples/s
+Step 7160/50462 | Loss: 2.8421 | LR: 4.80e-04 | 2835.8ms/step | 21.2 samples/s
+Step 7170/50462 | Loss: 2.8162 | LR: 4.80e-04 | 2836.7ms/step | 21.2 samples/s
+Step 7180/50462 | Loss: 2.8106 | LR: 4.80e-04 | 2834.7ms/step | 21.2 samples/s
+Step 7190/50462 | Loss: 2.8063 | LR: 4.80e-04 | 2843.9ms/step | 21.1 samples/s
+Step 7200/50462 | Loss: 2.8102 | LR: 4.80e-04 | 2832.8ms/step | 21.2 samples/s
+Step 7210/50462 | Loss: 2.8297 | LR: 4.79e-04 | 2832.2ms/step | 21.2 samples/s
+Step 7220/50462 | Loss: 2.8047 | LR: 4.79e-04 | 2833.6ms/step | 21.2 samples/s
+Step 7230/50462 | Loss: 2.7892 | LR: 4.79e-04 | 2829.4ms/step | 21.2 samples/s
+Step 7240/50462 | Loss: 2.8157 | LR: 4.79e-04 | 2832.2ms/step | 21.2 samples/s
+Step 7250/50462 | Loss: 2.8359 | LR: 4.79e-04 | 2835.0ms/step | 21.2 samples/s
+Validation at step 7250: Loss = 2.8303, Perplexity = 16.95
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-7250
+Step 7260/50462 | Loss: 2.8316 | LR: 4.79e-04 | 4728.0ms/step | 12.7 samples/s
+Step 7270/50462 | Loss: 2.8153 | LR: 4.79e-04 | 2829.0ms/step | 21.2 samples/s
+Step 7280/50462 | Loss: 2.8245 | LR: 4.79e-04 | 2831.0ms/step | 21.2 samples/s
+Step 7290/50462 | Loss: 2.8046 | LR: 4.79e-04 | 2832.9ms/step | 21.2 samples/s
+Step 7300/50462 | Loss: 2.8398 | LR: 4.79e-04 | 2836.7ms/step | 21.2 samples/s
+Step 7310/50462 | Loss: 2.8421 | LR: 4.79e-04 | 2832.1ms/step | 21.2 samples/s
+Step 7320/50462 | Loss: 2.8127 | LR: 4.79e-04 | 2832.3ms/step | 21.2 samples/s
+Step 7330/50462 | Loss: 2.8090 | LR: 4.79e-04 | 2832.4ms/step | 21.2 samples/s
+Step 7340/50462 | Loss: 2.8316 | LR: 4.79e-04 | 2831.0ms/step | 21.2 samples/s
+Step 7350/50462 | Loss: 2.8172 | LR: 4.79e-04 | 2834.3ms/step | 21.2 samples/s
+Step 7360/50462 | Loss: 2.7956 | LR: 4.79e-04 | 2831.1ms/step | 21.2 samples/s
+Step 7370/50462 | Loss: 2.8436 | LR: 4.78e-04 | 2830.1ms/step | 21.2 samples/s
+Step 7380/50462 | Loss: 2.7903 | LR: 4.78e-04 | 2833.2ms/step | 21.2 samples/s
+Step 7390/50462 | Loss: 2.7718 | LR: 4.78e-04 | 2832.8ms/step | 21.2 samples/s
+Step 7400/50462 | Loss: 2.8318 | LR: 4.78e-04 | 2832.7ms/step | 21.2 samples/s
+Step 7410/50462 | Loss: 2.7899 | LR: 4.78e-04 | 2832.4ms/step | 21.2 samples/s
+Step 7420/50462 | Loss: 2.8049 | LR: 4.78e-04 | 2832.6ms/step | 21.2 samples/s
+Step 7430/50462 | Loss: 2.8237 | LR: 4.78e-04 | 2832.6ms/step | 21.2 samples/s
+Step 7440/50462 | Loss: 2.7649 | LR: 4.78e-04 | 2832.7ms/step | 21.2 samples/s
+Step 7450/50462 | Loss: 2.8128 | LR: 4.78e-04 | 2830.2ms/step | 21.2 samples/s
+Step 7460/50462 | Loss: 2.7993 | LR: 4.78e-04 | 2831.4ms/step | 21.2 samples/s
+Step 7470/50462 | Loss: 2.8039 | LR: 4.78e-04 | 2828.4ms/step | 21.2 samples/s
+Step 7480/50462 | Loss: 2.8226 | LR: 4.78e-04 | 2834.6ms/step | 21.2 samples/s
+Step 7490/50462 | Loss: 2.8183 | LR: 4.78e-04 | 2833.3ms/step | 21.2 samples/s
+Step 7500/50462 | Loss: 2.8260 | LR: 4.78e-04 | 2835.8ms/step | 21.2 samples/s
+Validation at step 7500: Loss = 2.8226, Perplexity = 16.82
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-7500
+Step 7510/50462 | Loss: 2.7984 | LR: 4.78e-04 | 4804.0ms/step | 12.5 samples/s
+Step 7520/50462 | Loss: 2.8030 | LR: 4.77e-04 | 2832.1ms/step | 21.2 samples/s
+Step 7530/50462 | Loss: 2.8142 | LR: 4.77e-04 | 2833.8ms/step | 21.2 samples/s
+Step 7540/50462 | Loss: 2.8141 | LR: 4.77e-04 | 2829.7ms/step | 21.2 samples/s
+Step 7550/50462 | Loss: 2.7960 | LR: 4.77e-04 | 2832.5ms/step | 21.2 samples/s
+Step 7560/50462 | Loss: 2.8297 | LR: 4.77e-04 | 2831.0ms/step | 21.2 samples/s
+Step 7570/50462 | Loss: 2.7583 | LR: 4.77e-04 | 2834.3ms/step | 21.2 samples/s
+Step 7580/50462 | Loss: 2.8149 | LR: 4.77e-04 | 2839.6ms/step | 21.1 samples/s
+Step 7590/50462 | Loss: 2.8050 | LR: 4.77e-04 | 2834.0ms/step | 21.2 samples/s
+Step 7600/50462 | Loss: 2.7879 | LR: 4.77e-04 | 2834.0ms/step | 21.2 samples/s
+Step 7610/50462 | Loss: 2.7765 | LR: 4.77e-04 | 2832.4ms/step | 21.2 samples/s
+Step 7620/50462 | Loss: 2.8211 | LR: 4.77e-04 | 2831.4ms/step | 21.2 samples/s
+Step 7630/50462 | Loss: 2.7879 | LR: 4.77e-04 | 2834.5ms/step | 21.2 samples/s
+Step 7640/50462 | Loss: 2.8146 | LR: 4.77e-04 | 2831.1ms/step | 21.2 samples/s
+Step 7650/50462 | Loss: 2.7887 | LR: 4.77e-04 | 2826.6ms/step | 21.2 samples/s
+Step 7660/50462 | Loss: 2.7751 | LR: 4.77e-04 | 2832.6ms/step | 21.2 samples/s
+Step 7670/50462 | Loss: 2.7773 | LR: 4.76e-04 | 2826.7ms/step | 21.2 samples/s
+Step 7680/50462 | Loss: 2.8116 | LR: 4.76e-04 | 2831.2ms/step | 21.2 samples/s
+Step 7690/50462 | Loss: 2.7783 | LR: 4.76e-04 | 2833.0ms/step | 21.2 samples/s
+Step 7700/50462 | Loss: 2.7857 | LR: 4.76e-04 | 2829.2ms/step | 21.2 samples/s
+Step 7710/50462 | Loss: 2.8029 | LR: 4.76e-04 | 2835.2ms/step | 21.2 samples/s
+Step 7720/50462 | Loss: 2.7885 | LR: 4.76e-04 | 2832.6ms/step | 21.2 samples/s
+Step 7730/50462 | Loss: 2.8014 | LR: 4.76e-04 | 2830.9ms/step | 21.2 samples/s
+Step 7740/50462 | Loss: 2.8146 | LR: 4.76e-04 | 2831.2ms/step | 21.2 samples/s
+Step 7750/50462 | Loss: 2.8174 | LR: 4.76e-04 | 2829.3ms/step | 21.2 samples/s
+Validation at step 7750: Loss = 2.8131, Perplexity = 16.66
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-7750
+Step 7760/50462 | Loss: 2.7993 | LR: 4.76e-04 | 4732.3ms/step | 12.7 samples/s
+Step 7770/50462 | Loss: 2.7832 | LR: 4.76e-04 | 2832.5ms/step | 21.2 samples/s
+Step 7780/50462 | Loss: 2.7779 | LR: 4.76e-04 | 2832.1ms/step | 21.2 samples/s
+Step 7790/50462 | Loss: 2.7838 | LR: 4.76e-04 | 2834.5ms/step | 21.2 samples/s
+Step 7800/50462 | Loss: 2.7822 | LR: 4.76e-04 | 2836.6ms/step | 21.2 samples/s
+Step 7810/50462 | Loss: 2.7876 | LR: 4.76e-04 | 2830.9ms/step | 21.2 samples/s
+Step 7820/50462 | Loss: 2.8113 | LR: 4.75e-04 | 2834.8ms/step | 21.2 samples/s
+Step 7830/50462 | Loss: 2.8010 | LR: 4.75e-04 | 2832.0ms/step | 21.2 samples/s
+Step 7840/50462 | Loss: 2.8010 | LR: 4.75e-04 | 2832.6ms/step | 21.2 samples/s
+Step 7850/50462 | Loss: 2.8107 | LR: 4.75e-04 | 2835.3ms/step | 21.2 samples/s
+Step 7860/50462 | Loss: 2.8114 | LR: 4.75e-04 | 2829.9ms/step | 21.2 samples/s
+Step 7870/50462 | Loss: 2.7901 | LR: 4.75e-04 | 2827.4ms/step | 21.2 samples/s
+Step 7880/50462 | Loss: 2.8174 | LR: 4.75e-04 | 2839.2ms/step | 21.1 samples/s
+Step 7890/50462 | Loss: 2.7936 | LR: 4.75e-04 | 2829.6ms/step | 21.2 samples/s
+Step 7900/50462 | Loss: 2.7810 | LR: 4.75e-04 | 2847.7ms/step | 21.1 samples/s
+Step 7910/50462 | Loss: 2.7693 | LR: 4.75e-04 | 2837.5ms/step | 21.1 samples/s
+Step 7920/50462 | Loss: 2.8108 | LR: 4.75e-04 | 2831.0ms/step | 21.2 samples/s
+Step 7930/50462 | Loss: 2.7878 | LR: 4.75e-04 | 2835.8ms/step | 21.2 samples/s
+Step 7940/50462 | Loss: 2.7996 | LR: 4.75e-04 | 2835.8ms/step | 21.2 samples/s
+Step 7950/50462 | Loss: 2.7874 | LR: 4.75e-04 | 2832.5ms/step | 21.2 samples/s
+Step 7960/50462 | Loss: 2.7938 | LR: 4.74e-04 | 2833.2ms/step | 21.2 samples/s
+Step 7970/50462 | Loss: 2.7931 | LR: 4.74e-04 | 2832.0ms/step | 21.2 samples/s
+Step 7980/50462 | Loss: 2.7587 | LR: 4.74e-04 | 2831.3ms/step | 21.2 samples/s
+Step 7990/50462 | Loss: 2.7533 | LR: 4.74e-04 | 2828.2ms/step | 21.2 samples/s
+Step 8000/50462 | Loss: 2.7796 | LR: 4.74e-04 | 2829.2ms/step | 21.2 samples/s
+Validation at step 8000: Loss = 2.8065, Perplexity = 16.55
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-8000
+Step 8010/50462 | Loss: 2.8012 | LR: 4.74e-04 | 4733.1ms/step | 12.7 samples/s
+Step 8020/50462 | Loss: 2.7603 | LR: 4.74e-04 | 2824.5ms/step | 21.2 samples/s
+Step 8030/50462 | Loss: 2.7895 | LR: 4.74e-04 | 2826.0ms/step | 21.2 samples/s
+Step 8040/50462 | Loss: 2.7949 | LR: 4.74e-04 | 2826.9ms/step | 21.2 samples/s
+Step 8050/50462 | Loss: 2.8022 | LR: 4.74e-04 | 2830.8ms/step | 21.2 samples/s
+Step 8060/50462 | Loss: 2.8108 | LR: 4.74e-04 | 2828.6ms/step | 21.2 samples/s
+Step 8070/50462 | Loss: 2.7895 | LR: 4.74e-04 | 2831.8ms/step | 21.2 samples/s
+Step 8080/50462 | Loss: 2.8107 | LR: 4.74e-04 | 2829.3ms/step | 21.2 samples/s
+Step 8090/50462 | Loss: 2.8135 | LR: 4.74e-04 | 2828.4ms/step | 21.2 samples/s
+Step 8100/50462 | Loss: 2.7970 | LR: 4.74e-04 | 2828.4ms/step | 21.2 samples/s
+Step 8110/50462 | Loss: 2.7817 | LR: 4.73e-04 | 2826.3ms/step | 21.2 samples/s
+Step 8120/50462 | Loss: 2.7818 | LR: 4.73e-04 | 2826.8ms/step | 21.2 samples/s
+Step 8130/50462 | Loss: 2.7866 | LR: 4.73e-04 | 2829.5ms/step | 21.2 samples/s
+Step 8140/50462 | Loss: 2.7823 | LR: 4.73e-04 | 2827.8ms/step | 21.2 samples/s
+Step 8150/50462 | Loss: 2.7947 | LR: 4.73e-04 | 2824.0ms/step | 21.2 samples/s
+Step 8160/50462 | Loss: 2.8141 | LR: 4.73e-04 | 2827.9ms/step | 21.2 samples/s
+Step 8170/50462 | Loss: 2.7743 | LR: 4.73e-04 | 2828.2ms/step | 21.2 samples/s
+Step 8180/50462 | Loss: 2.7726 | LR: 4.73e-04 | 2827.4ms/step | 21.2 samples/s
+Step 8190/50462 | Loss: 2.7846 | LR: 4.73e-04 | 2839.0ms/step | 21.1 samples/s
+Step 8200/50462 | Loss: 2.7729 | LR: 4.73e-04 | 2824.9ms/step | 21.2 samples/s
+Step 8210/50462 | Loss: 2.8169 | LR: 4.73e-04 | 2828.4ms/step | 21.2 samples/s
+Step 8220/50462 | Loss: 2.7852 | LR: 4.73e-04 | 2839.1ms/step | 21.1 samples/s
+Step 8230/50462 | Loss: 2.7760 | LR: 4.73e-04 | 2829.6ms/step | 21.2 samples/s
+Step 8240/50462 | Loss: 2.7593 | LR: 4.73e-04 | 2828.5ms/step | 21.2 samples/s
+Step 8250/50462 | Loss: 2.7997 | LR: 4.72e-04 | 2828.1ms/step | 21.2 samples/s
+Validation at step 8250: Loss = 2.7999, Perplexity = 16.44
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-8250
+Step 8260/50462 | Loss: 2.7523 | LR: 4.72e-04 | 4773.8ms/step | 12.6 samples/s
+Step 8270/50462 | Loss: 2.7759 | LR: 4.72e-04 | 2824.9ms/step | 21.2 samples/s
+Step 8280/50462 | Loss: 2.7909 | LR: 4.72e-04 | 2828.9ms/step | 21.2 samples/s
+Step 8290/50462 | Loss: 2.7802 | LR: 4.72e-04 | 2827.2ms/step | 21.2 samples/s
+Step 8300/50462 | Loss: 2.7975 | LR: 4.72e-04 | 2824.6ms/step | 21.2 samples/s
+Step 8310/50462 | Loss: 2.7950 | LR: 4.72e-04 | 2829.5ms/step | 21.2 samples/s
+Step 8320/50462 | Loss: 2.7723 | LR: 4.72e-04 | 2827.7ms/step | 21.2 samples/s
+Step 8330/50462 | Loss: 2.7928 | LR: 4.72e-04 | 2823.9ms/step | 21.2 samples/s
+Step 8340/50462 | Loss: 2.7639 | LR: 4.72e-04 | 2846.1ms/step | 21.1 samples/s
+Step 8350/50462 | Loss: 2.7707 | LR: 4.72e-04 | 2831.0ms/step | 21.2 samples/s
+Step 8360/50462 | Loss: 2.7990 | LR: 4.72e-04 | 2829.4ms/step | 21.2 samples/s
+Step 8370/50462 | Loss: 2.8025 | LR: 4.72e-04 | 2831.3ms/step | 21.2 samples/s
+Step 8380/50462 | Loss: 2.8028 | LR: 4.71e-04 | 2832.1ms/step | 21.2 samples/s
+Step 8390/50462 | Loss: 2.7690 | LR: 4.71e-04 | 2828.4ms/step | 21.2 samples/s
+Step 8400/50462 | Loss: 2.7571 | LR: 4.71e-04 | 2827.9ms/step | 21.2 samples/s
+Step 8410/50462 | Loss: 2.7676 | LR: 4.71e-04 | 2830.2ms/step | 21.2 samples/s
+Step 8420/50462 | Loss: 2.7708 | LR: 4.71e-04 | 2827.5ms/step | 21.2 samples/s
+Step 8430/50462 | Loss: 2.7216 | LR: 4.71e-04 | 2825.5ms/step | 21.2 samples/s
+Step 8440/50462 | Loss: 2.7693 | LR: 4.71e-04 | 2826.9ms/step | 21.2 samples/s
+Step 8450/50462 | Loss: 2.7839 | LR: 4.71e-04 | 2826.9ms/step | 21.2 samples/s
+Step 8460/50462 | Loss: 2.7829 | LR: 4.71e-04 | 2834.4ms/step | 21.2 samples/s
+Step 8470/50462 | Loss: 2.8129 | LR: 4.71e-04 | 2828.9ms/step | 21.2 samples/s
+Step 8480/50462 | Loss: 2.8214 | LR: 4.71e-04 | 2827.1ms/step | 21.2 samples/s
+Step 8490/50462 | Loss: 2.7758 | LR: 4.71e-04 | 2825.1ms/step | 21.2 samples/s
+Step 8500/50462 | Loss: 2.7891 | LR: 4.71e-04 | 2831.6ms/step | 21.2 samples/s
+Validation at step 8500: Loss = 2.7920, Perplexity = 16.31
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-8500
+Step 8510/50462 | Loss: 2.8123 | LR: 4.71e-04 | 4720.9ms/step | 12.7 samples/s
+Step 8520/50462 | Loss: 2.7670 | LR: 4.70e-04 | 2828.4ms/step | 21.2 samples/s
+Step 8530/50462 | Loss: 2.7797 | LR: 4.70e-04 | 2828.3ms/step | 21.2 samples/s
+Step 8540/50462 | Loss: 2.7798 | LR: 4.70e-04 | 2827.7ms/step | 21.2 samples/s
+Step 8550/50462 | Loss: 2.8018 | LR: 4.70e-04 | 2841.3ms/step | 21.1 samples/s
+Step 8560/50462 | Loss: 2.7387 | LR: 4.70e-04 | 2828.4ms/step | 21.2 samples/s
+Step 8570/50462 | Loss: 2.8006 | LR: 4.70e-04 | 2827.3ms/step | 21.2 samples/s
+Step 8580/50462 | Loss: 2.7432 | LR: 4.70e-04 | 2829.6ms/step | 21.2 samples/s
+Step 8590/50462 | Loss: 2.7690 | LR: 4.70e-04 | 2836.0ms/step | 21.2 samples/s
+Step 8600/50462 | Loss: 2.7692 | LR: 4.70e-04 | 2831.0ms/step | 21.2 samples/s
+Step 8610/50462 | Loss: 2.7645 | LR: 4.70e-04 | 2830.1ms/step | 21.2 samples/s
+Step 8620/50462 | Loss: 2.7955 | LR: 4.70e-04 | 2831.2ms/step | 21.2 samples/s
+Step 8630/50462 | Loss: 2.7715 | LR: 4.70e-04 | 2830.8ms/step | 21.2 samples/s
+Step 8640/50462 | Loss: 2.7557 | LR: 4.70e-04 | 2827.9ms/step | 21.2 samples/s
+Step 8650/50462 | Loss: 2.7873 | LR: 4.69e-04 | 2827.3ms/step | 21.2 samples/s
+Step 8660/50462 | Loss: 2.7826 | LR: 4.69e-04 | 2827.2ms/step | 21.2 samples/s
+Step 8670/50462 | Loss: 2.7481 | LR: 4.69e-04 | 2828.3ms/step | 21.2 samples/s
+Step 8680/50462 | Loss: 2.7883 | LR: 4.69e-04 | 2828.5ms/step | 21.2 samples/s
+Step 8690/50462 | Loss: 2.7859 | LR: 4.69e-04 | 2827.6ms/step | 21.2 samples/s
+Step 8700/50462 | Loss: 2.7838 | LR: 4.69e-04 | 2824.2ms/step | 21.2 samples/s
+Step 8710/50462 | Loss: 2.7647 | LR: 4.69e-04 | 2826.2ms/step | 21.2 samples/s
+Step 8720/50462 | Loss: 2.7783 | LR: 4.69e-04 | 2821.9ms/step | 21.3 samples/s
+Step 8730/50462 | Loss: 2.7925 | LR: 4.69e-04 | 2818.3ms/step | 21.3 samples/s
+Step 8740/50462 | Loss: 2.7903 | LR: 4.69e-04 | 2819.6ms/step | 21.3 samples/s
+Step 8750/50462 | Loss: 2.7691 | LR: 4.69e-04 | 2819.6ms/step | 21.3 samples/s
+Validation at step 8750: Loss = 2.7840, Perplexity = 16.18
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-8750
+Step 8760/50462 | Loss: 2.7788 | LR: 4.69e-04 | 4684.9ms/step | 12.8 samples/s

epfl_final/SparseAtt/wandb/run-20260903_191410-7tfq6nwm/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260903_191410-7tfq6nwm/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-09-03T19:14:10.532910Z",
+  "args":  [
+    "--config",
+    "/tmp/p2_seed72_resume6000.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/SparseAtt.git",
+    "commit":  "5b79c4bdda9b10dca72a01f318ead9d505227eb5"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-train-4-gpu-0-56",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "7215713296384"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-c9ff8a50-9103-3e73-0066-6e7c0568cad4"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-e8ce0e9a-801b-03d2-c898-c0cc7d746d53"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-8b16a485-aa97-1545-8625-887d6079c32c"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-47444fc7-ea8c-cef6-aac8-ef408dc8047b"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "2bx5hh0drqprpievzzp1bd7jnlfcuqdq"
+}

epfl_final/SparseAtt/wandb/run-20260903_191410-7tfq6nwm/logs/debug.log

@@ -0,0 +1,20 @@
+2026-09-03 19:14:10,553 INFO    MainThread:371 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-03 19:14:10,553 INFO    MainThread:371 [wandb_setup.py:_flush():81] Configure stats pid to 371
+2026-09-03 19:14:10,553 INFO    MainThread:371 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-03 19:14:10,553 INFO    MainThread:371 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260903_191410-7tfq6nwm/logs/debug.log
+2026-09-03 19:14:10,553 INFO    MainThread:371 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260903_191410-7tfq6nwm/logs/debug-internal.log
+2026-09-03 19:14:10,553 INFO    MainThread:371 [wandb_init.py:init():850] calling init triggers
+2026-09-03 19:14:10,553 INFO    MainThread:371 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-6000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 15, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 72, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b dense 8k seed72 4gpu', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'dense', 'seed72'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-03 19:14:10,553 INFO    MainThread:371 [wandb_init.py:init():898] starting backend
+2026-09-03 19:14:10,832 INFO    MainThread:371 [wandb_init.py:init():913] sending inform_init request
+2026-09-03 19:14:11,204 INFO    MainThread:371 [wandb_init.py:init():918] backend started and connected
+2026-09-03 19:14:11,207 INFO    MainThread:371 [wandb_init.py:init():988] updated telemetry
+2026-09-03 19:14:11,266 INFO    MainThread:371 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-03 19:14:11,766 INFO    MainThread:371 [wandb_init.py:init():1051] run resumed
+2026-09-03 19:14:11,769 INFO    MainThread:371 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-03 19:14:12,044 INFO    MainThread:371 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-03 19:14:12,044 INFO    MainThread:371 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-03 19:14:12,044 INFO    MainThread:371 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-03 19:14:12,044 INFO    MainThread:371 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-03 19:14:12,052 INFO    MainThread:371 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/run-20260903_191410-7tfq6nwm/run-7tfq6nwm.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:038d19b3eaa10a7e5d88ad02da2202df1f2933b87b22650916ec7b6b58f1af7e
+size 2785280

epfl_final/SparseAtt/wandb/run-20260904_042057-7tfq6nwm/files/output.log

@@ -0,0 +1,78 @@
+Starting training...
+Validation at step 8750: Loss = 2.7840, Perplexity = 16.18
+wandb: WARNING Tried to log to step 8750 that is less than the current step 8751. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 8751: dense (dense step interval)
+Step 8760/50462 | Loss: 2.7789 | LR: 4.69e-04 | 30187.4ms/step | 2.0 samples/s
+Step 8770/50462 | Loss: 2.7657 | LR: 4.69e-04 | 2809.5ms/step | 21.4 samples/s
+Step 8780/50462 | Loss: 2.8037 | LR: 4.68e-04 | 2815.7ms/step | 21.3 samples/s
+Step 8790/50462 | Loss: 2.7473 | LR: 4.68e-04 | 2816.4ms/step | 21.3 samples/s
+Step 8800/50462 | Loss: 2.7436 | LR: 4.68e-04 | 2819.2ms/step | 21.3 samples/s
+Step 8810/50462 | Loss: 2.7757 | LR: 4.68e-04 | 2817.1ms/step | 21.3 samples/s
+Step 8820/50462 | Loss: 2.7557 | LR: 4.68e-04 | 2815.9ms/step | 21.3 samples/s
+Step 8830/50462 | Loss: 2.7731 | LR: 4.68e-04 | 2818.4ms/step | 21.3 samples/s
+Step 8840/50462 | Loss: 2.7812 | LR: 4.68e-04 | 2818.2ms/step | 21.3 samples/s
+Step 8850/50462 | Loss: 2.7587 | LR: 4.68e-04 | 2816.7ms/step | 21.3 samples/s
+Step 8860/50462 | Loss: 2.7233 | LR: 4.68e-04 | 2818.0ms/step | 21.3 samples/s
+Step 8870/50462 | Loss: 2.7967 | LR: 4.68e-04 | 2817.9ms/step | 21.3 samples/s
+Step 8880/50462 | Loss: 2.7658 | LR: 4.68e-04 | 2821.9ms/step | 21.3 samples/s
+Step 8890/50462 | Loss: 2.7572 | LR: 4.68e-04 | 2817.4ms/step | 21.3 samples/s
+Step 8900/50462 | Loss: 2.7699 | LR: 4.68e-04 | 2823.1ms/step | 21.3 samples/s
+Step 8910/50462 | Loss: 2.7876 | LR: 4.67e-04 | 2820.2ms/step | 21.3 samples/s
+Step 8920/50462 | Loss: 2.7481 | LR: 4.67e-04 | 2820.0ms/step | 21.3 samples/s
+Step 8930/50462 | Loss: 2.7610 | LR: 4.67e-04 | 2820.0ms/step | 21.3 samples/s
+Step 8940/50462 | Loss: 2.7630 | LR: 4.67e-04 | 2820.9ms/step | 21.3 samples/s
+Step 8950/50462 | Loss: 2.7806 | LR: 4.67e-04 | 2818.2ms/step | 21.3 samples/s
+Step 8960/50462 | Loss: 2.7853 | LR: 4.67e-04 | 2814.6ms/step | 21.3 samples/s
+Step 8970/50462 | Loss: 2.8039 | LR: 4.67e-04 | 2817.6ms/step | 21.3 samples/s
+Step 8980/50462 | Loss: 2.7285 | LR: 4.67e-04 | 2818.9ms/step | 21.3 samples/s
+Step 8990/50462 | Loss: 2.7869 | LR: 4.67e-04 | 2816.8ms/step | 21.3 samples/s
+Step 9000/50462 | Loss: 2.7825 | LR: 4.67e-04 | 2814.1ms/step | 21.3 samples/s
+Validation at step 9000: Loss = 2.7793, Perplexity = 16.11
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-9000
+Step 9010/50462 | Loss: 2.7555 | LR: 4.67e-04 | 4736.9ms/step | 12.7 samples/s
+Step 9020/50462 | Loss: 2.7954 | LR: 4.67e-04 | 2818.7ms/step | 21.3 samples/s
+Step 9030/50462 | Loss: 2.7633 | LR: 4.67e-04 | 2816.4ms/step | 21.3 samples/s
+Step 9040/50462 | Loss: 2.7525 | LR: 4.66e-04 | 2823.7ms/step | 21.2 samples/s
+Step 9050/50462 | Loss: 2.7646 | LR: 4.66e-04 | 2819.6ms/step | 21.3 samples/s
+Step 9060/50462 | Loss: 2.7677 | LR: 4.66e-04 | 2819.7ms/step | 21.3 samples/s
+Step 9070/50462 | Loss: 2.7845 | LR: 4.66e-04 | 2820.8ms/step | 21.3 samples/s
+Step 9080/50462 | Loss: 2.7606 | LR: 4.66e-04 | 2820.3ms/step | 21.3 samples/s
+Step 9090/50462 | Loss: 2.7926 | LR: 4.66e-04 | 2821.5ms/step | 21.3 samples/s
+Step 9100/50462 | Loss: 2.7167 | LR: 4.66e-04 | 2822.7ms/step | 21.3 samples/s
+Step 9110/50462 | Loss: 2.7777 | LR: 4.66e-04 | 2822.6ms/step | 21.3 samples/s
+Step 9120/50462 | Loss: 2.7821 | LR: 4.66e-04 | 2820.8ms/step | 21.3 samples/s
+Step 9130/50462 | Loss: 2.7547 | LR: 4.66e-04 | 2822.1ms/step | 21.3 samples/s
+Step 9140/50462 | Loss: 2.7391 | LR: 4.66e-04 | 2823.8ms/step | 21.2 samples/s
+Step 9150/50462 | Loss: 2.7436 | LR: 4.66e-04 | 2822.2ms/step | 21.3 samples/s
+Step 9160/50462 | Loss: 2.7022 | LR: 4.66e-04 | 2817.9ms/step | 21.3 samples/s
+Step 9170/50462 | Loss: 2.7746 | LR: 4.65e-04 | 2820.7ms/step | 21.3 samples/s
+Step 9180/50462 | Loss: 2.7842 | LR: 4.65e-04 | 2820.9ms/step | 21.3 samples/s
+Step 9190/50462 | Loss: 2.7826 | LR: 4.65e-04 | 2819.4ms/step | 21.3 samples/s
+Step 9200/50462 | Loss: 2.7867 | LR: 4.65e-04 | 2817.8ms/step | 21.3 samples/s
+Step 9210/50462 | Loss: 2.7495 | LR: 4.65e-04 | 2818.7ms/step | 21.3 samples/s
+Step 9220/50462 | Loss: 2.7309 | LR: 4.65e-04 | 2819.3ms/step | 21.3 samples/s
+Step 9230/50462 | Loss: 2.7350 | LR: 4.65e-04 | 2819.2ms/step | 21.3 samples/s
+Step 9240/50462 | Loss: 2.7455 | LR: 4.65e-04 | 2819.6ms/step | 21.3 samples/s
+Step 9250/50462 | Loss: 2.7629 | LR: 4.65e-04 | 2823.3ms/step | 21.3 samples/s
+Validation at step 9250: Loss = 2.7746, Perplexity = 16.03
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-9250
+Step 9260/50462 | Loss: 2.7622 | LR: 4.65e-04 | 4726.7ms/step | 12.7 samples/s
+Step 9270/50462 | Loss: 2.7572 | LR: 4.65e-04 | 2820.4ms/step | 21.3 samples/s
+Step 9280/50462 | Loss: 2.7671 | LR: 4.65e-04 | 2820.7ms/step | 21.3 samples/s
+Step 9290/50462 | Loss: 2.7625 | LR: 4.64e-04 | 2822.5ms/step | 21.3 samples/s
+Step 9300/50462 | Loss: 2.7504 | LR: 4.64e-04 | 2820.5ms/step | 21.3 samples/s
+Step 9310/50462 | Loss: 2.7484 | LR: 4.64e-04 | 2818.5ms/step | 21.3 samples/s
+Step 9320/50462 | Loss: 2.7604 | LR: 4.64e-04 | 2824.8ms/step | 21.2 samples/s
+Step 9330/50462 | Loss: 2.7670 | LR: 4.64e-04 | 2823.9ms/step | 21.2 samples/s
+Step 9340/50462 | Loss: 2.7511 | LR: 4.64e-04 | 2822.7ms/step | 21.3 samples/s
+Step 9350/50462 | Loss: 2.7556 | LR: 4.64e-04 | 2824.2ms/step | 21.2 samples/s
+Step 9360/50462 | Loss: 2.7818 | LR: 4.64e-04 | 2821.5ms/step | 21.3 samples/s
+Step 9370/50462 | Loss: 2.7846 | LR: 4.64e-04 | 2825.5ms/step | 21.2 samples/s
+Step 9380/50462 | Loss: 2.7812 | LR: 4.64e-04 | 2821.9ms/step | 21.3 samples/s
+Step 9390/50462 | Loss: 2.7779 | LR: 4.64e-04 | 2824.5ms/step | 21.2 samples/s
+Step 9400/50462 | Loss: 2.7706 | LR: 4.64e-04 | 2820.9ms/step | 21.3 samples/s
+Step 9410/50462 | Loss: 2.7315 | LR: 4.63e-04 | 2821.5ms/step | 21.3 samples/s
+Step 9420/50462 | Loss: 2.7492 | LR: 4.63e-04 | 2821.3ms/step | 21.3 samples/s
+Step 9430/50462 | Loss: 2.7498 | LR: 4.63e-04 | 2823.2ms/step | 21.3 samples/s
+Step 9440/50462 | Loss: 2.7252 | LR: 4.63e-04 | 2821.7ms/step | 21.3 samples/s
+Step 9450/50462 | Loss: 2.7576 | LR: 4.63e-04 | 2822.1ms/step | 21.3 samples/

epfl_final/SparseAtt/wandb/run-20260904_042057-7tfq6nwm/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260904_042057-7tfq6nwm/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-09-04T04:20:57.322929Z",
+  "args":  [
+    "--config",
+    "/tmp/p2_seed72_resume8750.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/SparseAtt.git",
+    "commit":  "5b79c4bdda9b10dca72a01f318ead9d505227eb5"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-train-4-gpu-0-61",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "6589085245440"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-fd3a94c5-cb60-deec-b7e0-5ab205a565ca"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-3cf31aa4-5f18-ea44-a9fc-50347a16f68f"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-a292d643-3028-83ee-9cfa-fac31861f139"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-0460dc7d-d83e-fd0f-7548-a3079803b79e"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "ykopj28t9opybfu3ebba6pv65pgfj0is"
+}

epfl_final/SparseAtt/wandb/run-20260904_042057-7tfq6nwm/logs/debug-internal.log

@@ -0,0 +1,314 @@
+{"time":"2026-09-04T04:20:58.134855927Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-09-04T04:20:58.140504878Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-09-04T04:20:58.543855008Z","level":"INFO","msg":"stream: created new stream","id":"7tfq6nwm"}
+{"time":"2026-09-04T04:20:58.543967797Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-09-04T04:20:58.554477095Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-09-04T04:20:58.556336977Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-09-04T04:20:58.556330046Z","level":"INFO","msg":"writer: started","stream_id":"7tfq6nwm"}
+{"time":"2026-09-04T04:20:59.137236226Z","level":"INFO","msg":"filestream: sending request","total_files":1}
+{"time":"2026-09-04T04:20:59.424111078Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:21:14.137977603Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1,"uploaded_len":2}
+{"time":"2026-09-04T04:21:14.354722137Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:21:29.137915675Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":0,"events_lines":2}
+{"time":"2026-09-04T04:21:29.330941479Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:21:44.137647937Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
+{"time":"2026-09-04T04:21:44.381326969Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:21:45.168141533Z","level":"WARN","msg":"handler: ignoring partial history record","step":8750,"current":8751}
+{"time":"2026-09-04T04:21:59.138302464Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":4,"events_lines":2,"console_offset":1,"console_lines":2}
+{"time":"2026-09-04T04:21:59.372813091Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:22:14.137188456Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":6,"events_lines":2}
+{"time":"2026-09-04T04:22:14.349384331Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:22:29.137621351Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":8,"events_lines":2}
+{"time":"2026-09-04T04:22:29.351918631Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:22:44.137612001Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":10,"events_lines":2}
+{"time":"2026-09-04T04:22:44.357229148Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:22:59.137495581Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":12,"events_lines":2}
+{"time":"2026-09-04T04:22:59.372800761Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:23:14.137641194Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":14,"events_lines":2}
+{"time":"2026-09-04T04:23:14.720364311Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:23:29.137385455Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":16,"events_lines":2}
+{"time":"2026-09-04T04:23:29.321980692Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:23:44.137940563Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":18,"events_lines":2}
+{"time":"2026-09-04T04:23:44.356793853Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:23:59.137574127Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":20,"events_lines":2}
+{"time":"2026-09-04T04:23:59.345104357Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:24:14.13812606Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":22,"events_lines":2}
+{"time":"2026-09-04T04:24:14.354131614Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:24:29.137682308Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":24,"events_lines":2}
+{"time":"2026-09-04T04:24:29.329664947Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:24:44.137564636Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":26,"events_lines":2}
+{"time":"2026-09-04T04:24:44.36436023Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:24:59.137674893Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":28,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-09-04T04:24:59.349132099Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:25:14.1375167Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":30,"events_lines":2}
+{"time":"2026-09-04T04:25:14.361921805Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:25:29.137722333Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":32,"events_lines":2}
+{"time":"2026-09-04T04:25:29.344325953Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:25:44.138144532Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":34,"events_lines":2}
+{"time":"2026-09-04T04:25:44.366738525Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:25:59.137803404Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":36,"events_lines":2}
+{"time":"2026-09-04T04:25:59.393690323Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:26:14.13768459Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":38,"events_lines":2,"console_offset":4,"console_lines":1}
+{"time":"2026-09-04T04:26:14.343685144Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:26:29.137850444Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":40,"events_lines":2}
+{"time":"2026-09-04T04:26:29.369825303Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:26:44.13747851Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":875,"history_lines":1,"events_offset":42,"events_lines":2,"console_offset":5,"console_lines":1}
+{"time":"2026-09-04T04:26:44.367296131Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:26:59.138203452Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":876,"history_lines":1,"events_offset":44,"events_lines":2,"console_offset":6,"console_lines":1}
+{"time":"2026-09-04T04:26:59.44204017Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:27:14.137397437Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":46,"events_lines":2}
+{"time":"2026-09-04T04:27:14.34909891Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:27:29.137784513Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":877,"history_lines":1,"events_offset":48,"events_lines":2,"console_offset":7,"console_lines":1}
+{"time":"2026-09-04T04:27:29.377623128Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:27:44.137207651Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":50,"events_lines":2}
+{"time":"2026-09-04T04:27:44.375426275Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:27:59.137402929Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":878,"history_lines":1,"events_offset":52,"events_lines":2,"console_offset":8,"console_lines":1}
+{"time":"2026-09-04T04:27:59.710718882Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:28:14.138307Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":54,"events_lines":2}
+{"time":"2026-09-04T04:28:14.349323775Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:28:29.138180022Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":879,"history_lines":1,"events_offset":56,"events_lines":2,"console_offset":9,"console_lines":1}
+{"time":"2026-09-04T04:28:29.382325394Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:28:44.137405894Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":58,"events_lines":2}
+{"time":"2026-09-04T04:28:44.374702753Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:28:59.137981501Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":880,"history_lines":1,"events_offset":60,"events_lines":2,"console_offset":10,"console_lines":1}
+{"time":"2026-09-04T04:28:59.357829694Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:29:14.137398299Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":62,"events_lines":2}
+{"time":"2026-09-04T04:29:14.352994868Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:29:29.137874918Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":881,"history_lines":1,"events_offset":64,"events_lines":2,"console_offset":11,"console_lines":1}
+{"time":"2026-09-04T04:29:29.372939035Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:29:44.137250944Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":66,"events_lines":2}
+{"time":"2026-09-04T04:29:44.363462221Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:29:59.137296376Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":882,"history_lines":1,"events_offset":68,"events_lines":2,"console_offset":12,"console_lines":1}
+{"time":"2026-09-04T04:29:59.338882751Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:30:14.137870011Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":70,"events_lines":2}
+{"time":"2026-09-04T04:30:14.326942668Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:30:29.137653519Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":883,"history_lines":1,"events_offset":72,"events_lines":2,"console_offset":13,"console_lines":1}
+{"time":"2026-09-04T04:30:29.38729354Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:30:44.138287554Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":884,"history_lines":1,"events_offset":74,"events_lines":2,"console_offset":14,"console_lines":1}
+{"time":"2026-09-04T04:30:44.390269993Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:30:59.137353375Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":76,"events_lines":2}
+{"time":"2026-09-04T04:30:59.348289087Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:31:14.138098227Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":885,"history_lines":1,"events_offset":78,"events_lines":2,"console_offset":15,"console_lines":1}
+{"time":"2026-09-04T04:31:14.372923817Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:31:29.137995937Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":80,"events_lines":2}
+{"time":"2026-09-04T04:31:29.376669844Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:31:44.137667207Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":886,"history_lines":1,"events_offset":82,"events_lines":2,"console_offset":16,"console_lines":1}
+{"time":"2026-09-04T04:31:44.383931009Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:31:59.137430365Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":84,"events_lines":2}
+{"time":"2026-09-04T04:31:59.356076453Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:32:14.137309277Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":887,"history_lines":1,"events_offset":86,"events_lines":2,"console_offset":17,"console_lines":1}
+{"time":"2026-09-04T04:32:14.388217992Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:32:29.13808861Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":88,"events_lines":2}
+{"time":"2026-09-04T04:32:29.341297782Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:32:44.137912689Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":888,"history_lines":1,"events_offset":90,"events_lines":2,"console_offset":18,"console_lines":1}
+{"time":"2026-09-04T04:32:44.412533065Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:32:59.137278691Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":92,"events_lines":2}
+{"time":"2026-09-04T04:32:59.34658107Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:33:14.138266918Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":889,"history_lines":1,"events_offset":94,"events_lines":2,"console_offset":19,"console_lines":1}
+{"time":"2026-09-04T04:33:14.381771918Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:33:29.137478678Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":96,"events_lines":2}
+{"time":"2026-09-04T04:33:29.329411059Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:33:44.137892743Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":890,"history_lines":1,"events_offset":98,"events_lines":2,"console_offset":20,"console_lines":1}
+{"time":"2026-09-04T04:33:44.45861179Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:33:59.137486096Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":100,"events_lines":2}
+{"time":"2026-09-04T04:33:59.344907141Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:34:14.138148023Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":891,"history_lines":1,"events_offset":102,"events_lines":2,"console_offset":21,"console_lines":1}
+{"time":"2026-09-04T04:34:14.39329342Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:34:29.137839193Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":892,"history_lines":1,"events_offset":104,"events_lines":2,"console_offset":22,"console_lines":1}
+{"time":"2026-09-04T04:34:29.374576733Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:34:44.137206386Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":106,"events_lines":2}
+{"time":"2026-09-04T04:34:44.343965498Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:34:59.137882284Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":893,"history_lines":1,"events_offset":108,"events_lines":2,"console_offset":23,"console_lines":1}
+{"time":"2026-09-04T04:34:59.394892141Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:35:14.137987897Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":110,"events_lines":2}
+{"time":"2026-09-04T04:35:14.361788834Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:35:29.138370575Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":894,"history_lines":1,"events_offset":112,"events_lines":2,"console_offset":24,"console_lines":1}
+{"time":"2026-09-04T04:35:29.371503803Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:35:44.138200883Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":114,"events_lines":2}
+{"time":"2026-09-04T04:35:44.375079084Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:35:59.137441977Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":895,"history_lines":1,"events_offset":116,"events_lines":2,"console_offset":25,"console_lines":1}
+{"time":"2026-09-04T04:35:59.376759851Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:36:14.137799998Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":118,"events_lines":2}
+{"time":"2026-09-04T04:36:14.373471548Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:36:29.137919521Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":896,"history_lines":1,"events_offset":120,"events_lines":2,"console_offset":26,"console_lines":1}
+{"time":"2026-09-04T04:36:29.374419104Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:36:44.137705874Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":122,"events_lines":2}
+{"time":"2026-09-04T04:36:44.344487889Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:36:59.137996693Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":897,"history_lines":1,"events_offset":124,"events_lines":2,"console_offset":27,"console_lines":1}
+{"time":"2026-09-04T04:36:59.378926134Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:37:14.137419662Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":126,"events_lines":2}
+{"time":"2026-09-04T04:37:14.329225983Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:37:29.138251737Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":898,"history_lines":1,"events_offset":128,"events_lines":2,"console_offset":28,"console_lines":2}
+{"time":"2026-09-04T04:37:29.388179229Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:37:44.137667264Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":130,"events_lines":2,"console_offset":30,"console_lines":1}
+{"time":"2026-09-04T04:37:44.331249742Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:37:59.137251064Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":132,"events_lines":2}
+{"time":"2026-09-04T04:37:59.333280346Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:38:14.137419692Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":899,"history_lines":1,"events_offset":134,"events_lines":2,"console_offset":31,"console_lines":1}
+{"time":"2026-09-04T04:38:14.368591999Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:38:29.13813454Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":136,"events_lines":2}
+{"time":"2026-09-04T04:38:29.342880673Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:38:44.137622406Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":900,"history_lines":1,"events_offset":138,"events_lines":2,"console_offset":32,"console_lines":1}
+{"time":"2026-09-04T04:38:44.393737501Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:38:59.137837914Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":140,"events_lines":2}
+{"time":"2026-09-04T04:38:59.34404383Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:39:14.137560012Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":901,"history_lines":1,"events_offset":142,"events_lines":2,"console_offset":33,"console_lines":1}
+{"time":"2026-09-04T04:39:14.377420998Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:39:29.137499647Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":144,"events_lines":2}
+{"time":"2026-09-04T04:39:29.353816856Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:39:44.137432701Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":902,"history_lines":1,"events_offset":146,"events_lines":2,"console_offset":34,"console_lines":1}
+{"time":"2026-09-04T04:39:44.34992753Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:39:59.1373821Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":903,"history_lines":1,"events_offset":148,"events_lines":2,"console_offset":35,"console_lines":1}
+{"time":"2026-09-04T04:39:59.389295565Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:40:14.137395625Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":150,"events_lines":2}
+{"time":"2026-09-04T04:40:14.417730732Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:40:29.138109263Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":904,"history_lines":1,"events_offset":152,"events_lines":2,"console_offset":36,"console_lines":1}
+{"time":"2026-09-04T04:40:29.364227521Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:40:44.137831841Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":154,"events_lines":2}
+{"time":"2026-09-04T04:40:44.348318294Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:40:59.137810644Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":905,"history_lines":1,"events_offset":156,"events_lines":2,"console_offset":37,"console_lines":1}
+{"time":"2026-09-04T04:40:59.374707849Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:41:14.137555115Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":158,"events_lines":2}
+{"time":"2026-09-04T04:41:14.361732008Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:41:29.138142482Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":906,"history_lines":1,"events_offset":160,"events_lines":2,"console_offset":38,"console_lines":1}
+{"time":"2026-09-04T04:41:29.381756214Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:41:44.13805758Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":162,"events_lines":2}
+{"time":"2026-09-04T04:41:44.384794061Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:41:59.137712637Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":907,"history_lines":1,"events_offset":164,"events_lines":2,"console_offset":39,"console_lines":1}
+{"time":"2026-09-04T04:41:59.369578286Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:42:14.13765841Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":166,"events_lines":2}
+{"time":"2026-09-04T04:42:14.354737664Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:42:29.138183504Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":908,"history_lines":1,"events_offset":168,"events_lines":2,"console_offset":40,"console_lines":1}
+{"time":"2026-09-04T04:42:29.434408012Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:42:44.137170307Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":170,"events_lines":2}
+{"time":"2026-09-04T04:42:44.349665276Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:42:59.13776187Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":909,"history_lines":1,"events_offset":172,"events_lines":2,"console_offset":41,"console_lines":1}
+{"time":"2026-09-04T04:42:59.383688895Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:43:14.137435966Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":174,"events_lines":2}
+{"time":"2026-09-04T04:43:14.351081944Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:43:29.13805489Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":910,"history_lines":1,"events_offset":176,"events_lines":2,"console_offset":42,"console_lines":1}
+{"time":"2026-09-04T04:43:29.363747129Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:43:44.137273522Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":911,"history_lines":1,"events_offset":178,"events_lines":2,"console_offset":43,"console_lines":1}
+{"time":"2026-09-04T04:43:44.397736837Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:43:59.137480848Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":180,"events_lines":2}
+{"time":"2026-09-04T04:43:59.348702175Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:44:14.137992071Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":912,"history_lines":1,"events_offset":182,"events_lines":2,"console_offset":44,"console_lines":1}
+{"time":"2026-09-04T04:44:14.388228758Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:44:29.138298135Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":184,"events_lines":2}
+{"time":"2026-09-04T04:44:29.345738554Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:44:44.137812791Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":913,"history_lines":1,"events_offset":186,"events_lines":2,"console_offset":45,"console_lines":1}
+{"time":"2026-09-04T04:44:44.394339715Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:44:59.137931735Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":188,"events_lines":2}
+{"time":"2026-09-04T04:44:59.37514901Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:45:14.13757894Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":914,"history_lines":1,"events_offset":190,"events_lines":2,"console_offset":46,"console_lines":1}
+{"time":"2026-09-04T04:45:14.361291175Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:45:29.137572766Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":192,"events_lines":2}
+{"time":"2026-09-04T04:45:29.363125829Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:45:44.137571953Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":915,"history_lines":1,"events_offset":194,"events_lines":2,"console_offset":47,"console_lines":1}
+{"time":"2026-09-04T04:45:44.419235585Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:45:59.138074637Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":196,"events_lines":2}
+{"time":"2026-09-04T04:45:59.346131157Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:46:14.138155949Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":916,"history_lines":1,"events_offset":198,"events_lines":2,"console_offset":48,"console_lines":1}
+{"time":"2026-09-04T04:46:14.400193179Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:46:29.137431971Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":200,"events_lines":2}
+{"time":"2026-09-04T04:46:29.366450909Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:46:44.137492851Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":917,"history_lines":1,"events_offset":202,"events_lines":2,"console_offset":49,"console_lines":1}
+{"time":"2026-09-04T04:46:44.400696503Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:46:59.137448003Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":204,"events_lines":2}
+{"time":"2026-09-04T04:46:59.367035735Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:47:14.137356014Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":918,"history_lines":1,"events_offset":206,"events_lines":2,"console_offset":50,"console_lines":1}
+{"time":"2026-09-04T04:47:14.368738241Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:47:29.13737372Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":919,"history_lines":1,"events_offset":208,"events_lines":2,"console_offset":51,"console_lines":1}
+{"time":"2026-09-04T04:47:29.360092938Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:47:44.13816709Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":210,"events_lines":2}
+{"time":"2026-09-04T04:47:44.329594529Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:47:59.138222273Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":920,"history_lines":1,"events_offset":212,"events_lines":2,"console_offset":52,"console_lines":1}
+{"time":"2026-09-04T04:47:59.387686903Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:48:14.137844845Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":214,"events_lines":2}
+{"time":"2026-09-04T04:48:14.365134817Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:48:29.137822951Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":921,"history_lines":1,"events_offset":216,"events_lines":2,"console_offset":53,"console_lines":1}
+{"time":"2026-09-04T04:48:29.383293825Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:48:44.138060948Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":218,"events_lines":2}
+{"time":"2026-09-04T04:48:44.380240048Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:48:59.138028799Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":922,"history_lines":1,"events_offset":220,"events_lines":2,"console_offset":54,"console_lines":1}
+{"time":"2026-09-04T04:48:59.399225387Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:49:14.138084151Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":222,"events_lines":2}
+{"time":"2026-09-04T04:49:14.342435647Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:49:29.138427196Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":923,"history_lines":1,"events_offset":224,"events_lines":2,"console_offset":55,"console_lines":1}
+{"time":"2026-09-04T04:49:29.383051898Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:49:44.137219772Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":226,"events_lines":2,"console_offset":56,"console_lines":2}
+{"time":"2026-09-04T04:49:44.375046756Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:49:59.137471419Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":228,"events_lines":2}
+{"time":"2026-09-04T04:49:59.364590043Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:50:14.138243107Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":924,"history_lines":1,"events_offset":230,"events_lines":2,"console_offset":58,"console_lines":1}
+{"time":"2026-09-04T04:50:14.376277603Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:50:29.137452529Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":232,"events_lines":2}
+{"time":"2026-09-04T04:50:29.382678524Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:50:44.13757407Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":925,"history_lines":1,"events_offset":234,"events_lines":2,"console_offset":59,"console_lines":1}
+{"time":"2026-09-04T04:50:44.378754481Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:50:59.137645998Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":236,"events_lines":2}
+{"time":"2026-09-04T04:50:59.363349737Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:51:14.13828072Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":926,"history_lines":1,"events_offset":238,"events_lines":2,"console_offset":60,"console_lines":1}
+{"time":"2026-09-04T04:51:14.405717408Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:51:29.137844564Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":240,"events_lines":2}
+{"time":"2026-09-04T04:51:29.341299955Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:51:44.138174238Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":927,"history_lines":1,"events_offset":242,"events_lines":2,"console_offset":61,"console_lines":1}
+{"time":"2026-09-04T04:51:44.44058073Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:51:59.137325202Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":244,"events_lines":2}
+{"time":"2026-09-04T04:51:59.368499127Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:52:14.137479584Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":928,"history_lines":1,"events_offset":246,"events_lines":2,"console_offset":62,"console_lines":1}
+{"time":"2026-09-04T04:52:14.394482885Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:52:29.137467044Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":248,"events_lines":2}
+{"time":"2026-09-04T04:52:29.351809082Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:52:44.138253855Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":929,"history_lines":1,"events_offset":250,"events_lines":2,"console_offset":63,"console_lines":1}
+{"time":"2026-09-04T04:52:44.372523137Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:52:59.137770527Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":930,"history_lines":1,"events_offset":252,"events_lines":2,"console_offset":64,"console_lines":1}
+{"time":"2026-09-04T04:52:59.364624364Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:53:14.137457977Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":254,"events_lines":2}
+{"time":"2026-09-04T04:53:14.346157392Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:53:29.138369065Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":931,"history_lines":1,"events_offset":256,"events_lines":2,"console_offset":65,"console_lines":1}
+{"time":"2026-09-04T04:53:29.382317101Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:53:44.137885378Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":258,"events_lines":2}
+{"time":"2026-09-04T04:53:44.330502663Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:53:59.1375746Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":932,"history_lines":1,"events_offset":260,"events_lines":2,"console_offset":66,"console_lines":1}
+{"time":"2026-09-04T04:53:59.353435069Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:54:14.137402433Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":262,"events_lines":2}
+{"time":"2026-09-04T04:54:14.47225443Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:54:29.138243397Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":933,"history_lines":1,"events_offset":264,"events_lines":2,"console_offset":67,"console_lines":1}
+{"time":"2026-09-04T04:54:29.397291183Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:54:44.138163668Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":266,"events_lines":2}
+{"time":"2026-09-04T04:54:44.364283558Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:54:59.138205631Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":934,"history_lines":1,"events_offset":268,"events_lines":2,"console_offset":68,"console_lines":1}
+{"time":"2026-09-04T04:54:59.388981185Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:55:14.137847724Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":270,"events_lines":2}
+{"time":"2026-09-04T04:55:14.370208223Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:55:29.137322204Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":935,"history_lines":1,"events_offset":272,"events_lines":2,"console_offset":69,"console_lines":1}
+{"time":"2026-09-04T04:55:29.371526624Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:55:44.137889026Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":274,"events_lines":2}
+{"time":"2026-09-04T04:55:44.383280239Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:55:59.137705801Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":936,"history_lines":1,"events_offset":276,"events_lines":2,"console_offset":70,"console_lines":1}
+{"time":"2026-09-04T04:55:59.380994504Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:56:14.137914816Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":278,"events_lines":2}
+{"time":"2026-09-04T04:56:14.378339975Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:56:29.137986404Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":937,"history_lines":1,"events_offset":280,"events_lines":2,"console_offset":71,"console_lines":1}
+{"time":"2026-09-04T04:56:29.369141021Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:56:44.13748448Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":282,"events_lines":2}
+{"time":"2026-09-04T04:56:44.350420367Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:56:59.137886935Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":938,"history_lines":1,"events_offset":284,"events_lines":2,"console_offset":72,"console_lines":1}
+{"time":"2026-09-04T04:56:59.398759763Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:57:14.137540775Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":939,"history_lines":1,"events_offset":286,"events_lines":2,"console_offset":73,"console_lines":1}
+{"time":"2026-09-04T04:57:14.487393275Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:57:29.137500836Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":288,"events_lines":2}
+{"time":"2026-09-04T04:57:29.351791208Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:57:44.137394316Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":940,"history_lines":1,"events_offset":290,"events_lines":2,"console_offset":74,"console_lines":1}
+{"time":"2026-09-04T04:57:44.359773084Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:57:59.138140428Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":292,"events_lines":2}
+{"time":"2026-09-04T04:57:59.377062968Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:58:14.137558604Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":941,"history_lines":1,"events_offset":294,"events_lines":2,"console_offset":75,"console_lines":1}
+{"time":"2026-09-04T04:58:14.497238552Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:58:29.138123393Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":296,"events_lines":2}
+{"time":"2026-09-04T04:58:29.349527618Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:58:44.138051765Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":942,"history_lines":1,"events_offset":298,"events_lines":2,"console_offset":76,"console_lines":1}
+{"time":"2026-09-04T04:58:44.373153935Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-09-04T04:58:59.137512305Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":300,"events_lines":2}
+{"time":"2026-09-04T04:58:59.338039744Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

epfl_final/SparseAtt/wandb/run-20260904_042057-7tfq6nwm/logs/debug.log

@@ -0,0 +1,20 @@
+2026-09-04 04:20:57,345 INFO    MainThread:229 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-09-04 04:20:57,345 INFO    MainThread:229 [wandb_setup.py:_flush():81] Configure stats pid to 229
+2026-09-04 04:20:57,346 INFO    MainThread:229 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-09-04 04:20:57,346 INFO    MainThread:229 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260904_042057-7tfq6nwm/logs/debug.log
+2026-09-04 04:20:57,346 INFO    MainThread:229 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260904_042057-7tfq6nwm/logs/debug-internal.log
+2026-09-04 04:20:57,346 INFO    MainThread:229 [wandb_init.py:init():850] calling init triggers
+2026-09-04 04:20:57,346 INFO    MainThread:229 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-8750', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 15, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 72, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b dense 8k seed72 4gpu', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'dense', 'seed72'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
+2026-09-04 04:20:57,346 INFO    MainThread:229 [wandb_init.py:init():898] starting backend
+2026-09-04 04:20:58,120 INFO    MainThread:229 [wandb_init.py:init():913] sending inform_init request
+2026-09-04 04:20:58,557 INFO    MainThread:229 [wandb_init.py:init():918] backend started and connected
+2026-09-04 04:20:58,559 INFO    MainThread:229 [wandb_init.py:init():988] updated telemetry
+2026-09-04 04:20:58,595 INFO    MainThread:229 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-09-04 04:20:59,133 INFO    MainThread:229 [wandb_init.py:init():1051] run resumed
+2026-09-04 04:20:59,135 INFO    MainThread:229 [wandb_init.py:init():1056] starting run threads in backend
+2026-09-04 04:20:59,385 INFO    MainThread:229 [wandb_run.py:_console_start():2554] atexit reg
+2026-09-04 04:20:59,385 INFO    MainThread:229 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-09-04 04:20:59,385 INFO    MainThread:229 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-09-04 04:20:59,385 INFO    MainThread:229 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-09-04 04:20:59,394 INFO    MainThread:229 [wandb_init.py:init():1094] run started, returning control to user process