diff --git a/.gitattributes b/.gitattributes index 6e168651c8f285d9ac62e45fc1daac5cf60b0f81..e1cd3ce48ef30079f8a8988c5d30b017fa65b0a4 100644 --- a/.gitattributes +++ b/.gitattributes @@ -176,3 +176,16 @@ epfl_final/SparseAtt/wandb/run-20260903_142704-7tfq6nwm/run-7tfq6nwm.wandb filte epfl_final/SparseAtt/wandb/run-20260903_191410-7tfq6nwm/run-7tfq6nwm.wandb filter=lfs diff=lfs merge=lfs -text epfl_final/SparseAtt/wandb/run-20260904_042057-7tfq6nwm/run-7tfq6nwm.wandb filter=lfs diff=lfs merge=lfs -text epfl_final/SparseAtt/wandb/offline-run-20260921_030515-1savcbqu/run-1savcbqu.wandb filter=lfs diff=lfs merge=lfs -text +epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ao5edbbv/run-ao5edbbv.wandb filter=lfs diff=lfs merge=lfs -text +epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ut49ok16/run-ut49ok16.wandb filter=lfs diff=lfs merge=lfs -text +epfl_final/SparseAtt/wandb/offline-run-20260921_030515-6k91ldtc/run-6k91ldtc.wandb filter=lfs diff=lfs merge=lfs -text +epfl_final/SparseAtt/wandb/offline-run-20260921_030515-hmdx78bp/run-hmdx78bp.wandb filter=lfs diff=lfs merge=lfs -text +epfl_final/SparseAtt/wandb/offline-run-20260921_030515-8km2sc23/run-8km2sc23.wandb filter=lfs diff=lfs merge=lfs -text +epfl_final/SparseAtt/wandb/offline-run-20260921_131249-ifgmkpzt/run-ifgmkpzt.wandb filter=lfs diff=lfs merge=lfs -text +epfl_final/SparseAtt/wandb/offline-run-20260921_131249-meicy3md/run-meicy3md.wandb filter=lfs diff=lfs merge=lfs -text +epfl_final/SparseAtt/wandb/offline-run-20260921_131249-n6s56yfr/run-n6s56yfr.wandb filter=lfs diff=lfs merge=lfs -text +epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3la36nzy/run-3la36nzy.wandb filter=lfs diff=lfs merge=lfs -text +epfl_final/SparseAtt/wandb/offline-run-20260921_131249-xc6c08ps/run-xc6c08ps.wandb filter=lfs diff=lfs merge=lfs -text +epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3uan03m3/run-3uan03m3.wandb filter=lfs diff=lfs merge=lfs -text +epfl_final/SparseAtt/wandb/offline-run-20260921_143024-kbn66izc/run-kbn66izc.wandb filter=lfs diff=lfs merge=lfs -text +epfl_final/SparseAtt/wandb/offline-run-20260921_143024-zgrsvkbi/run-zgrsvkbi.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-6k91ldtc/files/requirements.txt b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-6k91ldtc/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c3e38c4cce87cb53d52f45f6ac8cbf8688a7d92 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-6k91ldtc/files/requirements.txt @@ -0,0 +1,142 @@ +pip==26.0.1 +nvidia-cusparselt-cu13==0.8.0 +mpmath==1.3.0 +cuda-toolkit==13.0.2 +zstandard==0.25.0 +xxhash==3.7.0 +urllib3==2.6.3 +typing_extensions==4.15.0 +triton==3.6.0 +tqdm==4.67.3 +torchinfo==1.8.0 +sympy==1.14.0 +smmap==5.0.3 +six==1.17.0 +shellingham==1.5.4 +setuptools==81.0.0 +safetensors==0.7.0 +regex==2026.4.4 +PyYAML==6.0.3 +Pygments==2.20.0 +pyarrow==24.0.0 +psutil==7.2.2 +protobuf==7.34.1 +propcache==0.4.1 +platformdirs==4.9.6 +packaging==26.2 +nvidia-nvtx==13.0.85 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvjitlink==13.0.88 +nvidia-nccl-cu13==2.28.9 +nvidia-curand==10.4.0.35 +nvidia-cufile==1.15.1.6 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-cupti==13.0.85 +nvidia-cublas==13.1.0.3 +numpy==2.4.4 +networkx==3.6.1 +multidict==6.7.1 +mdurl==0.1.2 +MarkupSafe==3.0.3 +idna==3.13 +hf-xet==1.4.3 +h11==0.16.0 +fsspec==2026.2.0 +frozenlist==1.8.0 +filelock==3.29.0 +dill==0.4.1 +cuda-pathfinder==1.5.4 +click==8.3.3 +charset-normalizer==3.4.7 +certifi==2026.4.22 +attrs==26.1.0 +annotated-types==0.7.0 +annotated-doc==0.0.4 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +typing-inspection==0.4.2 +sentry-sdk==2.58.0 +requests==2.33.1 +python-dateutil==2.9.0.post0 +pydantic_core==2.46.3 +nvidia-cusparse==12.6.3.3 +nvidia-cufft==12.0.0.61 +nvidia-cudnn-cu13==9.19.0.56 +multiprocess==0.70.19 +markdown-it-py==4.0.0 +Jinja2==3.1.6 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==13.2.0 +anyio==4.13.0 +aiosignal==1.4.0 +rich==15.0.0 +pydantic==2.13.3 +pandas==3.0.2 +nvidia-cusolver==12.0.4.66 +httpx==0.28.1 +GitPython==3.1.49 +aiohttp==3.13.5 +wandb==0.26.1 +typer==0.25.1 +torch==2.11.0 +huggingface_hub==1.13.0 +tokenizers==0.22.2 +datasets==4.8.5 +accelerate==1.13.0 +transformers==5.7.0 +word2number==1.1 +sqlitedict==2.1.0 +pytz==2026.2 +threadpoolctl==3.6.0 +tcolorpy==0.1.7 +tabulate==0.10.0 +scipy==1.17.1 +portalocker==3.2.0 +pathvalidate==3.3.1 +narwhals==2.22.1 +more-itertools==11.1.0 +lxml==6.1.1 +joblib==1.5.3 +colorama==0.4.6 +chardet==6.0.0.post1 +absl-py==2.4.0 +scikit-learn==1.9.0 +sacrebleu==2.6.0 +nltk==3.9.4 +mbstrdecoder==1.1.5 +typepy==1.3.5 +rouge_score==0.1.2 +DataProperty==1.1.1 +tabledata==1.3.5 +evaluate==0.4.6 +pytablewriter==1.2.1 +lm_eval==0.4.12 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==9.1.0 +py-spy==0.4.2 +pillow==12.2.0 +tifffile==2026.6.1 +imagecodecs==2026.6.26 +zipp==4.1.0 +importlib_metadata==9.0.0 +derivative==0.6.3 +pysindy==2.1.0 +pyparsing==3.3.2 +kiwisolver==1.5.0 +fonttools==4.63.0 +cycler==0.12.1 +contourpy==1.3.3 +matplotlib==3.11.0 +sortedcontainers==2.4.0 +wcwidth==0.8.2 +sigfig==1.3.19 +importlib_resources==7.1.0 +enum-tools==0.13.0 +cramjam==2.11.0 +python-snappy==0.7.3 +compact-json==1.8.2 +numbers-parser==4.18.5 +entmax==1.3 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-6k91ldtc/logs/debug-internal.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-6k91ldtc/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..804cd2ca9215f13b60658ff0f69189463d4e5c43 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-6k91ldtc/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-09-21T03:05:15.627748328Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-09-21T03:05:15.627774557Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"} +{"time":"2026-09-21T03:05:15.977507774Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T03:05:15.977569276Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T03:05:15.977607514Z","level":"INFO","msg":"stream: created new stream","id":"6k91ldtc"} +{"time":"2026-09-21T03:05:15.977692221Z","level":"INFO","msg":"handler: started"} +{"time":"2026-09-21T03:05:15.986110077Z","level":"INFO","msg":"stream: started"} +{"time":"2026-09-21T03:05:15.986165781Z","level":"INFO","msg":"writer: started","stream_id":"6k91ldtc"} +{"time":"2026-09-21T03:05:15.986204038Z","level":"INFO","msg":"sender: started"} +{"time":"2026-09-21T03:05:16.013061227Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T03:05:16.0130764Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-6k91ldtc/logs/debug.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-6k91ldtc/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2fa84dc24784f812ee3c1c2cb71aa779b5fbb782 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-6k91ldtc/logs/debug.log @@ -0,0 +1,19 @@ +2026-09-21 03:05:15,374 INFO MainThread:4776 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1 +2026-09-21 03:05:15,374 INFO MainThread:4776 [wandb_setup.py:_flush():81] Configure stats pid to 4776 +2026-09-21 03:05:15,374 INFO MainThread:4776 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-09-21 03:05:15,374 INFO MainThread:4776 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_030515-6k91ldtc/logs/debug.log +2026-09-21 03:05:15,374 INFO MainThread:4776 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_030515-6k91ldtc/logs/debug-internal.log +2026-09-21 03:05:15,374 INFO MainThread:4776 [wandb_init.py:init():850] calling init triggers +2026-09-21 03:05:15,374 INFO MainThread:4776 [wandb_init.py:init():855] wandb.init called with sweep_config: {} +config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s44/run_20260921_030452', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 22000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 44, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s44', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed44'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}} +2026-09-21 03:05:15,374 INFO MainThread:4776 [wandb_init.py:init():898] starting backend +2026-09-21 03:05:15,616 INFO MainThread:4776 [wandb_init.py:init():913] sending inform_init request +2026-09-21 03:05:15,987 INFO MainThread:4776 [wandb_init.py:init():918] backend started and connected +2026-09-21 03:05:15,988 INFO MainThread:4776 [wandb_init.py:init():988] updated telemetry +2026-09-21 03:05:16,009 INFO MainThread:4776 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout +2026-09-21 03:05:16,014 INFO MainThread:4776 [wandb_init.py:init():1056] starting run threads in backend +2026-09-21 03:05:16,229 INFO MainThread:4776 [wandb_run.py:_console_start():2554] atexit reg +2026-09-21 03:05:16,229 INFO MainThread:4776 [wandb_run.py:_redirect():2403] redirect: wrap_raw +2026-09-21 03:05:16,229 INFO MainThread:4776 [wandb_run.py:_redirect():2472] Wrapping output streams. +2026-09-21 03:05:16,230 INFO MainThread:4776 [wandb_run.py:_redirect():2495] Redirects installed. +2026-09-21 03:05:16,235 INFO MainThread:4776 [wandb_init.py:init():1094] run started, returning control to user process diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-6k91ldtc/run-6k91ldtc.wandb b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-6k91ldtc/run-6k91ldtc.wandb new file mode 100644 index 0000000000000000000000000000000000000000..a95e74e3862b55a51f442733b44ca4132cebdac2 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-6k91ldtc/run-6k91ldtc.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9fa5a8893b0d64471e1060bb2634231d6cfeb06f87b5f0bdf963159327fa592 +size 9502720 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-8km2sc23/files/requirements.txt b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-8km2sc23/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c3e38c4cce87cb53d52f45f6ac8cbf8688a7d92 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-8km2sc23/files/requirements.txt @@ -0,0 +1,142 @@ +pip==26.0.1 +nvidia-cusparselt-cu13==0.8.0 +mpmath==1.3.0 +cuda-toolkit==13.0.2 +zstandard==0.25.0 +xxhash==3.7.0 +urllib3==2.6.3 +typing_extensions==4.15.0 +triton==3.6.0 +tqdm==4.67.3 +torchinfo==1.8.0 +sympy==1.14.0 +smmap==5.0.3 +six==1.17.0 +shellingham==1.5.4 +setuptools==81.0.0 +safetensors==0.7.0 +regex==2026.4.4 +PyYAML==6.0.3 +Pygments==2.20.0 +pyarrow==24.0.0 +psutil==7.2.2 +protobuf==7.34.1 +propcache==0.4.1 +platformdirs==4.9.6 +packaging==26.2 +nvidia-nvtx==13.0.85 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvjitlink==13.0.88 +nvidia-nccl-cu13==2.28.9 +nvidia-curand==10.4.0.35 +nvidia-cufile==1.15.1.6 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-cupti==13.0.85 +nvidia-cublas==13.1.0.3 +numpy==2.4.4 +networkx==3.6.1 +multidict==6.7.1 +mdurl==0.1.2 +MarkupSafe==3.0.3 +idna==3.13 +hf-xet==1.4.3 +h11==0.16.0 +fsspec==2026.2.0 +frozenlist==1.8.0 +filelock==3.29.0 +dill==0.4.1 +cuda-pathfinder==1.5.4 +click==8.3.3 +charset-normalizer==3.4.7 +certifi==2026.4.22 +attrs==26.1.0 +annotated-types==0.7.0 +annotated-doc==0.0.4 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +typing-inspection==0.4.2 +sentry-sdk==2.58.0 +requests==2.33.1 +python-dateutil==2.9.0.post0 +pydantic_core==2.46.3 +nvidia-cusparse==12.6.3.3 +nvidia-cufft==12.0.0.61 +nvidia-cudnn-cu13==9.19.0.56 +multiprocess==0.70.19 +markdown-it-py==4.0.0 +Jinja2==3.1.6 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==13.2.0 +anyio==4.13.0 +aiosignal==1.4.0 +rich==15.0.0 +pydantic==2.13.3 +pandas==3.0.2 +nvidia-cusolver==12.0.4.66 +httpx==0.28.1 +GitPython==3.1.49 +aiohttp==3.13.5 +wandb==0.26.1 +typer==0.25.1 +torch==2.11.0 +huggingface_hub==1.13.0 +tokenizers==0.22.2 +datasets==4.8.5 +accelerate==1.13.0 +transformers==5.7.0 +word2number==1.1 +sqlitedict==2.1.0 +pytz==2026.2 +threadpoolctl==3.6.0 +tcolorpy==0.1.7 +tabulate==0.10.0 +scipy==1.17.1 +portalocker==3.2.0 +pathvalidate==3.3.1 +narwhals==2.22.1 +more-itertools==11.1.0 +lxml==6.1.1 +joblib==1.5.3 +colorama==0.4.6 +chardet==6.0.0.post1 +absl-py==2.4.0 +scikit-learn==1.9.0 +sacrebleu==2.6.0 +nltk==3.9.4 +mbstrdecoder==1.1.5 +typepy==1.3.5 +rouge_score==0.1.2 +DataProperty==1.1.1 +tabledata==1.3.5 +evaluate==0.4.6 +pytablewriter==1.2.1 +lm_eval==0.4.12 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==9.1.0 +py-spy==0.4.2 +pillow==12.2.0 +tifffile==2026.6.1 +imagecodecs==2026.6.26 +zipp==4.1.0 +importlib_metadata==9.0.0 +derivative==0.6.3 +pysindy==2.1.0 +pyparsing==3.3.2 +kiwisolver==1.5.0 +fonttools==4.63.0 +cycler==0.12.1 +contourpy==1.3.3 +matplotlib==3.11.0 +sortedcontainers==2.4.0 +wcwidth==0.8.2 +sigfig==1.3.19 +importlib_resources==7.1.0 +enum-tools==0.13.0 +cramjam==2.11.0 +python-snappy==0.7.3 +compact-json==1.8.2 +numbers-parser==4.18.5 +entmax==1.3 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-8km2sc23/logs/debug-internal.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-8km2sc23/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..ef0c689b2102b9def426edebeda57c90b534d4e7 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-8km2sc23/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-09-21T03:05:15.647686351Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-09-21T03:05:15.647714233Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"} +{"time":"2026-09-21T03:05:16.001225716Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T03:05:16.001272596Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T03:05:16.001306587Z","level":"INFO","msg":"stream: created new stream","id":"8km2sc23"} +{"time":"2026-09-21T03:05:16.001376873Z","level":"INFO","msg":"handler: started"} +{"time":"2026-09-21T03:05:16.007121864Z","level":"INFO","msg":"stream: started"} +{"time":"2026-09-21T03:05:16.007230587Z","level":"INFO","msg":"writer: started","stream_id":"8km2sc23"} +{"time":"2026-09-21T03:05:16.007241514Z","level":"INFO","msg":"sender: started"} +{"time":"2026-09-21T03:05:16.028513461Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T03:05:16.028534102Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-8km2sc23/logs/debug.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-8km2sc23/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..05bc1bea5760dabfa11e2e202bbb0329709dd332 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-8km2sc23/logs/debug.log @@ -0,0 +1,19 @@ +2026-09-21 03:05:15,380 INFO MainThread:4779 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1 +2026-09-21 03:05:15,380 INFO MainThread:4779 [wandb_setup.py:_flush():81] Configure stats pid to 4779 +2026-09-21 03:05:15,380 INFO MainThread:4779 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-09-21 03:05:15,380 INFO MainThread:4779 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_030515-8km2sc23/logs/debug.log +2026-09-21 03:05:15,380 INFO MainThread:4779 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_030515-8km2sc23/logs/debug-internal.log +2026-09-21 03:05:15,380 INFO MainThread:4779 [wandb_init.py:init():850] calling init triggers +2026-09-21 03:05:15,381 INFO MainThread:4779 [wandb_init.py:init():855] wandb.init called with sweep_config: {} +config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s47/run_20260921_030452', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 22000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 47, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s47', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed47'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}} +2026-09-21 03:05:15,381 INFO MainThread:4779 [wandb_init.py:init():898] starting backend +2026-09-21 03:05:15,636 INFO MainThread:4779 [wandb_init.py:init():913] sending inform_init request +2026-09-21 03:05:16,008 INFO MainThread:4779 [wandb_init.py:init():918] backend started and connected +2026-09-21 03:05:16,010 INFO MainThread:4779 [wandb_init.py:init():988] updated telemetry +2026-09-21 03:05:16,025 INFO MainThread:4779 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout +2026-09-21 03:05:16,030 INFO MainThread:4779 [wandb_init.py:init():1056] starting run threads in backend +2026-09-21 03:05:16,260 INFO MainThread:4779 [wandb_run.py:_console_start():2554] atexit reg +2026-09-21 03:05:16,260 INFO MainThread:4779 [wandb_run.py:_redirect():2403] redirect: wrap_raw +2026-09-21 03:05:16,260 INFO MainThread:4779 [wandb_run.py:_redirect():2472] Wrapping output streams. +2026-09-21 03:05:16,260 INFO MainThread:4779 [wandb_run.py:_redirect():2495] Redirects installed. +2026-09-21 03:05:16,262 INFO MainThread:4779 [wandb_init.py:init():1094] run started, returning control to user process diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-8km2sc23/run-8km2sc23.wandb b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-8km2sc23/run-8km2sc23.wandb new file mode 100644 index 0000000000000000000000000000000000000000..1ece502b5f0770eb6e9018403288a050bbf613e2 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-8km2sc23/run-8km2sc23.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f20ea78ff25351a9482d7556532b12dfc04bb9feccf50b508a5d996379de376 +size 9502720 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ao5edbbv/run-ao5edbbv.wandb b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ao5edbbv/run-ao5edbbv.wandb new file mode 100644 index 0000000000000000000000000000000000000000..6f6850d6786d2c0076ef0f8c44036743b0766268 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ao5edbbv/run-ao5edbbv.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a47ccd47018e8925ebe18a6bcc6a903833a70cb17c477052d08acb8df1e90288 +size 9502720 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-hmdx78bp/files/requirements.txt b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-hmdx78bp/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c3e38c4cce87cb53d52f45f6ac8cbf8688a7d92 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-hmdx78bp/files/requirements.txt @@ -0,0 +1,142 @@ +pip==26.0.1 +nvidia-cusparselt-cu13==0.8.0 +mpmath==1.3.0 +cuda-toolkit==13.0.2 +zstandard==0.25.0 +xxhash==3.7.0 +urllib3==2.6.3 +typing_extensions==4.15.0 +triton==3.6.0 +tqdm==4.67.3 +torchinfo==1.8.0 +sympy==1.14.0 +smmap==5.0.3 +six==1.17.0 +shellingham==1.5.4 +setuptools==81.0.0 +safetensors==0.7.0 +regex==2026.4.4 +PyYAML==6.0.3 +Pygments==2.20.0 +pyarrow==24.0.0 +psutil==7.2.2 +protobuf==7.34.1 +propcache==0.4.1 +platformdirs==4.9.6 +packaging==26.2 +nvidia-nvtx==13.0.85 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvjitlink==13.0.88 +nvidia-nccl-cu13==2.28.9 +nvidia-curand==10.4.0.35 +nvidia-cufile==1.15.1.6 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-cupti==13.0.85 +nvidia-cublas==13.1.0.3 +numpy==2.4.4 +networkx==3.6.1 +multidict==6.7.1 +mdurl==0.1.2 +MarkupSafe==3.0.3 +idna==3.13 +hf-xet==1.4.3 +h11==0.16.0 +fsspec==2026.2.0 +frozenlist==1.8.0 +filelock==3.29.0 +dill==0.4.1 +cuda-pathfinder==1.5.4 +click==8.3.3 +charset-normalizer==3.4.7 +certifi==2026.4.22 +attrs==26.1.0 +annotated-types==0.7.0 +annotated-doc==0.0.4 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +typing-inspection==0.4.2 +sentry-sdk==2.58.0 +requests==2.33.1 +python-dateutil==2.9.0.post0 +pydantic_core==2.46.3 +nvidia-cusparse==12.6.3.3 +nvidia-cufft==12.0.0.61 +nvidia-cudnn-cu13==9.19.0.56 +multiprocess==0.70.19 +markdown-it-py==4.0.0 +Jinja2==3.1.6 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==13.2.0 +anyio==4.13.0 +aiosignal==1.4.0 +rich==15.0.0 +pydantic==2.13.3 +pandas==3.0.2 +nvidia-cusolver==12.0.4.66 +httpx==0.28.1 +GitPython==3.1.49 +aiohttp==3.13.5 +wandb==0.26.1 +typer==0.25.1 +torch==2.11.0 +huggingface_hub==1.13.0 +tokenizers==0.22.2 +datasets==4.8.5 +accelerate==1.13.0 +transformers==5.7.0 +word2number==1.1 +sqlitedict==2.1.0 +pytz==2026.2 +threadpoolctl==3.6.0 +tcolorpy==0.1.7 +tabulate==0.10.0 +scipy==1.17.1 +portalocker==3.2.0 +pathvalidate==3.3.1 +narwhals==2.22.1 +more-itertools==11.1.0 +lxml==6.1.1 +joblib==1.5.3 +colorama==0.4.6 +chardet==6.0.0.post1 +absl-py==2.4.0 +scikit-learn==1.9.0 +sacrebleu==2.6.0 +nltk==3.9.4 +mbstrdecoder==1.1.5 +typepy==1.3.5 +rouge_score==0.1.2 +DataProperty==1.1.1 +tabledata==1.3.5 +evaluate==0.4.6 +pytablewriter==1.2.1 +lm_eval==0.4.12 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==9.1.0 +py-spy==0.4.2 +pillow==12.2.0 +tifffile==2026.6.1 +imagecodecs==2026.6.26 +zipp==4.1.0 +importlib_metadata==9.0.0 +derivative==0.6.3 +pysindy==2.1.0 +pyparsing==3.3.2 +kiwisolver==1.5.0 +fonttools==4.63.0 +cycler==0.12.1 +contourpy==1.3.3 +matplotlib==3.11.0 +sortedcontainers==2.4.0 +wcwidth==0.8.2 +sigfig==1.3.19 +importlib_resources==7.1.0 +enum-tools==0.13.0 +cramjam==2.11.0 +python-snappy==0.7.3 +compact-json==1.8.2 +numbers-parser==4.18.5 +entmax==1.3 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-hmdx78bp/logs/debug-internal.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-hmdx78bp/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a86a41efdcfe7198836d3ab0ce52baafcaef871d --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-hmdx78bp/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-09-21T03:05:15.644133914Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-09-21T03:05:15.644161065Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"} +{"time":"2026-09-21T03:05:15.995507455Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T03:05:15.995566224Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T03:05:15.995608958Z","level":"INFO","msg":"stream: created new stream","id":"hmdx78bp"} +{"time":"2026-09-21T03:05:15.995691291Z","level":"INFO","msg":"handler: started"} +{"time":"2026-09-21T03:05:15.997829013Z","level":"INFO","msg":"stream: started"} +{"time":"2026-09-21T03:05:15.997901802Z","level":"INFO","msg":"writer: started","stream_id":"hmdx78bp"} +{"time":"2026-09-21T03:05:15.997918447Z","level":"INFO","msg":"sender: started"} +{"time":"2026-09-21T03:05:16.02085118Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T03:05:16.020873704Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-hmdx78bp/logs/debug.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-hmdx78bp/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f881e0721c0d28681cb29c8af629f29550f016b1 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-hmdx78bp/logs/debug.log @@ -0,0 +1,19 @@ +2026-09-21 03:05:15,378 INFO MainThread:4777 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1 +2026-09-21 03:05:15,378 INFO MainThread:4777 [wandb_setup.py:_flush():81] Configure stats pid to 4777 +2026-09-21 03:05:15,378 INFO MainThread:4777 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-09-21 03:05:15,378 INFO MainThread:4777 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_030515-hmdx78bp/logs/debug.log +2026-09-21 03:05:15,378 INFO MainThread:4777 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_030515-hmdx78bp/logs/debug-internal.log +2026-09-21 03:05:15,378 INFO MainThread:4777 [wandb_init.py:init():850] calling init triggers +2026-09-21 03:05:15,378 INFO MainThread:4777 [wandb_init.py:init():855] wandb.init called with sweep_config: {} +config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s45/run_20260921_030452', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 22000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 45, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s45', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed45'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}} +2026-09-21 03:05:15,378 INFO MainThread:4777 [wandb_init.py:init():898] starting backend +2026-09-21 03:05:15,632 INFO MainThread:4777 [wandb_init.py:init():913] sending inform_init request +2026-09-21 03:05:15,998 INFO MainThread:4777 [wandb_init.py:init():918] backend started and connected +2026-09-21 03:05:16,001 INFO MainThread:4777 [wandb_init.py:init():988] updated telemetry +2026-09-21 03:05:16,018 INFO MainThread:4777 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout +2026-09-21 03:05:16,023 INFO MainThread:4777 [wandb_init.py:init():1056] starting run threads in backend +2026-09-21 03:05:16,249 INFO MainThread:4777 [wandb_run.py:_console_start():2554] atexit reg +2026-09-21 03:05:16,249 INFO MainThread:4777 [wandb_run.py:_redirect():2403] redirect: wrap_raw +2026-09-21 03:05:16,249 INFO MainThread:4777 [wandb_run.py:_redirect():2472] Wrapping output streams. +2026-09-21 03:05:16,249 INFO MainThread:4777 [wandb_run.py:_redirect():2495] Redirects installed. +2026-09-21 03:05:16,251 INFO MainThread:4777 [wandb_init.py:init():1094] run started, returning control to user process diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-hmdx78bp/run-hmdx78bp.wandb b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-hmdx78bp/run-hmdx78bp.wandb new file mode 100644 index 0000000000000000000000000000000000000000..a04f94cd46774d94c3f3bf2e08dc5020dd213c3e --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-hmdx78bp/run-hmdx78bp.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:529c1489903aaab749fa0277af1d0a0149c7354fdf132a44620412bbaa1cc1d3 +size 9502720 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ut49ok16/files/requirements.txt b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ut49ok16/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c3e38c4cce87cb53d52f45f6ac8cbf8688a7d92 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ut49ok16/files/requirements.txt @@ -0,0 +1,142 @@ +pip==26.0.1 +nvidia-cusparselt-cu13==0.8.0 +mpmath==1.3.0 +cuda-toolkit==13.0.2 +zstandard==0.25.0 +xxhash==3.7.0 +urllib3==2.6.3 +typing_extensions==4.15.0 +triton==3.6.0 +tqdm==4.67.3 +torchinfo==1.8.0 +sympy==1.14.0 +smmap==5.0.3 +six==1.17.0 +shellingham==1.5.4 +setuptools==81.0.0 +safetensors==0.7.0 +regex==2026.4.4 +PyYAML==6.0.3 +Pygments==2.20.0 +pyarrow==24.0.0 +psutil==7.2.2 +protobuf==7.34.1 +propcache==0.4.1 +platformdirs==4.9.6 +packaging==26.2 +nvidia-nvtx==13.0.85 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvjitlink==13.0.88 +nvidia-nccl-cu13==2.28.9 +nvidia-curand==10.4.0.35 +nvidia-cufile==1.15.1.6 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-cupti==13.0.85 +nvidia-cublas==13.1.0.3 +numpy==2.4.4 +networkx==3.6.1 +multidict==6.7.1 +mdurl==0.1.2 +MarkupSafe==3.0.3 +idna==3.13 +hf-xet==1.4.3 +h11==0.16.0 +fsspec==2026.2.0 +frozenlist==1.8.0 +filelock==3.29.0 +dill==0.4.1 +cuda-pathfinder==1.5.4 +click==8.3.3 +charset-normalizer==3.4.7 +certifi==2026.4.22 +attrs==26.1.0 +annotated-types==0.7.0 +annotated-doc==0.0.4 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +typing-inspection==0.4.2 +sentry-sdk==2.58.0 +requests==2.33.1 +python-dateutil==2.9.0.post0 +pydantic_core==2.46.3 +nvidia-cusparse==12.6.3.3 +nvidia-cufft==12.0.0.61 +nvidia-cudnn-cu13==9.19.0.56 +multiprocess==0.70.19 +markdown-it-py==4.0.0 +Jinja2==3.1.6 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==13.2.0 +anyio==4.13.0 +aiosignal==1.4.0 +rich==15.0.0 +pydantic==2.13.3 +pandas==3.0.2 +nvidia-cusolver==12.0.4.66 +httpx==0.28.1 +GitPython==3.1.49 +aiohttp==3.13.5 +wandb==0.26.1 +typer==0.25.1 +torch==2.11.0 +huggingface_hub==1.13.0 +tokenizers==0.22.2 +datasets==4.8.5 +accelerate==1.13.0 +transformers==5.7.0 +word2number==1.1 +sqlitedict==2.1.0 +pytz==2026.2 +threadpoolctl==3.6.0 +tcolorpy==0.1.7 +tabulate==0.10.0 +scipy==1.17.1 +portalocker==3.2.0 +pathvalidate==3.3.1 +narwhals==2.22.1 +more-itertools==11.1.0 +lxml==6.1.1 +joblib==1.5.3 +colorama==0.4.6 +chardet==6.0.0.post1 +absl-py==2.4.0 +scikit-learn==1.9.0 +sacrebleu==2.6.0 +nltk==3.9.4 +mbstrdecoder==1.1.5 +typepy==1.3.5 +rouge_score==0.1.2 +DataProperty==1.1.1 +tabledata==1.3.5 +evaluate==0.4.6 +pytablewriter==1.2.1 +lm_eval==0.4.12 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==9.1.0 +py-spy==0.4.2 +pillow==12.2.0 +tifffile==2026.6.1 +imagecodecs==2026.6.26 +zipp==4.1.0 +importlib_metadata==9.0.0 +derivative==0.6.3 +pysindy==2.1.0 +pyparsing==3.3.2 +kiwisolver==1.5.0 +fonttools==4.63.0 +cycler==0.12.1 +contourpy==1.3.3 +matplotlib==3.11.0 +sortedcontainers==2.4.0 +wcwidth==0.8.2 +sigfig==1.3.19 +importlib_resources==7.1.0 +enum-tools==0.13.0 +cramjam==2.11.0 +python-snappy==0.7.3 +compact-json==1.8.2 +numbers-parser==4.18.5 +entmax==1.3 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ut49ok16/logs/debug-internal.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ut49ok16/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..3a364f5f08182ae725b5e4fe0c2f69505a4e04aa --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ut49ok16/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-09-21T03:05:15.626653773Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-09-21T03:05:15.626680303Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"} +{"time":"2026-09-21T03:05:15.981530157Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T03:05:15.981590878Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T03:05:15.981626161Z","level":"INFO","msg":"stream: created new stream","id":"ut49ok16"} +{"time":"2026-09-21T03:05:15.981707033Z","level":"INFO","msg":"handler: started"} +{"time":"2026-09-21T03:05:15.99192309Z","level":"INFO","msg":"stream: started"} +{"time":"2026-09-21T03:05:15.991947527Z","level":"INFO","msg":"writer: started","stream_id":"ut49ok16"} +{"time":"2026-09-21T03:05:15.991976481Z","level":"INFO","msg":"sender: started"} +{"time":"2026-09-21T03:05:16.024492801Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T03:05:16.0245186Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ut49ok16/logs/debug.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ut49ok16/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..9c8c9e3a709f161743eb6d94e0a0c73792d5de4b --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ut49ok16/logs/debug.log @@ -0,0 +1,19 @@ +2026-09-21 03:05:15,152 INFO MainThread:4775 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1 +2026-09-21 03:05:15,153 INFO MainThread:4775 [wandb_setup.py:_flush():81] Configure stats pid to 4775 +2026-09-21 03:05:15,153 INFO MainThread:4775 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-09-21 03:05:15,153 INFO MainThread:4775 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_030515-ut49ok16/logs/debug.log +2026-09-21 03:05:15,153 INFO MainThread:4775 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_030515-ut49ok16/logs/debug-internal.log +2026-09-21 03:05:15,153 INFO MainThread:4775 [wandb_init.py:init():850] calling init triggers +2026-09-21 03:05:15,153 INFO MainThread:4775 [wandb_init.py:init():855] wandb.init called with sweep_config: {} +config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s43/run_20260921_030452', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 22000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 43, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s43', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed43'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}} +2026-09-21 03:05:15,153 INFO MainThread:4775 [wandb_init.py:init():898] starting backend +2026-09-21 03:05:15,613 INFO MainThread:4775 [wandb_init.py:init():913] sending inform_init request +2026-09-21 03:05:15,992 INFO MainThread:4775 [wandb_init.py:init():918] backend started and connected +2026-09-21 03:05:15,998 INFO MainThread:4775 [wandb_init.py:init():988] updated telemetry +2026-09-21 03:05:16,020 INFO MainThread:4775 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout +2026-09-21 03:05:16,028 INFO MainThread:4775 [wandb_init.py:init():1056] starting run threads in backend +2026-09-21 03:05:16,251 INFO MainThread:4775 [wandb_run.py:_console_start():2554] atexit reg +2026-09-21 03:05:16,251 INFO MainThread:4775 [wandb_run.py:_redirect():2403] redirect: wrap_raw +2026-09-21 03:05:16,251 INFO MainThread:4775 [wandb_run.py:_redirect():2472] Wrapping output streams. +2026-09-21 03:05:16,251 INFO MainThread:4775 [wandb_run.py:_redirect():2495] Redirects installed. +2026-09-21 03:05:16,253 INFO MainThread:4775 [wandb_init.py:init():1094] run started, returning control to user process diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ut49ok16/run-ut49ok16.wandb b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ut49ok16/run-ut49ok16.wandb new file mode 100644 index 0000000000000000000000000000000000000000..cee1b031f54deb866cab8456254932acf235c2a8 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_030515-ut49ok16/run-ut49ok16.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:652398a1a92d2ce4c8f034a80d6119d7e6e8370b537dc3bfb78449df8b3d97dc +size 9502720 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3la36nzy/files/requirements.txt b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3la36nzy/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c3e38c4cce87cb53d52f45f6ac8cbf8688a7d92 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3la36nzy/files/requirements.txt @@ -0,0 +1,142 @@ +pip==26.0.1 +nvidia-cusparselt-cu13==0.8.0 +mpmath==1.3.0 +cuda-toolkit==13.0.2 +zstandard==0.25.0 +xxhash==3.7.0 +urllib3==2.6.3 +typing_extensions==4.15.0 +triton==3.6.0 +tqdm==4.67.3 +torchinfo==1.8.0 +sympy==1.14.0 +smmap==5.0.3 +six==1.17.0 +shellingham==1.5.4 +setuptools==81.0.0 +safetensors==0.7.0 +regex==2026.4.4 +PyYAML==6.0.3 +Pygments==2.20.0 +pyarrow==24.0.0 +psutil==7.2.2 +protobuf==7.34.1 +propcache==0.4.1 +platformdirs==4.9.6 +packaging==26.2 +nvidia-nvtx==13.0.85 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvjitlink==13.0.88 +nvidia-nccl-cu13==2.28.9 +nvidia-curand==10.4.0.35 +nvidia-cufile==1.15.1.6 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-cupti==13.0.85 +nvidia-cublas==13.1.0.3 +numpy==2.4.4 +networkx==3.6.1 +multidict==6.7.1 +mdurl==0.1.2 +MarkupSafe==3.0.3 +idna==3.13 +hf-xet==1.4.3 +h11==0.16.0 +fsspec==2026.2.0 +frozenlist==1.8.0 +filelock==3.29.0 +dill==0.4.1 +cuda-pathfinder==1.5.4 +click==8.3.3 +charset-normalizer==3.4.7 +certifi==2026.4.22 +attrs==26.1.0 +annotated-types==0.7.0 +annotated-doc==0.0.4 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +typing-inspection==0.4.2 +sentry-sdk==2.58.0 +requests==2.33.1 +python-dateutil==2.9.0.post0 +pydantic_core==2.46.3 +nvidia-cusparse==12.6.3.3 +nvidia-cufft==12.0.0.61 +nvidia-cudnn-cu13==9.19.0.56 +multiprocess==0.70.19 +markdown-it-py==4.0.0 +Jinja2==3.1.6 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==13.2.0 +anyio==4.13.0 +aiosignal==1.4.0 +rich==15.0.0 +pydantic==2.13.3 +pandas==3.0.2 +nvidia-cusolver==12.0.4.66 +httpx==0.28.1 +GitPython==3.1.49 +aiohttp==3.13.5 +wandb==0.26.1 +typer==0.25.1 +torch==2.11.0 +huggingface_hub==1.13.0 +tokenizers==0.22.2 +datasets==4.8.5 +accelerate==1.13.0 +transformers==5.7.0 +word2number==1.1 +sqlitedict==2.1.0 +pytz==2026.2 +threadpoolctl==3.6.0 +tcolorpy==0.1.7 +tabulate==0.10.0 +scipy==1.17.1 +portalocker==3.2.0 +pathvalidate==3.3.1 +narwhals==2.22.1 +more-itertools==11.1.0 +lxml==6.1.1 +joblib==1.5.3 +colorama==0.4.6 +chardet==6.0.0.post1 +absl-py==2.4.0 +scikit-learn==1.9.0 +sacrebleu==2.6.0 +nltk==3.9.4 +mbstrdecoder==1.1.5 +typepy==1.3.5 +rouge_score==0.1.2 +DataProperty==1.1.1 +tabledata==1.3.5 +evaluate==0.4.6 +pytablewriter==1.2.1 +lm_eval==0.4.12 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==9.1.0 +py-spy==0.4.2 +pillow==12.2.0 +tifffile==2026.6.1 +imagecodecs==2026.6.26 +zipp==4.1.0 +importlib_metadata==9.0.0 +derivative==0.6.3 +pysindy==2.1.0 +pyparsing==3.3.2 +kiwisolver==1.5.0 +fonttools==4.63.0 +cycler==0.12.1 +contourpy==1.3.3 +matplotlib==3.11.0 +sortedcontainers==2.4.0 +wcwidth==0.8.2 +sigfig==1.3.19 +importlib_resources==7.1.0 +enum-tools==0.13.0 +cramjam==2.11.0 +python-snappy==0.7.3 +compact-json==1.8.2 +numbers-parser==4.18.5 +entmax==1.3 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3la36nzy/logs/debug-internal.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3la36nzy/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6dbd66289cd71b673c9917ab36d34064cad86a79 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3la36nzy/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-09-21T13:12:50.483986586Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-09-21T13:12:50.484020637Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"} +{"time":"2026-09-21T13:12:50.838023892Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T13:12:50.838063321Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T13:12:50.838094006Z","level":"INFO","msg":"stream: created new stream","id":"3la36nzy"} +{"time":"2026-09-21T13:12:50.838213515Z","level":"INFO","msg":"handler: started"} +{"time":"2026-09-21T13:12:50.841655341Z","level":"INFO","msg":"stream: started"} +{"time":"2026-09-21T13:12:50.841683162Z","level":"INFO","msg":"writer: started","stream_id":"3la36nzy"} +{"time":"2026-09-21T13:12:50.841732275Z","level":"INFO","msg":"sender: started"} +{"time":"2026-09-21T13:12:50.870894371Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T13:12:50.87092038Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3la36nzy/logs/debug.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3la36nzy/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3d452be80fe3b45da18395735f61e48068523e4b --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3la36nzy/logs/debug.log @@ -0,0 +1,19 @@ +2026-09-21 13:12:49,924 INFO MainThread:356 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1 +2026-09-21 13:12:49,924 INFO MainThread:356 [wandb_setup.py:_flush():81] Configure stats pid to 356 +2026-09-21 13:12:49,924 INFO MainThread:356 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-09-21 13:12:49,924 INFO MainThread:356 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_131249-3la36nzy/logs/debug.log +2026-09-21 13:12:49,925 INFO MainThread:356 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_131249-3la36nzy/logs/debug-internal.log +2026-09-21 13:12:49,925 INFO MainThread:356 [wandb_init.py:init():850] calling init triggers +2026-09-21 13:12:49,925 INFO MainThread:356 [wandb_init.py:init():855] wandb.init called with sweep_config: {} +config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s45/run_20260921_131213', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 45, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s45', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed45'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}} +2026-09-21 13:12:49,925 INFO MainThread:356 [wandb_init.py:init():898] starting backend +2026-09-21 13:12:50,469 INFO MainThread:356 [wandb_init.py:init():913] sending inform_init request +2026-09-21 13:12:50,842 INFO MainThread:356 [wandb_init.py:init():918] backend started and connected +2026-09-21 13:12:50,847 INFO MainThread:356 [wandb_init.py:init():988] updated telemetry +2026-09-21 13:12:50,867 INFO MainThread:356 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout +2026-09-21 13:12:50,873 INFO MainThread:356 [wandb_init.py:init():1056] starting run threads in backend +2026-09-21 13:12:51,121 INFO MainThread:356 [wandb_run.py:_console_start():2554] atexit reg +2026-09-21 13:12:51,121 INFO MainThread:356 [wandb_run.py:_redirect():2403] redirect: wrap_raw +2026-09-21 13:12:51,121 INFO MainThread:356 [wandb_run.py:_redirect():2472] Wrapping output streams. +2026-09-21 13:12:51,121 INFO MainThread:356 [wandb_run.py:_redirect():2495] Redirects installed. +2026-09-21 13:12:51,127 INFO MainThread:356 [wandb_init.py:init():1094] run started, returning control to user process diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3la36nzy/run-3la36nzy.wandb b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3la36nzy/run-3la36nzy.wandb new file mode 100644 index 0000000000000000000000000000000000000000..93cb543682f700d49599f614fd57f698caf25f11 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3la36nzy/run-3la36nzy.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50d40438b665e32cf8402c79131dd92f63553951050e3e7266dc0c35d6e0b96c +size 622592 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3uan03m3/files/requirements.txt b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3uan03m3/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c3e38c4cce87cb53d52f45f6ac8cbf8688a7d92 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3uan03m3/files/requirements.txt @@ -0,0 +1,142 @@ +pip==26.0.1 +nvidia-cusparselt-cu13==0.8.0 +mpmath==1.3.0 +cuda-toolkit==13.0.2 +zstandard==0.25.0 +xxhash==3.7.0 +urllib3==2.6.3 +typing_extensions==4.15.0 +triton==3.6.0 +tqdm==4.67.3 +torchinfo==1.8.0 +sympy==1.14.0 +smmap==5.0.3 +six==1.17.0 +shellingham==1.5.4 +setuptools==81.0.0 +safetensors==0.7.0 +regex==2026.4.4 +PyYAML==6.0.3 +Pygments==2.20.0 +pyarrow==24.0.0 +psutil==7.2.2 +protobuf==7.34.1 +propcache==0.4.1 +platformdirs==4.9.6 +packaging==26.2 +nvidia-nvtx==13.0.85 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvjitlink==13.0.88 +nvidia-nccl-cu13==2.28.9 +nvidia-curand==10.4.0.35 +nvidia-cufile==1.15.1.6 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-cupti==13.0.85 +nvidia-cublas==13.1.0.3 +numpy==2.4.4 +networkx==3.6.1 +multidict==6.7.1 +mdurl==0.1.2 +MarkupSafe==3.0.3 +idna==3.13 +hf-xet==1.4.3 +h11==0.16.0 +fsspec==2026.2.0 +frozenlist==1.8.0 +filelock==3.29.0 +dill==0.4.1 +cuda-pathfinder==1.5.4 +click==8.3.3 +charset-normalizer==3.4.7 +certifi==2026.4.22 +attrs==26.1.0 +annotated-types==0.7.0 +annotated-doc==0.0.4 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +typing-inspection==0.4.2 +sentry-sdk==2.58.0 +requests==2.33.1 +python-dateutil==2.9.0.post0 +pydantic_core==2.46.3 +nvidia-cusparse==12.6.3.3 +nvidia-cufft==12.0.0.61 +nvidia-cudnn-cu13==9.19.0.56 +multiprocess==0.70.19 +markdown-it-py==4.0.0 +Jinja2==3.1.6 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==13.2.0 +anyio==4.13.0 +aiosignal==1.4.0 +rich==15.0.0 +pydantic==2.13.3 +pandas==3.0.2 +nvidia-cusolver==12.0.4.66 +httpx==0.28.1 +GitPython==3.1.49 +aiohttp==3.13.5 +wandb==0.26.1 +typer==0.25.1 +torch==2.11.0 +huggingface_hub==1.13.0 +tokenizers==0.22.2 +datasets==4.8.5 +accelerate==1.13.0 +transformers==5.7.0 +word2number==1.1 +sqlitedict==2.1.0 +pytz==2026.2 +threadpoolctl==3.6.0 +tcolorpy==0.1.7 +tabulate==0.10.0 +scipy==1.17.1 +portalocker==3.2.0 +pathvalidate==3.3.1 +narwhals==2.22.1 +more-itertools==11.1.0 +lxml==6.1.1 +joblib==1.5.3 +colorama==0.4.6 +chardet==6.0.0.post1 +absl-py==2.4.0 +scikit-learn==1.9.0 +sacrebleu==2.6.0 +nltk==3.9.4 +mbstrdecoder==1.1.5 +typepy==1.3.5 +rouge_score==0.1.2 +DataProperty==1.1.1 +tabledata==1.3.5 +evaluate==0.4.6 +pytablewriter==1.2.1 +lm_eval==0.4.12 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==9.1.0 +py-spy==0.4.2 +pillow==12.2.0 +tifffile==2026.6.1 +imagecodecs==2026.6.26 +zipp==4.1.0 +importlib_metadata==9.0.0 +derivative==0.6.3 +pysindy==2.1.0 +pyparsing==3.3.2 +kiwisolver==1.5.0 +fonttools==4.63.0 +cycler==0.12.1 +contourpy==1.3.3 +matplotlib==3.11.0 +sortedcontainers==2.4.0 +wcwidth==0.8.2 +sigfig==1.3.19 +importlib_resources==7.1.0 +enum-tools==0.13.0 +cramjam==2.11.0 +python-snappy==0.7.3 +compact-json==1.8.2 +numbers-parser==4.18.5 +entmax==1.3 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3uan03m3/logs/debug-internal.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3uan03m3/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6052bd44f6f4cbd95bb91cba634887988992c971 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3uan03m3/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-09-21T13:12:50.483874479Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-09-21T13:12:50.483901889Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"} +{"time":"2026-09-21T13:12:50.841088435Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T13:12:50.841137017Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T13:12:50.841184688Z","level":"INFO","msg":"stream: created new stream","id":"3uan03m3"} +{"time":"2026-09-21T13:12:50.841265799Z","level":"INFO","msg":"handler: started"} +{"time":"2026-09-21T13:12:50.84313623Z","level":"INFO","msg":"stream: started"} +{"time":"2026-09-21T13:12:50.843167918Z","level":"INFO","msg":"writer: started","stream_id":"3uan03m3"} +{"time":"2026-09-21T13:12:50.84320998Z","level":"INFO","msg":"sender: started"} +{"time":"2026-09-21T13:12:50.870785509Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T13:12:50.870814782Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3uan03m3/logs/debug.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3uan03m3/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e4b4d38cdce8ab2aa5a1476967682db05693a87c --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3uan03m3/logs/debug.log @@ -0,0 +1,19 @@ +2026-09-21 13:12:49,929 INFO MainThread:353 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1 +2026-09-21 13:12:49,929 INFO MainThread:353 [wandb_setup.py:_flush():81] Configure stats pid to 353 +2026-09-21 13:12:49,929 INFO MainThread:353 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-09-21 13:12:49,929 INFO MainThread:353 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_131249-3uan03m3/logs/debug.log +2026-09-21 13:12:49,930 INFO MainThread:353 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_131249-3uan03m3/logs/debug-internal.log +2026-09-21 13:12:49,930 INFO MainThread:353 [wandb_init.py:init():850] calling init triggers +2026-09-21 13:12:49,930 INFO MainThread:353 [wandb_init.py:init():855] wandb.init called with sweep_config: {} +config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s42/run_20260921_131213', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s42', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed42'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}} +2026-09-21 13:12:49,930 INFO MainThread:353 [wandb_init.py:init():898] starting backend +2026-09-21 13:12:50,469 INFO MainThread:353 [wandb_init.py:init():913] sending inform_init request +2026-09-21 13:12:50,844 INFO MainThread:353 [wandb_init.py:init():918] backend started and connected +2026-09-21 13:12:50,847 INFO MainThread:353 [wandb_init.py:init():988] updated telemetry +2026-09-21 13:12:50,867 INFO MainThread:353 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout +2026-09-21 13:12:50,873 INFO MainThread:353 [wandb_init.py:init():1056] starting run threads in backend +2026-09-21 13:12:51,108 INFO MainThread:353 [wandb_run.py:_console_start():2554] atexit reg +2026-09-21 13:12:51,108 INFO MainThread:353 [wandb_run.py:_redirect():2403] redirect: wrap_raw +2026-09-21 13:12:51,108 INFO MainThread:353 [wandb_run.py:_redirect():2472] Wrapping output streams. +2026-09-21 13:12:51,108 INFO MainThread:353 [wandb_run.py:_redirect():2495] Redirects installed. +2026-09-21 13:12:51,115 INFO MainThread:353 [wandb_init.py:init():1094] run started, returning control to user process diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3uan03m3/run-3uan03m3.wandb b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3uan03m3/run-3uan03m3.wandb new file mode 100644 index 0000000000000000000000000000000000000000..32ade0ed68cfb4e82ac7e2920ac33cb9d45c188e --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3uan03m3/run-3uan03m3.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8131887d0f9a50fc1d08591b27b346f6ac4586d89dfbf1fd0cf022321587af0f +size 622592 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-ifgmkpzt/files/requirements.txt b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-ifgmkpzt/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c3e38c4cce87cb53d52f45f6ac8cbf8688a7d92 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-ifgmkpzt/files/requirements.txt @@ -0,0 +1,142 @@ +pip==26.0.1 +nvidia-cusparselt-cu13==0.8.0 +mpmath==1.3.0 +cuda-toolkit==13.0.2 +zstandard==0.25.0 +xxhash==3.7.0 +urllib3==2.6.3 +typing_extensions==4.15.0 +triton==3.6.0 +tqdm==4.67.3 +torchinfo==1.8.0 +sympy==1.14.0 +smmap==5.0.3 +six==1.17.0 +shellingham==1.5.4 +setuptools==81.0.0 +safetensors==0.7.0 +regex==2026.4.4 +PyYAML==6.0.3 +Pygments==2.20.0 +pyarrow==24.0.0 +psutil==7.2.2 +protobuf==7.34.1 +propcache==0.4.1 +platformdirs==4.9.6 +packaging==26.2 +nvidia-nvtx==13.0.85 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvjitlink==13.0.88 +nvidia-nccl-cu13==2.28.9 +nvidia-curand==10.4.0.35 +nvidia-cufile==1.15.1.6 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-cupti==13.0.85 +nvidia-cublas==13.1.0.3 +numpy==2.4.4 +networkx==3.6.1 +multidict==6.7.1 +mdurl==0.1.2 +MarkupSafe==3.0.3 +idna==3.13 +hf-xet==1.4.3 +h11==0.16.0 +fsspec==2026.2.0 +frozenlist==1.8.0 +filelock==3.29.0 +dill==0.4.1 +cuda-pathfinder==1.5.4 +click==8.3.3 +charset-normalizer==3.4.7 +certifi==2026.4.22 +attrs==26.1.0 +annotated-types==0.7.0 +annotated-doc==0.0.4 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +typing-inspection==0.4.2 +sentry-sdk==2.58.0 +requests==2.33.1 +python-dateutil==2.9.0.post0 +pydantic_core==2.46.3 +nvidia-cusparse==12.6.3.3 +nvidia-cufft==12.0.0.61 +nvidia-cudnn-cu13==9.19.0.56 +multiprocess==0.70.19 +markdown-it-py==4.0.0 +Jinja2==3.1.6 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==13.2.0 +anyio==4.13.0 +aiosignal==1.4.0 +rich==15.0.0 +pydantic==2.13.3 +pandas==3.0.2 +nvidia-cusolver==12.0.4.66 +httpx==0.28.1 +GitPython==3.1.49 +aiohttp==3.13.5 +wandb==0.26.1 +typer==0.25.1 +torch==2.11.0 +huggingface_hub==1.13.0 +tokenizers==0.22.2 +datasets==4.8.5 +accelerate==1.13.0 +transformers==5.7.0 +word2number==1.1 +sqlitedict==2.1.0 +pytz==2026.2 +threadpoolctl==3.6.0 +tcolorpy==0.1.7 +tabulate==0.10.0 +scipy==1.17.1 +portalocker==3.2.0 +pathvalidate==3.3.1 +narwhals==2.22.1 +more-itertools==11.1.0 +lxml==6.1.1 +joblib==1.5.3 +colorama==0.4.6 +chardet==6.0.0.post1 +absl-py==2.4.0 +scikit-learn==1.9.0 +sacrebleu==2.6.0 +nltk==3.9.4 +mbstrdecoder==1.1.5 +typepy==1.3.5 +rouge_score==0.1.2 +DataProperty==1.1.1 +tabledata==1.3.5 +evaluate==0.4.6 +pytablewriter==1.2.1 +lm_eval==0.4.12 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==9.1.0 +py-spy==0.4.2 +pillow==12.2.0 +tifffile==2026.6.1 +imagecodecs==2026.6.26 +zipp==4.1.0 +importlib_metadata==9.0.0 +derivative==0.6.3 +pysindy==2.1.0 +pyparsing==3.3.2 +kiwisolver==1.5.0 +fonttools==4.63.0 +cycler==0.12.1 +contourpy==1.3.3 +matplotlib==3.11.0 +sortedcontainers==2.4.0 +wcwidth==0.8.2 +sigfig==1.3.19 +importlib_resources==7.1.0 +enum-tools==0.13.0 +cramjam==2.11.0 +python-snappy==0.7.3 +compact-json==1.8.2 +numbers-parser==4.18.5 +entmax==1.3 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-ifgmkpzt/logs/debug-internal.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-ifgmkpzt/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..4ee35a45ec3f804f33036b25a49a1bf21231234e --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-ifgmkpzt/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-09-21T13:12:50.483216577Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-09-21T13:12:50.483239Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"} +{"time":"2026-09-21T13:12:50.836243114Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T13:12:50.836287921Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T13:12:50.836332528Z","level":"INFO","msg":"stream: created new stream","id":"ifgmkpzt"} +{"time":"2026-09-21T13:12:50.836402232Z","level":"INFO","msg":"handler: started"} +{"time":"2026-09-21T13:12:50.839011786Z","level":"INFO","msg":"stream: started"} +{"time":"2026-09-21T13:12:50.839030384Z","level":"INFO","msg":"writer: started","stream_id":"ifgmkpzt"} +{"time":"2026-09-21T13:12:50.839085496Z","level":"INFO","msg":"sender: started"} +{"time":"2026-09-21T13:12:50.870954671Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T13:12:50.870977545Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-ifgmkpzt/logs/debug.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-ifgmkpzt/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..0c97690140f548b64d817d1022dbb36abdf38c99 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-ifgmkpzt/logs/debug.log @@ -0,0 +1,19 @@ +2026-09-21 13:12:49,902 INFO MainThread:355 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1 +2026-09-21 13:12:49,902 INFO MainThread:355 [wandb_setup.py:_flush():81] Configure stats pid to 355 +2026-09-21 13:12:49,902 INFO MainThread:355 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-09-21 13:12:49,902 INFO MainThread:355 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_131249-ifgmkpzt/logs/debug.log +2026-09-21 13:12:49,902 INFO MainThread:355 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_131249-ifgmkpzt/logs/debug-internal.log +2026-09-21 13:12:49,902 INFO MainThread:355 [wandb_init.py:init():850] calling init triggers +2026-09-21 13:12:49,902 INFO MainThread:355 [wandb_init.py:init():855] wandb.init called with sweep_config: {} +config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s44/run_20260921_131213', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 44, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s44', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed44'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}} +2026-09-21 13:12:49,902 INFO MainThread:355 [wandb_init.py:init():898] starting backend +2026-09-21 13:12:50,469 INFO MainThread:355 [wandb_init.py:init():913] sending inform_init request +2026-09-21 13:12:50,839 INFO MainThread:355 [wandb_init.py:init():918] backend started and connected +2026-09-21 13:12:50,843 INFO MainThread:355 [wandb_init.py:init():988] updated telemetry +2026-09-21 13:12:50,867 INFO MainThread:355 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout +2026-09-21 13:12:50,873 INFO MainThread:355 [wandb_init.py:init():1056] starting run threads in backend +2026-09-21 13:12:51,130 INFO MainThread:355 [wandb_run.py:_console_start():2554] atexit reg +2026-09-21 13:12:51,130 INFO MainThread:355 [wandb_run.py:_redirect():2403] redirect: wrap_raw +2026-09-21 13:12:51,130 INFO MainThread:355 [wandb_run.py:_redirect():2472] Wrapping output streams. +2026-09-21 13:12:51,131 INFO MainThread:355 [wandb_run.py:_redirect():2495] Redirects installed. +2026-09-21 13:12:51,133 INFO MainThread:355 [wandb_init.py:init():1094] run started, returning control to user process diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-ifgmkpzt/run-ifgmkpzt.wandb b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-ifgmkpzt/run-ifgmkpzt.wandb new file mode 100644 index 0000000000000000000000000000000000000000..c5ad1805705f8dc064d21452ee39cf844ba75fd5 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-ifgmkpzt/run-ifgmkpzt.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b50e684727aea5e38ea564a5e13082ab3e25791c61b34c009b307cc1a7eb8bbd +size 622592 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-meicy3md/files/requirements.txt b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-meicy3md/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c3e38c4cce87cb53d52f45f6ac8cbf8688a7d92 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-meicy3md/files/requirements.txt @@ -0,0 +1,142 @@ +pip==26.0.1 +nvidia-cusparselt-cu13==0.8.0 +mpmath==1.3.0 +cuda-toolkit==13.0.2 +zstandard==0.25.0 +xxhash==3.7.0 +urllib3==2.6.3 +typing_extensions==4.15.0 +triton==3.6.0 +tqdm==4.67.3 +torchinfo==1.8.0 +sympy==1.14.0 +smmap==5.0.3 +six==1.17.0 +shellingham==1.5.4 +setuptools==81.0.0 +safetensors==0.7.0 +regex==2026.4.4 +PyYAML==6.0.3 +Pygments==2.20.0 +pyarrow==24.0.0 +psutil==7.2.2 +protobuf==7.34.1 +propcache==0.4.1 +platformdirs==4.9.6 +packaging==26.2 +nvidia-nvtx==13.0.85 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvjitlink==13.0.88 +nvidia-nccl-cu13==2.28.9 +nvidia-curand==10.4.0.35 +nvidia-cufile==1.15.1.6 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-cupti==13.0.85 +nvidia-cublas==13.1.0.3 +numpy==2.4.4 +networkx==3.6.1 +multidict==6.7.1 +mdurl==0.1.2 +MarkupSafe==3.0.3 +idna==3.13 +hf-xet==1.4.3 +h11==0.16.0 +fsspec==2026.2.0 +frozenlist==1.8.0 +filelock==3.29.0 +dill==0.4.1 +cuda-pathfinder==1.5.4 +click==8.3.3 +charset-normalizer==3.4.7 +certifi==2026.4.22 +attrs==26.1.0 +annotated-types==0.7.0 +annotated-doc==0.0.4 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +typing-inspection==0.4.2 +sentry-sdk==2.58.0 +requests==2.33.1 +python-dateutil==2.9.0.post0 +pydantic_core==2.46.3 +nvidia-cusparse==12.6.3.3 +nvidia-cufft==12.0.0.61 +nvidia-cudnn-cu13==9.19.0.56 +multiprocess==0.70.19 +markdown-it-py==4.0.0 +Jinja2==3.1.6 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==13.2.0 +anyio==4.13.0 +aiosignal==1.4.0 +rich==15.0.0 +pydantic==2.13.3 +pandas==3.0.2 +nvidia-cusolver==12.0.4.66 +httpx==0.28.1 +GitPython==3.1.49 +aiohttp==3.13.5 +wandb==0.26.1 +typer==0.25.1 +torch==2.11.0 +huggingface_hub==1.13.0 +tokenizers==0.22.2 +datasets==4.8.5 +accelerate==1.13.0 +transformers==5.7.0 +word2number==1.1 +sqlitedict==2.1.0 +pytz==2026.2 +threadpoolctl==3.6.0 +tcolorpy==0.1.7 +tabulate==0.10.0 +scipy==1.17.1 +portalocker==3.2.0 +pathvalidate==3.3.1 +narwhals==2.22.1 +more-itertools==11.1.0 +lxml==6.1.1 +joblib==1.5.3 +colorama==0.4.6 +chardet==6.0.0.post1 +absl-py==2.4.0 +scikit-learn==1.9.0 +sacrebleu==2.6.0 +nltk==3.9.4 +mbstrdecoder==1.1.5 +typepy==1.3.5 +rouge_score==0.1.2 +DataProperty==1.1.1 +tabledata==1.3.5 +evaluate==0.4.6 +pytablewriter==1.2.1 +lm_eval==0.4.12 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==9.1.0 +py-spy==0.4.2 +pillow==12.2.0 +tifffile==2026.6.1 +imagecodecs==2026.6.26 +zipp==4.1.0 +importlib_metadata==9.0.0 +derivative==0.6.3 +pysindy==2.1.0 +pyparsing==3.3.2 +kiwisolver==1.5.0 +fonttools==4.63.0 +cycler==0.12.1 +contourpy==1.3.3 +matplotlib==3.11.0 +sortedcontainers==2.4.0 +wcwidth==0.8.2 +sigfig==1.3.19 +importlib_resources==7.1.0 +enum-tools==0.13.0 +cramjam==2.11.0 +python-snappy==0.7.3 +compact-json==1.8.2 +numbers-parser==4.18.5 +entmax==1.3 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-meicy3md/logs/debug-internal.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-meicy3md/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7467a6580ae86dd2afe496b6c60514cd33eb274e --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-meicy3md/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-09-21T13:12:50.483553329Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-09-21T13:12:50.48358054Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"} +{"time":"2026-09-21T13:12:50.839693533Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T13:12:50.839788525Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T13:12:50.839831339Z","level":"INFO","msg":"stream: created new stream","id":"meicy3md"} +{"time":"2026-09-21T13:12:50.83995932Z","level":"INFO","msg":"handler: started"} +{"time":"2026-09-21T13:12:50.843120577Z","level":"INFO","msg":"stream: started"} +{"time":"2026-09-21T13:12:50.843209209Z","level":"INFO","msg":"writer: started","stream_id":"meicy3md"} +{"time":"2026-09-21T13:12:50.843275598Z","level":"INFO","msg":"sender: started"} +{"time":"2026-09-21T13:12:50.87087387Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T13:12:50.870909333Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-meicy3md/logs/debug.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-meicy3md/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..6a36d4cad9af8242dbbeb4bfccc8c8a371cbeb58 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-meicy3md/logs/debug.log @@ -0,0 +1,19 @@ +2026-09-21 13:12:49,918 INFO MainThread:357 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1 +2026-09-21 13:12:49,918 INFO MainThread:357 [wandb_setup.py:_flush():81] Configure stats pid to 357 +2026-09-21 13:12:49,918 INFO MainThread:357 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-09-21 13:12:49,919 INFO MainThread:357 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_131249-meicy3md/logs/debug.log +2026-09-21 13:12:49,919 INFO MainThread:357 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_131249-meicy3md/logs/debug-internal.log +2026-09-21 13:12:49,919 INFO MainThread:357 [wandb_init.py:init():850] calling init triggers +2026-09-21 13:12:49,919 INFO MainThread:357 [wandb_init.py:init():855] wandb.init called with sweep_config: {} +config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s46/run_20260921_131213', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 46, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s46', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed46'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}} +2026-09-21 13:12:49,919 INFO MainThread:357 [wandb_init.py:init():898] starting backend +2026-09-21 13:12:50,470 INFO MainThread:357 [wandb_init.py:init():913] sending inform_init request +2026-09-21 13:12:50,844 INFO MainThread:357 [wandb_init.py:init():918] backend started and connected +2026-09-21 13:12:50,849 INFO MainThread:357 [wandb_init.py:init():988] updated telemetry +2026-09-21 13:12:50,867 INFO MainThread:357 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout +2026-09-21 13:12:50,873 INFO MainThread:357 [wandb_init.py:init():1056] starting run threads in backend +2026-09-21 13:12:51,111 INFO MainThread:357 [wandb_run.py:_console_start():2554] atexit reg +2026-09-21 13:12:51,111 INFO MainThread:357 [wandb_run.py:_redirect():2403] redirect: wrap_raw +2026-09-21 13:12:51,111 INFO MainThread:357 [wandb_run.py:_redirect():2472] Wrapping output streams. +2026-09-21 13:12:51,111 INFO MainThread:357 [wandb_run.py:_redirect():2495] Redirects installed. +2026-09-21 13:12:51,115 INFO MainThread:357 [wandb_init.py:init():1094] run started, returning control to user process diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-meicy3md/run-meicy3md.wandb b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-meicy3md/run-meicy3md.wandb new file mode 100644 index 0000000000000000000000000000000000000000..4ab467cc95500e1dca49c367a4046d52109a27fb --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-meicy3md/run-meicy3md.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78693de0a766c541b0b8c83847e80ef9f3fb9a094f3b14113b5310df63b79b4c +size 622592 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-n6s56yfr/files/requirements.txt b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-n6s56yfr/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c3e38c4cce87cb53d52f45f6ac8cbf8688a7d92 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-n6s56yfr/files/requirements.txt @@ -0,0 +1,142 @@ +pip==26.0.1 +nvidia-cusparselt-cu13==0.8.0 +mpmath==1.3.0 +cuda-toolkit==13.0.2 +zstandard==0.25.0 +xxhash==3.7.0 +urllib3==2.6.3 +typing_extensions==4.15.0 +triton==3.6.0 +tqdm==4.67.3 +torchinfo==1.8.0 +sympy==1.14.0 +smmap==5.0.3 +six==1.17.0 +shellingham==1.5.4 +setuptools==81.0.0 +safetensors==0.7.0 +regex==2026.4.4 +PyYAML==6.0.3 +Pygments==2.20.0 +pyarrow==24.0.0 +psutil==7.2.2 +protobuf==7.34.1 +propcache==0.4.1 +platformdirs==4.9.6 +packaging==26.2 +nvidia-nvtx==13.0.85 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvjitlink==13.0.88 +nvidia-nccl-cu13==2.28.9 +nvidia-curand==10.4.0.35 +nvidia-cufile==1.15.1.6 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-cupti==13.0.85 +nvidia-cublas==13.1.0.3 +numpy==2.4.4 +networkx==3.6.1 +multidict==6.7.1 +mdurl==0.1.2 +MarkupSafe==3.0.3 +idna==3.13 +hf-xet==1.4.3 +h11==0.16.0 +fsspec==2026.2.0 +frozenlist==1.8.0 +filelock==3.29.0 +dill==0.4.1 +cuda-pathfinder==1.5.4 +click==8.3.3 +charset-normalizer==3.4.7 +certifi==2026.4.22 +attrs==26.1.0 +annotated-types==0.7.0 +annotated-doc==0.0.4 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +typing-inspection==0.4.2 +sentry-sdk==2.58.0 +requests==2.33.1 +python-dateutil==2.9.0.post0 +pydantic_core==2.46.3 +nvidia-cusparse==12.6.3.3 +nvidia-cufft==12.0.0.61 +nvidia-cudnn-cu13==9.19.0.56 +multiprocess==0.70.19 +markdown-it-py==4.0.0 +Jinja2==3.1.6 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==13.2.0 +anyio==4.13.0 +aiosignal==1.4.0 +rich==15.0.0 +pydantic==2.13.3 +pandas==3.0.2 +nvidia-cusolver==12.0.4.66 +httpx==0.28.1 +GitPython==3.1.49 +aiohttp==3.13.5 +wandb==0.26.1 +typer==0.25.1 +torch==2.11.0 +huggingface_hub==1.13.0 +tokenizers==0.22.2 +datasets==4.8.5 +accelerate==1.13.0 +transformers==5.7.0 +word2number==1.1 +sqlitedict==2.1.0 +pytz==2026.2 +threadpoolctl==3.6.0 +tcolorpy==0.1.7 +tabulate==0.10.0 +scipy==1.17.1 +portalocker==3.2.0 +pathvalidate==3.3.1 +narwhals==2.22.1 +more-itertools==11.1.0 +lxml==6.1.1 +joblib==1.5.3 +colorama==0.4.6 +chardet==6.0.0.post1 +absl-py==2.4.0 +scikit-learn==1.9.0 +sacrebleu==2.6.0 +nltk==3.9.4 +mbstrdecoder==1.1.5 +typepy==1.3.5 +rouge_score==0.1.2 +DataProperty==1.1.1 +tabledata==1.3.5 +evaluate==0.4.6 +pytablewriter==1.2.1 +lm_eval==0.4.12 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==9.1.0 +py-spy==0.4.2 +pillow==12.2.0 +tifffile==2026.6.1 +imagecodecs==2026.6.26 +zipp==4.1.0 +importlib_metadata==9.0.0 +derivative==0.6.3 +pysindy==2.1.0 +pyparsing==3.3.2 +kiwisolver==1.5.0 +fonttools==4.63.0 +cycler==0.12.1 +contourpy==1.3.3 +matplotlib==3.11.0 +sortedcontainers==2.4.0 +wcwidth==0.8.2 +sigfig==1.3.19 +importlib_resources==7.1.0 +enum-tools==0.13.0 +cramjam==2.11.0 +python-snappy==0.7.3 +compact-json==1.8.2 +numbers-parser==4.18.5 +entmax==1.3 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-n6s56yfr/logs/debug-internal.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-n6s56yfr/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a627086646b28501e739e86da2bb50d52a2dfefc --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-n6s56yfr/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-09-21T13:12:50.484598128Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-09-21T13:12:50.484624147Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"} +{"time":"2026-09-21T13:12:50.831760545Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T13:12:50.831852012Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T13:12:50.831921986Z","level":"INFO","msg":"stream: created new stream","id":"n6s56yfr"} +{"time":"2026-09-21T13:12:50.832074734Z","level":"INFO","msg":"handler: started"} +{"time":"2026-09-21T13:12:50.834780502Z","level":"INFO","msg":"stream: started"} +{"time":"2026-09-21T13:12:50.83493327Z","level":"INFO","msg":"writer: started","stream_id":"n6s56yfr"} +{"time":"2026-09-21T13:12:50.834970566Z","level":"INFO","msg":"sender: started"} +{"time":"2026-09-21T13:12:50.870640873Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T13:12:50.870664909Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-n6s56yfr/logs/debug.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-n6s56yfr/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..45458f9aac14c4eda018c6c128776a948c51739e --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-n6s56yfr/logs/debug.log @@ -0,0 +1,19 @@ +2026-09-21 13:12:49,926 INFO MainThread:354 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1 +2026-09-21 13:12:49,927 INFO MainThread:354 [wandb_setup.py:_flush():81] Configure stats pid to 354 +2026-09-21 13:12:49,927 INFO MainThread:354 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-09-21 13:12:49,927 INFO MainThread:354 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_131249-n6s56yfr/logs/debug.log +2026-09-21 13:12:49,927 INFO MainThread:354 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_131249-n6s56yfr/logs/debug-internal.log +2026-09-21 13:12:49,927 INFO MainThread:354 [wandb_init.py:init():850] calling init triggers +2026-09-21 13:12:49,927 INFO MainThread:354 [wandb_init.py:init():855] wandb.init called with sweep_config: {} +config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s43/run_20260921_131213', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 43, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s43', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed43'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}} +2026-09-21 13:12:49,927 INFO MainThread:354 [wandb_init.py:init():898] starting backend +2026-09-21 13:12:50,469 INFO MainThread:354 [wandb_init.py:init():913] sending inform_init request +2026-09-21 13:12:50,836 INFO MainThread:354 [wandb_init.py:init():918] backend started and connected +2026-09-21 13:12:50,839 INFO MainThread:354 [wandb_init.py:init():988] updated telemetry +2026-09-21 13:12:50,867 INFO MainThread:354 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout +2026-09-21 13:12:50,873 INFO MainThread:354 [wandb_init.py:init():1056] starting run threads in backend +2026-09-21 13:12:51,107 INFO MainThread:354 [wandb_run.py:_console_start():2554] atexit reg +2026-09-21 13:12:51,107 INFO MainThread:354 [wandb_run.py:_redirect():2403] redirect: wrap_raw +2026-09-21 13:12:51,107 INFO MainThread:354 [wandb_run.py:_redirect():2472] Wrapping output streams. +2026-09-21 13:12:51,107 INFO MainThread:354 [wandb_run.py:_redirect():2495] Redirects installed. +2026-09-21 13:12:51,115 INFO MainThread:354 [wandb_init.py:init():1094] run started, returning control to user process diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-n6s56yfr/run-n6s56yfr.wandb b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-n6s56yfr/run-n6s56yfr.wandb new file mode 100644 index 0000000000000000000000000000000000000000..141b94d07e90195ddc7c6df4787e9aba198c790d --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-n6s56yfr/run-n6s56yfr.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c2e5252108c00a233fb04e5c68db422bc976a5a19e24447d297f1d4ab141507 +size 622592 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-xc6c08ps/files/requirements.txt b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-xc6c08ps/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c3e38c4cce87cb53d52f45f6ac8cbf8688a7d92 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-xc6c08ps/files/requirements.txt @@ -0,0 +1,142 @@ +pip==26.0.1 +nvidia-cusparselt-cu13==0.8.0 +mpmath==1.3.0 +cuda-toolkit==13.0.2 +zstandard==0.25.0 +xxhash==3.7.0 +urllib3==2.6.3 +typing_extensions==4.15.0 +triton==3.6.0 +tqdm==4.67.3 +torchinfo==1.8.0 +sympy==1.14.0 +smmap==5.0.3 +six==1.17.0 +shellingham==1.5.4 +setuptools==81.0.0 +safetensors==0.7.0 +regex==2026.4.4 +PyYAML==6.0.3 +Pygments==2.20.0 +pyarrow==24.0.0 +psutil==7.2.2 +protobuf==7.34.1 +propcache==0.4.1 +platformdirs==4.9.6 +packaging==26.2 +nvidia-nvtx==13.0.85 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvjitlink==13.0.88 +nvidia-nccl-cu13==2.28.9 +nvidia-curand==10.4.0.35 +nvidia-cufile==1.15.1.6 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-cupti==13.0.85 +nvidia-cublas==13.1.0.3 +numpy==2.4.4 +networkx==3.6.1 +multidict==6.7.1 +mdurl==0.1.2 +MarkupSafe==3.0.3 +idna==3.13 +hf-xet==1.4.3 +h11==0.16.0 +fsspec==2026.2.0 +frozenlist==1.8.0 +filelock==3.29.0 +dill==0.4.1 +cuda-pathfinder==1.5.4 +click==8.3.3 +charset-normalizer==3.4.7 +certifi==2026.4.22 +attrs==26.1.0 +annotated-types==0.7.0 +annotated-doc==0.0.4 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +typing-inspection==0.4.2 +sentry-sdk==2.58.0 +requests==2.33.1 +python-dateutil==2.9.0.post0 +pydantic_core==2.46.3 +nvidia-cusparse==12.6.3.3 +nvidia-cufft==12.0.0.61 +nvidia-cudnn-cu13==9.19.0.56 +multiprocess==0.70.19 +markdown-it-py==4.0.0 +Jinja2==3.1.6 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==13.2.0 +anyio==4.13.0 +aiosignal==1.4.0 +rich==15.0.0 +pydantic==2.13.3 +pandas==3.0.2 +nvidia-cusolver==12.0.4.66 +httpx==0.28.1 +GitPython==3.1.49 +aiohttp==3.13.5 +wandb==0.26.1 +typer==0.25.1 +torch==2.11.0 +huggingface_hub==1.13.0 +tokenizers==0.22.2 +datasets==4.8.5 +accelerate==1.13.0 +transformers==5.7.0 +word2number==1.1 +sqlitedict==2.1.0 +pytz==2026.2 +threadpoolctl==3.6.0 +tcolorpy==0.1.7 +tabulate==0.10.0 +scipy==1.17.1 +portalocker==3.2.0 +pathvalidate==3.3.1 +narwhals==2.22.1 +more-itertools==11.1.0 +lxml==6.1.1 +joblib==1.5.3 +colorama==0.4.6 +chardet==6.0.0.post1 +absl-py==2.4.0 +scikit-learn==1.9.0 +sacrebleu==2.6.0 +nltk==3.9.4 +mbstrdecoder==1.1.5 +typepy==1.3.5 +rouge_score==0.1.2 +DataProperty==1.1.1 +tabledata==1.3.5 +evaluate==0.4.6 +pytablewriter==1.2.1 +lm_eval==0.4.12 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==9.1.0 +py-spy==0.4.2 +pillow==12.2.0 +tifffile==2026.6.1 +imagecodecs==2026.6.26 +zipp==4.1.0 +importlib_metadata==9.0.0 +derivative==0.6.3 +pysindy==2.1.0 +pyparsing==3.3.2 +kiwisolver==1.5.0 +fonttools==4.63.0 +cycler==0.12.1 +contourpy==1.3.3 +matplotlib==3.11.0 +sortedcontainers==2.4.0 +wcwidth==0.8.2 +sigfig==1.3.19 +importlib_resources==7.1.0 +enum-tools==0.13.0 +cramjam==2.11.0 +python-snappy==0.7.3 +compact-json==1.8.2 +numbers-parser==4.18.5 +entmax==1.3 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-xc6c08ps/logs/debug-internal.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-xc6c08ps/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b7bae8467e07aa7b305183cb4857a66c3ee7b0f5 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-xc6c08ps/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-09-21T13:12:50.484437969Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-09-21T13:12:50.484464449Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"} +{"time":"2026-09-21T13:12:50.838624218Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T13:12:50.838691207Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T13:12:50.838730746Z","level":"INFO","msg":"stream: created new stream","id":"xc6c08ps"} +{"time":"2026-09-21T13:12:50.838795473Z","level":"INFO","msg":"handler: started"} +{"time":"2026-09-21T13:12:50.840941876Z","level":"INFO","msg":"stream: started"} +{"time":"2026-09-21T13:12:50.840979752Z","level":"INFO","msg":"writer: started","stream_id":"xc6c08ps"} +{"time":"2026-09-21T13:12:50.841010919Z","level":"INFO","msg":"sender: started"} +{"time":"2026-09-21T13:12:50.870744397Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T13:12:50.870771448Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-xc6c08ps/logs/debug.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-xc6c08ps/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1f33c416de2c4150c999d02fc3e38601fb7f75da --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-xc6c08ps/logs/debug.log @@ -0,0 +1,19 @@ +2026-09-21 13:12:49,928 INFO MainThread:358 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1 +2026-09-21 13:12:49,928 INFO MainThread:358 [wandb_setup.py:_flush():81] Configure stats pid to 358 +2026-09-21 13:12:49,928 INFO MainThread:358 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-09-21 13:12:49,928 INFO MainThread:358 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_131249-xc6c08ps/logs/debug.log +2026-09-21 13:12:49,928 INFO MainThread:358 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_131249-xc6c08ps/logs/debug-internal.log +2026-09-21 13:12:49,928 INFO MainThread:358 [wandb_init.py:init():850] calling init triggers +2026-09-21 13:12:49,928 INFO MainThread:358 [wandb_init.py:init():855] wandb.init called with sweep_config: {} +config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s47/run_20260921_131213', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 47, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s47', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed47'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}} +2026-09-21 13:12:49,928 INFO MainThread:358 [wandb_init.py:init():898] starting backend +2026-09-21 13:12:50,469 INFO MainThread:358 [wandb_init.py:init():913] sending inform_init request +2026-09-21 13:12:50,841 INFO MainThread:358 [wandb_init.py:init():918] backend started and connected +2026-09-21 13:12:50,846 INFO MainThread:358 [wandb_init.py:init():988] updated telemetry +2026-09-21 13:12:50,867 INFO MainThread:358 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout +2026-09-21 13:12:50,873 INFO MainThread:358 [wandb_init.py:init():1056] starting run threads in backend +2026-09-21 13:12:51,127 INFO MainThread:358 [wandb_run.py:_console_start():2554] atexit reg +2026-09-21 13:12:51,127 INFO MainThread:358 [wandb_run.py:_redirect():2403] redirect: wrap_raw +2026-09-21 13:12:51,127 INFO MainThread:358 [wandb_run.py:_redirect():2472] Wrapping output streams. +2026-09-21 13:12:51,127 INFO MainThread:358 [wandb_run.py:_redirect():2495] Redirects installed. +2026-09-21 13:12:51,129 INFO MainThread:358 [wandb_init.py:init():1094] run started, returning control to user process diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-xc6c08ps/run-xc6c08ps.wandb b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-xc6c08ps/run-xc6c08ps.wandb new file mode 100644 index 0000000000000000000000000000000000000000..2bb2e40304364d68b315fc3e4cc13aef70ed2ba5 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_131249-xc6c08ps/run-xc6c08ps.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9483abf7a5154cc7067a0597415e245fd9c190499f29c23aa5f133ed608d76c +size 622592 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-ghfbqjah/logs/debug.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-ghfbqjah/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f7a5dd0ad41e1546e270ea88971acefa3c89fade --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-ghfbqjah/logs/debug.log @@ -0,0 +1,19 @@ +2026-09-21 14:30:24,657 INFO MainThread:70 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1 +2026-09-21 14:30:24,657 INFO MainThread:70 [wandb_setup.py:_flush():81] Configure stats pid to 70 +2026-09-21 14:30:24,658 INFO MainThread:70 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-09-21 14:30:24,658 INFO MainThread:70 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_143024-ghfbqjah/logs/debug.log +2026-09-21 14:30:24,658 INFO MainThread:70 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_143024-ghfbqjah/logs/debug-internal.log +2026-09-21 14:30:24,658 INFO MainThread:70 [wandb_init.py:init():850] calling init triggers +2026-09-21 14:30:24,658 INFO MainThread:70 [wandb_init.py:init():855] wandb.init called with sweep_config: {} +config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s45/run_20260921_142951', 'resume_from_checkpoint': None, 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 45, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s45', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed45'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}} +2026-09-21 14:30:24,658 INFO MainThread:70 [wandb_init.py:init():898] starting backend +2026-09-21 14:30:25,191 INFO MainThread:70 [wandb_init.py:init():913] sending inform_init request +2026-09-21 14:30:25,568 INFO MainThread:70 [wandb_init.py:init():918] backend started and connected +2026-09-21 14:30:25,570 INFO MainThread:70 [wandb_init.py:init():988] updated telemetry +2026-09-21 14:30:25,584 INFO MainThread:70 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout +2026-09-21 14:30:25,589 INFO MainThread:70 [wandb_init.py:init():1056] starting run threads in backend +2026-09-21 14:30:25,756 INFO MainThread:70 [wandb_run.py:_console_start():2554] atexit reg +2026-09-21 14:30:25,756 INFO MainThread:70 [wandb_run.py:_redirect():2403] redirect: wrap_raw +2026-09-21 14:30:25,757 INFO MainThread:70 [wandb_run.py:_redirect():2472] Wrapping output streams. +2026-09-21 14:30:25,757 INFO MainThread:70 [wandb_run.py:_redirect():2495] Redirects installed. +2026-09-21 14:30:25,763 INFO MainThread:70 [wandb_init.py:init():1094] run started, returning control to user process diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-kbn66izc/files/requirements.txt b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-kbn66izc/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c3e38c4cce87cb53d52f45f6ac8cbf8688a7d92 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-kbn66izc/files/requirements.txt @@ -0,0 +1,142 @@ +pip==26.0.1 +nvidia-cusparselt-cu13==0.8.0 +mpmath==1.3.0 +cuda-toolkit==13.0.2 +zstandard==0.25.0 +xxhash==3.7.0 +urllib3==2.6.3 +typing_extensions==4.15.0 +triton==3.6.0 +tqdm==4.67.3 +torchinfo==1.8.0 +sympy==1.14.0 +smmap==5.0.3 +six==1.17.0 +shellingham==1.5.4 +setuptools==81.0.0 +safetensors==0.7.0 +regex==2026.4.4 +PyYAML==6.0.3 +Pygments==2.20.0 +pyarrow==24.0.0 +psutil==7.2.2 +protobuf==7.34.1 +propcache==0.4.1 +platformdirs==4.9.6 +packaging==26.2 +nvidia-nvtx==13.0.85 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvjitlink==13.0.88 +nvidia-nccl-cu13==2.28.9 +nvidia-curand==10.4.0.35 +nvidia-cufile==1.15.1.6 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-cupti==13.0.85 +nvidia-cublas==13.1.0.3 +numpy==2.4.4 +networkx==3.6.1 +multidict==6.7.1 +mdurl==0.1.2 +MarkupSafe==3.0.3 +idna==3.13 +hf-xet==1.4.3 +h11==0.16.0 +fsspec==2026.2.0 +frozenlist==1.8.0 +filelock==3.29.0 +dill==0.4.1 +cuda-pathfinder==1.5.4 +click==8.3.3 +charset-normalizer==3.4.7 +certifi==2026.4.22 +attrs==26.1.0 +annotated-types==0.7.0 +annotated-doc==0.0.4 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +typing-inspection==0.4.2 +sentry-sdk==2.58.0 +requests==2.33.1 +python-dateutil==2.9.0.post0 +pydantic_core==2.46.3 +nvidia-cusparse==12.6.3.3 +nvidia-cufft==12.0.0.61 +nvidia-cudnn-cu13==9.19.0.56 +multiprocess==0.70.19 +markdown-it-py==4.0.0 +Jinja2==3.1.6 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==13.2.0 +anyio==4.13.0 +aiosignal==1.4.0 +rich==15.0.0 +pydantic==2.13.3 +pandas==3.0.2 +nvidia-cusolver==12.0.4.66 +httpx==0.28.1 +GitPython==3.1.49 +aiohttp==3.13.5 +wandb==0.26.1 +typer==0.25.1 +torch==2.11.0 +huggingface_hub==1.13.0 +tokenizers==0.22.2 +datasets==4.8.5 +accelerate==1.13.0 +transformers==5.7.0 +word2number==1.1 +sqlitedict==2.1.0 +pytz==2026.2 +threadpoolctl==3.6.0 +tcolorpy==0.1.7 +tabulate==0.10.0 +scipy==1.17.1 +portalocker==3.2.0 +pathvalidate==3.3.1 +narwhals==2.22.1 +more-itertools==11.1.0 +lxml==6.1.1 +joblib==1.5.3 +colorama==0.4.6 +chardet==6.0.0.post1 +absl-py==2.4.0 +scikit-learn==1.9.0 +sacrebleu==2.6.0 +nltk==3.9.4 +mbstrdecoder==1.1.5 +typepy==1.3.5 +rouge_score==0.1.2 +DataProperty==1.1.1 +tabledata==1.3.5 +evaluate==0.4.6 +pytablewriter==1.2.1 +lm_eval==0.4.12 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==9.1.0 +py-spy==0.4.2 +pillow==12.2.0 +tifffile==2026.6.1 +imagecodecs==2026.6.26 +zipp==4.1.0 +importlib_metadata==9.0.0 +derivative==0.6.3 +pysindy==2.1.0 +pyparsing==3.3.2 +kiwisolver==1.5.0 +fonttools==4.63.0 +cycler==0.12.1 +contourpy==1.3.3 +matplotlib==3.11.0 +sortedcontainers==2.4.0 +wcwidth==0.8.2 +sigfig==1.3.19 +importlib_resources==7.1.0 +enum-tools==0.13.0 +cramjam==2.11.0 +python-snappy==0.7.3 +compact-json==1.8.2 +numbers-parser==4.18.5 +entmax==1.3 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-kbn66izc/logs/debug-internal.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-kbn66izc/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..26c08890597d39d76e353493c0daafc1d332483c --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-kbn66izc/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-09-21T14:30:25.206945921Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-09-21T14:30:25.206970507Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"} +{"time":"2026-09-21T14:30:25.559404152Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T14:30:25.559465233Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T14:30:25.559507276Z","level":"INFO","msg":"stream: created new stream","id":"kbn66izc"} +{"time":"2026-09-21T14:30:25.559588297Z","level":"INFO","msg":"handler: started"} +{"time":"2026-09-21T14:30:25.561924124Z","level":"INFO","msg":"stream: started"} +{"time":"2026-09-21T14:30:25.561956583Z","level":"INFO","msg":"writer: started","stream_id":"kbn66izc"} +{"time":"2026-09-21T14:30:25.562002852Z","level":"INFO","msg":"sender: started"} +{"time":"2026-09-21T14:30:25.588409287Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T14:30:25.588434474Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-kbn66izc/logs/debug.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-kbn66izc/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..983c8622329136a6291fd3d6814e2acdfb12e4b1 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-kbn66izc/logs/debug.log @@ -0,0 +1,19 @@ +2026-09-21 14:30:24,644 INFO MainThread:69 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1 +2026-09-21 14:30:24,644 INFO MainThread:69 [wandb_setup.py:_flush():81] Configure stats pid to 69 +2026-09-21 14:30:24,644 INFO MainThread:69 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-09-21 14:30:24,644 INFO MainThread:69 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_143024-kbn66izc/logs/debug.log +2026-09-21 14:30:24,644 INFO MainThread:69 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_143024-kbn66izc/logs/debug-internal.log +2026-09-21 14:30:24,644 INFO MainThread:69 [wandb_init.py:init():850] calling init triggers +2026-09-21 14:30:24,644 INFO MainThread:69 [wandb_init.py:init():855] wandb.init called with sweep_config: {} +config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s44/run_20260921_142951', 'resume_from_checkpoint': None, 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 44, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s44', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed44'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}} +2026-09-21 14:30:24,644 INFO MainThread:69 [wandb_init.py:init():898] starting backend +2026-09-21 14:30:25,191 INFO MainThread:69 [wandb_init.py:init():913] sending inform_init request +2026-09-21 14:30:25,562 INFO MainThread:69 [wandb_init.py:init():918] backend started and connected +2026-09-21 14:30:25,565 INFO MainThread:69 [wandb_init.py:init():988] updated telemetry +2026-09-21 14:30:25,585 INFO MainThread:69 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout +2026-09-21 14:30:25,591 INFO MainThread:69 [wandb_init.py:init():1056] starting run threads in backend +2026-09-21 14:30:25,790 INFO MainThread:69 [wandb_run.py:_console_start():2554] atexit reg +2026-09-21 14:30:25,790 INFO MainThread:69 [wandb_run.py:_redirect():2403] redirect: wrap_raw +2026-09-21 14:30:25,790 INFO MainThread:69 [wandb_run.py:_redirect():2472] Wrapping output streams. +2026-09-21 14:30:25,791 INFO MainThread:69 [wandb_run.py:_redirect():2495] Redirects installed. +2026-09-21 14:30:25,792 INFO MainThread:69 [wandb_init.py:init():1094] run started, returning control to user process diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-kbn66izc/run-kbn66izc.wandb b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-kbn66izc/run-kbn66izc.wandb new file mode 100644 index 0000000000000000000000000000000000000000..3be52500c1432632cf1e6c08cf11b1f5f8723b1c --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-kbn66izc/run-kbn66izc.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f8d7767727a9fb0c9c652f571b990bbae7a5074ddfc36a5ebeed9b64e493455 +size 1376256 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-zgrsvkbi/files/requirements.txt b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-zgrsvkbi/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c3e38c4cce87cb53d52f45f6ac8cbf8688a7d92 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-zgrsvkbi/files/requirements.txt @@ -0,0 +1,142 @@ +pip==26.0.1 +nvidia-cusparselt-cu13==0.8.0 +mpmath==1.3.0 +cuda-toolkit==13.0.2 +zstandard==0.25.0 +xxhash==3.7.0 +urllib3==2.6.3 +typing_extensions==4.15.0 +triton==3.6.0 +tqdm==4.67.3 +torchinfo==1.8.0 +sympy==1.14.0 +smmap==5.0.3 +six==1.17.0 +shellingham==1.5.4 +setuptools==81.0.0 +safetensors==0.7.0 +regex==2026.4.4 +PyYAML==6.0.3 +Pygments==2.20.0 +pyarrow==24.0.0 +psutil==7.2.2 +protobuf==7.34.1 +propcache==0.4.1 +platformdirs==4.9.6 +packaging==26.2 +nvidia-nvtx==13.0.85 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvjitlink==13.0.88 +nvidia-nccl-cu13==2.28.9 +nvidia-curand==10.4.0.35 +nvidia-cufile==1.15.1.6 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-cupti==13.0.85 +nvidia-cublas==13.1.0.3 +numpy==2.4.4 +networkx==3.6.1 +multidict==6.7.1 +mdurl==0.1.2 +MarkupSafe==3.0.3 +idna==3.13 +hf-xet==1.4.3 +h11==0.16.0 +fsspec==2026.2.0 +frozenlist==1.8.0 +filelock==3.29.0 +dill==0.4.1 +cuda-pathfinder==1.5.4 +click==8.3.3 +charset-normalizer==3.4.7 +certifi==2026.4.22 +attrs==26.1.0 +annotated-types==0.7.0 +annotated-doc==0.0.4 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +typing-inspection==0.4.2 +sentry-sdk==2.58.0 +requests==2.33.1 +python-dateutil==2.9.0.post0 +pydantic_core==2.46.3 +nvidia-cusparse==12.6.3.3 +nvidia-cufft==12.0.0.61 +nvidia-cudnn-cu13==9.19.0.56 +multiprocess==0.70.19 +markdown-it-py==4.0.0 +Jinja2==3.1.6 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==13.2.0 +anyio==4.13.0 +aiosignal==1.4.0 +rich==15.0.0 +pydantic==2.13.3 +pandas==3.0.2 +nvidia-cusolver==12.0.4.66 +httpx==0.28.1 +GitPython==3.1.49 +aiohttp==3.13.5 +wandb==0.26.1 +typer==0.25.1 +torch==2.11.0 +huggingface_hub==1.13.0 +tokenizers==0.22.2 +datasets==4.8.5 +accelerate==1.13.0 +transformers==5.7.0 +word2number==1.1 +sqlitedict==2.1.0 +pytz==2026.2 +threadpoolctl==3.6.0 +tcolorpy==0.1.7 +tabulate==0.10.0 +scipy==1.17.1 +portalocker==3.2.0 +pathvalidate==3.3.1 +narwhals==2.22.1 +more-itertools==11.1.0 +lxml==6.1.1 +joblib==1.5.3 +colorama==0.4.6 +chardet==6.0.0.post1 +absl-py==2.4.0 +scikit-learn==1.9.0 +sacrebleu==2.6.0 +nltk==3.9.4 +mbstrdecoder==1.1.5 +typepy==1.3.5 +rouge_score==0.1.2 +DataProperty==1.1.1 +tabledata==1.3.5 +evaluate==0.4.6 +pytablewriter==1.2.1 +lm_eval==0.4.12 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==9.1.0 +py-spy==0.4.2 +pillow==12.2.0 +tifffile==2026.6.1 +imagecodecs==2026.6.26 +zipp==4.1.0 +importlib_metadata==9.0.0 +derivative==0.6.3 +pysindy==2.1.0 +pyparsing==3.3.2 +kiwisolver==1.5.0 +fonttools==4.63.0 +cycler==0.12.1 +contourpy==1.3.3 +matplotlib==3.11.0 +sortedcontainers==2.4.0 +wcwidth==0.8.2 +sigfig==1.3.19 +importlib_resources==7.1.0 +enum-tools==0.13.0 +cramjam==2.11.0 +python-snappy==0.7.3 +compact-json==1.8.2 +numbers-parser==4.18.5 +entmax==1.3 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-zgrsvkbi/logs/debug-internal.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-zgrsvkbi/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6afa466abe830dc5404e30d74634f12555bd55b4 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-zgrsvkbi/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-09-21T14:30:25.206626644Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-09-21T14:30:25.20665084Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"} +{"time":"2026-09-21T14:30:25.554203978Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T14:30:25.554264238Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T14:30:25.554298309Z","level":"INFO","msg":"stream: created new stream","id":"zgrsvkbi"} +{"time":"2026-09-21T14:30:25.554400832Z","level":"INFO","msg":"handler: started"} +{"time":"2026-09-21T14:30:25.556590301Z","level":"INFO","msg":"stream: started"} +{"time":"2026-09-21T14:30:25.556631613Z","level":"INFO","msg":"writer: started","stream_id":"zgrsvkbi"} +{"time":"2026-09-21T14:30:25.556676189Z","level":"INFO","msg":"sender: started"} +{"time":"2026-09-21T14:30:25.588291171Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T14:30:25.588316949Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-zgrsvkbi/logs/debug.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-zgrsvkbi/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..9cca0611f1176af97355fa1b100925f28a51ed9f --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-zgrsvkbi/logs/debug.log @@ -0,0 +1,19 @@ +2026-09-21 14:30:24,658 INFO MainThread:68 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1 +2026-09-21 14:30:24,659 INFO MainThread:68 [wandb_setup.py:_flush():81] Configure stats pid to 68 +2026-09-21 14:30:24,659 INFO MainThread:68 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-09-21 14:30:24,659 INFO MainThread:68 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_143024-zgrsvkbi/logs/debug.log +2026-09-21 14:30:24,659 INFO MainThread:68 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_143024-zgrsvkbi/logs/debug-internal.log +2026-09-21 14:30:24,659 INFO MainThread:68 [wandb_init.py:init():850] calling init triggers +2026-09-21 14:30:24,659 INFO MainThread:68 [wandb_init.py:init():855] wandb.init called with sweep_config: {} +config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s43/run_20260921_142951', 'resume_from_checkpoint': None, 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 43, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s43', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed43'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}} +2026-09-21 14:30:24,659 INFO MainThread:68 [wandb_init.py:init():898] starting backend +2026-09-21 14:30:25,191 INFO MainThread:68 [wandb_init.py:init():913] sending inform_init request +2026-09-21 14:30:25,557 INFO MainThread:68 [wandb_init.py:init():918] backend started and connected +2026-09-21 14:30:25,558 INFO MainThread:68 [wandb_init.py:init():988] updated telemetry +2026-09-21 14:30:25,585 INFO MainThread:68 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout +2026-09-21 14:30:25,590 INFO MainThread:68 [wandb_init.py:init():1056] starting run threads in backend +2026-09-21 14:30:25,804 INFO MainThread:68 [wandb_run.py:_console_start():2554] atexit reg +2026-09-21 14:30:25,805 INFO MainThread:68 [wandb_run.py:_redirect():2403] redirect: wrap_raw +2026-09-21 14:30:25,805 INFO MainThread:68 [wandb_run.py:_redirect():2472] Wrapping output streams. +2026-09-21 14:30:25,805 INFO MainThread:68 [wandb_run.py:_redirect():2495] Redirects installed. +2026-09-21 14:30:25,806 INFO MainThread:68 [wandb_init.py:init():1094] run started, returning control to user process diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-zgrsvkbi/run-zgrsvkbi.wandb b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-zgrsvkbi/run-zgrsvkbi.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e2e6e17f865b87b0d259553c174169cb08975752 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-zgrsvkbi/run-zgrsvkbi.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1eea381c292d4f55126e4059e9539425ef002b8e337b808efadfaf23faaf0ee +size 1376256