diff --git a/.gitattributes b/.gitattributes index e1cd3ce48ef30079f8a8988c5d30b017fa65b0a4..59b97c028746d368d3b10f174a6ebfc0f1a761bb 100644 --- a/.gitattributes +++ b/.gitattributes @@ -189,3 +189,15 @@ epfl_final/SparseAtt/wandb/offline-run-20260921_131249-xc6c08ps/run-xc6c08ps.wan epfl_final/SparseAtt/wandb/offline-run-20260921_131249-3uan03m3/run-3uan03m3.wandb filter=lfs diff=lfs merge=lfs -text epfl_final/SparseAtt/wandb/offline-run-20260921_143024-kbn66izc/run-kbn66izc.wandb filter=lfs diff=lfs merge=lfs -text epfl_final/SparseAtt/wandb/offline-run-20260921_143024-zgrsvkbi/run-zgrsvkbi.wandb filter=lfs diff=lfs merge=lfs -text +epfl_final/SparseAtt/wandb/offline-run-20260921_143024-ghfbqjah/run-ghfbqjah.wandb filter=lfs diff=lfs merge=lfs -text +epfl_final/SparseAtt/wandb/offline-run-20260921_143024-6ijtsdl5/run-6ijtsdl5.wandb filter=lfs diff=lfs merge=lfs -text +epfl_final/SparseAtt/wandb/offline-run-20260921_143024-wk3rt7ae/run-wk3rt7ae.wandb filter=lfs diff=lfs merge=lfs -text +epfl_final/SparseAtt/wandb/offline-run-20260921_143024-nihe6o4e/run-nihe6o4e.wandb filter=lfs diff=lfs merge=lfs -text +epfl_final/SparseAtt/wandb/offline-run-20260921_154845-ghfbqjah/run-ghfbqjah.wandb filter=lfs diff=lfs merge=lfs -text +epfl_final/SparseAtt/wandb/offline-run-20260921_154845-zgrsvkbi/run-zgrsvkbi.wandb filter=lfs diff=lfs merge=lfs -text +epfl_final/SparseAtt/wandb/offline-run-20260921_154845-nihe6o4e/run-nihe6o4e.wandb filter=lfs diff=lfs merge=lfs -text +epfl_final/SparseAtt/wandb/offline-run-20260921_154845-wk3rt7ae/run-wk3rt7ae.wandb filter=lfs diff=lfs merge=lfs -text +epfl_final/SparseAtt/wandb/offline-run-20260921_154846-kbn66izc/run-kbn66izc.wandb filter=lfs diff=lfs merge=lfs -text +epfl_final/SparseAtt/wandb/offline-run-20260921_163219-nihe6o4e/run-nihe6o4e.wandb filter=lfs diff=lfs merge=lfs -text +epfl_final/SparseAtt/wandb/offline-run-20260921_164533-kbn66izc/run-kbn66izc.wandb filter=lfs diff=lfs merge=lfs -text +epfl_final/SparseAtt/wandb/offline-run-20260921_164537-zgrsvkbi/run-zgrsvkbi.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-6ijtsdl5/files/requirements.txt b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-6ijtsdl5/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c3e38c4cce87cb53d52f45f6ac8cbf8688a7d92 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-6ijtsdl5/files/requirements.txt @@ -0,0 +1,142 @@ +pip==26.0.1 +nvidia-cusparselt-cu13==0.8.0 +mpmath==1.3.0 +cuda-toolkit==13.0.2 +zstandard==0.25.0 +xxhash==3.7.0 +urllib3==2.6.3 +typing_extensions==4.15.0 +triton==3.6.0 +tqdm==4.67.3 +torchinfo==1.8.0 +sympy==1.14.0 +smmap==5.0.3 +six==1.17.0 +shellingham==1.5.4 +setuptools==81.0.0 +safetensors==0.7.0 +regex==2026.4.4 +PyYAML==6.0.3 +Pygments==2.20.0 +pyarrow==24.0.0 +psutil==7.2.2 +protobuf==7.34.1 +propcache==0.4.1 +platformdirs==4.9.6 +packaging==26.2 +nvidia-nvtx==13.0.85 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvjitlink==13.0.88 +nvidia-nccl-cu13==2.28.9 +nvidia-curand==10.4.0.35 +nvidia-cufile==1.15.1.6 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-cupti==13.0.85 +nvidia-cublas==13.1.0.3 +numpy==2.4.4 +networkx==3.6.1 +multidict==6.7.1 +mdurl==0.1.2 +MarkupSafe==3.0.3 +idna==3.13 +hf-xet==1.4.3 +h11==0.16.0 +fsspec==2026.2.0 +frozenlist==1.8.0 +filelock==3.29.0 +dill==0.4.1 +cuda-pathfinder==1.5.4 +click==8.3.3 +charset-normalizer==3.4.7 +certifi==2026.4.22 +attrs==26.1.0 +annotated-types==0.7.0 +annotated-doc==0.0.4 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +typing-inspection==0.4.2 +sentry-sdk==2.58.0 +requests==2.33.1 +python-dateutil==2.9.0.post0 +pydantic_core==2.46.3 +nvidia-cusparse==12.6.3.3 +nvidia-cufft==12.0.0.61 +nvidia-cudnn-cu13==9.19.0.56 +multiprocess==0.70.19 +markdown-it-py==4.0.0 +Jinja2==3.1.6 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==13.2.0 +anyio==4.13.0 +aiosignal==1.4.0 +rich==15.0.0 +pydantic==2.13.3 +pandas==3.0.2 +nvidia-cusolver==12.0.4.66 +httpx==0.28.1 +GitPython==3.1.49 +aiohttp==3.13.5 +wandb==0.26.1 +typer==0.25.1 +torch==2.11.0 +huggingface_hub==1.13.0 +tokenizers==0.22.2 +datasets==4.8.5 +accelerate==1.13.0 +transformers==5.7.0 +word2number==1.1 +sqlitedict==2.1.0 +pytz==2026.2 +threadpoolctl==3.6.0 +tcolorpy==0.1.7 +tabulate==0.10.0 +scipy==1.17.1 +portalocker==3.2.0 +pathvalidate==3.3.1 +narwhals==2.22.1 +more-itertools==11.1.0 +lxml==6.1.1 +joblib==1.5.3 +colorama==0.4.6 +chardet==6.0.0.post1 +absl-py==2.4.0 +scikit-learn==1.9.0 +sacrebleu==2.6.0 +nltk==3.9.4 +mbstrdecoder==1.1.5 +typepy==1.3.5 +rouge_score==0.1.2 +DataProperty==1.1.1 +tabledata==1.3.5 +evaluate==0.4.6 +pytablewriter==1.2.1 +lm_eval==0.4.12 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==9.1.0 +py-spy==0.4.2 +pillow==12.2.0 +tifffile==2026.6.1 +imagecodecs==2026.6.26 +zipp==4.1.0 +importlib_metadata==9.0.0 +derivative==0.6.3 +pysindy==2.1.0 +pyparsing==3.3.2 +kiwisolver==1.5.0 +fonttools==4.63.0 +cycler==0.12.1 +contourpy==1.3.3 +matplotlib==3.11.0 +sortedcontainers==2.4.0 +wcwidth==0.8.2 +sigfig==1.3.19 +importlib_resources==7.1.0 +enum-tools==0.13.0 +cramjam==2.11.0 +python-snappy==0.7.3 +compact-json==1.8.2 +numbers-parser==4.18.5 +entmax==1.3 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-6ijtsdl5/logs/debug-internal.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-6ijtsdl5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f8ad71260c7fad909d4d07219839ec3a5e297d85 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-6ijtsdl5/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-09-21T14:30:25.205688293Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-09-21T14:30:25.205700601Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"} +{"time":"2026-09-21T14:30:25.544942881Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T14:30:25.545041428Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T14:30:25.545090001Z","level":"INFO","msg":"stream: created new stream","id":"6ijtsdl5"} +{"time":"2026-09-21T14:30:25.545224552Z","level":"INFO","msg":"handler: started"} +{"time":"2026-09-21T14:30:25.547588391Z","level":"INFO","msg":"stream: started"} +{"time":"2026-09-21T14:30:25.548874862Z","level":"INFO","msg":"sender: started"} +{"time":"2026-09-21T14:30:25.548875933Z","level":"INFO","msg":"writer: started","stream_id":"6ijtsdl5"} +{"time":"2026-09-21T14:30:25.588698359Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T14:30:25.58873973Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-6ijtsdl5/logs/debug.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-6ijtsdl5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..03572e6599c04a0e3ccb6de29439491d556942d6 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-6ijtsdl5/logs/debug.log @@ -0,0 +1,19 @@ +2026-09-21 14:30:24,663 INFO MainThread:72 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1 +2026-09-21 14:30:24,663 INFO MainThread:72 [wandb_setup.py:_flush():81] Configure stats pid to 72 +2026-09-21 14:30:24,663 INFO MainThread:72 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-09-21 14:30:24,663 INFO MainThread:72 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_143024-6ijtsdl5/logs/debug.log +2026-09-21 14:30:24,663 INFO MainThread:72 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_143024-6ijtsdl5/logs/debug-internal.log +2026-09-21 14:30:24,663 INFO MainThread:72 [wandb_init.py:init():850] calling init triggers +2026-09-21 14:30:24,664 INFO MainThread:72 [wandb_init.py:init():855] wandb.init called with sweep_config: {} +config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951', 'resume_from_checkpoint': None, 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 47, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s47', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed47'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}} +2026-09-21 14:30:24,664 INFO MainThread:72 [wandb_init.py:init():898] starting backend +2026-09-21 14:30:25,192 INFO MainThread:72 [wandb_init.py:init():913] sending inform_init request +2026-09-21 14:30:25,548 INFO MainThread:72 [wandb_init.py:init():918] backend started and connected +2026-09-21 14:30:25,552 INFO MainThread:72 [wandb_init.py:init():988] updated telemetry +2026-09-21 14:30:25,585 INFO MainThread:72 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout +2026-09-21 14:30:25,591 INFO MainThread:72 [wandb_init.py:init():1056] starting run threads in backend +2026-09-21 14:30:25,795 INFO MainThread:72 [wandb_run.py:_console_start():2554] atexit reg +2026-09-21 14:30:25,795 INFO MainThread:72 [wandb_run.py:_redirect():2403] redirect: wrap_raw +2026-09-21 14:30:25,795 INFO MainThread:72 [wandb_run.py:_redirect():2472] Wrapping output streams. +2026-09-21 14:30:25,796 INFO MainThread:72 [wandb_run.py:_redirect():2495] Redirects installed. +2026-09-21 14:30:25,797 INFO MainThread:72 [wandb_init.py:init():1094] run started, returning control to user process diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-6ijtsdl5/run-6ijtsdl5.wandb b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-6ijtsdl5/run-6ijtsdl5.wandb new file mode 100644 index 0000000000000000000000000000000000000000..5bfc1ca142cb5c61fe3b86957f1c87f54edef478 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-6ijtsdl5/run-6ijtsdl5.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cc5d982f52825586bc97397d7069a333bd09a3ba50406a9b0266cfef88e8421 +size 1376256 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-ghfbqjah/files/requirements.txt b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-ghfbqjah/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c3e38c4cce87cb53d52f45f6ac8cbf8688a7d92 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-ghfbqjah/files/requirements.txt @@ -0,0 +1,142 @@ +pip==26.0.1 +nvidia-cusparselt-cu13==0.8.0 +mpmath==1.3.0 +cuda-toolkit==13.0.2 +zstandard==0.25.0 +xxhash==3.7.0 +urllib3==2.6.3 +typing_extensions==4.15.0 +triton==3.6.0 +tqdm==4.67.3 +torchinfo==1.8.0 +sympy==1.14.0 +smmap==5.0.3 +six==1.17.0 +shellingham==1.5.4 +setuptools==81.0.0 +safetensors==0.7.0 +regex==2026.4.4 +PyYAML==6.0.3 +Pygments==2.20.0 +pyarrow==24.0.0 +psutil==7.2.2 +protobuf==7.34.1 +propcache==0.4.1 +platformdirs==4.9.6 +packaging==26.2 +nvidia-nvtx==13.0.85 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvjitlink==13.0.88 +nvidia-nccl-cu13==2.28.9 +nvidia-curand==10.4.0.35 +nvidia-cufile==1.15.1.6 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-cupti==13.0.85 +nvidia-cublas==13.1.0.3 +numpy==2.4.4 +networkx==3.6.1 +multidict==6.7.1 +mdurl==0.1.2 +MarkupSafe==3.0.3 +idna==3.13 +hf-xet==1.4.3 +h11==0.16.0 +fsspec==2026.2.0 +frozenlist==1.8.0 +filelock==3.29.0 +dill==0.4.1 +cuda-pathfinder==1.5.4 +click==8.3.3 +charset-normalizer==3.4.7 +certifi==2026.4.22 +attrs==26.1.0 +annotated-types==0.7.0 +annotated-doc==0.0.4 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +typing-inspection==0.4.2 +sentry-sdk==2.58.0 +requests==2.33.1 +python-dateutil==2.9.0.post0 +pydantic_core==2.46.3 +nvidia-cusparse==12.6.3.3 +nvidia-cufft==12.0.0.61 +nvidia-cudnn-cu13==9.19.0.56 +multiprocess==0.70.19 +markdown-it-py==4.0.0 +Jinja2==3.1.6 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==13.2.0 +anyio==4.13.0 +aiosignal==1.4.0 +rich==15.0.0 +pydantic==2.13.3 +pandas==3.0.2 +nvidia-cusolver==12.0.4.66 +httpx==0.28.1 +GitPython==3.1.49 +aiohttp==3.13.5 +wandb==0.26.1 +typer==0.25.1 +torch==2.11.0 +huggingface_hub==1.13.0 +tokenizers==0.22.2 +datasets==4.8.5 +accelerate==1.13.0 +transformers==5.7.0 +word2number==1.1 +sqlitedict==2.1.0 +pytz==2026.2 +threadpoolctl==3.6.0 +tcolorpy==0.1.7 +tabulate==0.10.0 +scipy==1.17.1 +portalocker==3.2.0 +pathvalidate==3.3.1 +narwhals==2.22.1 +more-itertools==11.1.0 +lxml==6.1.1 +joblib==1.5.3 +colorama==0.4.6 +chardet==6.0.0.post1 +absl-py==2.4.0 +scikit-learn==1.9.0 +sacrebleu==2.6.0 +nltk==3.9.4 +mbstrdecoder==1.1.5 +typepy==1.3.5 +rouge_score==0.1.2 +DataProperty==1.1.1 +tabledata==1.3.5 +evaluate==0.4.6 +pytablewriter==1.2.1 +lm_eval==0.4.12 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==9.1.0 +py-spy==0.4.2 +pillow==12.2.0 +tifffile==2026.6.1 +imagecodecs==2026.6.26 +zipp==4.1.0 +importlib_metadata==9.0.0 +derivative==0.6.3 +pysindy==2.1.0 +pyparsing==3.3.2 +kiwisolver==1.5.0 +fonttools==4.63.0 +cycler==0.12.1 +contourpy==1.3.3 +matplotlib==3.11.0 +sortedcontainers==2.4.0 +wcwidth==0.8.2 +sigfig==1.3.19 +importlib_resources==7.1.0 +enum-tools==0.13.0 +cramjam==2.11.0 +python-snappy==0.7.3 +compact-json==1.8.2 +numbers-parser==4.18.5 +entmax==1.3 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-ghfbqjah/logs/debug-internal.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-ghfbqjah/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..15baf7f636373b85e4dfa9223e15743a46ff691f --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-ghfbqjah/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-09-21T14:30:25.207855399Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-09-21T14:30:25.207885784Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"} +{"time":"2026-09-21T14:30:25.565662376Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T14:30:25.565698019Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T14:30:25.565724288Z","level":"INFO","msg":"stream: created new stream","id":"ghfbqjah"} +{"time":"2026-09-21T14:30:25.565760121Z","level":"INFO","msg":"handler: started"} +{"time":"2026-09-21T14:30:25.567608171Z","level":"INFO","msg":"stream: started"} +{"time":"2026-09-21T14:30:25.567633909Z","level":"INFO","msg":"writer: started","stream_id":"ghfbqjah"} +{"time":"2026-09-21T14:30:25.567661911Z","level":"INFO","msg":"sender: started"} +{"time":"2026-09-21T14:30:25.588444309Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T14:30:25.588477479Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-ghfbqjah/run-ghfbqjah.wandb b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-ghfbqjah/run-ghfbqjah.wandb new file mode 100644 index 0000000000000000000000000000000000000000..a200f79ed421533e5c03642ec8feaa422d8674a7 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-ghfbqjah/run-ghfbqjah.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85bd743fed8b7029fefeb99cd0ad6c4ae0e10464be9097780d5aaef5bf31dc7f +size 1376256 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-nihe6o4e/files/requirements.txt b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-nihe6o4e/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c3e38c4cce87cb53d52f45f6ac8cbf8688a7d92 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-nihe6o4e/files/requirements.txt @@ -0,0 +1,142 @@ +pip==26.0.1 +nvidia-cusparselt-cu13==0.8.0 +mpmath==1.3.0 +cuda-toolkit==13.0.2 +zstandard==0.25.0 +xxhash==3.7.0 +urllib3==2.6.3 +typing_extensions==4.15.0 +triton==3.6.0 +tqdm==4.67.3 +torchinfo==1.8.0 +sympy==1.14.0 +smmap==5.0.3 +six==1.17.0 +shellingham==1.5.4 +setuptools==81.0.0 +safetensors==0.7.0 +regex==2026.4.4 +PyYAML==6.0.3 +Pygments==2.20.0 +pyarrow==24.0.0 +psutil==7.2.2 +protobuf==7.34.1 +propcache==0.4.1 +platformdirs==4.9.6 +packaging==26.2 +nvidia-nvtx==13.0.85 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvjitlink==13.0.88 +nvidia-nccl-cu13==2.28.9 +nvidia-curand==10.4.0.35 +nvidia-cufile==1.15.1.6 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-cupti==13.0.85 +nvidia-cublas==13.1.0.3 +numpy==2.4.4 +networkx==3.6.1 +multidict==6.7.1 +mdurl==0.1.2 +MarkupSafe==3.0.3 +idna==3.13 +hf-xet==1.4.3 +h11==0.16.0 +fsspec==2026.2.0 +frozenlist==1.8.0 +filelock==3.29.0 +dill==0.4.1 +cuda-pathfinder==1.5.4 +click==8.3.3 +charset-normalizer==3.4.7 +certifi==2026.4.22 +attrs==26.1.0 +annotated-types==0.7.0 +annotated-doc==0.0.4 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +typing-inspection==0.4.2 +sentry-sdk==2.58.0 +requests==2.33.1 +python-dateutil==2.9.0.post0 +pydantic_core==2.46.3 +nvidia-cusparse==12.6.3.3 +nvidia-cufft==12.0.0.61 +nvidia-cudnn-cu13==9.19.0.56 +multiprocess==0.70.19 +markdown-it-py==4.0.0 +Jinja2==3.1.6 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==13.2.0 +anyio==4.13.0 +aiosignal==1.4.0 +rich==15.0.0 +pydantic==2.13.3 +pandas==3.0.2 +nvidia-cusolver==12.0.4.66 +httpx==0.28.1 +GitPython==3.1.49 +aiohttp==3.13.5 +wandb==0.26.1 +typer==0.25.1 +torch==2.11.0 +huggingface_hub==1.13.0 +tokenizers==0.22.2 +datasets==4.8.5 +accelerate==1.13.0 +transformers==5.7.0 +word2number==1.1 +sqlitedict==2.1.0 +pytz==2026.2 +threadpoolctl==3.6.0 +tcolorpy==0.1.7 +tabulate==0.10.0 +scipy==1.17.1 +portalocker==3.2.0 +pathvalidate==3.3.1 +narwhals==2.22.1 +more-itertools==11.1.0 +lxml==6.1.1 +joblib==1.5.3 +colorama==0.4.6 +chardet==6.0.0.post1 +absl-py==2.4.0 +scikit-learn==1.9.0 +sacrebleu==2.6.0 +nltk==3.9.4 +mbstrdecoder==1.1.5 +typepy==1.3.5 +rouge_score==0.1.2 +DataProperty==1.1.1 +tabledata==1.3.5 +evaluate==0.4.6 +pytablewriter==1.2.1 +lm_eval==0.4.12 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==9.1.0 +py-spy==0.4.2 +pillow==12.2.0 +tifffile==2026.6.1 +imagecodecs==2026.6.26 +zipp==4.1.0 +importlib_metadata==9.0.0 +derivative==0.6.3 +pysindy==2.1.0 +pyparsing==3.3.2 +kiwisolver==1.5.0 +fonttools==4.63.0 +cycler==0.12.1 +contourpy==1.3.3 +matplotlib==3.11.0 +sortedcontainers==2.4.0 +wcwidth==0.8.2 +sigfig==1.3.19 +importlib_resources==7.1.0 +enum-tools==0.13.0 +cramjam==2.11.0 +python-snappy==0.7.3 +compact-json==1.8.2 +numbers-parser==4.18.5 +entmax==1.3 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-nihe6o4e/logs/debug-internal.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-nihe6o4e/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f6788f77a877862b0a409e9335cfdcc8024f9849 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-nihe6o4e/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-09-21T14:30:25.207122284Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-09-21T14:30:25.207148273Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"} +{"time":"2026-09-21T14:30:25.563932403Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T14:30:25.563969739Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T14:30:25.564003799Z","level":"INFO","msg":"stream: created new stream","id":"nihe6o4e"} +{"time":"2026-09-21T14:30:25.564132572Z","level":"INFO","msg":"handler: started"} +{"time":"2026-09-21T14:30:25.566124576Z","level":"INFO","msg":"stream: started"} +{"time":"2026-09-21T14:30:25.566219718Z","level":"INFO","msg":"sender: started"} +{"time":"2026-09-21T14:30:25.566224274Z","level":"INFO","msg":"writer: started","stream_id":"nihe6o4e"} +{"time":"2026-09-21T14:30:25.598552731Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T14:30:25.598580863Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-nihe6o4e/logs/debug.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-nihe6o4e/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..abd3fb3892f9ce2104201df03d3bcbcc6ee94410 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-nihe6o4e/logs/debug.log @@ -0,0 +1,19 @@ +2026-09-21 14:30:24,666 INFO MainThread:67 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1 +2026-09-21 14:30:24,666 INFO MainThread:67 [wandb_setup.py:_flush():81] Configure stats pid to 67 +2026-09-21 14:30:24,666 INFO MainThread:67 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-09-21 14:30:24,666 INFO MainThread:67 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_143024-nihe6o4e/logs/debug.log +2026-09-21 14:30:24,667 INFO MainThread:67 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_143024-nihe6o4e/logs/debug-internal.log +2026-09-21 14:30:24,667 INFO MainThread:67 [wandb_init.py:init():850] calling init triggers +2026-09-21 14:30:24,667 INFO MainThread:67 [wandb_init.py:init():855] wandb.init called with sweep_config: {} +config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s42/run_20260921_142951', 'resume_from_checkpoint': None, 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s42', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed42'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}} +2026-09-21 14:30:24,667 INFO MainThread:67 [wandb_init.py:init():898] starting backend +2026-09-21 14:30:25,192 INFO MainThread:67 [wandb_init.py:init():913] sending inform_init request +2026-09-21 14:30:25,567 INFO MainThread:67 [wandb_init.py:init():918] backend started and connected +2026-09-21 14:30:25,571 INFO MainThread:67 [wandb_init.py:init():988] updated telemetry +2026-09-21 14:30:25,595 INFO MainThread:67 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout +2026-09-21 14:30:25,601 INFO MainThread:67 [wandb_init.py:init():1056] starting run threads in backend +2026-09-21 14:30:25,802 INFO MainThread:67 [wandb_run.py:_console_start():2554] atexit reg +2026-09-21 14:30:25,802 INFO MainThread:67 [wandb_run.py:_redirect():2403] redirect: wrap_raw +2026-09-21 14:30:25,802 INFO MainThread:67 [wandb_run.py:_redirect():2472] Wrapping output streams. +2026-09-21 14:30:25,802 INFO MainThread:67 [wandb_run.py:_redirect():2495] Redirects installed. +2026-09-21 14:30:25,804 INFO MainThread:67 [wandb_init.py:init():1094] run started, returning control to user process diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-nihe6o4e/run-nihe6o4e.wandb b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-nihe6o4e/run-nihe6o4e.wandb new file mode 100644 index 0000000000000000000000000000000000000000..584d9e9319d66995379d59266caff7db4533a9a1 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-nihe6o4e/run-nihe6o4e.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96a013ff2d1155a019f60391503db1a304dae2d90dfd1ded3ca4ff9e1ecb270e +size 1376256 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-wk3rt7ae/files/requirements.txt b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-wk3rt7ae/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c3e38c4cce87cb53d52f45f6ac8cbf8688a7d92 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-wk3rt7ae/files/requirements.txt @@ -0,0 +1,142 @@ +pip==26.0.1 +nvidia-cusparselt-cu13==0.8.0 +mpmath==1.3.0 +cuda-toolkit==13.0.2 +zstandard==0.25.0 +xxhash==3.7.0 +urllib3==2.6.3 +typing_extensions==4.15.0 +triton==3.6.0 +tqdm==4.67.3 +torchinfo==1.8.0 +sympy==1.14.0 +smmap==5.0.3 +six==1.17.0 +shellingham==1.5.4 +setuptools==81.0.0 +safetensors==0.7.0 +regex==2026.4.4 +PyYAML==6.0.3 +Pygments==2.20.0 +pyarrow==24.0.0 +psutil==7.2.2 +protobuf==7.34.1 +propcache==0.4.1 +platformdirs==4.9.6 +packaging==26.2 +nvidia-nvtx==13.0.85 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvjitlink==13.0.88 +nvidia-nccl-cu13==2.28.9 +nvidia-curand==10.4.0.35 +nvidia-cufile==1.15.1.6 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-cupti==13.0.85 +nvidia-cublas==13.1.0.3 +numpy==2.4.4 +networkx==3.6.1 +multidict==6.7.1 +mdurl==0.1.2 +MarkupSafe==3.0.3 +idna==3.13 +hf-xet==1.4.3 +h11==0.16.0 +fsspec==2026.2.0 +frozenlist==1.8.0 +filelock==3.29.0 +dill==0.4.1 +cuda-pathfinder==1.5.4 +click==8.3.3 +charset-normalizer==3.4.7 +certifi==2026.4.22 +attrs==26.1.0 +annotated-types==0.7.0 +annotated-doc==0.0.4 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +typing-inspection==0.4.2 +sentry-sdk==2.58.0 +requests==2.33.1 +python-dateutil==2.9.0.post0 +pydantic_core==2.46.3 +nvidia-cusparse==12.6.3.3 +nvidia-cufft==12.0.0.61 +nvidia-cudnn-cu13==9.19.0.56 +multiprocess==0.70.19 +markdown-it-py==4.0.0 +Jinja2==3.1.6 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==13.2.0 +anyio==4.13.0 +aiosignal==1.4.0 +rich==15.0.0 +pydantic==2.13.3 +pandas==3.0.2 +nvidia-cusolver==12.0.4.66 +httpx==0.28.1 +GitPython==3.1.49 +aiohttp==3.13.5 +wandb==0.26.1 +typer==0.25.1 +torch==2.11.0 +huggingface_hub==1.13.0 +tokenizers==0.22.2 +datasets==4.8.5 +accelerate==1.13.0 +transformers==5.7.0 +word2number==1.1 +sqlitedict==2.1.0 +pytz==2026.2 +threadpoolctl==3.6.0 +tcolorpy==0.1.7 +tabulate==0.10.0 +scipy==1.17.1 +portalocker==3.2.0 +pathvalidate==3.3.1 +narwhals==2.22.1 +more-itertools==11.1.0 +lxml==6.1.1 +joblib==1.5.3 +colorama==0.4.6 +chardet==6.0.0.post1 +absl-py==2.4.0 +scikit-learn==1.9.0 +sacrebleu==2.6.0 +nltk==3.9.4 +mbstrdecoder==1.1.5 +typepy==1.3.5 +rouge_score==0.1.2 +DataProperty==1.1.1 +tabledata==1.3.5 +evaluate==0.4.6 +pytablewriter==1.2.1 +lm_eval==0.4.12 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==9.1.0 +py-spy==0.4.2 +pillow==12.2.0 +tifffile==2026.6.1 +imagecodecs==2026.6.26 +zipp==4.1.0 +importlib_metadata==9.0.0 +derivative==0.6.3 +pysindy==2.1.0 +pyparsing==3.3.2 +kiwisolver==1.5.0 +fonttools==4.63.0 +cycler==0.12.1 +contourpy==1.3.3 +matplotlib==3.11.0 +sortedcontainers==2.4.0 +wcwidth==0.8.2 +sigfig==1.3.19 +importlib_resources==7.1.0 +enum-tools==0.13.0 +cramjam==2.11.0 +python-snappy==0.7.3 +compact-json==1.8.2 +numbers-parser==4.18.5 +entmax==1.3 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-wk3rt7ae/logs/debug-internal.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-wk3rt7ae/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..9e0f1147f719b3ab80f1907eb4ab43fd9b447957 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-wk3rt7ae/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-09-21T14:30:25.207082344Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-09-21T14:30:25.207109204Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"} +{"time":"2026-09-21T14:30:25.554324889Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T14:30:25.554389605Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T14:30:25.554438769Z","level":"INFO","msg":"stream: created new stream","id":"wk3rt7ae"} +{"time":"2026-09-21T14:30:25.554579739Z","level":"INFO","msg":"handler: started"} +{"time":"2026-09-21T14:30:25.556817811Z","level":"INFO","msg":"stream: started"} +{"time":"2026-09-21T14:30:25.556913373Z","level":"INFO","msg":"writer: started","stream_id":"wk3rt7ae"} +{"time":"2026-09-21T14:30:25.556961245Z","level":"INFO","msg":"sender: started"} +{"time":"2026-09-21T14:30:25.58822386Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T14:30:25.588239193Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-wk3rt7ae/logs/debug.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-wk3rt7ae/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..62f65320940cb9a307a29f5b670cf21f0d756d85 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-wk3rt7ae/logs/debug.log @@ -0,0 +1,19 @@ +2026-09-21 14:30:24,664 INFO MainThread:71 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1 +2026-09-21 14:30:24,664 INFO MainThread:71 [wandb_setup.py:_flush():81] Configure stats pid to 71 +2026-09-21 14:30:24,664 INFO MainThread:71 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-09-21 14:30:24,664 INFO MainThread:71 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_143024-wk3rt7ae/logs/debug.log +2026-09-21 14:30:24,664 INFO MainThread:71 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_143024-wk3rt7ae/logs/debug-internal.log +2026-09-21 14:30:24,665 INFO MainThread:71 [wandb_init.py:init():850] calling init triggers +2026-09-21 14:30:24,665 INFO MainThread:71 [wandb_init.py:init():855] wandb.init called with sweep_config: {} +config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s46/run_20260921_142951', 'resume_from_checkpoint': None, 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 46, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s46', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed46'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}} +2026-09-21 14:30:24,665 INFO MainThread:71 [wandb_init.py:init():898] starting backend +2026-09-21 14:30:25,192 INFO MainThread:71 [wandb_init.py:init():913] sending inform_init request +2026-09-21 14:30:25,557 INFO MainThread:71 [wandb_init.py:init():918] backend started and connected +2026-09-21 14:30:25,561 INFO MainThread:71 [wandb_init.py:init():988] updated telemetry +2026-09-21 14:30:25,585 INFO MainThread:71 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout +2026-09-21 14:30:25,590 INFO MainThread:71 [wandb_init.py:init():1056] starting run threads in backend +2026-09-21 14:30:25,799 INFO MainThread:71 [wandb_run.py:_console_start():2554] atexit reg +2026-09-21 14:30:25,799 INFO MainThread:71 [wandb_run.py:_redirect():2403] redirect: wrap_raw +2026-09-21 14:30:25,799 INFO MainThread:71 [wandb_run.py:_redirect():2472] Wrapping output streams. +2026-09-21 14:30:25,799 INFO MainThread:71 [wandb_run.py:_redirect():2495] Redirects installed. +2026-09-21 14:30:25,801 INFO MainThread:71 [wandb_init.py:init():1094] run started, returning control to user process diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-wk3rt7ae/run-wk3rt7ae.wandb b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-wk3rt7ae/run-wk3rt7ae.wandb new file mode 100644 index 0000000000000000000000000000000000000000..f950e6161b7d21369e29c14f48aaa0589aac4e2f --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_143024-wk3rt7ae/run-wk3rt7ae.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eaa8febec31ee92d4896cc02cd635856a829e30a54c8794baf3d7b91821413ac +size 1376256 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-ghfbqjah/files/requirements.txt b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-ghfbqjah/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c3e38c4cce87cb53d52f45f6ac8cbf8688a7d92 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-ghfbqjah/files/requirements.txt @@ -0,0 +1,142 @@ +pip==26.0.1 +nvidia-cusparselt-cu13==0.8.0 +mpmath==1.3.0 +cuda-toolkit==13.0.2 +zstandard==0.25.0 +xxhash==3.7.0 +urllib3==2.6.3 +typing_extensions==4.15.0 +triton==3.6.0 +tqdm==4.67.3 +torchinfo==1.8.0 +sympy==1.14.0 +smmap==5.0.3 +six==1.17.0 +shellingham==1.5.4 +setuptools==81.0.0 +safetensors==0.7.0 +regex==2026.4.4 +PyYAML==6.0.3 +Pygments==2.20.0 +pyarrow==24.0.0 +psutil==7.2.2 +protobuf==7.34.1 +propcache==0.4.1 +platformdirs==4.9.6 +packaging==26.2 +nvidia-nvtx==13.0.85 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvjitlink==13.0.88 +nvidia-nccl-cu13==2.28.9 +nvidia-curand==10.4.0.35 +nvidia-cufile==1.15.1.6 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-cupti==13.0.85 +nvidia-cublas==13.1.0.3 +numpy==2.4.4 +networkx==3.6.1 +multidict==6.7.1 +mdurl==0.1.2 +MarkupSafe==3.0.3 +idna==3.13 +hf-xet==1.4.3 +h11==0.16.0 +fsspec==2026.2.0 +frozenlist==1.8.0 +filelock==3.29.0 +dill==0.4.1 +cuda-pathfinder==1.5.4 +click==8.3.3 +charset-normalizer==3.4.7 +certifi==2026.4.22 +attrs==26.1.0 +annotated-types==0.7.0 +annotated-doc==0.0.4 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +typing-inspection==0.4.2 +sentry-sdk==2.58.0 +requests==2.33.1 +python-dateutil==2.9.0.post0 +pydantic_core==2.46.3 +nvidia-cusparse==12.6.3.3 +nvidia-cufft==12.0.0.61 +nvidia-cudnn-cu13==9.19.0.56 +multiprocess==0.70.19 +markdown-it-py==4.0.0 +Jinja2==3.1.6 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==13.2.0 +anyio==4.13.0 +aiosignal==1.4.0 +rich==15.0.0 +pydantic==2.13.3 +pandas==3.0.2 +nvidia-cusolver==12.0.4.66 +httpx==0.28.1 +GitPython==3.1.49 +aiohttp==3.13.5 +wandb==0.26.1 +typer==0.25.1 +torch==2.11.0 +huggingface_hub==1.13.0 +tokenizers==0.22.2 +datasets==4.8.5 +accelerate==1.13.0 +transformers==5.7.0 +word2number==1.1 +sqlitedict==2.1.0 +pytz==2026.2 +threadpoolctl==3.6.0 +tcolorpy==0.1.7 +tabulate==0.10.0 +scipy==1.17.1 +portalocker==3.2.0 +pathvalidate==3.3.1 +narwhals==2.22.1 +more-itertools==11.1.0 +lxml==6.1.1 +joblib==1.5.3 +colorama==0.4.6 +chardet==6.0.0.post1 +absl-py==2.4.0 +scikit-learn==1.9.0 +sacrebleu==2.6.0 +nltk==3.9.4 +mbstrdecoder==1.1.5 +typepy==1.3.5 +rouge_score==0.1.2 +DataProperty==1.1.1 +tabledata==1.3.5 +evaluate==0.4.6 +pytablewriter==1.2.1 +lm_eval==0.4.12 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==9.1.0 +py-spy==0.4.2 +pillow==12.2.0 +tifffile==2026.6.1 +imagecodecs==2026.6.26 +zipp==4.1.0 +importlib_metadata==9.0.0 +derivative==0.6.3 +pysindy==2.1.0 +pyparsing==3.3.2 +kiwisolver==1.5.0 +fonttools==4.63.0 +cycler==0.12.1 +contourpy==1.3.3 +matplotlib==3.11.0 +sortedcontainers==2.4.0 +wcwidth==0.8.2 +sigfig==1.3.19 +importlib_resources==7.1.0 +enum-tools==0.13.0 +cramjam==2.11.0 +python-snappy==0.7.3 +compact-json==1.8.2 +numbers-parser==4.18.5 +entmax==1.3 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-ghfbqjah/logs/debug-internal.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-ghfbqjah/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c14c0553b65c9ad4615cd4f463eb3eb04caca8e0 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-ghfbqjah/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-09-21T15:48:45.97315144Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-09-21T15:48:45.973175436Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"} +{"time":"2026-09-21T15:48:46.317654694Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T15:48:46.317697138Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T15:48:46.317754213Z","level":"INFO","msg":"stream: created new stream","id":"ghfbqjah"} +{"time":"2026-09-21T15:48:46.317790477Z","level":"INFO","msg":"handler: started"} +{"time":"2026-09-21T15:48:46.319985701Z","level":"INFO","msg":"stream: started"} +{"time":"2026-09-21T15:48:46.32000528Z","level":"INFO","msg":"writer: started","stream_id":"ghfbqjah"} +{"time":"2026-09-21T15:48:46.320027153Z","level":"INFO","msg":"sender: started"} +{"time":"2026-09-21T15:48:46.353007627Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T15:48:46.353039114Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-ghfbqjah/logs/debug.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-ghfbqjah/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e234eb4ac399205413c3253c8fa4980ef105f250 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-ghfbqjah/logs/debug.log @@ -0,0 +1,19 @@ +2026-09-21 15:48:45,417 INFO MainThread:189 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1 +2026-09-21 15:48:45,417 INFO MainThread:189 [wandb_setup.py:_flush():81] Configure stats pid to 189 +2026-09-21 15:48:45,418 INFO MainThread:189 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-09-21 15:48:45,418 INFO MainThread:189 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_154845-ghfbqjah/logs/debug.log +2026-09-21 15:48:45,418 INFO MainThread:189 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_154845-ghfbqjah/logs/debug-internal.log +2026-09-21 15:48:45,418 INFO MainThread:189 [wandb_init.py:init():850] calling init triggers +2026-09-21 15:48:45,418 INFO MainThread:189 [wandb_init.py:init():855] wandb.init called with sweep_config: {} +config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s45/run_20260921_142951', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s45/run_20260921_142951/checkpoint-2000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 45, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s45', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed45'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}} +2026-09-21 15:48:45,418 INFO MainThread:189 [wandb_init.py:init():898] starting backend +2026-09-21 15:48:45,958 INFO MainThread:189 [wandb_init.py:init():913] sending inform_init request +2026-09-21 15:48:46,320 INFO MainThread:189 [wandb_init.py:init():918] backend started and connected +2026-09-21 15:48:46,324 INFO MainThread:189 [wandb_init.py:init():988] updated telemetry +2026-09-21 15:48:46,350 INFO MainThread:189 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout +2026-09-21 15:48:46,355 INFO MainThread:189 [wandb_init.py:init():1056] starting run threads in backend +2026-09-21 15:48:46,601 INFO MainThread:189 [wandb_run.py:_console_start():2554] atexit reg +2026-09-21 15:48:46,601 INFO MainThread:189 [wandb_run.py:_redirect():2403] redirect: wrap_raw +2026-09-21 15:48:46,601 INFO MainThread:189 [wandb_run.py:_redirect():2472] Wrapping output streams. +2026-09-21 15:48:46,601 INFO MainThread:189 [wandb_run.py:_redirect():2495] Redirects installed. +2026-09-21 15:48:46,603 INFO MainThread:189 [wandb_init.py:init():1094] run started, returning control to user process diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-ghfbqjah/run-ghfbqjah.wandb b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-ghfbqjah/run-ghfbqjah.wandb new file mode 100644 index 0000000000000000000000000000000000000000..cf7e46421716b3151e9fce56933bfaf1b1de2ad9 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-ghfbqjah/run-ghfbqjah.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86e0ecf363da86a6737dc4e5bad2001c7af3f31850e9439be4b2d951331841e3 +size 557056 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-nihe6o4e/files/requirements.txt b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-nihe6o4e/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c3e38c4cce87cb53d52f45f6ac8cbf8688a7d92 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-nihe6o4e/files/requirements.txt @@ -0,0 +1,142 @@ +pip==26.0.1 +nvidia-cusparselt-cu13==0.8.0 +mpmath==1.3.0 +cuda-toolkit==13.0.2 +zstandard==0.25.0 +xxhash==3.7.0 +urllib3==2.6.3 +typing_extensions==4.15.0 +triton==3.6.0 +tqdm==4.67.3 +torchinfo==1.8.0 +sympy==1.14.0 +smmap==5.0.3 +six==1.17.0 +shellingham==1.5.4 +setuptools==81.0.0 +safetensors==0.7.0 +regex==2026.4.4 +PyYAML==6.0.3 +Pygments==2.20.0 +pyarrow==24.0.0 +psutil==7.2.2 +protobuf==7.34.1 +propcache==0.4.1 +platformdirs==4.9.6 +packaging==26.2 +nvidia-nvtx==13.0.85 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvjitlink==13.0.88 +nvidia-nccl-cu13==2.28.9 +nvidia-curand==10.4.0.35 +nvidia-cufile==1.15.1.6 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-cupti==13.0.85 +nvidia-cublas==13.1.0.3 +numpy==2.4.4 +networkx==3.6.1 +multidict==6.7.1 +mdurl==0.1.2 +MarkupSafe==3.0.3 +idna==3.13 +hf-xet==1.4.3 +h11==0.16.0 +fsspec==2026.2.0 +frozenlist==1.8.0 +filelock==3.29.0 +dill==0.4.1 +cuda-pathfinder==1.5.4 +click==8.3.3 +charset-normalizer==3.4.7 +certifi==2026.4.22 +attrs==26.1.0 +annotated-types==0.7.0 +annotated-doc==0.0.4 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +typing-inspection==0.4.2 +sentry-sdk==2.58.0 +requests==2.33.1 +python-dateutil==2.9.0.post0 +pydantic_core==2.46.3 +nvidia-cusparse==12.6.3.3 +nvidia-cufft==12.0.0.61 +nvidia-cudnn-cu13==9.19.0.56 +multiprocess==0.70.19 +markdown-it-py==4.0.0 +Jinja2==3.1.6 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==13.2.0 +anyio==4.13.0 +aiosignal==1.4.0 +rich==15.0.0 +pydantic==2.13.3 +pandas==3.0.2 +nvidia-cusolver==12.0.4.66 +httpx==0.28.1 +GitPython==3.1.49 +aiohttp==3.13.5 +wandb==0.26.1 +typer==0.25.1 +torch==2.11.0 +huggingface_hub==1.13.0 +tokenizers==0.22.2 +datasets==4.8.5 +accelerate==1.13.0 +transformers==5.7.0 +word2number==1.1 +sqlitedict==2.1.0 +pytz==2026.2 +threadpoolctl==3.6.0 +tcolorpy==0.1.7 +tabulate==0.10.0 +scipy==1.17.1 +portalocker==3.2.0 +pathvalidate==3.3.1 +narwhals==2.22.1 +more-itertools==11.1.0 +lxml==6.1.1 +joblib==1.5.3 +colorama==0.4.6 +chardet==6.0.0.post1 +absl-py==2.4.0 +scikit-learn==1.9.0 +sacrebleu==2.6.0 +nltk==3.9.4 +mbstrdecoder==1.1.5 +typepy==1.3.5 +rouge_score==0.1.2 +DataProperty==1.1.1 +tabledata==1.3.5 +evaluate==0.4.6 +pytablewriter==1.2.1 +lm_eval==0.4.12 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==9.1.0 +py-spy==0.4.2 +pillow==12.2.0 +tifffile==2026.6.1 +imagecodecs==2026.6.26 +zipp==4.1.0 +importlib_metadata==9.0.0 +derivative==0.6.3 +pysindy==2.1.0 +pyparsing==3.3.2 +kiwisolver==1.5.0 +fonttools==4.63.0 +cycler==0.12.1 +contourpy==1.3.3 +matplotlib==3.11.0 +sortedcontainers==2.4.0 +wcwidth==0.8.2 +sigfig==1.3.19 +importlib_resources==7.1.0 +enum-tools==0.13.0 +cramjam==2.11.0 +python-snappy==0.7.3 +compact-json==1.8.2 +numbers-parser==4.18.5 +entmax==1.3 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-nihe6o4e/logs/debug-internal.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-nihe6o4e/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b5359d5536cd5b6b0b4d81ff085274f811f697af --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-nihe6o4e/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-09-21T15:48:45.973155927Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-09-21T15:48:45.97317795Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"} +{"time":"2026-09-21T15:48:46.224512758Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T15:48:46.224589834Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T15:48:46.224627911Z","level":"INFO","msg":"stream: created new stream","id":"nihe6o4e"} +{"time":"2026-09-21T15:48:46.224692076Z","level":"INFO","msg":"handler: started"} +{"time":"2026-09-21T15:48:46.226575624Z","level":"INFO","msg":"stream: started"} +{"time":"2026-09-21T15:48:46.226611888Z","level":"INFO","msg":"writer: started","stream_id":"nihe6o4e"} +{"time":"2026-09-21T15:48:46.22665338Z","level":"INFO","msg":"sender: started"} +{"time":"2026-09-21T15:48:46.290800955Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T15:48:46.290818902Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-nihe6o4e/logs/debug.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-nihe6o4e/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..02c18e7b7d49685b4da59a4219ec7053265b365f --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-nihe6o4e/logs/debug.log @@ -0,0 +1,19 @@ +2026-09-21 15:48:45,723 INFO MainThread:174 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1 +2026-09-21 15:48:45,723 INFO MainThread:174 [wandb_setup.py:_flush():81] Configure stats pid to 174 +2026-09-21 15:48:45,723 INFO MainThread:174 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-09-21 15:48:45,723 INFO MainThread:174 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_154845-nihe6o4e/logs/debug.log +2026-09-21 15:48:45,723 INFO MainThread:174 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_154845-nihe6o4e/logs/debug-internal.log +2026-09-21 15:48:45,723 INFO MainThread:174 [wandb_init.py:init():850] calling init triggers +2026-09-21 15:48:45,723 INFO MainThread:174 [wandb_init.py:init():855] wandb.init called with sweep_config: {} +config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s42/run_20260921_142951', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s42/run_20260921_142951/checkpoint-2000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s42', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed42'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}} +2026-09-21 15:48:45,723 INFO MainThread:174 [wandb_init.py:init():898] starting backend +2026-09-21 15:48:45,955 INFO MainThread:174 [wandb_init.py:init():913] sending inform_init request +2026-09-21 15:48:46,227 INFO MainThread:174 [wandb_init.py:init():918] backend started and connected +2026-09-21 15:48:46,230 INFO MainThread:174 [wandb_init.py:init():988] updated telemetry +2026-09-21 15:48:46,285 INFO MainThread:174 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout +2026-09-21 15:48:46,291 INFO MainThread:174 [wandb_init.py:init():1056] starting run threads in backend +2026-09-21 15:48:46,522 INFO MainThread:174 [wandb_run.py:_console_start():2554] atexit reg +2026-09-21 15:48:46,522 INFO MainThread:174 [wandb_run.py:_redirect():2403] redirect: wrap_raw +2026-09-21 15:48:46,522 INFO MainThread:174 [wandb_run.py:_redirect():2472] Wrapping output streams. +2026-09-21 15:48:46,523 INFO MainThread:174 [wandb_run.py:_redirect():2495] Redirects installed. +2026-09-21 15:48:46,529 INFO MainThread:174 [wandb_init.py:init():1094] run started, returning control to user process diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-nihe6o4e/run-nihe6o4e.wandb b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-nihe6o4e/run-nihe6o4e.wandb new file mode 100644 index 0000000000000000000000000000000000000000..9dc8e9ce75efd5144587cfe6c3115ef49820ac6d --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-nihe6o4e/run-nihe6o4e.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca6c43495cd064bdb2d9b5967a5b6e3949b7d97bcb3d1dc8781de46e4089ea91 +size 557056 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-wk3rt7ae/files/requirements.txt b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-wk3rt7ae/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c3e38c4cce87cb53d52f45f6ac8cbf8688a7d92 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-wk3rt7ae/files/requirements.txt @@ -0,0 +1,142 @@ +pip==26.0.1 +nvidia-cusparselt-cu13==0.8.0 +mpmath==1.3.0 +cuda-toolkit==13.0.2 +zstandard==0.25.0 +xxhash==3.7.0 +urllib3==2.6.3 +typing_extensions==4.15.0 +triton==3.6.0 +tqdm==4.67.3 +torchinfo==1.8.0 +sympy==1.14.0 +smmap==5.0.3 +six==1.17.0 +shellingham==1.5.4 +setuptools==81.0.0 +safetensors==0.7.0 +regex==2026.4.4 +PyYAML==6.0.3 +Pygments==2.20.0 +pyarrow==24.0.0 +psutil==7.2.2 +protobuf==7.34.1 +propcache==0.4.1 +platformdirs==4.9.6 +packaging==26.2 +nvidia-nvtx==13.0.85 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvjitlink==13.0.88 +nvidia-nccl-cu13==2.28.9 +nvidia-curand==10.4.0.35 +nvidia-cufile==1.15.1.6 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-cupti==13.0.85 +nvidia-cublas==13.1.0.3 +numpy==2.4.4 +networkx==3.6.1 +multidict==6.7.1 +mdurl==0.1.2 +MarkupSafe==3.0.3 +idna==3.13 +hf-xet==1.4.3 +h11==0.16.0 +fsspec==2026.2.0 +frozenlist==1.8.0 +filelock==3.29.0 +dill==0.4.1 +cuda-pathfinder==1.5.4 +click==8.3.3 +charset-normalizer==3.4.7 +certifi==2026.4.22 +attrs==26.1.0 +annotated-types==0.7.0 +annotated-doc==0.0.4 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +typing-inspection==0.4.2 +sentry-sdk==2.58.0 +requests==2.33.1 +python-dateutil==2.9.0.post0 +pydantic_core==2.46.3 +nvidia-cusparse==12.6.3.3 +nvidia-cufft==12.0.0.61 +nvidia-cudnn-cu13==9.19.0.56 +multiprocess==0.70.19 +markdown-it-py==4.0.0 +Jinja2==3.1.6 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==13.2.0 +anyio==4.13.0 +aiosignal==1.4.0 +rich==15.0.0 +pydantic==2.13.3 +pandas==3.0.2 +nvidia-cusolver==12.0.4.66 +httpx==0.28.1 +GitPython==3.1.49 +aiohttp==3.13.5 +wandb==0.26.1 +typer==0.25.1 +torch==2.11.0 +huggingface_hub==1.13.0 +tokenizers==0.22.2 +datasets==4.8.5 +accelerate==1.13.0 +transformers==5.7.0 +word2number==1.1 +sqlitedict==2.1.0 +pytz==2026.2 +threadpoolctl==3.6.0 +tcolorpy==0.1.7 +tabulate==0.10.0 +scipy==1.17.1 +portalocker==3.2.0 +pathvalidate==3.3.1 +narwhals==2.22.1 +more-itertools==11.1.0 +lxml==6.1.1 +joblib==1.5.3 +colorama==0.4.6 +chardet==6.0.0.post1 +absl-py==2.4.0 +scikit-learn==1.9.0 +sacrebleu==2.6.0 +nltk==3.9.4 +mbstrdecoder==1.1.5 +typepy==1.3.5 +rouge_score==0.1.2 +DataProperty==1.1.1 +tabledata==1.3.5 +evaluate==0.4.6 +pytablewriter==1.2.1 +lm_eval==0.4.12 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==9.1.0 +py-spy==0.4.2 +pillow==12.2.0 +tifffile==2026.6.1 +imagecodecs==2026.6.26 +zipp==4.1.0 +importlib_metadata==9.0.0 +derivative==0.6.3 +pysindy==2.1.0 +pyparsing==3.3.2 +kiwisolver==1.5.0 +fonttools==4.63.0 +cycler==0.12.1 +contourpy==1.3.3 +matplotlib==3.11.0 +sortedcontainers==2.4.0 +wcwidth==0.8.2 +sigfig==1.3.19 +importlib_resources==7.1.0 +enum-tools==0.13.0 +cramjam==2.11.0 +python-snappy==0.7.3 +compact-json==1.8.2 +numbers-parser==4.18.5 +entmax==1.3 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-wk3rt7ae/logs/debug-internal.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-wk3rt7ae/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..2cdb8404b4c9acbb052083dfbe55ff97c3886d38 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-wk3rt7ae/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-09-21T15:48:45.973165451Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-09-21T15:48:45.973200123Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"} +{"time":"2026-09-21T15:48:46.220742849Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T15:48:46.220815608Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T15:48:46.220868327Z","level":"INFO","msg":"stream: created new stream","id":"wk3rt7ae"} +{"time":"2026-09-21T15:48:46.220971972Z","level":"INFO","msg":"handler: started"} +{"time":"2026-09-21T15:48:46.222811433Z","level":"INFO","msg":"stream: started"} +{"time":"2026-09-21T15:48:46.223799912Z","level":"INFO","msg":"sender: started"} +{"time":"2026-09-21T15:48:46.223838379Z","level":"INFO","msg":"writer: started","stream_id":"wk3rt7ae"} +{"time":"2026-09-21T15:48:46.291016127Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T15:48:46.291045461Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-wk3rt7ae/logs/debug.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-wk3rt7ae/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..14cd2b0d7a835de7e5b3ec5cb1670894e22547ee --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-wk3rt7ae/logs/debug.log @@ -0,0 +1,19 @@ +2026-09-21 15:48:45,731 INFO MainThread:194 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1 +2026-09-21 15:48:45,731 INFO MainThread:194 [wandb_setup.py:_flush():81] Configure stats pid to 194 +2026-09-21 15:48:45,731 INFO MainThread:194 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-09-21 15:48:45,731 INFO MainThread:194 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_154845-wk3rt7ae/logs/debug.log +2026-09-21 15:48:45,731 INFO MainThread:194 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_154845-wk3rt7ae/logs/debug-internal.log +2026-09-21 15:48:45,731 INFO MainThread:194 [wandb_init.py:init():850] calling init triggers +2026-09-21 15:48:45,732 INFO MainThread:194 [wandb_init.py:init():855] wandb.init called with sweep_config: {} +config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s46/run_20260921_142951', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s46/run_20260921_142951/checkpoint-2000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 46, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s46', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed46'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}} +2026-09-21 15:48:45,732 INFO MainThread:194 [wandb_init.py:init():898] starting backend +2026-09-21 15:48:45,961 INFO MainThread:194 [wandb_init.py:init():913] sending inform_init request +2026-09-21 15:48:46,223 INFO MainThread:194 [wandb_init.py:init():918] backend started and connected +2026-09-21 15:48:46,228 INFO MainThread:194 [wandb_init.py:init():988] updated telemetry +2026-09-21 15:48:46,285 INFO MainThread:194 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout +2026-09-21 15:48:46,293 INFO MainThread:194 [wandb_init.py:init():1056] starting run threads in backend +2026-09-21 15:48:46,546 INFO MainThread:194 [wandb_run.py:_console_start():2554] atexit reg +2026-09-21 15:48:46,546 INFO MainThread:194 [wandb_run.py:_redirect():2403] redirect: wrap_raw +2026-09-21 15:48:46,546 INFO MainThread:194 [wandb_run.py:_redirect():2472] Wrapping output streams. +2026-09-21 15:48:46,546 INFO MainThread:194 [wandb_run.py:_redirect():2495] Redirects installed. +2026-09-21 15:48:46,548 INFO MainThread:194 [wandb_init.py:init():1094] run started, returning control to user process diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-wk3rt7ae/run-wk3rt7ae.wandb b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-wk3rt7ae/run-wk3rt7ae.wandb new file mode 100644 index 0000000000000000000000000000000000000000..fb52bc90874d02bea8c437d4426b7ab3833104b5 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-wk3rt7ae/run-wk3rt7ae.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:977d8eb231aae9cd8adfc08e4b2a0c9d75d7151ebde1202a3b518e3f51c1fc2b +size 557056 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-zgrsvkbi/files/requirements.txt b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-zgrsvkbi/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c3e38c4cce87cb53d52f45f6ac8cbf8688a7d92 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-zgrsvkbi/files/requirements.txt @@ -0,0 +1,142 @@ +pip==26.0.1 +nvidia-cusparselt-cu13==0.8.0 +mpmath==1.3.0 +cuda-toolkit==13.0.2 +zstandard==0.25.0 +xxhash==3.7.0 +urllib3==2.6.3 +typing_extensions==4.15.0 +triton==3.6.0 +tqdm==4.67.3 +torchinfo==1.8.0 +sympy==1.14.0 +smmap==5.0.3 +six==1.17.0 +shellingham==1.5.4 +setuptools==81.0.0 +safetensors==0.7.0 +regex==2026.4.4 +PyYAML==6.0.3 +Pygments==2.20.0 +pyarrow==24.0.0 +psutil==7.2.2 +protobuf==7.34.1 +propcache==0.4.1 +platformdirs==4.9.6 +packaging==26.2 +nvidia-nvtx==13.0.85 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvjitlink==13.0.88 +nvidia-nccl-cu13==2.28.9 +nvidia-curand==10.4.0.35 +nvidia-cufile==1.15.1.6 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-cupti==13.0.85 +nvidia-cublas==13.1.0.3 +numpy==2.4.4 +networkx==3.6.1 +multidict==6.7.1 +mdurl==0.1.2 +MarkupSafe==3.0.3 +idna==3.13 +hf-xet==1.4.3 +h11==0.16.0 +fsspec==2026.2.0 +frozenlist==1.8.0 +filelock==3.29.0 +dill==0.4.1 +cuda-pathfinder==1.5.4 +click==8.3.3 +charset-normalizer==3.4.7 +certifi==2026.4.22 +attrs==26.1.0 +annotated-types==0.7.0 +annotated-doc==0.0.4 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +typing-inspection==0.4.2 +sentry-sdk==2.58.0 +requests==2.33.1 +python-dateutil==2.9.0.post0 +pydantic_core==2.46.3 +nvidia-cusparse==12.6.3.3 +nvidia-cufft==12.0.0.61 +nvidia-cudnn-cu13==9.19.0.56 +multiprocess==0.70.19 +markdown-it-py==4.0.0 +Jinja2==3.1.6 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==13.2.0 +anyio==4.13.0 +aiosignal==1.4.0 +rich==15.0.0 +pydantic==2.13.3 +pandas==3.0.2 +nvidia-cusolver==12.0.4.66 +httpx==0.28.1 +GitPython==3.1.49 +aiohttp==3.13.5 +wandb==0.26.1 +typer==0.25.1 +torch==2.11.0 +huggingface_hub==1.13.0 +tokenizers==0.22.2 +datasets==4.8.5 +accelerate==1.13.0 +transformers==5.7.0 +word2number==1.1 +sqlitedict==2.1.0 +pytz==2026.2 +threadpoolctl==3.6.0 +tcolorpy==0.1.7 +tabulate==0.10.0 +scipy==1.17.1 +portalocker==3.2.0 +pathvalidate==3.3.1 +narwhals==2.22.1 +more-itertools==11.1.0 +lxml==6.1.1 +joblib==1.5.3 +colorama==0.4.6 +chardet==6.0.0.post1 +absl-py==2.4.0 +scikit-learn==1.9.0 +sacrebleu==2.6.0 +nltk==3.9.4 +mbstrdecoder==1.1.5 +typepy==1.3.5 +rouge_score==0.1.2 +DataProperty==1.1.1 +tabledata==1.3.5 +evaluate==0.4.6 +pytablewriter==1.2.1 +lm_eval==0.4.12 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==9.1.0 +py-spy==0.4.2 +pillow==12.2.0 +tifffile==2026.6.1 +imagecodecs==2026.6.26 +zipp==4.1.0 +importlib_metadata==9.0.0 +derivative==0.6.3 +pysindy==2.1.0 +pyparsing==3.3.2 +kiwisolver==1.5.0 +fonttools==4.63.0 +cycler==0.12.1 +contourpy==1.3.3 +matplotlib==3.11.0 +sortedcontainers==2.4.0 +wcwidth==0.8.2 +sigfig==1.3.19 +importlib_resources==7.1.0 +enum-tools==0.13.0 +cramjam==2.11.0 +python-snappy==0.7.3 +compact-json==1.8.2 +numbers-parser==4.18.5 +entmax==1.3 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-zgrsvkbi/logs/debug-internal.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-zgrsvkbi/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..94c763c7918229203606b118c46307ee786d1fe3 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-zgrsvkbi/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-09-21T15:48:45.976437516Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-09-21T15:48:45.976465287Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"} +{"time":"2026-09-21T15:48:46.229963081Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T15:48:46.230010913Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T15:48:46.230057112Z","level":"INFO","msg":"stream: created new stream","id":"zgrsvkbi"} +{"time":"2026-09-21T15:48:46.230134417Z","level":"INFO","msg":"handler: started"} +{"time":"2026-09-21T15:48:46.2381986Z","level":"INFO","msg":"stream: started"} +{"time":"2026-09-21T15:48:46.238231209Z","level":"INFO","msg":"writer: started","stream_id":"zgrsvkbi"} +{"time":"2026-09-21T15:48:46.238269336Z","level":"INFO","msg":"sender: started"} +{"time":"2026-09-21T15:48:46.291039061Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T15:48:46.291076247Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-zgrsvkbi/logs/debug.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-zgrsvkbi/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..07a5aae2ec0aad91117cdbc9e2f3637e5b915238 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-zgrsvkbi/logs/debug.log @@ -0,0 +1,19 @@ +2026-09-21 15:48:45,718 INFO MainThread:179 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1 +2026-09-21 15:48:45,718 INFO MainThread:179 [wandb_setup.py:_flush():81] Configure stats pid to 179 +2026-09-21 15:48:45,718 INFO MainThread:179 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-09-21 15:48:45,719 INFO MainThread:179 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_154845-zgrsvkbi/logs/debug.log +2026-09-21 15:48:45,719 INFO MainThread:179 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_154845-zgrsvkbi/logs/debug-internal.log +2026-09-21 15:48:45,719 INFO MainThread:179 [wandb_init.py:init():850] calling init triggers +2026-09-21 15:48:45,719 INFO MainThread:179 [wandb_init.py:init():855] wandb.init called with sweep_config: {} +config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s43/run_20260921_142951', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s43/run_20260921_142951/checkpoint-2000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 43, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s43', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed43'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}} +2026-09-21 15:48:45,719 INFO MainThread:179 [wandb_init.py:init():898] starting backend +2026-09-21 15:48:45,965 INFO MainThread:179 [wandb_init.py:init():913] sending inform_init request +2026-09-21 15:48:46,239 INFO MainThread:179 [wandb_init.py:init():918] backend started and connected +2026-09-21 15:48:46,243 INFO MainThread:179 [wandb_init.py:init():988] updated telemetry +2026-09-21 15:48:46,285 INFO MainThread:179 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout +2026-09-21 15:48:46,292 INFO MainThread:179 [wandb_init.py:init():1056] starting run threads in backend +2026-09-21 15:48:46,527 INFO MainThread:179 [wandb_run.py:_console_start():2554] atexit reg +2026-09-21 15:48:46,527 INFO MainThread:179 [wandb_run.py:_redirect():2403] redirect: wrap_raw +2026-09-21 15:48:46,527 INFO MainThread:179 [wandb_run.py:_redirect():2472] Wrapping output streams. +2026-09-21 15:48:46,527 INFO MainThread:179 [wandb_run.py:_redirect():2495] Redirects installed. +2026-09-21 15:48:46,529 INFO MainThread:179 [wandb_init.py:init():1094] run started, returning control to user process diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-zgrsvkbi/run-zgrsvkbi.wandb b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-zgrsvkbi/run-zgrsvkbi.wandb new file mode 100644 index 0000000000000000000000000000000000000000..994b1abbb5409418b8ce408303ea137a5c320dbf --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_154845-zgrsvkbi/run-zgrsvkbi.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7211ab74941b2c2fbc03f25eda120604bf257819452998b7a19429fc7814243 +size 557056 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_154846-kbn66izc/files/requirements.txt b/epfl_final/SparseAtt/wandb/offline-run-20260921_154846-kbn66izc/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c3e38c4cce87cb53d52f45f6ac8cbf8688a7d92 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_154846-kbn66izc/files/requirements.txt @@ -0,0 +1,142 @@ +pip==26.0.1 +nvidia-cusparselt-cu13==0.8.0 +mpmath==1.3.0 +cuda-toolkit==13.0.2 +zstandard==0.25.0 +xxhash==3.7.0 +urllib3==2.6.3 +typing_extensions==4.15.0 +triton==3.6.0 +tqdm==4.67.3 +torchinfo==1.8.0 +sympy==1.14.0 +smmap==5.0.3 +six==1.17.0 +shellingham==1.5.4 +setuptools==81.0.0 +safetensors==0.7.0 +regex==2026.4.4 +PyYAML==6.0.3 +Pygments==2.20.0 +pyarrow==24.0.0 +psutil==7.2.2 +protobuf==7.34.1 +propcache==0.4.1 +platformdirs==4.9.6 +packaging==26.2 +nvidia-nvtx==13.0.85 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvjitlink==13.0.88 +nvidia-nccl-cu13==2.28.9 +nvidia-curand==10.4.0.35 +nvidia-cufile==1.15.1.6 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-cupti==13.0.85 +nvidia-cublas==13.1.0.3 +numpy==2.4.4 +networkx==3.6.1 +multidict==6.7.1 +mdurl==0.1.2 +MarkupSafe==3.0.3 +idna==3.13 +hf-xet==1.4.3 +h11==0.16.0 +fsspec==2026.2.0 +frozenlist==1.8.0 +filelock==3.29.0 +dill==0.4.1 +cuda-pathfinder==1.5.4 +click==8.3.3 +charset-normalizer==3.4.7 +certifi==2026.4.22 +attrs==26.1.0 +annotated-types==0.7.0 +annotated-doc==0.0.4 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +typing-inspection==0.4.2 +sentry-sdk==2.58.0 +requests==2.33.1 +python-dateutil==2.9.0.post0 +pydantic_core==2.46.3 +nvidia-cusparse==12.6.3.3 +nvidia-cufft==12.0.0.61 +nvidia-cudnn-cu13==9.19.0.56 +multiprocess==0.70.19 +markdown-it-py==4.0.0 +Jinja2==3.1.6 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==13.2.0 +anyio==4.13.0 +aiosignal==1.4.0 +rich==15.0.0 +pydantic==2.13.3 +pandas==3.0.2 +nvidia-cusolver==12.0.4.66 +httpx==0.28.1 +GitPython==3.1.49 +aiohttp==3.13.5 +wandb==0.26.1 +typer==0.25.1 +torch==2.11.0 +huggingface_hub==1.13.0 +tokenizers==0.22.2 +datasets==4.8.5 +accelerate==1.13.0 +transformers==5.7.0 +word2number==1.1 +sqlitedict==2.1.0 +pytz==2026.2 +threadpoolctl==3.6.0 +tcolorpy==0.1.7 +tabulate==0.10.0 +scipy==1.17.1 +portalocker==3.2.0 +pathvalidate==3.3.1 +narwhals==2.22.1 +more-itertools==11.1.0 +lxml==6.1.1 +joblib==1.5.3 +colorama==0.4.6 +chardet==6.0.0.post1 +absl-py==2.4.0 +scikit-learn==1.9.0 +sacrebleu==2.6.0 +nltk==3.9.4 +mbstrdecoder==1.1.5 +typepy==1.3.5 +rouge_score==0.1.2 +DataProperty==1.1.1 +tabledata==1.3.5 +evaluate==0.4.6 +pytablewriter==1.2.1 +lm_eval==0.4.12 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==9.1.0 +py-spy==0.4.2 +pillow==12.2.0 +tifffile==2026.6.1 +imagecodecs==2026.6.26 +zipp==4.1.0 +importlib_metadata==9.0.0 +derivative==0.6.3 +pysindy==2.1.0 +pyparsing==3.3.2 +kiwisolver==1.5.0 +fonttools==4.63.0 +cycler==0.12.1 +contourpy==1.3.3 +matplotlib==3.11.0 +sortedcontainers==2.4.0 +wcwidth==0.8.2 +sigfig==1.3.19 +importlib_resources==7.1.0 +enum-tools==0.13.0 +cramjam==2.11.0 +python-snappy==0.7.3 +compact-json==1.8.2 +numbers-parser==4.18.5 +entmax==1.3 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_154846-kbn66izc/logs/debug-internal.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_154846-kbn66izc/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..042c157b17d82715aa9c7fbda05f8646c2103d53 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_154846-kbn66izc/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-09-21T15:48:46.438986648Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-09-21T15:48:46.43901461Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"} +{"time":"2026-09-21T15:48:46.574888667Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T15:48:46.574951741Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T15:48:46.574980875Z","level":"INFO","msg":"stream: created new stream","id":"kbn66izc"} +{"time":"2026-09-21T15:48:46.5750312Z","level":"INFO","msg":"handler: started"} +{"time":"2026-09-21T15:48:46.577122839Z","level":"INFO","msg":"stream: started"} +{"time":"2026-09-21T15:48:46.57714365Z","level":"INFO","msg":"writer: started","stream_id":"kbn66izc"} +{"time":"2026-09-21T15:48:46.577159904Z","level":"INFO","msg":"sender: started"} +{"time":"2026-09-21T15:48:46.596521556Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T15:48:46.596543008Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_154846-kbn66izc/logs/debug.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_154846-kbn66izc/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d483f8a4ab3521f84ce6f8aecf9fe9d30a8fa75d --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_154846-kbn66izc/logs/debug.log @@ -0,0 +1,19 @@ +2026-09-21 15:48:46,197 INFO MainThread:184 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1 +2026-09-21 15:48:46,197 INFO MainThread:184 [wandb_setup.py:_flush():81] Configure stats pid to 184 +2026-09-21 15:48:46,197 INFO MainThread:184 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-09-21 15:48:46,197 INFO MainThread:184 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_154846-kbn66izc/logs/debug.log +2026-09-21 15:48:46,197 INFO MainThread:184 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_154846-kbn66izc/logs/debug-internal.log +2026-09-21 15:48:46,197 INFO MainThread:184 [wandb_init.py:init():850] calling init triggers +2026-09-21 15:48:46,197 INFO MainThread:184 [wandb_init.py:init():855] wandb.init called with sweep_config: {} +config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s44/run_20260921_142951', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s44/run_20260921_142951/checkpoint-2000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 44, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s44', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed44'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}} +2026-09-21 15:48:46,197 INFO MainThread:184 [wandb_init.py:init():898] starting backend +2026-09-21 15:48:46,427 INFO MainThread:184 [wandb_init.py:init():913] sending inform_init request +2026-09-21 15:48:46,577 INFO MainThread:184 [wandb_init.py:init():918] backend started and connected +2026-09-21 15:48:46,580 INFO MainThread:184 [wandb_init.py:init():988] updated telemetry +2026-09-21 15:48:46,594 INFO MainThread:184 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout +2026-09-21 15:48:46,597 INFO MainThread:184 [wandb_init.py:init():1056] starting run threads in backend +2026-09-21 15:48:46,811 INFO MainThread:184 [wandb_run.py:_console_start():2554] atexit reg +2026-09-21 15:48:46,811 INFO MainThread:184 [wandb_run.py:_redirect():2403] redirect: wrap_raw +2026-09-21 15:48:46,811 INFO MainThread:184 [wandb_run.py:_redirect():2472] Wrapping output streams. +2026-09-21 15:48:46,811 INFO MainThread:184 [wandb_run.py:_redirect():2495] Redirects installed. +2026-09-21 15:48:46,813 INFO MainThread:184 [wandb_init.py:init():1094] run started, returning control to user process diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_154846-kbn66izc/run-kbn66izc.wandb b/epfl_final/SparseAtt/wandb/offline-run-20260921_154846-kbn66izc/run-kbn66izc.wandb new file mode 100644 index 0000000000000000000000000000000000000000..22c9905c2e5cff79e24428d47326b425802a76de --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_154846-kbn66izc/run-kbn66izc.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aec2d054f22f5949d7d9d256bc1cfbad70df2d5f04db8d4745b8d208897ca7d2 +size 557056 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_163219-nihe6o4e/files/requirements.txt b/epfl_final/SparseAtt/wandb/offline-run-20260921_163219-nihe6o4e/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c3e38c4cce87cb53d52f45f6ac8cbf8688a7d92 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_163219-nihe6o4e/files/requirements.txt @@ -0,0 +1,142 @@ +pip==26.0.1 +nvidia-cusparselt-cu13==0.8.0 +mpmath==1.3.0 +cuda-toolkit==13.0.2 +zstandard==0.25.0 +xxhash==3.7.0 +urllib3==2.6.3 +typing_extensions==4.15.0 +triton==3.6.0 +tqdm==4.67.3 +torchinfo==1.8.0 +sympy==1.14.0 +smmap==5.0.3 +six==1.17.0 +shellingham==1.5.4 +setuptools==81.0.0 +safetensors==0.7.0 +regex==2026.4.4 +PyYAML==6.0.3 +Pygments==2.20.0 +pyarrow==24.0.0 +psutil==7.2.2 +protobuf==7.34.1 +propcache==0.4.1 +platformdirs==4.9.6 +packaging==26.2 +nvidia-nvtx==13.0.85 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvjitlink==13.0.88 +nvidia-nccl-cu13==2.28.9 +nvidia-curand==10.4.0.35 +nvidia-cufile==1.15.1.6 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-cupti==13.0.85 +nvidia-cublas==13.1.0.3 +numpy==2.4.4 +networkx==3.6.1 +multidict==6.7.1 +mdurl==0.1.2 +MarkupSafe==3.0.3 +idna==3.13 +hf-xet==1.4.3 +h11==0.16.0 +fsspec==2026.2.0 +frozenlist==1.8.0 +filelock==3.29.0 +dill==0.4.1 +cuda-pathfinder==1.5.4 +click==8.3.3 +charset-normalizer==3.4.7 +certifi==2026.4.22 +attrs==26.1.0 +annotated-types==0.7.0 +annotated-doc==0.0.4 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +typing-inspection==0.4.2 +sentry-sdk==2.58.0 +requests==2.33.1 +python-dateutil==2.9.0.post0 +pydantic_core==2.46.3 +nvidia-cusparse==12.6.3.3 +nvidia-cufft==12.0.0.61 +nvidia-cudnn-cu13==9.19.0.56 +multiprocess==0.70.19 +markdown-it-py==4.0.0 +Jinja2==3.1.6 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==13.2.0 +anyio==4.13.0 +aiosignal==1.4.0 +rich==15.0.0 +pydantic==2.13.3 +pandas==3.0.2 +nvidia-cusolver==12.0.4.66 +httpx==0.28.1 +GitPython==3.1.49 +aiohttp==3.13.5 +wandb==0.26.1 +typer==0.25.1 +torch==2.11.0 +huggingface_hub==1.13.0 +tokenizers==0.22.2 +datasets==4.8.5 +accelerate==1.13.0 +transformers==5.7.0 +word2number==1.1 +sqlitedict==2.1.0 +pytz==2026.2 +threadpoolctl==3.6.0 +tcolorpy==0.1.7 +tabulate==0.10.0 +scipy==1.17.1 +portalocker==3.2.0 +pathvalidate==3.3.1 +narwhals==2.22.1 +more-itertools==11.1.0 +lxml==6.1.1 +joblib==1.5.3 +colorama==0.4.6 +chardet==6.0.0.post1 +absl-py==2.4.0 +scikit-learn==1.9.0 +sacrebleu==2.6.0 +nltk==3.9.4 +mbstrdecoder==1.1.5 +typepy==1.3.5 +rouge_score==0.1.2 +DataProperty==1.1.1 +tabledata==1.3.5 +evaluate==0.4.6 +pytablewriter==1.2.1 +lm_eval==0.4.12 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==9.1.0 +py-spy==0.4.2 +pillow==12.2.0 +tifffile==2026.6.1 +imagecodecs==2026.6.26 +zipp==4.1.0 +importlib_metadata==9.0.0 +derivative==0.6.3 +pysindy==2.1.0 +pyparsing==3.3.2 +kiwisolver==1.5.0 +fonttools==4.63.0 +cycler==0.12.1 +contourpy==1.3.3 +matplotlib==3.11.0 +sortedcontainers==2.4.0 +wcwidth==0.8.2 +sigfig==1.3.19 +importlib_resources==7.1.0 +enum-tools==0.13.0 +cramjam==2.11.0 +python-snappy==0.7.3 +compact-json==1.8.2 +numbers-parser==4.18.5 +entmax==1.3 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_163219-nihe6o4e/logs/debug-internal.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_163219-nihe6o4e/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..aa28545decdeda5df3d4482c1bfaf3db8debd721 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_163219-nihe6o4e/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-09-21T16:32:19.893591207Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-09-21T16:32:19.893619479Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"} +{"time":"2026-09-21T16:32:20.046817338Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T16:32:20.046876216Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T16:32:20.046930778Z","level":"INFO","msg":"stream: created new stream","id":"nihe6o4e"} +{"time":"2026-09-21T16:32:20.047043416Z","level":"INFO","msg":"handler: started"} +{"time":"2026-09-21T16:32:20.049160733Z","level":"INFO","msg":"stream: started"} +{"time":"2026-09-21T16:32:20.04919915Z","level":"INFO","msg":"writer: started","stream_id":"nihe6o4e"} +{"time":"2026-09-21T16:32:20.049243957Z","level":"INFO","msg":"sender: started"} +{"time":"2026-09-21T16:32:20.089660068Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T16:32:20.089689632Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_163219-nihe6o4e/logs/debug.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_163219-nihe6o4e/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..74f0e5a7ceafd1eed309f502a0074389422d33f0 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_163219-nihe6o4e/logs/debug.log @@ -0,0 +1,19 @@ +2026-09-21 16:32:19,612 INFO MainThread:268 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1 +2026-09-21 16:32:19,612 INFO MainThread:268 [wandb_setup.py:_flush():81] Configure stats pid to 268 +2026-09-21 16:32:19,612 INFO MainThread:268 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-09-21 16:32:19,612 INFO MainThread:268 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_163219-nihe6o4e/logs/debug.log +2026-09-21 16:32:19,613 INFO MainThread:268 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_163219-nihe6o4e/logs/debug-internal.log +2026-09-21 16:32:19,613 INFO MainThread:268 [wandb_init.py:init():850] calling init triggers +2026-09-21 16:32:19,613 INFO MainThread:268 [wandb_init.py:init():855] wandb.init called with sweep_config: {} +config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s42/run_20260921_142951', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s42/run_20260921_142951/checkpoint-2000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 5, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': 10000, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s42', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed42'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}} +2026-09-21 16:32:19,613 INFO MainThread:268 [wandb_init.py:init():898] starting backend +2026-09-21 16:32:19,880 INFO MainThread:268 [wandb_init.py:init():913] sending inform_init request +2026-09-21 16:32:20,050 INFO MainThread:268 [wandb_init.py:init():918] backend started and connected +2026-09-21 16:32:20,055 INFO MainThread:268 [wandb_init.py:init():988] updated telemetry +2026-09-21 16:32:20,086 INFO MainThread:268 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout +2026-09-21 16:32:20,100 INFO MainThread:268 [wandb_init.py:init():1056] starting run threads in backend +2026-09-21 16:32:20,398 INFO MainThread:268 [wandb_run.py:_console_start():2554] atexit reg +2026-09-21 16:32:20,398 INFO MainThread:268 [wandb_run.py:_redirect():2403] redirect: wrap_raw +2026-09-21 16:32:20,398 INFO MainThread:268 [wandb_run.py:_redirect():2472] Wrapping output streams. +2026-09-21 16:32:20,399 INFO MainThread:268 [wandb_run.py:_redirect():2495] Redirects installed. +2026-09-21 16:32:20,402 INFO MainThread:268 [wandb_init.py:init():1094] run started, returning control to user process diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_163219-nihe6o4e/run-nihe6o4e.wandb b/epfl_final/SparseAtt/wandb/offline-run-20260921_163219-nihe6o4e/run-nihe6o4e.wandb new file mode 100644 index 0000000000000000000000000000000000000000..1b67ad15c5dc6101362200957768f97f598ddb4a --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_163219-nihe6o4e/run-nihe6o4e.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d55a048cb5bc4a9201d9644da58d274285914becd036f1bff4e4470fe88abf43 +size 2392064 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_164533-kbn66izc/files/requirements.txt b/epfl_final/SparseAtt/wandb/offline-run-20260921_164533-kbn66izc/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c3e38c4cce87cb53d52f45f6ac8cbf8688a7d92 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_164533-kbn66izc/files/requirements.txt @@ -0,0 +1,142 @@ +pip==26.0.1 +nvidia-cusparselt-cu13==0.8.0 +mpmath==1.3.0 +cuda-toolkit==13.0.2 +zstandard==0.25.0 +xxhash==3.7.0 +urllib3==2.6.3 +typing_extensions==4.15.0 +triton==3.6.0 +tqdm==4.67.3 +torchinfo==1.8.0 +sympy==1.14.0 +smmap==5.0.3 +six==1.17.0 +shellingham==1.5.4 +setuptools==81.0.0 +safetensors==0.7.0 +regex==2026.4.4 +PyYAML==6.0.3 +Pygments==2.20.0 +pyarrow==24.0.0 +psutil==7.2.2 +protobuf==7.34.1 +propcache==0.4.1 +platformdirs==4.9.6 +packaging==26.2 +nvidia-nvtx==13.0.85 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvjitlink==13.0.88 +nvidia-nccl-cu13==2.28.9 +nvidia-curand==10.4.0.35 +nvidia-cufile==1.15.1.6 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-cupti==13.0.85 +nvidia-cublas==13.1.0.3 +numpy==2.4.4 +networkx==3.6.1 +multidict==6.7.1 +mdurl==0.1.2 +MarkupSafe==3.0.3 +idna==3.13 +hf-xet==1.4.3 +h11==0.16.0 +fsspec==2026.2.0 +frozenlist==1.8.0 +filelock==3.29.0 +dill==0.4.1 +cuda-pathfinder==1.5.4 +click==8.3.3 +charset-normalizer==3.4.7 +certifi==2026.4.22 +attrs==26.1.0 +annotated-types==0.7.0 +annotated-doc==0.0.4 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +typing-inspection==0.4.2 +sentry-sdk==2.58.0 +requests==2.33.1 +python-dateutil==2.9.0.post0 +pydantic_core==2.46.3 +nvidia-cusparse==12.6.3.3 +nvidia-cufft==12.0.0.61 +nvidia-cudnn-cu13==9.19.0.56 +multiprocess==0.70.19 +markdown-it-py==4.0.0 +Jinja2==3.1.6 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==13.2.0 +anyio==4.13.0 +aiosignal==1.4.0 +rich==15.0.0 +pydantic==2.13.3 +pandas==3.0.2 +nvidia-cusolver==12.0.4.66 +httpx==0.28.1 +GitPython==3.1.49 +aiohttp==3.13.5 +wandb==0.26.1 +typer==0.25.1 +torch==2.11.0 +huggingface_hub==1.13.0 +tokenizers==0.22.2 +datasets==4.8.5 +accelerate==1.13.0 +transformers==5.7.0 +word2number==1.1 +sqlitedict==2.1.0 +pytz==2026.2 +threadpoolctl==3.6.0 +tcolorpy==0.1.7 +tabulate==0.10.0 +scipy==1.17.1 +portalocker==3.2.0 +pathvalidate==3.3.1 +narwhals==2.22.1 +more-itertools==11.1.0 +lxml==6.1.1 +joblib==1.5.3 +colorama==0.4.6 +chardet==6.0.0.post1 +absl-py==2.4.0 +scikit-learn==1.9.0 +sacrebleu==2.6.0 +nltk==3.9.4 +mbstrdecoder==1.1.5 +typepy==1.3.5 +rouge_score==0.1.2 +DataProperty==1.1.1 +tabledata==1.3.5 +evaluate==0.4.6 +pytablewriter==1.2.1 +lm_eval==0.4.12 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==9.1.0 +py-spy==0.4.2 +pillow==12.2.0 +tifffile==2026.6.1 +imagecodecs==2026.6.26 +zipp==4.1.0 +importlib_metadata==9.0.0 +derivative==0.6.3 +pysindy==2.1.0 +pyparsing==3.3.2 +kiwisolver==1.5.0 +fonttools==4.63.0 +cycler==0.12.1 +contourpy==1.3.3 +matplotlib==3.11.0 +sortedcontainers==2.4.0 +wcwidth==0.8.2 +sigfig==1.3.19 +importlib_resources==7.1.0 +enum-tools==0.13.0 +cramjam==2.11.0 +python-snappy==0.7.3 +compact-json==1.8.2 +numbers-parser==4.18.5 +entmax==1.3 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_164533-kbn66izc/logs/debug-internal.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_164533-kbn66izc/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..71ccd28d3d28a8590ac5838fd5cad3a8cdbd2c4c --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_164533-kbn66izc/logs/debug-internal.log @@ -0,0 +1,16 @@ +{"time":"2026-09-21T16:45:34.951715029Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-09-21T16:45:34.951729165Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"} +{"time":"2026-09-21T16:45:35.478503057Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T16:45:35.478524787Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T16:45:35.478555224Z","level":"INFO","msg":"stream: created new stream","id":"kbn66izc"} +{"time":"2026-09-21T16:45:35.478593225Z","level":"INFO","msg":"handler: started"} +{"time":"2026-09-21T16:45:35.501714375Z","level":"INFO","msg":"stream: started"} +{"time":"2026-09-21T16:45:35.505389607Z","level":"INFO","msg":"sender: started"} +{"time":"2026-09-21T16:45:35.505394967Z","level":"INFO","msg":"writer: started","stream_id":"kbn66izc"} +{"time":"2026-09-21T16:45:35.549255731Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T16:45:35.549268585Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-09-22T03:50:08.983849971Z","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-09-22T03:50:08.990497594Z","level":"INFO","msg":"stream: finishing up"} +{"time":"2026-09-22T03:50:08.99051679Z","level":"INFO","msg":"handler: closed"} +{"time":"2026-09-22T03:50:08.991565127Z","level":"INFO","msg":"sender: closed"} +{"time":"2026-09-22T03:50:08.99157224Z","level":"INFO","msg":"stream: all finished"} diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_164533-kbn66izc/logs/debug.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_164533-kbn66izc/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4136a6253423466673e638de2142568b06464996 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_164533-kbn66izc/logs/debug.log @@ -0,0 +1,23 @@ +2026-09-21 16:45:33,941 INFO MainThread:238 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1 +2026-09-21 16:45:33,942 INFO MainThread:238 [wandb_setup.py:_flush():81] Configure stats pid to 238 +2026-09-21 16:45:33,942 INFO MainThread:238 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-09-21 16:45:33,942 INFO MainThread:238 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_164533-kbn66izc/logs/debug.log +2026-09-21 16:45:33,942 INFO MainThread:238 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_164533-kbn66izc/logs/debug-internal.log +2026-09-21 16:45:33,942 INFO MainThread:238 [wandb_init.py:init():850] calling init triggers +2026-09-21 16:45:33,942 INFO MainThread:238 [wandb_init.py:init():855] wandb.init called with sweep_config: {} +config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s44/run_20260921_142951', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s44/run_20260921_142951/checkpoint-2000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 4, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 44, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': 8000, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s44', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed44'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}} +2026-09-21 16:45:33,942 INFO MainThread:238 [wandb_init.py:init():898] starting backend +2026-09-21 16:45:34,935 INFO MainThread:238 [wandb_init.py:init():913] sending inform_init request +2026-09-21 16:45:35,502 INFO MainThread:238 [wandb_init.py:init():918] backend started and connected +2026-09-21 16:45:35,506 INFO MainThread:238 [wandb_init.py:init():988] updated telemetry +2026-09-21 16:45:35,546 INFO MainThread:238 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout +2026-09-21 16:45:35,550 INFO MainThread:238 [wandb_init.py:init():1056] starting run threads in backend +2026-09-21 16:45:35,784 INFO MainThread:238 [wandb_run.py:_console_start():2554] atexit reg +2026-09-21 16:45:35,784 INFO MainThread:238 [wandb_run.py:_redirect():2403] redirect: wrap_raw +2026-09-21 16:45:35,784 INFO MainThread:238 [wandb_run.py:_redirect():2472] Wrapping output streams. +2026-09-21 16:45:35,784 INFO MainThread:238 [wandb_run.py:_redirect():2495] Redirects installed. +2026-09-21 16:45:35,799 INFO MainThread:238 [wandb_init.py:init():1094] run started, returning control to user process +2026-09-22 03:50:08,977 INFO MainThread:238 [wandb_run.py:_finish():2321] finishing run stochastic_sparsity/kbn66izc +2026-09-22 03:50:08,978 INFO MainThread:238 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0 +2026-09-22 03:50:08,980 INFO MainThread:238 [wandb_run.py:_restore():2502] restore +2026-09-22 03:50:08,980 INFO MainThread:238 [wandb_run.py:_restore():2508] restore done diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_164533-kbn66izc/run-kbn66izc.wandb b/epfl_final/SparseAtt/wandb/offline-run-20260921_164533-kbn66izc/run-kbn66izc.wandb new file mode 100644 index 0000000000000000000000000000000000000000..4b851b046c4846f47b2abb0f416b35419fa521ff --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_164533-kbn66izc/run-kbn66izc.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d917d1f5ecf537741b1509bda27a6cf13f48cb81298670cc7a75e03721986608 +size 7641216 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_164537-zgrsvkbi/files/requirements.txt b/epfl_final/SparseAtt/wandb/offline-run-20260921_164537-zgrsvkbi/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c3e38c4cce87cb53d52f45f6ac8cbf8688a7d92 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_164537-zgrsvkbi/files/requirements.txt @@ -0,0 +1,142 @@ +pip==26.0.1 +nvidia-cusparselt-cu13==0.8.0 +mpmath==1.3.0 +cuda-toolkit==13.0.2 +zstandard==0.25.0 +xxhash==3.7.0 +urllib3==2.6.3 +typing_extensions==4.15.0 +triton==3.6.0 +tqdm==4.67.3 +torchinfo==1.8.0 +sympy==1.14.0 +smmap==5.0.3 +six==1.17.0 +shellingham==1.5.4 +setuptools==81.0.0 +safetensors==0.7.0 +regex==2026.4.4 +PyYAML==6.0.3 +Pygments==2.20.0 +pyarrow==24.0.0 +psutil==7.2.2 +protobuf==7.34.1 +propcache==0.4.1 +platformdirs==4.9.6 +packaging==26.2 +nvidia-nvtx==13.0.85 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvjitlink==13.0.88 +nvidia-nccl-cu13==2.28.9 +nvidia-curand==10.4.0.35 +nvidia-cufile==1.15.1.6 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-cupti==13.0.85 +nvidia-cublas==13.1.0.3 +numpy==2.4.4 +networkx==3.6.1 +multidict==6.7.1 +mdurl==0.1.2 +MarkupSafe==3.0.3 +idna==3.13 +hf-xet==1.4.3 +h11==0.16.0 +fsspec==2026.2.0 +frozenlist==1.8.0 +filelock==3.29.0 +dill==0.4.1 +cuda-pathfinder==1.5.4 +click==8.3.3 +charset-normalizer==3.4.7 +certifi==2026.4.22 +attrs==26.1.0 +annotated-types==0.7.0 +annotated-doc==0.0.4 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +typing-inspection==0.4.2 +sentry-sdk==2.58.0 +requests==2.33.1 +python-dateutil==2.9.0.post0 +pydantic_core==2.46.3 +nvidia-cusparse==12.6.3.3 +nvidia-cufft==12.0.0.61 +nvidia-cudnn-cu13==9.19.0.56 +multiprocess==0.70.19 +markdown-it-py==4.0.0 +Jinja2==3.1.6 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==13.2.0 +anyio==4.13.0 +aiosignal==1.4.0 +rich==15.0.0 +pydantic==2.13.3 +pandas==3.0.2 +nvidia-cusolver==12.0.4.66 +httpx==0.28.1 +GitPython==3.1.49 +aiohttp==3.13.5 +wandb==0.26.1 +typer==0.25.1 +torch==2.11.0 +huggingface_hub==1.13.0 +tokenizers==0.22.2 +datasets==4.8.5 +accelerate==1.13.0 +transformers==5.7.0 +word2number==1.1 +sqlitedict==2.1.0 +pytz==2026.2 +threadpoolctl==3.6.0 +tcolorpy==0.1.7 +tabulate==0.10.0 +scipy==1.17.1 +portalocker==3.2.0 +pathvalidate==3.3.1 +narwhals==2.22.1 +more-itertools==11.1.0 +lxml==6.1.1 +joblib==1.5.3 +colorama==0.4.6 +chardet==6.0.0.post1 +absl-py==2.4.0 +scikit-learn==1.9.0 +sacrebleu==2.6.0 +nltk==3.9.4 +mbstrdecoder==1.1.5 +typepy==1.3.5 +rouge_score==0.1.2 +DataProperty==1.1.1 +tabledata==1.3.5 +evaluate==0.4.6 +pytablewriter==1.2.1 +lm_eval==0.4.12 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==9.1.0 +py-spy==0.4.2 +pillow==12.2.0 +tifffile==2026.6.1 +imagecodecs==2026.6.26 +zipp==4.1.0 +importlib_metadata==9.0.0 +derivative==0.6.3 +pysindy==2.1.0 +pyparsing==3.3.2 +kiwisolver==1.5.0 +fonttools==4.63.0 +cycler==0.12.1 +contourpy==1.3.3 +matplotlib==3.11.0 +sortedcontainers==2.4.0 +wcwidth==0.8.2 +sigfig==1.3.19 +importlib_resources==7.1.0 +enum-tools==0.13.0 +cramjam==2.11.0 +python-snappy==0.7.3 +compact-json==1.8.2 +numbers-parser==4.18.5 +entmax==1.3 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_164537-zgrsvkbi/logs/debug-internal.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_164537-zgrsvkbi/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6a514443eb143a539a9720252b7d81ca10f6116c --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_164537-zgrsvkbi/logs/debug-internal.log @@ -0,0 +1,16 @@ +{"time":"2026-09-21T16:45:37.803741511Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-09-21T16:45:37.803757971Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"} +{"time":"2026-09-21T16:45:38.139478759Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T16:45:38.139514896Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T16:45:38.139546384Z","level":"INFO","msg":"stream: created new stream","id":"zgrsvkbi"} +{"time":"2026-09-21T16:45:38.139586388Z","level":"INFO","msg":"handler: started"} +{"time":"2026-09-21T16:45:38.141813423Z","level":"INFO","msg":"stream: started"} +{"time":"2026-09-21T16:45:38.142565807Z","level":"INFO","msg":"sender: started"} +{"time":"2026-09-21T16:45:38.142566037Z","level":"INFO","msg":"writer: started","stream_id":"zgrsvkbi"} +{"time":"2026-09-21T16:45:38.193714108Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T16:45:38.193729426Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-09-22T04:08:06.511878708Z","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-09-22T04:08:06.522226378Z","level":"INFO","msg":"stream: finishing up"} +{"time":"2026-09-22T04:08:06.52224334Z","level":"INFO","msg":"handler: closed"} +{"time":"2026-09-22T04:08:06.523497333Z","level":"INFO","msg":"sender: closed"} +{"time":"2026-09-22T04:08:06.523505869Z","level":"INFO","msg":"stream: all finished"} diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_164537-zgrsvkbi/logs/debug.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_164537-zgrsvkbi/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..8f2b2eb455b184810d20de144bd4d4dc5e53de07 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_164537-zgrsvkbi/logs/debug.log @@ -0,0 +1,23 @@ +2026-09-21 16:45:37,291 INFO MainThread:235 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1 +2026-09-21 16:45:37,291 INFO MainThread:235 [wandb_setup.py:_flush():81] Configure stats pid to 235 +2026-09-21 16:45:37,291 INFO MainThread:235 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-09-21 16:45:37,291 INFO MainThread:235 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_164537-zgrsvkbi/logs/debug.log +2026-09-21 16:45:37,291 INFO MainThread:235 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_164537-zgrsvkbi/logs/debug-internal.log +2026-09-21 16:45:37,291 INFO MainThread:235 [wandb_init.py:init():850] calling init triggers +2026-09-21 16:45:37,291 INFO MainThread:235 [wandb_init.py:init():855] wandb.init called with sweep_config: {} +config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s43/run_20260921_142951', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s43/run_20260921_142951/checkpoint-2000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 4, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 43, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': 8000, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s43', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed43'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}} +2026-09-21 16:45:37,291 INFO MainThread:235 [wandb_init.py:init():898] starting backend +2026-09-21 16:45:37,788 INFO MainThread:235 [wandb_init.py:init():913] sending inform_init request +2026-09-21 16:45:38,142 INFO MainThread:235 [wandb_init.py:init():918] backend started and connected +2026-09-21 16:45:38,147 INFO MainThread:235 [wandb_init.py:init():988] updated telemetry +2026-09-21 16:45:38,190 INFO MainThread:235 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout +2026-09-21 16:45:38,195 INFO MainThread:235 [wandb_init.py:init():1056] starting run threads in backend +2026-09-21 16:45:38,433 INFO MainThread:235 [wandb_run.py:_console_start():2554] atexit reg +2026-09-21 16:45:38,433 INFO MainThread:235 [wandb_run.py:_redirect():2403] redirect: wrap_raw +2026-09-21 16:45:38,433 INFO MainThread:235 [wandb_run.py:_redirect():2472] Wrapping output streams. +2026-09-21 16:45:38,433 INFO MainThread:235 [wandb_run.py:_redirect():2495] Redirects installed. +2026-09-21 16:45:38,439 INFO MainThread:235 [wandb_init.py:init():1094] run started, returning control to user process +2026-09-22 04:08:06,508 INFO MainThread:235 [wandb_run.py:_finish():2321] finishing run stochastic_sparsity/zgrsvkbi +2026-09-22 04:08:06,509 INFO MainThread:235 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0 +2026-09-22 04:08:06,510 INFO MainThread:235 [wandb_run.py:_restore():2502] restore +2026-09-22 04:08:06,510 INFO MainThread:235 [wandb_run.py:_restore():2508] restore done diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_164537-zgrsvkbi/run-zgrsvkbi.wandb b/epfl_final/SparseAtt/wandb/offline-run-20260921_164537-zgrsvkbi/run-zgrsvkbi.wandb new file mode 100644 index 0000000000000000000000000000000000000000..338338bf0de342a7ff9c9c51f07adc510af877e7 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_164537-zgrsvkbi/run-zgrsvkbi.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4000d30c7617d887accba66cddef90a2450a99e75aa1cd604be87994ec3a8f6 +size 7726453 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_195241-nihe6o4e/files/requirements.txt b/epfl_final/SparseAtt/wandb/offline-run-20260921_195241-nihe6o4e/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c3e38c4cce87cb53d52f45f6ac8cbf8688a7d92 --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_195241-nihe6o4e/files/requirements.txt @@ -0,0 +1,142 @@ +pip==26.0.1 +nvidia-cusparselt-cu13==0.8.0 +mpmath==1.3.0 +cuda-toolkit==13.0.2 +zstandard==0.25.0 +xxhash==3.7.0 +urllib3==2.6.3 +typing_extensions==4.15.0 +triton==3.6.0 +tqdm==4.67.3 +torchinfo==1.8.0 +sympy==1.14.0 +smmap==5.0.3 +six==1.17.0 +shellingham==1.5.4 +setuptools==81.0.0 +safetensors==0.7.0 +regex==2026.4.4 +PyYAML==6.0.3 +Pygments==2.20.0 +pyarrow==24.0.0 +psutil==7.2.2 +protobuf==7.34.1 +propcache==0.4.1 +platformdirs==4.9.6 +packaging==26.2 +nvidia-nvtx==13.0.85 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvjitlink==13.0.88 +nvidia-nccl-cu13==2.28.9 +nvidia-curand==10.4.0.35 +nvidia-cufile==1.15.1.6 +nvidia-cuda-runtime==13.0.96 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-cupti==13.0.85 +nvidia-cublas==13.1.0.3 +numpy==2.4.4 +networkx==3.6.1 +multidict==6.7.1 +mdurl==0.1.2 +MarkupSafe==3.0.3 +idna==3.13 +hf-xet==1.4.3 +h11==0.16.0 +fsspec==2026.2.0 +frozenlist==1.8.0 +filelock==3.29.0 +dill==0.4.1 +cuda-pathfinder==1.5.4 +click==8.3.3 +charset-normalizer==3.4.7 +certifi==2026.4.22 +attrs==26.1.0 +annotated-types==0.7.0 +annotated-doc==0.0.4 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +typing-inspection==0.4.2 +sentry-sdk==2.58.0 +requests==2.33.1 +python-dateutil==2.9.0.post0 +pydantic_core==2.46.3 +nvidia-cusparse==12.6.3.3 +nvidia-cufft==12.0.0.61 +nvidia-cudnn-cu13==9.19.0.56 +multiprocess==0.70.19 +markdown-it-py==4.0.0 +Jinja2==3.1.6 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==13.2.0 +anyio==4.13.0 +aiosignal==1.4.0 +rich==15.0.0 +pydantic==2.13.3 +pandas==3.0.2 +nvidia-cusolver==12.0.4.66 +httpx==0.28.1 +GitPython==3.1.49 +aiohttp==3.13.5 +wandb==0.26.1 +typer==0.25.1 +torch==2.11.0 +huggingface_hub==1.13.0 +tokenizers==0.22.2 +datasets==4.8.5 +accelerate==1.13.0 +transformers==5.7.0 +word2number==1.1 +sqlitedict==2.1.0 +pytz==2026.2 +threadpoolctl==3.6.0 +tcolorpy==0.1.7 +tabulate==0.10.0 +scipy==1.17.1 +portalocker==3.2.0 +pathvalidate==3.3.1 +narwhals==2.22.1 +more-itertools==11.1.0 +lxml==6.1.1 +joblib==1.5.3 +colorama==0.4.6 +chardet==6.0.0.post1 +absl-py==2.4.0 +scikit-learn==1.9.0 +sacrebleu==2.6.0 +nltk==3.9.4 +mbstrdecoder==1.1.5 +typepy==1.3.5 +rouge_score==0.1.2 +DataProperty==1.1.1 +tabledata==1.3.5 +evaluate==0.4.6 +pytablewriter==1.2.1 +lm_eval==0.4.12 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==9.1.0 +py-spy==0.4.2 +pillow==12.2.0 +tifffile==2026.6.1 +imagecodecs==2026.6.26 +zipp==4.1.0 +importlib_metadata==9.0.0 +derivative==0.6.3 +pysindy==2.1.0 +pyparsing==3.3.2 +kiwisolver==1.5.0 +fonttools==4.63.0 +cycler==0.12.1 +contourpy==1.3.3 +matplotlib==3.11.0 +sortedcontainers==2.4.0 +wcwidth==0.8.2 +sigfig==1.3.19 +importlib_resources==7.1.0 +enum-tools==0.13.0 +cramjam==2.11.0 +python-snappy==0.7.3 +compact-json==1.8.2 +numbers-parser==4.18.5 +entmax==1.3 diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_195241-nihe6o4e/logs/debug-internal.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_195241-nihe6o4e/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..0fad14c1d6ce0d2657ee79488280bb10b50d6cbc --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_195241-nihe6o4e/logs/debug-internal.log @@ -0,0 +1,16 @@ +{"time":"2026-09-21T19:52:42.319490993Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-09-21T19:52:42.319520437Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"} +{"time":"2026-09-21T19:52:42.58442705Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T19:52:42.584504607Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T19:52:42.584554181Z","level":"INFO","msg":"stream: created new stream","id":"nihe6o4e"} +{"time":"2026-09-21T19:52:42.584644176Z","level":"INFO","msg":"handler: started"} +{"time":"2026-09-21T19:52:42.586982479Z","level":"INFO","msg":"stream: started"} +{"time":"2026-09-21T19:52:42.588333569Z","level":"INFO","msg":"sender: started"} +{"time":"2026-09-21T19:52:42.588362913Z","level":"INFO","msg":"writer: started","stream_id":"nihe6o4e"} +{"time":"2026-09-21T19:52:42.666823239Z","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"} +{"time":"2026-09-21T19:52:42.666853084Z","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"} +{"time":"2026-09-21T23:17:02.832069883Z","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-09-21T23:17:02.836913023Z","level":"INFO","msg":"stream: finishing up"} +{"time":"2026-09-21T23:17:02.836921235Z","level":"INFO","msg":"handler: closed"} +{"time":"2026-09-21T23:17:02.838136842Z","level":"INFO","msg":"sender: closed"} +{"time":"2026-09-21T23:17:02.838143012Z","level":"INFO","msg":"stream: all finished"} diff --git a/epfl_final/SparseAtt/wandb/offline-run-20260921_195241-nihe6o4e/logs/debug.log b/epfl_final/SparseAtt/wandb/offline-run-20260921_195241-nihe6o4e/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..36d689dafa76c1e8e5aeb5b0ac0874ce594da44b --- /dev/null +++ b/epfl_final/SparseAtt/wandb/offline-run-20260921_195241-nihe6o4e/logs/debug.log @@ -0,0 +1,23 @@ +2026-09-21 19:52:41,555 INFO MainThread:216 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1 +2026-09-21 19:52:41,555 INFO MainThread:216 [wandb_setup.py:_flush():81] Configure stats pid to 216 +2026-09-21 19:52:41,555 INFO MainThread:216 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-09-21 19:52:41,556 INFO MainThread:216 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_195241-nihe6o4e/logs/debug.log +2026-09-21 19:52:41,556 INFO MainThread:216 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/offline-run-20260921_195241-nihe6o4e/logs/debug-internal.log +2026-09-21 19:52:41,556 INFO MainThread:216 [wandb_init.py:init():850] calling init triggers +2026-09-21 19:52:41,556 INFO MainThread:216 [wandb_init.py:init():855] wandb.init called with sweep_config: {} +config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s42/run_20260921_142951', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/models/ALIBI_20260921/365m_alibi_s42/run_20260921_142951/checkpoint-17000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 5, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 100, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'resume_micro_step_override': None, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m alibi 8k s42', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'alibi', 'seed42'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'positional': 'alibi', 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}} +2026-09-21 19:52:41,556 INFO MainThread:216 [wandb_init.py:init():898] starting backend +2026-09-21 19:52:42,301 INFO MainThread:216 [wandb_init.py:init():913] sending inform_init request +2026-09-21 19:52:42,587 INFO MainThread:216 [wandb_init.py:init():918] backend started and connected +2026-09-21 19:52:42,593 INFO MainThread:216 [wandb_init.py:init():988] updated telemetry +2026-09-21 19:52:42,662 INFO MainThread:216 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout +2026-09-21 19:52:42,668 INFO MainThread:216 [wandb_init.py:init():1056] starting run threads in backend +2026-09-21 19:52:42,947 INFO MainThread:216 [wandb_run.py:_console_start():2554] atexit reg +2026-09-21 19:52:42,947 INFO MainThread:216 [wandb_run.py:_redirect():2403] redirect: wrap_raw +2026-09-21 19:52:42,947 INFO MainThread:216 [wandb_run.py:_redirect():2472] Wrapping output streams. +2026-09-21 19:52:42,947 INFO MainThread:216 [wandb_run.py:_redirect():2495] Redirects installed. +2026-09-21 19:52:42,973 INFO MainThread:216 [wandb_init.py:init():1094] run started, returning control to user process +2026-09-21 23:17:02,827 INFO MainThread:216 [wandb_run.py:_finish():2321] finishing run stochastic_sparsity/nihe6o4e +2026-09-21 23:17:02,828 INFO MainThread:216 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0 +2026-09-21 23:17:02,829 INFO MainThread:216 [wandb_run.py:_restore():2502] restore +2026-09-21 23:17:02,829 INFO MainThread:216 [wandb_run.py:_restore():2508] restore done