EPFL ALiBi and training provenance 201-250 of 1184

.gitattributes

@@ -217,3 +217,9 @@ epfl_final/wandb/run-20260506_200450-z06f2580/run-z06f2580.wandb filter=lfs diff
 epfl_final/wandb/run-20260509_201725-v6634orv/run-v6634orv.wandb filter=lfs diff=lfs merge=lfs -text
 epfl_final/wandb/run-20260511_155411-fii64tue/run-fii64tue.wandb filter=lfs diff=lfs merge=lfs -text
 epfl_final/wandb/run-20260511_201933-jzi5hfrk/run-jzi5hfrk.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/wandb/run-20260512_052740-cn25b1jk/run-cn25b1jk.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/wandb/run-20260513_051915-x63i12u5/run-x63i12u5.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/wandb/run-20260513_052440-i0f0xwcq/run-i0f0xwcq.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/wandb/run-20260513_151725-qudv4nmq/run-qudv4nmq.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/wandb/run-20260513_164613-40cb81kv/run-40cb81kv.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/wandb/run-20260517_023932-hi2z3jh6/run-hi2z3jh6.wandb filter=lfs diff=lfs merge=lfs -text

epfl_final/wandb/run-20260511_215604-377k1je4/files/config.yaml

@@ -0,0 +1,134 @@
+_wandb:
+    value:
+        cli_version: 0.26.1
+        e:
+            jhi8chi0zzs19xazmkzvwdnmzvij10cx:
+                codePath: SparseAtt/train.py
+                codePathLocal: SparseAtt/train.py
+                cpu_count: 96
+                cpu_count_logical: 192
+                cudaVersion: "13.0"
+                disk:
+                    /:
+                        total: "9374931091456"
+                        used: "7086919069696"
+                email: ruben.navasardyan@mail.utoronto.ca
+                executable: /usr/bin/python3
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 1
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-5cb9f8b7-4a13-4f85-d73e-6bcbebebefb8
+                host: h100-train-13-0-5
+                memory:
+                    total: "1623005741056"
+                os: Linux-6.8.0-59-generic-x86_64-with-glibc2.35
+                program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
+                python: CPython 3.10.12
+                root: /mloscratch/homes/navasard/sparse
+                startedAt: "2026-05-11T21:56:04.033845Z"
+                writerId: jhi8chi0zzs19xazmkzvwdnmzvij10cx
+        m: []
+        python_version: 3.10.12
+        t:
+            "1":
+                - 1
+                - 11
+                - 49
+                - 51
+                - 71
+            "2":
+                - 1
+                - 11
+                - 49
+                - 51
+                - 71
+            "3":
+                - 15
+                - 16
+            "4": 3.10.12
+            "5": 0.26.1
+            "6": 5.8.0
+            "12": 0.26.1
+            "13": linux-x86_64
+attention_type:
+    value: sparse_stochastic
+checkpoint_activations:
+    value: false
+eval_batches:
+    value: 200
+eval_steps:
+    value: 250
+gradient_accumulation_steps:
+    value: 8
+init_from_scratch:
+    value: true
+learning_rate:
+    value: 0.0005
+logging_steps:
+    value: 10
+lr_scheduler:
+    value: cosine
+max_grad_norm:
+    value: 1
+max_seq_length:
+    value: 8192
+max_steps:
+    value: 20005
+mixed_precision:
+    value: bf16
+model_dtype:
+    value: bfloat16
+model_path:
+    value: /mloscratch/homes/navasard/qwen3_06b_base
+num_epochs:
+    value: 1
+num_workers:
+    value: 8
+output_dir:
+    value: ./checkpoints/run_20260511_215307
+per_device_batch_size:
+    value: 3
+resume_from_checkpoint:
+    value: null
+save_steps:
+    value: 1000
+seed:
+    value: 42
+sparse_block_size:
+    value: 128
+sparse_layer_indices:
+    value: null
+sparse_mask_cache_dir:
+    value: /mloscratch/homes/navasard/sparse/mask_cache
+sparse_mask_pool_size:
+    value: 10000
+sparse_num_random_blocks:
+    value: 31
+sparse_val_dense_compare:
+    value: false
+strategy:
+    value: ddp
+train_path:
+    value: /mloscratch/homes/navasard/data/fineweb/tokenized
+val_path:
+    value: null
+verbose:
+    value: true
+wandb_enabled:
+    value: true
+wandb_name:
+    value: null
+wandb_project:
+    value: stochastic_sparsity
+wandb_tags:
+    value:
+        - qwen3
+        - 0.6b
+warmup_steps:
+    value: 250
+weight_decay:
+    value: 0.01

epfl_final/wandb/run-20260512_052740-cn25b1jk/files/config.yaml

@@ -0,0 +1,136 @@
+_wandb:
+    value:
+        cli_version: 0.26.1
+        e:
+            n6qlcy60609ow7u1awe6lzorqadjusj0:
+                codePath: SparseAtt/train.py
+                codePathLocal: SparseAtt/train.py
+                cpu_count: 96
+                cpu_count_logical: 192
+                cudaVersion: "13.0"
+                disk:
+                    /:
+                        total: "9374931091456"
+                        used: "6480959242240"
+                email: ruben.navasardyan@mail.utoronto.ca
+                executable: /usr/bin/python3
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 1
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-cd5c7d97-0f62-2f41-36c8-233c8f57b814
+                host: h100-train-1-0-3
+                memory:
+                    total: "1623005847552"
+                os: Linux-6.8.0-59-generic-x86_64-with-glibc2.35
+                program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
+                python: CPython 3.10.12
+                root: /mloscratch/homes/navasard/sparse
+                startedAt: "2026-05-12T05:27:40.216251Z"
+                writerId: n6qlcy60609ow7u1awe6lzorqadjusj0
+        m: []
+        python_version: 3.10.12
+        t:
+            "1":
+                - 1
+                - 11
+                - 49
+                - 51
+                - 71
+            "2":
+                - 1
+                - 11
+                - 49
+                - 51
+                - 71
+            "3":
+                - 2
+                - 15
+                - 16
+                - 61
+            "4": 3.10.12
+            "5": 0.26.1
+            "6": 5.8.0
+            "12": 0.26.1
+            "13": linux-x86_64
+attention_type:
+    value: sparse_stochastic
+checkpoint_activations:
+    value: false
+eval_batches:
+    value: 200
+eval_steps:
+    value: 250
+gradient_accumulation_steps:
+    value: 8
+init_from_scratch:
+    value: true
+learning_rate:
+    value: 0.0005
+logging_steps:
+    value: 10
+lr_scheduler:
+    value: cosine
+max_grad_norm:
+    value: 1
+max_seq_length:
+    value: 8192
+max_steps:
+    value: 20005
+mixed_precision:
+    value: bf16
+model_dtype:
+    value: bfloat16
+model_path:
+    value: /mloscratch/homes/navasard/qwen3_06b_base
+num_epochs:
+    value: 1
+num_workers:
+    value: 8
+output_dir:
+    value: ./checkpoints/run_20260512_052551
+per_device_batch_size:
+    value: 3
+resume_from_checkpoint:
+    value: null
+save_steps:
+    value: 1000
+seed:
+    value: 42
+sparse_block_size:
+    value: 128
+sparse_layer_indices:
+    value: null
+sparse_mask_cache_dir:
+    value: /mloscratch/homes/navasard/sparse/mask_cache
+sparse_mask_pool_size:
+    value: 10000
+sparse_num_random_blocks:
+    value: 31
+sparse_val_dense_compare:
+    value: false
+strategy:
+    value: ddp
+train_path:
+    value: /mloscratch/homes/navasard/data/fineweb/tokenized
+val_path:
+    value: null
+verbose:
+    value: true
+wandb_enabled:
+    value: true
+wandb_name:
+    value: null
+wandb_project:
+    value: stochastic_sparsity
+wandb_tags:
+    value:
+        - qwen3
+        - 0.6b
+warmup_steps:
+    value: 250
+weight_decay:
+    value: 0.01

epfl_final/wandb/run-20260512_052740-cn25b1jk/files/requirements.txt

@@ -0,0 +1,97 @@
+pytz==2026.2
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.7.0
+tzdata==2026.2
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+safetensors==0.7.0
+regex==2026.5.9
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.5.2
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.2.6
+networkx==3.4.2
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.14
+hf-xet==1.5.0
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+async-timeout==5.0.1
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+typing-inspection==0.4.2
+sentry-sdk==2.59.0
+requests==2.34.0
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.4
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+multidict==6.7.1
+markdown-it-py==4.2.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+exceptiongroup==1.3.1
+cuda-bindings==13.2.0
+aiosignal==1.4.0
+yarl==1.23.0
+rich==15.0.0
+pydantic==2.13.4
+pandas==2.3.3
+nvidia-cusolver==12.0.4.66
+GitPython==3.1.50
+anyio==4.13.0
+wandb==0.26.1
+typer==0.25.1
+httpx==0.28.1
+aiohttp==3.13.5
+torch==2.11.0
+huggingface_hub==1.14.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.8.0
+PyGObject==3.42.1
+dbus-python==1.2.18
+distro==1.7.0
+pip==22.0.2
+setuptools==59.6.0
+ssh-import-id==5.11
+wheel==0.37.1

epfl_final/wandb/run-20260512_052740-cn25b1jk/files/wandb-metadata.json

@@ -0,0 +1,36 @@
+{
+  "os":  "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.10.12",
+  "startedAt":  "2026-05-12T05:27:40.216251Z",
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "SparseAtt/train.py",
+  "codePathLocal":  "SparseAtt/train.py",
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse",
+  "host":  "h100-train-1-0-3",
+  "executable":  "/usr/bin/python3",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  1,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "6480959242240"
+    }
+  },
+  "memory":  {
+    "total":  "1623005847552"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-cd5c7d97-0f62-2f41-36c8-233c8f57b814"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "n6qlcy60609ow7u1awe6lzorqadjusj0"
+}

epfl_final/wandb/run-20260512_052740-cn25b1jk/files/wandb-summary.json

@@ -0,0 +1 @@
+{"speed/samples_per_sec":9.039362638940517,"_timestamp":1.7786318543712018e+09,"train/loss":2.9195768535137177,"speed/steps_per_sec":0.37664010995585484,"eval/loss":2.923399579823017,"speed/ms_per_step":2655.054450035095,"train/global_step":20000,"_runtime":68219.90925666,"train/learning_rate":7.903067225645266e-11,"_wandb":{"runtime":68219},"train/epoch":0.4716981132075472,"_step":20000,"eval/perplexity":18.60442731015885}

epfl_final/wandb/run-20260512_052740-cn25b1jk/logs/debug.log

@@ -0,0 +1,24 @@
+2026-05-12 05:27:40,235 INFO    MainThread:25915 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-05-12 05:27:40,235 INFO    MainThread:25915 [wandb_setup.py:_flush():81] Configure stats pid to 25915
+2026-05-12 05:27:40,236 INFO    MainThread:25915 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-12 05:27:40,237 INFO    MainThread:25915 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260512_052740-cn25b1jk/logs/debug.log
+2026-05-12 05:27:40,237 INFO    MainThread:25915 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260512_052740-cn25b1jk/logs/debug-internal.log
+2026-05-12 05:27:40,238 INFO    MainThread:25915 [wandb_init.py:init():850] calling init triggers
+2026-05-12 05:27:40,239 INFO    MainThread:25915 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': './checkpoints/run_20260512_052551', 'resume_from_checkpoint': None, 'num_epochs': 1, 'max_steps': 20005, 'per_device_batch_size': 3, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 250, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 200, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': None, 'wandb_tags': ['qwen3', '0.6b'], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache', '_wandb': {}}
+2026-05-12 05:27:40,239 INFO    MainThread:25915 [wandb_init.py:init():898] starting backend
+2026-05-12 05:27:40,470 INFO    MainThread:25915 [wandb_init.py:init():913] sending inform_init request
+2026-05-12 05:27:40,782 INFO    MainThread:25915 [wandb_init.py:init():918] backend started and connected
+2026-05-12 05:27:40,784 INFO    MainThread:25915 [wandb_init.py:init():988] updated telemetry
+2026-05-12 05:27:40,785 INFO    MainThread:25915 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-05-12 05:27:41,484 INFO    MainThread:25915 [wandb_init.py:init():1056] starting run threads in backend
+2026-05-12 05:27:41,540 INFO    MainThread:25915 [wandb_run.py:_console_start():2554] atexit reg
+2026-05-12 05:27:41,540 INFO    MainThread:25915 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-05-12 05:27:41,541 INFO    MainThread:25915 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-05-12 05:27:41,541 INFO    MainThread:25915 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-05-12 05:27:41,545 INFO    MainThread:25915 [wandb_init.py:init():1094] run started, returning control to user process
+2026-05-13 00:24:41,391 INFO    MainThread:25915 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/cn25b1jk
+2026-05-13 00:24:41,392 INFO    MainThread:25915 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
+2026-05-13 00:24:41,393 INFO    MainThread:25915 [wandb_run.py:_restore():2502] restore
+2026-05-13 00:24:41,393 INFO    MainThread:25915 [wandb_run.py:_restore():2508] restore done
+2026-05-13 00:24:42,807 INFO    MainThread:25915 [wandb_run.py:_footer_sync_info():3889] logging synced files

epfl_final/wandb/run-20260512_052740-cn25b1jk/run-cn25b1jk.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a15e4afbef688195317652e83d6a7c117d31d8c53b18ec7377ecd78d8c959641
+size 11223915

epfl_final/wandb/run-20260513_051915-x63i12u5/files/config.yaml

@@ -0,0 +1,140 @@
+_wandb:
+    value:
+        cli_version: 0.26.1
+        e:
+            rwkqonaudhuzcez0jkojd2bi7cq7j6so:
+                codePath: SparseAtt/train.py
+                codePathLocal: SparseAtt/train.py
+                cpu_count: 96
+                cpu_count_logical: 192
+                cudaVersion: "13.0"
+                disk:
+                    /:
+                        total: "9374931091456"
+                        used: "7268479995904"
+                email: ruben.navasardyan@mail.utoronto.ca
+                executable: /usr/bin/python3
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 1
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-3ea6c840-350f-c68c-7562-6e0affa4e153
+                host: h100-workspace-7-0-4
+                memory:
+                    total: "1623005741056"
+                os: Linux-6.8.0-59-generic-x86_64-with-glibc2.35
+                program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
+                python: CPython 3.10.12
+                root: /mloscratch/homes/navasard/sparse
+                startedAt: "2026-05-13T05:19:15.392571Z"
+                writerId: rwkqonaudhuzcez0jkojd2bi7cq7j6so
+        m: []
+        python_version: 3.10.12
+        t:
+            "1":
+                - 1
+                - 11
+                - 49
+                - 51
+                - 71
+            "2":
+                - 1
+                - 11
+                - 49
+                - 51
+                - 71
+            "3":
+                - 2
+                - 13
+                - 15
+                - 16
+                - 61
+            "4": 3.10.12
+            "5": 0.26.1
+            "6": 5.8.0
+            "12": 0.26.1
+            "13": linux-x86_64
+attention_type:
+    value: sparse_stochastic
+checkpoint_activations:
+    value: false
+dense_mode:
+    value: true
+eval_batches:
+    value: 200
+eval_steps:
+    value: 250
+gradient_accumulation_steps:
+    value: 8
+init_from_scratch:
+    value: true
+learning_rate:
+    value: 0.0005
+logging_steps:
+    value: 10
+lr_scheduler:
+    value: cosine
+max_grad_norm:
+    value: 1
+max_seq_length:
+    value: 8192
+max_steps:
+    value: 20005
+mixed_precision:
+    value: bf16
+model_dtype:
+    value: bfloat16
+model_path:
+    value: /mloscratch/homes/navasard/qwen3_06b_base
+num_epochs:
+    value: 1
+num_workers:
+    value: 8
+output_dir:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723
+per_device_batch_size:
+    value: 3
+resume_from_checkpoint:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-18000
+save_steps:
+    value: 1000
+seed:
+    value: 42
+sparse_block_size:
+    value: 128
+sparse_layer_indices:
+    value: null
+sparse_mask_cache_dir:
+    value: null
+sparse_mask_pool_size:
+    value: 10
+sparse_num_random_blocks:
+    value: 31
+sparse_val_dense_compare:
+    value: false
+strategy:
+    value: ddp
+train_path:
+    value: /mloscratch/homes/navasard/data/fineweb/tokenized
+val_path:
+    value: null
+verbose:
+    value: true
+wandb_enabled:
+    value: true
+wandb_name:
+    value: dense-resume-18k
+wandb_project:
+    value: stochastic_sparsity
+wandb_tags:
+    value:
+        - qwen3
+        - 0.6b
+        - dense-resume
+warmup_steps:
+    value: 250
+weight_decay:
+    value: 0.01

epfl_final/wandb/run-20260513_051915-x63i12u5/files/output.log

@@ -0,0 +1,314 @@
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1024
+  num_hidden_layers: 28
+  num_attention_heads: 16
+  num_key_value_heads: 8
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 10
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 596,049,920
+  Trainable parameters: 596,049,920
+  Non-trainable parameters: 0
+  Model size (BF16): 1.19 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [3, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [3, 8192, 1024]           --                        True
+│    └─Embedding: 2-1                                        [3, 8192]                 [3, 8192, 1024]           155,582,464               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    └─Qwen3RMSNorm: 2-3                                     [3, 8192, 1024]           [3, 8192, 1024]           1,024                     True
+├─Linear: 1-2                                                [3, 8192, 1024]           [3, 8192, 151936]         155,582,464               True
+================================================================================================================================================================
+Total params: 751,632,384
+Trainable params: 751,632,384
+Non-trainable params: 0
+Total mult-adds (G): 2.25
+================================================================================================================================================================
+Input size (MB): 0.20
+Forward/backward pass size (MB): 63052.97
+Params size (MB): 1503.26
+Estimated Total Size (MB): 64556.43
+================================================================================================================================================================
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Loaded 5088012 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1272003)
+Dataset split: 1017602 train, 254401 validation
+Training examples (packed, train split):  1017602
+Validation examples (packed, val split):  254401
+Per-device batch size:                    3
+World size (GPUs):                        1
+Gradient accumulation steps:              8
+Effective batch size (seqs/step):         24
+Sequence length:                          8192
+Tokens per optimizer step:                196,608
+Micro-batches per epoch (per rank):       339200
+Optimizer steps per epoch:                42400
+Total optimizer steps:                    20005  (capped by max_steps; 0.47 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-18000...
+Resumed from checkpoint at step 17999 (epoch 0, micro_step 8)
+Resuming training from step 17999
+Starting training...
+Step 18000/20005 | Loss: 0.2852 | LR: 1.26e-05 | 10700.3ms/step | 2.2 samples/s
+Validation at step 18000: Loss = 2.9081, Perplexity = 18.32
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-18000
+Step 18010/20005 | Loss: 2.9110 | LR: 1.25e-05 | 41505.6ms/step | 0.6 samples/s
+Step 18020/20005 | Loss: 2.9064 | LR: 1.23e-05 | 5760.1ms/step | 4.2 samples/s
+Step 18030/20005 | Loss: 2.9086 | LR: 1.22e-05 | 5760.0ms/step | 4.2 samples/s
+Step 18040/20005 | Loss: 2.8658 | LR: 1.21e-05 | 5759.9ms/step | 4.2 samples/s
+Step 18050/20005 | Loss: 2.8887 | LR: 1.20e-05 | 5759.9ms/step | 4.2 samples/s
+Step 18060/20005 | Loss: 2.9105 | LR: 1.18e-05 | 5760.0ms/step | 4.2 samples/s
+Step 18070/20005 | Loss: 2.9040 | LR: 1.17e-05 | 5759.7ms/step | 4.2 samples/s
+Step 18080/20005 | Loss: 2.8859 | LR: 1.16e-05 | 5760.2ms/step | 4.2 samples/s
+Step 18090/20005 | Loss: 2.8690 | LR: 1.15e-05 | 5764.3ms/step | 4.2 samples/s
+Step 18100/20005 | Loss: 2.9167 | LR: 1.14e-05 | 5766.5ms/step | 4.2 samples/s
+Step 18110/20005 | Loss: 2.8816 | LR: 1.12e-05 | 5762.6ms/step | 4.2 samples/s
+Step 18120/20005 | Loss: 2.9037 | LR: 1.11e-05 | 5760.5ms/step | 4.2 samples/s
+Step 18130/20005 | Loss: 2.8764 | LR: 1.10e-05 | 5759.8ms/step | 4.2 samples/s
+Step 18140/20005 | Loss: 2.8728 | LR: 1.09e-05 | 5760.0ms/step | 4.2 samples/s
+Step 18150/20005 | Loss: 2.8748 | LR: 1.08e-05 | 5759.9ms/step | 4.2 samples/s
+Step 18160/20005 | Loss: 2.8974 | LR: 1.07e-05 | 5759.9ms/step | 4.2 samples/s
+Step 18170/20005 | Loss: 2.8732 | LR: 1.05e-05 | 5759.7ms/step | 4.2 samples/s
+Step 18180/20005 | Loss: 2.9163 | LR: 1.04e-05 | 5760.3ms/step | 4.2 samples/s
+Step 18190/20005 | Loss: 2.8905 | LR: 1.03e-05 | 5762.0ms/step | 4.2 samples/s
+Step 18200/20005 | Loss: 2.8774 | LR: 1.02e-05 | 5766.3ms/step | 4.2 samples/s
+Step 18210/20005 | Loss: 2.8885 | LR: 1.01e-05 | 5778.2ms/step | 4.2 samples/s
+Step 18220/20005 | Loss: 2.8888 | LR: 9.98e-06 | 5759.5ms/step | 4.2 samples/s
+Step 18230/20005 | Loss: 2.8564 | LR: 9.87e-06 | 5759.8ms/step | 4.2 samples/s
+Step 18240/20005 | Loss: 2.8919 | LR: 9.76e-06 | 5759.9ms/step | 4.2 samples/s
+Step 18250/20005 | Loss: 2.8810 | LR: 9.65e-06 | 5760.6ms/step | 4.2 samples/s
+Validation at step 18250: Loss = 2.8985, Perplexity = 18.15
+Step 18260/20005 | Loss: 2.8656 | LR: 9.54e-06 | 40647.7ms/step | 0.6 samples/s
+Step 18270/20005 | Loss: 2.9053 | LR: 9.43e-06 | 5771.9ms/step | 4.2 samples/s
+Step 18280/20005 | Loss: 2.8800 | LR: 9.33e-06 | 5786.6ms/step | 4.1 samples/s
+Step 18290/20005 | Loss: 2.8887 | LR: 9.22e-06 | 5774.1ms/step | 4.2 samples/s
+Step 18300/20005 | Loss: 2.8839 | LR: 9.11e-06 | 5776.8ms/step | 4.2 samples/s
+Step 18310/20005 | Loss: 2.8669 | LR: 9.01e-06 | 5778.4ms/step | 4.2 samples/s
+Step 18320/20005 | Loss: 2.8687 | LR: 8.90e-06 | 5774.5ms/step | 4.2 samples/s
+Step 18330/20005 | Loss: 2.8891 | LR: 8.80e-06 | 5775.6ms/step | 4.2 samples/s
+Step 18340/20005 | Loss: 2.8904 | LR: 8.69e-06 | 5771.7ms/step | 4.2 samples/s
+Step 18350/20005 | Loss: 2.8801 | LR: 8.59e-06 | 5774.2ms/step | 4.2 samples/s
+Step 18360/20005 | Loss: 2.8828 | LR: 8.49e-06 | 5775.4ms/step | 4.2 samples/s
+Step 18370/20005 | Loss: 2.8928 | LR: 8.38e-06 | 5777.1ms/step | 4.2 samples/s
+Step 18380/20005 | Loss: 2.9098 | LR: 8.28e-06 | 5774.0ms/step | 4.2 samples/s
+Step 18390/20005 | Loss: 2.9067 | LR: 8.18e-06 | 5775.2ms/step | 4.2 samples/s
+Step 18400/20005 | Loss: 2.8925 | LR: 8.08e-06 | 5773.9ms/step | 4.2 samples/s
+Step 18410/20005 | Loss: 2.8638 | LR: 7.98e-06 | 5771.0ms/step | 4.2 samples/s
+Step 18420/20005 | Loss: 2.8978 | LR: 7.88e-06 | 5771.7ms/step | 4.2 samples/s
+Step 18430/20005 | Loss: 2.8809 | LR: 7.78e-06 | 5772.0ms/step | 4.2 samples/s
+Step 18440/20005 | Loss: 2.8783 | LR: 7.68e-06 | 5775.7ms/step | 4.2 samples/s
+Step 18450/20005 | Loss: 2.9109 | LR: 7.59e-06 | 5776.0ms/step | 4.2 samples/s
+Step 18460/20005 | Loss: 2.9144 | LR: 7.49e-06 | 5775.8ms/step | 4.2 samples/s
+Step 18470/20005 | Loss: 2.8866 | LR: 7.39e-06 | 5776.2ms/step | 4.2 samples/s
+Step 18480/20005 | Loss: 2.8802 | LR: 7.30e-06 | 5775.8ms/step | 4.2 samples/s
+Step 18490/20005 | Loss: 2.8752 | LR: 7.20e-06 | 5775.6ms/step | 4.2 samples/s
+Step 18500/20005 | Loss: 2.8748 | LR: 7.11e-06 | 5774.0ms/step | 4.2 samples/s
+Validation at step 18500: Loss = 2.8981, Perplexity = 18.14
+Step 18510/20005 | Loss: 2.8879 | LR: 7.01e-06 | 40721.3ms/step | 0.6 samples/s
+Step 18520/20005 | Loss: 2.8966 | LR: 6.92e-06 | 5782.4ms/step | 4.2 samples/s
+Step 18530/20005 | Loss: 2.8946 | LR: 6.83e-06 | 5778.6ms/step | 4.2 samples/s
+Step 18540/20005 | Loss: 2.8748 | LR: 6.74e-06 | 5780.1ms/step | 4.2 samples/s
+Step 18550/20005 | Loss: 2.8823 | LR: 6.64e-06 | 5779.3ms/step | 4.2 samples/s
+Step 18560/20005 | Loss: 2.8972 | LR: 6.55e-06 | 5818.9ms/step | 4.1 samples/s
+Step 18570/20005 | Loss: 2.9034 | LR: 6.46e-06 | 5780.9ms/step | 4.2 samples/s
+Step 18580/20005 | Loss: 2.8880 | LR: 6.37e-06 | 5780.6ms/step | 4.2 samples/s
+Step 18590/20005 | Loss: 2.9004 | LR: 6.29e-06 | 5775.3ms/step | 4.2 samples/s
+Step 18600/20005 | Loss: 2.8733 | LR: 6.20e-06 | 5780.6ms/step | 4.2 samples/s
+Step 18610/20005 | Loss: 2.8637 | LR: 6.11e-06 | 5775.6ms/step | 4.2 samples/s
+Step 18620/20005 | Loss: 2.8966 | LR: 6.02e-06 | 5775.9ms/step | 4.2 samples/s
+Step 18630/20005 | Loss: 2.8995 | LR: 5.94e-06 | 5777.4ms/step | 4.2 samples/s
+Step 18640/20005 | Loss: 2.8848 | LR: 5.85e-06 | 5772.6ms/step | 4.2 samples/s
+Step 18650/20005 | Loss: 2.9146 | LR: 5.76e-06 | 5778.0ms/step | 4.2 samples/s
+Step 18660/20005 | Loss: 2.8737 | LR: 5.68e-06 | 5778.2ms/step | 4.2 samples/s
+Step 18670/20005 | Loss: 2.8748 | LR: 5.60e-06 | 5778.0ms/step | 4.2 samples/s
+Step 18680/20005 | Loss: 2.8967 | LR: 5.51e-06 | 5777.8ms/step | 4.2 samples/s
+Step 18690/20005 | Loss: 2.8899 | LR: 5.43e-06 | 5780.1ms/step | 4.2 samples/s
+Step 18700/20005 | Loss: 2.8876 | LR: 5.35e-06 | 5780.2ms/step | 4.2 samples/s
+Step 18710/20005 | Loss: 2.8914 | LR: 5.27e-06 | 5777.7ms/step | 4.2 samples/s
+Step 18720/20005 | Loss: 2.8691 | LR: 5.19e-06 | 5778.1ms/step | 4.2 samples/s
+Step 18730/20005 | Loss: 2.8760 | LR: 5.11e-06 | 5781.6ms/step | 4.2 samples/s
+Step 18740/20005 | Loss: 2.8732 | LR: 5.03e-06 | 5780.2ms/step | 4.2 samples/s
+Step 18750/20005 | Loss: 2.8755 | LR: 4.95e-06 | 5779.0ms/step | 4.2 samples/s
+Validation at step 18750: Loss = 2.8979, Perplexity = 18.14
+Step 18760/20005 | Loss: 2.8886 | LR: 4.87e-06 | 40736.6ms/step | 0.6 samples/s
+Step 18770/20005 | Loss: 2.8898 | LR: 4.79e-06 | 5778.0ms/step | 4.2 samples/s
+Step 18780/20005 | Loss: 2.9159 | LR: 4.71e-06 | 5776.5ms/step | 4.2 samples/s
+Step 18790/20005 | Loss: 2.8839 | LR: 4.64e-06 | 5780.8ms/step | 4.2 samples/s
+Step 18800/20005 | Loss: 2.8888 | LR: 4.56e-06 | 5782.9ms/step | 4.2 samples/s
+Step 18810/20005 | Loss: 2.8654 | LR: 4.49e-06 | 5780.0ms/step | 4.2 samples/s
+Step 18820/20005 | Loss: 2.8871 | LR: 4.41e-06 | 5779.5ms/step | 4.2 samples/s
+Step 18830/20005 | Loss: 2.8760 | LR: 4.34e-06 | 5773.8ms/step | 4.2 samples/s
+Step 18840/20005 | Loss: 2.8729 | LR: 4.26e-06 | 5776.2ms/step | 4.2 samples/s
+Step 18850/20005 | Loss: 2.8442 | LR: 4.19e-06 | 5791.7ms/step | 4.1 samples/s
+Step 18860/20005 | Loss: 2.8648 | LR: 4.12e-06 | 5769.8ms/step | 4.2 samples/s
+Step 18870/20005 | Loss: 2.8708 | LR: 4.05e-06 | 5763.9ms/step | 4.2 samples/s
+Step 18880/20005 | Loss: 2.8830 | LR: 3.98e-06 | 5760.7ms/step | 4.2 samples/s
+Step 18890/20005 | Loss: 2.8680 | LR: 3.91e-06 | 5777.8ms/step | 4.2 samples/s
+Step 18900/20005 | Loss: 2.8684 | LR: 3.84e-06 | 5775.8ms/step | 4.2 samples/s
+Step 18910/20005 | Loss: 2.8863 | LR: 3.77e-06 | 5773.6ms/step | 4.2 samples/s
+Step 18920/20005 | Loss: 2.8779 | LR: 3.70e-06 | 5760.1ms/step | 4.2 samples/s
+Step 18930/20005 | Loss: 2.8561 | LR: 3.63e-06 | 5774.8ms/step | 4.2 samples/s
+Step 18940/20005 | Loss: 2.8433 | LR: 3.56e-06 | 5778.9ms/step | 4.2 samples/s
+Step 18950/20005 | Loss: 2.8701 | LR: 3.50e-06 | 5773.9ms/step | 4.2 samples/s
+Step 18960/20005 | Loss: 2.8910 | LR: 3.43e-06 | 5780.5ms/step | 4.2 samples/s
+Step 18970/20005 | Loss: 2.8687 | LR: 3.37e-06 | 5775.0ms/step | 4.2 samples/s
+Step 18980/20005 | Loss: 2.8965 | LR: 3.30e-06 | 5775.0ms/step | 4.2 samples/s
+Step 18990/20005 | Loss: 2.8651 | LR: 3.24e-06 | 5778.0ms/step | 4.2 samples/s
+Step 19000/20005 | Loss: 2.8824 | LR: 3.17e-06 | 5776.1ms/step | 4.2 samples/s
+Validation at step 19000: Loss = 2.8979, Perplexity = 18.14
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-19000
+Step 19010/20005 | Loss: 2.8664 | LR: 3.11e-06 | 41143.6ms/step | 0.6 samples/s
+Step 19020/20005 | Loss: 2.8876 | LR: 3.05e-06 | 5779.8ms/step | 4.2 samples/s
+Step 19030/20005 | Loss: 2.8684 | LR: 2.99e-06 | 5774.5ms/step | 4.2 samples/s
+Step 19040/20005 | Loss: 2.8736 | LR: 2.93e-06 | 5781.5ms/step | 4.2 samples/s
+Step 19050/20005 | Loss: 2.8690 | LR: 2.87e-06 | 5774.6ms/step | 4.2 samples/s
+Step 19060/20005 | Loss: 2.8869 | LR: 2.81e-06 | 5771.3ms/step | 4.2 samples/s
+Step 19070/20005 | Loss: 2.8845 | LR: 2.75e-06 | 5764.2ms/step | 4.2 samples/s
+Step 19080/20005 | Loss: 2.8591 | LR: 2.69e-06 | 5769.6ms/step | 4.2 samples/s
+Step 19090/20005 | Loss: 2.8939 | LR: 2.63e-06 | 5763.8ms/step | 4.2 samples/s
+Step 19100/20005 | Loss: 2.8573 | LR: 2.57e-06 | 5759.8ms/step | 4.2 samples/s
+Step 19110/20005 | Loss: 2.8781 | LR: 2.52e-06 | 5773.0ms/step | 4.2 samples/s
+Step 19120/20005 | Loss: 2.8942 | LR: 2.46e-06 | 5791.6ms/step | 4.1 samples/s
+Step 19130/20005 | Loss: 2.8947 | LR: 2.41e-06 | 5778.0ms/step | 4.2 samples/s
+Step 19140/20005 | Loss: 2.8891 | LR: 2.35e-06 | 5776.2ms/step | 4.2 samples/s
+Step 19150/20005 | Loss: 2.8893 | LR: 2.30e-06 | 5775.6ms/step | 4.2 samples/s
+Step 19160/20005 | Loss: 2.8665 | LR: 2.24e-06 | 5776.2ms/step | 4.2 samples/s
+Step 19170/20005 | Loss: 2.9181 | LR: 2.19e-06 | 5779.7ms/step | 4.2 samples/s
+Step 19180/20005 | Loss: 2.8777 | LR: 2.14e-06 | 5778.1ms/step | 4.2 samples/s
+Step 19190/20005 | Loss: 2.8752 | LR: 2.09e-06 | 5777.8ms/step | 4.2 samples/s
+Step 19200/20005 | Loss: 2.8909 | LR: 2.04e-06 | 5779.3ms/step | 4.2 samples/s
+Step 19210/20005 | Loss: 2.8683 | LR: 1.99e-06 | 5777.0ms/step | 4.2 samples/s
+Step 19220/20005 | Loss: 2.8592 | LR: 1.94e-06 | 5775.7ms/step | 4.2 samples/s
+Step 19230/20005 | Loss: 2.9080 | LR: 1.89e-06 | 5777.4ms/step | 4.2 samples/s
+Step 19240/20005 | Loss: 2.8627 | LR: 1.84e-06 | 5775.9ms/step | 4.2 samples/s
+Step 19250/20005 | Loss: 2.8805 | LR: 1.79e-06 | 5776.8ms/step | 4.2 samples/s
+Validation at step 19250: Loss = 2.8979, Perplexity = 18.14
+Step 19260/20005 | Loss: 2.8908 | LR: 1.74e-06 | 40705.5ms/step | 0.6 samples/s
+Step 19270/20005 | Loss: 2.9098 | LR: 1.70e-06 | 5778.0ms/step | 4.2 samples/s
+Step 19280/20005 | Loss: 2.8604 | LR: 1.65e-06 | 5776.3ms/step | 4.2 samples/s
+Step 19290/20005 | Loss: 2.8455 | LR: 1.61e-06 | 5776.0ms/step | 4.2 samples/s
+Step 19300/20005 | Loss: 2.8978 | LR: 1.56e-06 | 5777.9ms/step | 4.2 samples/s
+Step 19310/20005 | Loss: 2.8934 | LR: 1.52e-06 | 5779.6ms/step | 4.2 samples/s
+Step 19320/20005 | Loss: 2.8838 | LR: 1.47e-06 | 5773.6ms/step | 4.2 samples/s
+Step 19330/20005 | Loss: 2.8824 | LR: 1.43e-06 | 5776.8ms/step | 4.2 samples/s
+Step 19340/20005 | Loss: 2.8992 | LR: 1.39e-06 | 5780.0ms/step | 4.2 samples/s
+Step 19350/20005 | Loss: 2.8552 | LR: 1.35e-06 | 5777.8ms/step | 4.2 samples/s
+Step 19360/20005 | Loss: 2.9096 | LR: 1.31e-06 | 5778.1ms/step | 4.2 samples/s
+Step 19370/20005 | Loss: 2.8786 | LR: 1.27e-06 | 5773.7ms/step | 4.2 samples/s
+Step 19380/20005 | Loss: 2.8628 | LR: 1.23e-06 | 5770.1ms/step | 4.2 samples/s
+Step 19390/20005 | Loss: 2.8680 | LR: 1.19e-06 | 5777.8ms/step | 4.2 samples/s
+Step 19400/20005 | Loss: 2.9039 | LR: 1.15e-06 | 5787.8ms/step | 4.1 samples/s
+Step 19410/20005 | Loss: 2.8735 | LR: 1.11e-06 | 5780.3ms/step | 4.2 samples/s
+Step 19420/20005 | Loss: 2.8843 | LR: 1.07e-06 | 5777.6ms/step | 4.2 samples/s
+Step 19430/20005 | Loss: 2.8824 | LR: 1.04e-06 | 5776.6ms/step | 4.2 samples/s
+Step 19440/20005 | Loss: 2.8936 | LR: 1.00e-06 | 5773.9ms/step | 4.2 samples/s
+Step 19450/20005 | Loss: 2.8645 | LR: 9.66e-07 | 5776.2ms/step | 4.2 samples/s
+Step 19460/20005 | Loss: 2.8658 | LR: 9.32e-07 | 5775.3ms/step | 4.2 samples/s
+Step 19470/20005 | Loss: 2.9018 | LR: 8.98e-07 | 5775.5ms/step | 4.2 samples/s
+Step 19480/20005 | Loss: 2.8946 | LR: 8.64e-07 | 5774.4ms/step | 4.2 samples/s
+Step 19490/20005 | Loss: 2.8730 | LR: 8.31e-07 | 5778.1ms/step | 4.2 samples/s
+Step 19500/20005 | Loss: 2.9203 | LR: 7.99e-07 | 5777.7ms/step | 4.2 samples/s
+Validation at step 19500: Loss = 2.8979, Perplexity = 18.14
+Step 19510/20005 | Loss: 2.8908 | LR: 7.68e-07 | 40711.9ms/step | 0.6 samples/s
+Step 19520/20005 | Loss: 2.8764 | LR: 7.37e-07 | 5778.0ms/step | 4.2 samples/s
+Step 19530/20005 | Loss: 2.8918 | LR: 7.07e-07 | 5777.3ms/step | 4.2 samples/s
+Step 19540/20005 | Loss: 2.8953 | LR: 6.77e-07 | 5776.2ms/step | 4.2 samples/s
+Step 19550/20005 | Loss: 2.8620 | LR: 6.48e-07 | 5779.5ms/step | 4.2 samples/s
+Step 19560/20005 | Loss: 2.8739 | LR: 6.20e-07 | 5778.6ms/step | 4.2 samples/s
+Step 19570/20005 | Loss: 2.9002 | LR: 5.92e-07 | 5777.4ms/step | 4.2 samples/s
+Step 19580/20005 | Loss: 2.8928 | LR: 5.65e-07 | 5776.2ms/step | 4.2 samples/s
+Step 19590/20005 | Loss: 2.8798 | LR: 5.39e-07 | 5774.3ms/step | 4.2 samples/s
+Step 19600/20005 | Loss: 2.8677 | LR: 5.13e-07 | 5779.9ms/step | 4.2 samples/s
+Step 19610/20005 | Loss: 2.8904 | LR: 4.88e-07 | 5780.2ms/step | 4.2 samples/s
+Step 19620/20005 | Loss: 2.9101 | LR: 4.64e-07 | 5777.7ms/step | 4.2 samples/s
+Step 19630/20005 | Loss: 2.8670 | LR: 4.40e-07 | 5777.9ms/step | 4.2 samples/s
+Step 19640/20005 | Loss: 2.8891 | LR: 4.16e-07 | 5778.0ms/step | 4.2 samples/s
+Step 19650/20005 | Loss: 2.8858 | LR: 3.94e-07 | 5780.1ms/step | 4.2 samples/s
+Step 19660/20005 | Loss: 2.8628 | LR: 3.72e-07 | 5775.7ms/step | 4.2 samples/s
+Step 19670/20005 | Loss: 2.9036 | LR: 3.50e-07 | 5778.7ms/step | 4.2 samples/s
+Step 19680/20005 | Loss: 2.8706 | LR: 3.30e-07 | 5779.3ms/step | 4.2 samples/s
+Step 19690/20005 | Loss: 2.8739 | LR: 3.10e-07 | 5778.1ms/step | 4.2 samples/s
+Step 19700/20005 | Loss: 2.8742 | LR: 2.90e-07 | 5771.9ms/step | 4.2 samples/s
+Step 19710/20005 | Loss: 2.8850 | LR: 2.71e-07 | 5775.1ms/step | 4.2 samples/s
+Step 19720/20005 | Loss: 2.8914 | LR: 2.53e-07 | 5776.7ms/step | 4.2 samples/s
+Step 19730/20005 | Loss: 2.8632 | LR: 2.36e-07 | 5772.3ms/step | 4.2 samples/s
+Step 19740/20005 | Loss: 2.8995 | LR: 2.19e-07 | 5777.8ms/step | 4.2 samples/s
+Step 19750/20005 | Loss: 2.8975 | LR: 2.02e-07 | 5777.7ms/step | 4.2 samples/s
+Validation at step 19750: Loss = 2.8979, Perplexity = 18.14
+Step 19760/20005 | Loss: 2.9018 | LR: 1.87e-07 | 40710.2ms/step | 0.6 samples/s
+Step 19770/20005 | Loss: 2.8736 | LR: 1.72e-07 | 5777.7ms/step | 4.2 samples/s
+Step 19780/20005 | Loss: 2.8970 | LR: 1.57e-07 | 5779.6ms/step | 4.2 samples/s
+Step 19790/20005 | Loss: 2.9008 | LR: 1.43e-07 | 5776.4ms/step | 4.2 samples/s
+Step 19800/20005 | Loss: 2.8631 | LR: 1.30e-07 | 5778.7ms/step | 4.2 samples/s
+Step 19810/20005 | Loss: 2.8762 | LR: 1.18e-07 | 5777.3ms/step | 4.2 samples/s
+Step 19820/20005 | Loss: 2.8971 | LR: 1.06e-07 | 5777.5ms/step | 4.2 samples/s
+Step 19830/20005 | Loss: 2.8783 | LR: 9.46e-08 | 5776.2ms/step | 4.2 samples/s
+Step 19840/20005 | Loss: 2.8528 | LR: 8.40e-08 | 5776.0ms/step | 4.2 samples/s
+Step 19850/20005 | Loss: 2.8973 | LR: 7.40e-08 | 5780.0ms/step | 4.2 samples/s
+Step 19860/20005 | Loss: 2.9007 | LR: 6.46e-08 | 5777.9ms/step | 4.2 samples/s
+Step 19870/20005 | Loss: 2.8989 | LR: 5.59e-08 | 5779.7ms/step | 4.2 samples/s
+Step 19880/20005 | Loss: 2.8650 | LR: 4.78e-08 | 5776.2ms/step | 4.2 samples/s
+Step 19890/20005 | Loss: 2.8813 | LR: 4.04e-08 | 5779.7ms/step | 4.2 samples/s
+Step 19900/20005 | Loss: 2.8916 | LR: 3.35e-08 | 5777.9ms/step | 4.2 samples/s
+Step 19910/20005 | Loss: 2.8695 | LR: 2.73e-08 | 5774.5ms/step | 4.2 samples/s
+Step 19920/20005 | Loss: 2.8699 | LR: 2.18e-08 | 5779.9ms/step | 4.2 samples/s
+Step 19930/20005 | Loss: 2.8845 | LR: 1.68e-08 | 5775.9ms/step | 4.2 samples/s
+Step 19940/20005 | Loss: 2.8951 | LR: 1.25e-08 | 5776.4ms/step | 4.2 samples/s
+Step 19950/20005 | Loss: 2.8941 | LR: 8.88e-09 | 5779.3ms/step | 4.2 samples/s
+Step 19960/20005 | Loss: 2.9041 | LR: 5.85e-09 | 5774.6ms/step | 4.2 samples/s
+Step 19970/20005 | Loss: 2.8624 | LR: 3.44e-09 | 5787.5ms/step | 4.1 samples/s
+Step 19980/20005 | Loss: 2.8843 | LR: 1.67e-09 | 5781.1ms/step | 4.2 samples/s
+Step 19990/20005 | Loss: 2.8803 | LR: 5.34e-10 | 5777.1ms/step | 4.2 samples/s
+Step 20000/20005 | Loss: 2.9004 | LR: 2.85e-11 | 5774.7ms/step | 4.2 samples/s
+Validation at step 20000: Loss = 2.8979, Perplexity = 18.14
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-20000
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-20005

epfl_final/wandb/run-20260513_051915-x63i12u5/files/requirements.txt

@@ -0,0 +1,97 @@
+pytz==2026.2
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.7.0
+tzdata==2026.2
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+safetensors==0.7.0
+regex==2026.5.9
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.5.2
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.2.6
+networkx==3.4.2
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.15
+hf-xet==1.5.0
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+async-timeout==5.0.1
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+typing-inspection==0.4.2
+sentry-sdk==2.59.0
+requests==2.34.0
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.4
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+multidict==6.7.1
+markdown-it-py==4.2.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+exceptiongroup==1.3.1
+cuda-bindings==13.2.0
+aiosignal==1.4.0
+yarl==1.23.0
+rich==15.0.0
+pydantic==2.13.4
+pandas==2.3.3
+nvidia-cusolver==12.0.4.66
+GitPython==3.1.50
+anyio==4.13.0
+wandb==0.26.1
+typer==0.25.1
+httpx==0.28.1
+aiohttp==3.13.5
+torch==2.11.0
+huggingface_hub==1.14.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.8.0
+PyGObject==3.42.1
+dbus-python==1.2.18
+distro==1.7.0
+pip==22.0.2
+setuptools==59.6.0
+ssh-import-id==5.11
+wheel==0.37.1

epfl_final/wandb/run-20260513_051915-x63i12u5/files/wandb-metadata.json

@@ -0,0 +1,36 @@
+{
+  "os":  "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.10.12",
+  "startedAt":  "2026-05-13T05:19:15.392571Z",
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "SparseAtt/train.py",
+  "codePathLocal":  "SparseAtt/train.py",
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse",
+  "host":  "h100-workspace-7-0-4",
+  "executable":  "/usr/bin/python3",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  1,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "7268479995904"
+    }
+  },
+  "memory":  {
+    "total":  "1623005741056"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-3ea6c840-350f-c68c-7562-6e0affa4e153"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "rwkqonaudhuzcez0jkojd2bi7cq7j6so"
+}

epfl_final/wandb/run-20260513_051915-x63i12u5/files/wandb-summary.json

@@ -0,0 +1 @@
+{"eval/perplexity":18.135404591913275,"_runtime":14766.281703552,"eval/loss":2.8978660824894904,"train/global_step":20000,"_step":20000,"speed/steps_per_sec":0.17316923694522243,"speed/samples_per_sec":4.156061686685338,"speed/ms_per_step":5774.697732925415,"_timestamp":1.7786642833359153e+09,"train/learning_rate":2.8451042966004538e-11,"_wandb":{"runtime":14766},"train/epoch":0.4716981132075472,"train/loss":2.900362515449524}

epfl_final/wandb/run-20260513_051915-x63i12u5/logs/debug.log

@@ -0,0 +1,24 @@
+2026-05-13 05:19:15,409 INFO    MainThread:37729 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-05-13 05:19:15,409 INFO    MainThread:37729 [wandb_setup.py:_flush():81] Configure stats pid to 37729
+2026-05-13 05:19:15,410 INFO    MainThread:37729 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-13 05:19:15,410 INFO    MainThread:37729 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260513_051915-x63i12u5/logs/debug.log
+2026-05-13 05:19:15,411 INFO    MainThread:37729 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260513_051915-x63i12u5/logs/debug-internal.log
+2026-05-13 05:19:15,411 INFO    MainThread:37729 [wandb_init.py:init():850] calling init triggers
+2026-05-13 05:19:15,412 INFO    MainThread:37729 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-18000', 'num_epochs': 1, 'max_steps': 20005, 'per_device_batch_size': 3, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 250, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 200, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'dense-resume-18k', 'wandb_tags': ['qwen3', '0.6b', 'dense-resume'], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 10, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'dense_mode': True, '_wandb': {}}
+2026-05-13 05:19:15,412 INFO    MainThread:37729 [wandb_init.py:init():898] starting backend
+2026-05-13 05:19:15,628 INFO    MainThread:37729 [wandb_init.py:init():913] sending inform_init request
+2026-05-13 05:19:15,932 INFO    MainThread:37729 [wandb_init.py:init():918] backend started and connected
+2026-05-13 05:19:15,934 INFO    MainThread:37729 [wandb_init.py:init():988] updated telemetry
+2026-05-13 05:19:15,934 INFO    MainThread:37729 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-05-13 05:19:16,334 INFO    MainThread:37729 [wandb_init.py:init():1056] starting run threads in backend
+2026-05-13 05:19:16,389 INFO    MainThread:37729 [wandb_run.py:_console_start():2554] atexit reg
+2026-05-13 05:19:16,389 INFO    MainThread:37729 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-05-13 05:19:16,389 INFO    MainThread:37729 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-05-13 05:19:16,389 INFO    MainThread:37729 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-05-13 05:19:16,391 INFO    MainThread:37729 [wandb_init.py:init():1094] run started, returning control to user process
+2026-05-13 09:25:22,614 INFO    MainThread:37729 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/x63i12u5
+2026-05-13 09:25:22,615 INFO    MainThread:37729 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
+2026-05-13 09:25:22,615 INFO    MainThread:37729 [wandb_run.py:_restore():2502] restore
+2026-05-13 09:25:22,616 INFO    MainThread:37729 [wandb_run.py:_restore():2508] restore done
+2026-05-13 09:25:23,635 INFO    MainThread:37729 [wandb_run.py:_footer_sync_info():3889] logging synced files

epfl_final/wandb/run-20260513_051915-x63i12u5/run-x63i12u5.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd79cea41e5d4174d9a111f6137133e8d48ce83aa7f2ffa88f85b75e57badf7c
+size 2146561

epfl_final/wandb/run-20260513_052440-i0f0xwcq/files/config.yaml

@@ -0,0 +1,140 @@
+_wandb:
+    value:
+        cli_version: 0.26.1
+        e:
+            oi35laauinf857cg8y9o9liqwb40t030:
+                codePath: SparseAtt/train.py
+                codePathLocal: SparseAtt/train.py
+                cpu_count: 96
+                cpu_count_logical: 192
+                cudaVersion: "13.0"
+                disk:
+                    /:
+                        total: "9374931091456"
+                        used: "7271060475904"
+                email: ruben.navasardyan@mail.utoronto.ca
+                executable: /usr/bin/python3
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 1
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-572b71c8-9d82-4b1c-e4f6-ed5616fd870a
+                host: h100-workspace-6-0-7
+                memory:
+                    total: "1623005741056"
+                os: Linux-6.8.0-59-generic-x86_64-with-glibc2.35
+                program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
+                python: CPython 3.10.12
+                root: /mloscratch/homes/navasard/sparse
+                startedAt: "2026-05-13T05:24:40.371637Z"
+                writerId: oi35laauinf857cg8y9o9liqwb40t030
+        m: []
+        python_version: 3.10.12
+        t:
+            "1":
+                - 1
+                - 11
+                - 49
+                - 51
+                - 71
+            "2":
+                - 1
+                - 11
+                - 49
+                - 51
+                - 71
+            "3":
+                - 2
+                - 13
+                - 15
+                - 16
+                - 61
+            "4": 3.10.12
+            "5": 0.26.1
+            "6": 5.8.1
+            "12": 0.26.1
+            "13": linux-x86_64
+attention_type:
+    value: sparse_stochastic
+checkpoint_activations:
+    value: false
+dense_mode:
+    value: true
+eval_batches:
+    value: 200
+eval_steps:
+    value: 250
+gradient_accumulation_steps:
+    value: 8
+init_from_scratch:
+    value: true
+learning_rate:
+    value: 0.0005
+logging_steps:
+    value: 10
+lr_scheduler:
+    value: cosine
+max_grad_norm:
+    value: 1
+max_seq_length:
+    value: 8192
+max_steps:
+    value: 20005
+mixed_precision:
+    value: bf16
+model_dtype:
+    value: bfloat16
+model_path:
+    value: /mloscratch/homes/navasard/qwen3_06b_base
+num_epochs:
+    value: 1
+num_workers:
+    value: 8
+output_dir:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723
+per_device_batch_size:
+    value: 3
+resume_from_checkpoint:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-19000
+save_steps:
+    value: 1000
+seed:
+    value: 42
+sparse_block_size:
+    value: 128
+sparse_layer_indices:
+    value: null
+sparse_mask_cache_dir:
+    value: null
+sparse_mask_pool_size:
+    value: 10
+sparse_num_random_blocks:
+    value: 31
+sparse_val_dense_compare:
+    value: false
+strategy:
+    value: ddp
+train_path:
+    value: /mloscratch/homes/navasard/data/fineweb/tokenized
+val_path:
+    value: null
+verbose:
+    value: true
+wandb_enabled:
+    value: true
+wandb_name:
+    value: dense-resume-18k
+wandb_project:
+    value: stochastic_sparsity
+wandb_tags:
+    value:
+        - qwen3
+        - 0.6b
+        - dense-resume
+warmup_steps:
+    value: 250
+weight_decay:
+    value: 0.01

epfl_final/wandb/run-20260513_052440-i0f0xwcq/files/output.log

@@ -0,0 +1,211 @@
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1024
+  num_hidden_layers: 28
+  num_attention_heads: 16
+  num_key_value_heads: 8
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 10
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 596,049,920
+  Trainable parameters: 596,049,920
+  Non-trainable parameters: 0
+  Model size (BF16): 1.19 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [3, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [3, 8192, 1024]           --                        True
+│    └─Embedding: 2-1                                        [3, 8192]                 [3, 8192, 1024]           155,582,464               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    └─Qwen3RMSNorm: 2-3                                     [3, 8192, 1024]           [3, 8192, 1024]           1,024                     True
+├─Linear: 1-2                                                [3, 8192, 1024]           [3, 8192, 151936]         155,582,464               True
+================================================================================================================================================================
+Total params: 751,632,384
+Trainable params: 751,632,384
+Non-trainable params: 0
+Total mult-adds (G): 2.25
+================================================================================================================================================================
+Input size (MB): 0.20
+Forward/backward pass size (MB): 63052.97
+Params size (MB): 1503.26
+Estimated Total Size (MB): 64556.43
+================================================================================================================================================================
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Loaded 5088012 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1272003)
+Dataset split: 1017602 train, 254401 validation
+Training examples (packed, train split):  1017602
+Validation examples (packed, val split):  254401
+Per-device batch size:                    3
+World size (GPUs):                        1
+Gradient accumulation steps:              8
+Effective batch size (seqs/step):         24
+Sequence length:                          8192
+Tokens per optimizer step:                196,608
+Micro-batches per epoch (per rank):       339200
+Optimizer steps per epoch:                42400
+Total optimizer steps:                    20005  (capped by max_steps; 0.47 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-19000...
+Warning: no RNG state file found for rank 0; RNG will not be restored.
+Resumed from checkpoint at step 18999 (epoch 0, micro_step 0)
+Resuming training from step 18999
+Starting training...
+Warning: no loader_gen_state file found; DataLoader shuffle order on resume may differ from original run.
+Step 19000/20005 | Loss: 0.2856 | LR: 3.18e-06 | 64856.9ms/step | 0.4 samples/s
+Validation at step 19000: Loss = 2.9082, Perplexity = 18.32
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-19000
+Step 19010/20005 | Loss: 2.8841 | LR: 3.12e-06 | 43506.1ms/step | 0.6 samples/s
+Step 19020/20005 | Loss: 2.9121 | LR: 3.05e-06 | 5798.6ms/step | 4.1 samples/s
+Step 19030/20005 | Loss: 2.8848 | LR: 2.99e-06 | 5799.2ms/step | 4.1 samples/s
+Step 19040/20005 | Loss: 2.9030 | LR: 2.93e-06 | 5796.6ms/step | 4.1 samples/s
+Step 19050/20005 | Loss: 2.9076 | LR: 2.87e-06 | 5801.1ms/step | 4.1 samples/s
+Step 19060/20005 | Loss: 2.9055 | LR: 2.81e-06 | 5802.7ms/step | 4.1 samples/s
+Step 19070/20005 | Loss: 2.9181 | LR: 2.75e-06 | 5800.7ms/step | 4.1 samples/s
+Step 19080/20005 | Loss: 2.8634 | LR: 2.69e-06 | 5803.0ms/step | 4.1 samples/s
+Step 19090/20005 | Loss: 2.9011 | LR: 2.64e-06 | 5803.1ms/step | 4.1 samples/s
+Step 19100/20005 | Loss: 2.8946 | LR: 2.58e-06 | 5801.6ms/step | 4.1 samples/s
+Step 19110/20005 | Loss: 2.8855 | LR: 2.52e-06 | 5846.1ms/step | 4.1 samples/s
+Step 19120/20005 | Loss: 2.8977 | LR: 2.47e-06 | 5802.8ms/step | 4.1 samples/s
+Step 19130/20005 | Loss: 2.8775 | LR: 2.41e-06 | 5799.4ms/step | 4.1 samples/s
+Step 19140/20005 | Loss: 2.8940 | LR: 2.36e-06 | 5801.6ms/step | 4.1 samples/s
+Step 19150/20005 | Loss: 2.8891 | LR: 2.30e-06 | 5801.3ms/step | 4.1 samples/s
+Step 19160/20005 | Loss: 2.8665 | LR: 2.25e-06 | 5803.0ms/step | 4.1 samples/s
+Step 19170/20005 | Loss: 2.8587 | LR: 2.20e-06 | 5802.4ms/step | 4.1 samples/s
+Step 19180/20005 | Loss: 2.8930 | LR: 2.14e-06 | 5798.3ms/step | 4.1 samples/s
+Step 19190/20005 | Loss: 2.9038 | LR: 2.09e-06 | 5801.6ms/step | 4.1 samples/s
+Step 19200/20005 | Loss: 2.8835 | LR: 2.04e-06 | 5799.8ms/step | 4.1 samples/s
+Step 19210/20005 | Loss: 2.9115 | LR: 1.99e-06 | 5801.6ms/step | 4.1 samples/s
+Step 19220/20005 | Loss: 2.9010 | LR: 1.94e-06 | 5800.3ms/step | 4.1 samples/s
+Step 19230/20005 | Loss: 2.9071 | LR: 1.89e-06 | 5800.3ms/step | 4.1 samples/s
+Step 19240/20005 | Loss: 2.9033 | LR: 1.84e-06 | 5802.5ms/step | 4.1 samples/s
+Step 19250/20005 | Loss: 2.8618 | LR: 1.80e-06 | 5799.7ms/step | 4.1 samples/s
+Validation at step 19250: Loss = 2.9042, Perplexity = 18.25
+Step 19260/20005 | Loss: 2.8789 | LR: 1.75e-06 | 40859.3ms/step | 0.6 samples/s
+Step 19270/20005 | Loss: 2.9005 | LR: 1.70e-06 | 5798.0ms/step | 4.1 samples/s
+Step 19280/20005 | Loss: 2.9064 | LR: 1.66e-06 | 5802.0ms/step | 4.1 samples/s
+Step 19290/20005 | Loss: 2.9035 | LR: 1.61e-06 | 5802.4ms/step | 4.1 samples/s
+Step 19300/20005 | Loss: 2.8814 | LR: 1.57e-06 | 5800.4ms/step | 4.1 samples/s
+Step 19310/20005 | Loss: 2.9034 | LR: 1.52e-06 | 5799.9ms/step | 4.1 samples/s
+Step 19320/20005 | Loss: 2.8996 | LR: 1.48e-06 | 5800.9ms/step | 4.1 samples/s
+Step 19330/20005 | Loss: 2.8856 | LR: 1.43e-06 | 5802.4ms/step | 4.1 samples/s
+Step 19340/20005 | Loss: 2.8643 | LR: 1.39e-06 | 5798.2ms/step | 4.1 samples/s
+Step 19350/20005 | Loss: 2.8999 | LR: 1.35e-06 | 5801.1ms/step | 4.1 samples/s
+Step 19360/20005 | Loss: 2.8943 | LR: 1.31e-06 | 5803.3ms/step | 4.1 samples/s
+Step 19370/20005 | Loss: 2.8913 | LR: 1.27e-06 | 5801.3ms/step | 4.1 samples/s
+Step 19380/20005 | Loss: 2.8962 | LR: 1.23e-06 | 5800.5ms/step | 4.1 samples/s
+Step 19390/20005 | Loss: 2.8758 | LR: 1.19e-06 | 5802.7ms/step | 4.1 samples/s
+Step 19400/20005 | Loss: 2.8741 | LR: 1.15e-06 | 5800.3ms/step | 4.1 samples/s
+Step 19410/20005 | Loss: 2.9121 | LR: 1.11e-06 | 5798.2ms/step | 4.1 samples/s
+Step 19420/20005 | Loss: 2.8689 | LR: 1.08e-06 | 5798.5ms/step | 4.1 samples/s
+Step 19430/20005 | Loss: 2.8710 | LR: 1.04e-06 | 5798.4ms/step | 4.1 samples/s
+Step 19440/20005 | Loss: 2.8682 | LR: 1.00e-06 | 5801.2ms/step | 4.1 samples/s
+Step 19450/20005 | Loss: 2.8935 | LR: 9.70e-07 | 5801.5ms/step | 4.1 samples/s
+Step 19460/20005 | Loss: 2.8957 | LR: 9.35e-07 | 5801.1ms/step | 4.1 samples/s
+Step 19470/20005 | Loss: 2.8909 | LR: 9.01e-07 | 5798.9ms/step | 4.1 samples/s
+Step 19480/20005 | Loss: 2.8730 | LR: 8.67e-07 | 5796.8ms/step | 4.1 samples/s
+Step 19490/20005 | Loss: 2.8959 | LR: 8.35e-07 | 5797.7ms/step | 4.1 samples/s
+Step 19500/20005 | Loss: 2.8824 | LR: 8.03e-07 | 5797.8ms/step | 4.1 samples/s
+Validation at step 19500: Loss = 2.9036, Perplexity = 18.24
+Step 19510/20005 | Loss: 2.8928 | LR: 7.71e-07 | 40869.8ms/step | 0.6 samples/s
+Step 19520/20005 | Loss: 2.8678 | LR: 7.40e-07 | 5801.5ms/step | 4.1 samples/s
+Step 19530/20005 | Loss: 2.8681 | LR: 7.10e-07 | 5800.2ms/step | 4.1 samples/s
+Step 19540/20005 | Loss: 2.8642 | LR: 6.80e-07 | 5798.5ms/step | 4.1 samples/s
+Step 19550/20005 | Loss: 2.9356 | LR: 6.51e-07 | 5801.8ms/step | 4.1 samples/s
+Step 19560/20005 | Loss: 2.8972 | LR: 6.23e-07 | 5802.1ms/step | 4.1 samples/s
+Step 19570/20005 | Loss: 2.9023 | LR: 5.95e-07 | 5798.3ms/step | 4.1 samples/s
+Step 19580/20005 | Loss: 2.8574 | LR: 5.68e-07 | 5836.4ms/step | 4.1 samples/s
+Step 19590/20005 | Loss: 2.8756 | LR: 5.42e-07 | 5807.5ms/step | 4.1 samples/s
+Step 19600/20005 | Loss: 2.8952 | LR: 5.16e-07 | 5800.3ms/step | 4.1 samples/s
+Step 19610/20005 | Loss: 2.8886 | LR: 4.91e-07 | 5799.3ms/step | 4.1 samples/s
+Step 19620/20005 | Loss: 2.8894 | LR: 4.66e-07 | 5803.7ms/step | 4.1 samples/s
+Step 19630/20005 | Loss: 2.8946 | LR: 4.42e-07 | 5803.9ms/step | 4.1 samples/s
+Step 19640/20005 | Loss: 2.9110 | LR: 4.19e-07 | 5802.1ms/step | 4.1 samples/s
+Step 19650/20005 | Loss: 2.9165 | LR: 3.96e-07 | 5800.4ms/step | 4.1 samples/s
+Step 19660/20005 | Loss: 2.8830 | LR: 3.74e-07 | 5804.4ms/step | 4.1 samples/s
+Step 19670/20005 | Loss: 2.8803 | LR: 3.53e-07 | 5801.1ms/step | 4.1 samples/s
+Step 19680/20005 | Loss: 2.8821 | LR: 3.32e-07 | 5802.8ms/step | 4.1 samples/s
+Step 19690/20005 | Loss: 2.8748 | LR: 3.12e-07 | 5801.4ms/step | 4.1 samples/s
+Step 19700/20005 | Loss: 2.8839 | LR: 2.92e-07 | 5802.6ms/step | 4.1 samples/s
+Step 19710/20005 | Loss: 2.8690 | LR: 2.73e-07 | 5803.1ms/step | 4.1 samples/s
+Step 19720/20005 | Loss: 2.8843 | LR: 2.55e-07 | 5799.9ms/step | 4.1 samples/s
+Step 19730/20005 | Loss: 2.8619 | LR: 2.37e-07 | 5802.0ms/step | 4.1 samples/s
+Step 19740/20005 | Loss: 2.8787 | LR: 2.20e-07 | 5803.5ms/step | 4.1 samples/s
+Step 19750/20005 | Loss: 2.8705 | LR: 2.04e-07 | 5801.0ms/step | 4.1 samples/s
+Validation at step 19750: Loss = 2.9035, Perplexity = 18.24
+Step 19760/20005 | Loss: 2.8728 | LR: 1.88e-07 | 40861.9ms/step | 0.6 samples/s
+Step 19770/20005 | Loss: 2.9028 | LR: 1.73e-07 | 5803.2ms/step | 4.1 samples/s
+Step 19780/20005 | Loss: 2.8959 | LR: 1.59e-07 | 5801.9ms/step | 4.1 samples/s
+Step 19790/20005 | Loss: 2.8860 | LR: 1.45e-07 | 5800.9ms/step | 4.1 samples/s
+Step 19800/20005 | Loss: 2.8817 | LR: 1.32e-07 | 5799.4ms/step | 4.1 samples/s
+Step 19810/20005 | Loss: 2.8840 | LR: 1.19e-07 | 5798.4ms/step | 4.1 samples/s
+Step 19820/20005 | Loss: 2.8737 | LR: 1.07e-07 | 5797.4ms/step | 4.1 samples/s
+Step 19830/20005 | Loss: 2.8726 | LR: 9.57e-08 | 5800.9ms/step | 4.1 samples/s
+Step 19840/20005 | Loss: 2.8749 | LR: 8.50e-08 | 5800.1ms/step | 4.1 samples/s
+Step 19850/20005 | Loss: 2.8855 | LR: 7.50e-08 | 5798.7ms/step | 4.1 samples/s
+Step 19860/20005 | Loss: 2.8639 | LR: 6.55e-08 | 5798.3ms/step | 4.1 samples/s
+Step 19870/20005 | Loss: 2.9065 | LR: 5.68e-08 | 5798.5ms/step | 4.1 samples/s
+Step 19880/20005 | Loss: 2.8865 | LR: 4.86e-08 | 5799.3ms/step | 4.1 samples/s
+Step 19890/20005 | Loss: 2.8910 | LR: 4.11e-08 | 5801.9ms/step | 4.1 samples/s
+Step 19900/20005 | Loss: 2.8658 | LR: 3.42e-08 | 5806.0ms/step | 4.1 samples/s
+Step 19910/20005 | Loss: 2.8996 | LR: 2.79e-08 | 5799.8ms/step | 4.1 samples/s
+Step 19920/20005 | Loss: 2.8895 | LR: 2.23e-08 | 5799.6ms/step | 4.1 samples/s
+Step 19930/20005 | Loss: 2.8606 | LR: 1.73e-08 | 5802.8ms/step | 4.1 samples/s
+Step 19940/20005 | Loss: 2.8804 | LR: 1.29e-08 | 5800.1ms/step | 4.1 samples/s
+Step 19950/20005 | Loss: 2.8914 | LR: 9.22e-09 | 5803.8ms/step | 4.1 samples/s
+Step 19960/20005 | Loss: 2.9020 | LR: 6.12e-09 | 5804.7ms/step | 4.1 samples/s
+Step 19970/20005 | Loss: 2.9103 | LR: 3.65e-09 | 5801.9ms/step | 4.1 samples/s
+Step 19980/20005 | Loss: 2.9041 | LR: 1.82e-09 | 5802.7ms/step | 4.1 samples/s
+Step 19990/20005 | Loss: 2.8692 | LR: 6.20e-10 | 5801.4ms/step | 4.1 samples/s
+Step 20000/20005 | Loss: 2.8643 | LR: 5.06e-11 | 5798.9ms/step | 4.1 samples/s
+Validation at step 20000: Loss = 2.9035, Perplexity = 18.24
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-20000
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-20005

epfl_final/wandb/run-20260513_052440-i0f0xwcq/files/requirements.txt

@@ -0,0 +1,97 @@
+pytz==2026.2
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.7.0
+tzdata==2026.2
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+safetensors==0.7.0
+regex==2026.5.9
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.5.2
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.2.6
+networkx==3.4.2
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.15
+hf-xet==1.5.0
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+async-timeout==5.0.1
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+typing-inspection==0.4.2
+sentry-sdk==2.59.0
+requests==2.34.0
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.4
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+multidict==6.7.1
+markdown-it-py==4.2.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+exceptiongroup==1.3.1
+cuda-bindings==13.2.0
+aiosignal==1.4.0
+yarl==1.23.0
+rich==15.0.0
+pydantic==2.13.4
+pandas==2.3.3
+nvidia-cusolver==12.0.4.66
+GitPython==3.1.50
+anyio==4.13.0
+wandb==0.26.1
+typer==0.25.1
+httpx==0.28.1
+aiohttp==3.13.5
+torch==2.11.0
+huggingface_hub==1.14.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.8.1
+PyGObject==3.42.1
+dbus-python==1.2.18
+distro==1.7.0
+pip==22.0.2
+setuptools==59.6.0
+ssh-import-id==5.11
+wheel==0.37.1

epfl_final/wandb/run-20260513_052440-i0f0xwcq/files/wandb-metadata.json

@@ -0,0 +1,36 @@
+{
+  "os":  "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.10.12",
+  "startedAt":  "2026-05-13T05:24:40.371637Z",
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "SparseAtt/train.py",
+  "codePathLocal":  "SparseAtt/train.py",
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse",
+  "host":  "h100-workspace-6-0-7",
+  "executable":  "/usr/bin/python3",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  1,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "7271060475904"
+    }
+  },
+  "memory":  {
+    "total":  "1623005741056"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-572b71c8-9d82-4b1c-e4f6-ed5616fd870a"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "oi35laauinf857cg8y9o9liqwb40t030"
+}

epfl_final/wandb/run-20260513_052440-i0f0xwcq/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/loss":2.864294093847275,"speed/ms_per_step":5798.852062225342,"train/global_step":20000,"_timestamp":1.77865753894026e+09,"eval/loss":2.903532140851021,"_wandb":{"runtime":7696},"eval/perplexity":18.238452514824385,"_step":20000,"train/epoch":0.4716981132075472,"train/learning_rate":5.0579631205582844e-11,"speed/steps_per_sec":0.17244792404934098,"speed/samples_per_sec":4.138750177184184,"_runtime":7696.841809526}

epfl_final/wandb/run-20260513_052440-i0f0xwcq/logs/debug.log

@@ -0,0 +1,24 @@
+2026-05-13 05:24:40,389 INFO    MainThread:2274 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-05-13 05:24:40,389 INFO    MainThread:2274 [wandb_setup.py:_flush():81] Configure stats pid to 2274
+2026-05-13 05:24:40,390 INFO    MainThread:2274 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-13 05:24:40,390 INFO    MainThread:2274 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260513_052440-i0f0xwcq/logs/debug.log
+2026-05-13 05:24:40,391 INFO    MainThread:2274 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260513_052440-i0f0xwcq/logs/debug-internal.log
+2026-05-13 05:24:40,391 INFO    MainThread:2274 [wandb_init.py:init():850] calling init triggers
+2026-05-13 05:24:40,392 INFO    MainThread:2274 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-19000', 'num_epochs': 1, 'max_steps': 20005, 'per_device_batch_size': 3, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 250, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 200, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'dense-resume-18k', 'wandb_tags': ['qwen3', '0.6b', 'dense-resume'], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 10, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'dense_mode': True, '_wandb': {}}
+2026-05-13 05:24:40,392 INFO    MainThread:2274 [wandb_init.py:init():898] starting backend
+2026-05-13 05:24:40,608 INFO    MainThread:2274 [wandb_init.py:init():913] sending inform_init request
+2026-05-13 05:24:40,902 INFO    MainThread:2274 [wandb_init.py:init():918] backend started and connected
+2026-05-13 05:24:40,904 INFO    MainThread:2274 [wandb_init.py:init():988] updated telemetry
+2026-05-13 05:24:40,904 INFO    MainThread:2274 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-05-13 05:24:41,233 INFO    MainThread:2274 [wandb_init.py:init():1056] starting run threads in backend
+2026-05-13 05:24:41,289 INFO    MainThread:2274 [wandb_run.py:_console_start():2554] atexit reg
+2026-05-13 05:24:41,289 INFO    MainThread:2274 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-05-13 05:24:41,289 INFO    MainThread:2274 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-05-13 05:24:41,289 INFO    MainThread:2274 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-05-13 05:24:41,290 INFO    MainThread:2274 [wandb_init.py:init():1094] run started, returning control to user process
+2026-05-13 07:32:58,073 INFO    MainThread:2274 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/i0f0xwcq
+2026-05-13 07:32:58,074 INFO    MainThread:2274 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
+2026-05-13 07:32:58,074 INFO    MainThread:2274 [wandb_run.py:_restore():2502] restore
+2026-05-13 07:32:58,075 INFO    MainThread:2274 [wandb_run.py:_restore():2508] restore done
+2026-05-13 07:32:59,635 INFO    MainThread:2274 [wandb_run.py:_footer_sync_info():3889] logging synced files

epfl_final/wandb/run-20260513_052440-i0f0xwcq/run-i0f0xwcq.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f86037cf88e114dae924fecec941a332ce5737d02df2b556d277a42e51fd162f
+size 1114772

epfl_final/wandb/run-20260513_151725-qudv4nmq/files/output.log

@@ -0,0 +1,236 @@
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1024
+  num_hidden_layers: 28
+  num_attention_heads: 16
+  num_key_value_heads: 8
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 10
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 596,049,920
+  Trainable parameters: 596,049,920
+  Non-trainable parameters: 0
+  Model size (BF16): 1.19 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [3, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [3, 8192, 1024]           --                        True
+│    └─Embedding: 2-1                                        [3, 8192]                 [3, 8192, 1024]           155,582,464               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    └─Qwen3RMSNorm: 2-3                                     [3, 8192, 1024]           [3, 8192, 1024]           1,024                     True
+├─Linear: 1-2                                                [3, 8192, 1024]           [3, 8192, 151936]         155,582,464               True
+================================================================================================================================================================
+Total params: 751,632,384
+Trainable params: 751,632,384
+Non-trainable params: 0
+Total mult-adds (G): 2.25
+================================================================================================================================================================
+Input size (MB): 0.20
+Forward/backward pass size (MB): 63052.97
+Params size (MB): 1503.26
+Estimated Total Size (MB): 64556.43
+================================================================================================================================================================
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Loaded 5088012 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1272003)
+Dataset split: 1017602 train, 254401 validation
+Training examples (packed, train split):  1017602
+Validation examples (packed, val split):  254401
+Per-device batch size:                    3
+World size (GPUs):                        1
+Gradient accumulation steps:              8
+Effective batch size (seqs/step):         24
+Sequence length:                          8192
+Tokens per optimizer step:                196,608
+Micro-batches per epoch (per rank):       339200
+Optimizer steps per epoch:                42400
+Total optimizer steps:                    20005  (capped by max_steps; 0.47 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-16000...
+Warning: no RNG state file found for rank 0; RNG will not be restored.
+Resumed from checkpoint at step 15999 (epoch 0, micro_step 0)
+Resuming training from step 15999
+Starting training...
+Warning: no loader_gen_state file found; DataLoader shuffle order on resume may differ from original run.
+Step 16000/20005 | Loss: 0.2859 | LR: 4.90e-05 | 61450.0ms/step | 0.4 samples/s
+Validation at step 16000: Loss = 2.9095, Perplexity = 18.35
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-16000
+Step 16010/20005 | Loss: 2.8805 | LR: 4.88e-05 | 42322.3ms/step | 0.6 samples/s
+Step 16020/20005 | Loss: 2.9052 | LR: 4.85e-05 | 5797.6ms/step | 4.1 samples/s
+Step 16030/20005 | Loss: 2.8774 | LR: 4.83e-05 | 5801.3ms/step | 4.1 samples/s
+Step 16040/20005 | Loss: 2.8951 | LR: 4.80e-05 | 5783.8ms/step | 4.1 samples/s
+Step 16050/20005 | Loss: 2.8999 | LR: 4.78e-05 | 5774.8ms/step | 4.2 samples/s
+Step 16060/20005 | Loss: 2.8975 | LR: 4.76e-05 | 5774.1ms/step | 4.2 samples/s
+Step 16070/20005 | Loss: 2.9103 | LR: 4.73e-05 | 5786.3ms/step | 4.1 samples/s
+Step 16080/20005 | Loss: 2.8558 | LR: 4.71e-05 | 5793.9ms/step | 4.1 samples/s
+Step 16090/20005 | Loss: 2.8936 | LR: 4.69e-05 | 5796.0ms/step | 4.1 samples/s
+Step 16100/20005 | Loss: 2.8871 | LR: 4.67e-05 | 5782.7ms/step | 4.2 samples/s
+Step 16110/20005 | Loss: 2.8781 | LR: 4.64e-05 | 5777.7ms/step | 4.2 samples/s
+Step 16120/20005 | Loss: 2.8904 | LR: 4.62e-05 | 5775.9ms/step | 4.2 samples/s
+Step 16130/20005 | Loss: 2.8697 | LR: 4.60e-05 | 5791.7ms/step | 4.1 samples/s
+Step 16140/20005 | Loss: 2.8868 | LR: 4.57e-05 | 5801.7ms/step | 4.1 samples/s
+Step 16150/20005 | Loss: 2.8816 | LR: 4.55e-05 | 5800.1ms/step | 4.1 samples/s
+Step 16160/20005 | Loss: 2.8591 | LR: 4.53e-05 | 5784.0ms/step | 4.1 samples/s
+Step 16170/20005 | Loss: 2.8515 | LR: 4.50e-05 | 5782.4ms/step | 4.2 samples/s
+Step 16180/20005 | Loss: 2.8859 | LR: 4.48e-05 | 5784.0ms/step | 4.1 samples/s
+Step 16190/20005 | Loss: 2.8966 | LR: 4.46e-05 | 5787.5ms/step | 4.1 samples/s
+Step 16200/20005 | Loss: 2.8766 | LR: 4.44e-05 | 5802.8ms/step | 4.1 samples/s
+Step 16210/20005 | Loss: 2.9043 | LR: 4.41e-05 | 5805.9ms/step | 4.1 samples/s
+Step 16220/20005 | Loss: 2.8939 | LR: 4.39e-05 | 5783.9ms/step | 4.1 samples/s
+Step 16230/20005 | Loss: 2.8999 | LR: 4.37e-05 | 5777.6ms/step | 4.2 samples/s
+Step 16240/20005 | Loss: 2.8962 | LR: 4.35e-05 | 5775.0ms/step | 4.2 samples/s
+Step 16250/20005 | Loss: 2.8544 | LR: 4.32e-05 | 5791.8ms/step | 4.1 samples/s
+Validation at step 16250: Loss = 2.8974, Perplexity = 18.13
+Step 16260/20005 | Loss: 2.8719 | LR: 4.30e-05 | 39967.3ms/step | 0.6 samples/s
+Step 16270/20005 | Loss: 2.8932 | LR: 4.28e-05 | 5794.1ms/step | 4.1 samples/s
+Step 16280/20005 | Loss: 2.8991 | LR: 4.26e-05 | 5778.4ms/step | 4.2 samples/s
+Step 16290/20005 | Loss: 2.8963 | LR: 4.24e-05 | 5782.1ms/step | 4.2 samples/s
+Step 16300/20005 | Loss: 2.8744 | LR: 4.21e-05 | 5779.4ms/step | 4.2 samples/s
+Step 16310/20005 | Loss: 2.8959 | LR: 4.19e-05 | 5794.2ms/step | 4.1 samples/s
+Step 16320/20005 | Loss: 2.8922 | LR: 4.17e-05 | 5805.9ms/step | 4.1 samples/s
+Step 16330/20005 | Loss: 2.8783 | LR: 4.15e-05 | 5792.6ms/step | 4.1 samples/s
+Step 16340/20005 | Loss: 2.8571 | LR: 4.13e-05 | 5780.2ms/step | 4.2 samples/s
+Step 16350/20005 | Loss: 2.8927 | LR: 4.10e-05 | 5782.3ms/step | 4.2 samples/s
+Step 16360/20005 | Loss: 2.8868 | LR: 4.08e-05 | 5774.0ms/step | 4.2 samples/s
+Step 16370/20005 | Loss: 2.8835 | LR: 4.06e-05 | 5793.8ms/step | 4.1 samples/s
+Step 16380/20005 | Loss: 2.8890 | LR: 4.04e-05 | 5805.2ms/step | 4.1 samples/s
+Step 16390/20005 | Loss: 2.8688 | LR: 4.02e-05 | 5792.8ms/step | 4.1 samples/s
+Step 16400/20005 | Loss: 2.8665 | LR: 3.99e-05 | 5775.8ms/step | 4.2 samples/s
+Step 16410/20005 | Loss: 2.9046 | LR: 3.97e-05 | 5798.5ms/step | 4.1 samples/s
+Step 16420/20005 | Loss: 2.8618 | LR: 3.95e-05 | 5777.6ms/step | 4.2 samples/s
+Step 16430/20005 | Loss: 2.8637 | LR: 3.93e-05 | 5798.4ms/step | 4.1 samples/s
+Step 16440/20005 | Loss: 2.8608 | LR: 3.91e-05 | 5808.3ms/step | 4.1 samples/s
+Step 16450/20005 | Loss: 2.8862 | LR: 3.89e-05 | 5787.8ms/step | 4.1 samples/s
+Step 16460/20005 | Loss: 2.8885 | LR: 3.87e-05 | 5775.7ms/step | 4.2 samples/s
+Step 16470/20005 | Loss: 2.8833 | LR: 3.85e-05 | 5771.0ms/step | 4.2 samples/s
+Step 16480/20005 | Loss: 2.8654 | LR: 3.82e-05 | 5777.6ms/step | 4.2 samples/s
+Step 16490/20005 | Loss: 2.8883 | LR: 3.80e-05 | 5805.9ms/step | 4.1 samples/s
+Step 16500/20005 | Loss: 2.8750 | LR: 3.78e-05 | 5847.9ms/step | 4.1 samples/s
+Validation at step 16500: Loss = 2.8964, Perplexity = 18.11
+Step 16510/20005 | Loss: 2.8852 | LR: 3.76e-05 | 39988.2ms/step | 0.6 samples/s
+Step 16520/20005 | Loss: 2.8601 | LR: 3.74e-05 | 5779.8ms/step | 4.2 samples/s
+Step 16530/20005 | Loss: 2.8606 | LR: 3.72e-05 | 5778.4ms/step | 4.2 samples/s
+Step 16540/20005 | Loss: 2.8567 | LR: 3.70e-05 | 5788.9ms/step | 4.1 samples/s
+Step 16550/20005 | Loss: 2.9279 | LR: 3.68e-05 | 5799.2ms/step | 4.1 samples/s
+Step 16560/20005 | Loss: 2.8896 | LR: 3.66e-05 | 5806.0ms/step | 4.1 samples/s
+Step 16570/20005 | Loss: 2.8944 | LR: 3.64e-05 | 5789.4ms/step | 4.1 samples/s
+Step 16580/20005 | Loss: 2.8497 | LR: 3.62e-05 | 5778.2ms/step | 4.2 samples/s
+Step 16590/20005 | Loss: 2.8682 | LR: 3.59e-05 | 5787.9ms/step | 4.1 samples/s
+Step 16600/20005 | Loss: 2.8877 | LR: 3.57e-05 | 5789.2ms/step | 4.1 samples/s
+Step 16610/20005 | Loss: 2.8810 | LR: 3.55e-05 | 5805.3ms/step | 4.1 samples/s
+Step 16620/20005 | Loss: 2.8817 | LR: 3.53e-05 | 5806.6ms/step | 4.1 samples/s
+Step 16630/20005 | Loss: 2.8868 | LR: 3.51e-05 | 5785.3ms/step | 4.1 samples/s
+Step 16640/20005 | Loss: 2.9033 | LR: 3.49e-05 | 5778.0ms/step | 4.2 samples/s
+Step 16650/20005 | Loss: 2.9087 | LR: 3.47e-05 | 5774.1ms/step | 4.2 samples/s
+Step 16660/20005 | Loss: 2.8753 | LR: 3.45e-05 | 5783.8ms/step | 4.1 samples/s
+Step 16670/20005 | Loss: 2.8726 | LR: 3.43e-05 | 5806.0ms/step | 4.1 samples/s
+Step 16680/20005 | Loss: 2.8741 | LR: 3.41e-05 | 5808.2ms/step | 4.1 samples/s
+Step 16690/20005 | Loss: 2.8671 | LR: 3.39e-05 | 5792.3ms/step | 4.1 samples/s
+Step 16700/20005 | Loss: 2.8762 | LR: 3.37e-05 | 5802.2ms/step | 4.1 samples/s
+Step 16710/20005 | Loss: 2.8609 | LR: 3.35e-05 | 5801.5ms/step | 4.1 samples/s
+Step 16720/20005 | Loss: 2.8764 | LR: 3.33e-05 | 5798.4ms/step | 4.1 samples/s
+Step 16730/20005 | Loss: 2.8537 | LR: 3.31e-05 | 5804.1ms/step | 4.1 samples/s
+Step 16740/20005 | Loss: 2.8709 | LR: 3.29e-05 | 5801.8ms/step | 4.1 samples/s
+Step 16750/20005 | Loss: 2.8625 | LR: 3.27e-05 | 5785.3ms/step | 4.1 samples/s
+Validation at step 16750: Loss = 2.8958, Perplexity = 18.10
+Step 16760/20005 | Loss: 2.8645 | LR: 3.25e-05 | 40056.8ms/step | 0.6 samples/s
+Step 16770/20005 | Loss: 2.8947 | LR: 3.23e-05 | 5803.4ms/step | 4.1 samples/s
+Step 16780/20005 | Loss: 2.8879 | LR: 3.21e-05 | 5797.2ms/step | 4.1 samples/s
+Step 16790/20005 | Loss: 2.8779 | LR: 3.20e-05 | 5802.3ms/step | 4.1 samples/s
+Step 16800/20005 | Loss: 2.8738 | LR: 3.18e-05 | 5812.2ms/step | 4.1 samples/s
+Step 16810/20005 | Loss: 2.8759 | LR: 3.16e-05 | 5789.0ms/step | 4.1 samples/s
+Step 16820/20005 | Loss: 2.8653 | LR: 3.14e-05 | 5787.6ms/step | 4.1 samples/s
+Step 16830/20005 | Loss: 2.8641 | LR: 3.12e-05 | 5796.3ms/step | 4.1 samples/s
+Step 16840/20005 | Loss: 2.8667 | LR: 3.10e-05 | 5800.3ms/step | 4.1 samples/s
+Step 16850/20005 | Loss: 2.8774 | LR: 3.08e-05 | 5800.0ms/step | 4.1 samples/s
+Step 16860/20005 | Loss: 2.8557 | LR: 3.06e-05 | 5810.7ms/step | 4.1 samples/s
+Step 16870/20005 | Loss: 2.8982 | LR: 3.04e-05 | 5787.4ms/step | 4.1 samples/s
+Step 16880/20005 | Loss: 2.8782 | LR: 3.02e-05 | 5794.7ms/step | 4.1 samples/s
+Step 16890/20005 | Loss: 2.8829 | LR: 3.00e-05 | 5801.1ms/step | 4.1 samples/s
+Step 16900/20005 | Loss: 2.8573 | LR: 2.98e-05 | 5800.6ms/step | 4.1 samples/s
+Step 16910/20005 | Loss: 2.8912 | LR: 2.97e-05 | 5796.6ms/step | 4.1 samples/s
+Step 16920/20005 | Loss: 2.8813 | LR: 2.95e-05 | 5808.5ms/step | 4.1 samples/s
+Step 16930/20005 | Loss: 2.8525 | LR: 2.93e-05 | 5787.2ms/step | 4.1 samples/s
+Step 16940/20005 | Loss: 2.8718 | LR: 2.91e-05 | 5777.5ms/step | 4.2 samples/s
+Step 16950/20005 | Loss: 2.8829 | LR: 2.89e-05 | 5788.7ms/step | 4.1 samples/s
+Step 16960/20005 | Loss: 2.8935 | LR: 2.87e-05 | 5801.4ms/step | 4.1 samples/s
+Step 16970/20005 | Loss: 2.9022 | LR: 2.85e-05 | 5795.8ms/step | 4.1 samples/s
+Step 16980/20005 | Loss: 2.8955 | LR: 2.84e-05 | 5799.3ms/step | 4.1 samples/s
+Step 16990/20005 | Loss: 2.8606 | LR: 2.82e-05 | 5796.4ms/step | 4.1 samples/s
+Step 17000/20005 | Loss: 2.8556 | LR: 2.80e-05 | 5822.8ms/step | 4.1 samples/s
+Validation at step 17000: Loss = 2.8954, Perplexity = 18.09
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-17000
+Step 17010/20005 | Loss: 2.8932 | LR: 2.78e-05 | 40472.1ms/step | 0.6 samples/s
+Step 17020/20005 | Loss: 2.8715 | LR: 2.76e-05 | 5795.8ms/step | 4.1 samples/s
+Step 17030/20005 | Loss: 2.8630 | LR: 2.74e-05 | 5784.2ms/step | 4.1 samples/s
+Step 17040/20005 | Loss: 2.8649 | LR: 2.73e-05 | 5804.2ms/step | 4.1 samples/s
+Step 17050/20005 | Loss: 2.8631 | LR: 2.71e-05 | 5803.9ms/step | 4.1 samples/s
+Step 17060/20005 | Loss: 2.8951 | LR: 2.69e-05 | 5785.9ms/step | 4.1 samples/s
+Step 17070/20005 | Loss: 2.8709 | LR: 2.67e-05 | 5773.8ms/step | 4.2 samples/s
+Step 17080/20005 | Loss: 2.9004 | LR: 2.65e-05 | 5774.1ms/step | 4.2 samples/s
+Step 17090/20005 | Loss: 2.8492 | LR: 2.64e-05 | 5778.0ms/step | 4.2 samples/s
+Step 17100/20005 | Loss: 2.8993 | LR: 2.62e-05 | 5799.8ms/step | 4.1 samples/s
+Step 17110/20005 | Loss: 2.9000 | LR: 2.60e-05 | 5806.2ms/step | 4.1 samples/s
+Step 17120/20005 | Loss: 2.8823 | LR: 2.58e-05 | 5780.5ms/step | 4.2 samples/s
+Step 17130/20005 | Loss: 2.8930 | LR: 2.57e-05 | 5777.5ms/step | 4.2 samples/s
+Step 17140/20005 | Loss: 2.8838 | LR: 2.55e-05 | 5772.2ms/step | 4.2 samples/s
+Step 17150/20005 | Loss: 2.8710 | LR: 2.53e-05 | 5776.7ms/step | 4.2 samples/s
+Step 17160/20005 | Loss: 2.8906 | LR: 2.51e-05 | 5799.7ms/step | 4.1 samples/s
+Step 17170/20005 | Loss: 2.8886 | LR: 2.50e-05 | 5802.9ms/step | 4.1 samples/s
+Step 17180/20005 | Loss: 2.8920 | LR: 2.48e-05 | 5786.9ms/step | 4.1 samples/s
+Step 17190/20005 | Loss: 2.8665 | LR: 2.46e-05 | 5775.9ms/step | 4.2 samples/s
+Step 17200/20005 | Loss: 2.8719 | LR: 2.44e-05 | 5776.3ms/step | 4.2 samples/s
+Step 17210/20005 | Loss: 2.8837 | LR: 2.43e-05 | 5777.9ms/step | 4.2 samples/s
+Step 17220/20005 | Loss: 2.8843 | LR: 2.41e-05 | 5796.4ms/step | 4.1 samples/s
+Step 17230/20005 | Loss: 2.8992 | LR: 2.39e-05 | 5804.3ms/step | 4.1 samples/s
+Step 17240/20005 | Loss: 2.8704 | LR: 2.38e-05 | 5788.7ms/step | 4.1 samples/s
+Step 17250/20005 | Loss: 2.8868 | LR: 2.36e-05 | 5776.6ms/step | 4.2 samples/s
+Validation at step 17250: Loss = 2.8951, Perplexity = 18.08

epfl_final/wandb/run-20260513_151725-qudv4nmq/files/requirements.txt

@@ -0,0 +1,97 @@
+pytz==2026.2
+nvidia-cusparselt-cu13==0.8.1
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.7.0
+tzdata==2026.2
+typing_extensions==4.15.0
+triton==3.7.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+safetensors==0.7.0
+regex==2026.5.9
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.5.2
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.29.7
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+numpy==2.2.6
+networkx==3.4.2
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.15
+hf-xet==1.5.0
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+async-timeout==5.0.1
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+typing-inspection==0.4.2
+sentry-sdk==2.60.0
+requests==2.34.0
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.4
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cublas==13.1.1.3
+multiprocess==0.70.19
+multidict==6.7.1
+markdown-it-py==4.2.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+exceptiongroup==1.3.1
+cuda-bindings==13.2.0
+aiosignal==1.4.0
+yarl==1.23.0
+rich==15.0.0
+pydantic==2.13.4
+pandas==2.3.3
+nvidia-cusolver==12.0.4.66
+nvidia-cudnn-cu13==9.20.0.48
+GitPython==3.1.50
+anyio==4.13.0
+wandb==0.26.1
+typer==0.25.1
+httpx==0.28.1
+aiohttp==3.13.5
+torch==2.12.0
+huggingface_hub==1.14.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.8.1
+PyGObject==3.42.1
+dbus-python==1.2.18
+distro==1.7.0
+pip==22.0.2
+setuptools==59.6.0
+ssh-import-id==5.11
+wheel==0.37.1

epfl_final/wandb/run-20260513_151725-qudv4nmq/files/wandb-metadata.json

@@ -0,0 +1,36 @@
+{
+  "os":  "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.10.12",
+  "startedAt":  "2026-05-13T15:17:25.943605Z",
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "SparseAtt/train.py",
+  "codePathLocal":  "SparseAtt/train.py",
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse",
+  "host":  "h100-workspace-7-0-9",
+  "executable":  "/usr/bin/python3",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  1,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "7303405711360"
+    }
+  },
+  "memory":  {
+    "total":  "1623005847552"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-a51146ae-8325-4166-8144-f51a61307b63"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "xg8gah9pknow75h6lckr9ubkw9v8aljx"
+}

epfl_final/wandb/run-20260513_151725-qudv4nmq/logs/debug.log

@@ -0,0 +1,19 @@
+2026-05-13 15:17:25,959 INFO    MainThread:2444 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-05-13 15:17:25,959 INFO    MainThread:2444 [wandb_setup.py:_flush():81] Configure stats pid to 2444
+2026-05-13 15:17:25,960 INFO    MainThread:2444 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-13 15:17:25,960 INFO    MainThread:2444 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260513_151725-qudv4nmq/logs/debug.log
+2026-05-13 15:17:25,961 INFO    MainThread:2444 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260513_151725-qudv4nmq/logs/debug-internal.log
+2026-05-13 15:17:25,961 INFO    MainThread:2444 [wandb_init.py:init():850] calling init triggers
+2026-05-13 15:17:25,962 INFO    MainThread:2444 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-16000', 'num_epochs': 1, 'max_steps': 20005, 'per_device_batch_size': 3, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 250, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 200, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'dense-resume-18k', 'wandb_tags': ['qwen3', '0.6b', 'dense-resume'], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 10, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'dense_mode': True, '_wandb': {}}
+2026-05-13 15:17:25,962 INFO    MainThread:2444 [wandb_init.py:init():898] starting backend
+2026-05-13 15:17:26,183 INFO    MainThread:2444 [wandb_init.py:init():913] sending inform_init request
+2026-05-13 15:17:26,496 INFO    MainThread:2444 [wandb_init.py:init():918] backend started and connected
+2026-05-13 15:17:26,498 INFO    MainThread:2444 [wandb_init.py:init():988] updated telemetry
+2026-05-13 15:17:26,499 INFO    MainThread:2444 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-05-13 15:17:27,411 INFO    MainThread:2444 [wandb_init.py:init():1056] starting run threads in backend
+2026-05-13 15:17:27,498 INFO    MainThread:2444 [wandb_run.py:_console_start():2554] atexit reg
+2026-05-13 15:17:27,498 INFO    MainThread:2444 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-05-13 15:17:27,499 INFO    MainThread:2444 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-05-13 15:17:27,499 INFO    MainThread:2444 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-05-13 15:17:27,502 INFO    MainThread:2444 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/wandb/run-20260513_151725-qudv4nmq/run-qudv4nmq.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f84bb4e4d56a929d091c6aa686abea9252247e728aa2ea4e4ed5da90bee64333
+size 1376256

epfl_final/wandb/run-20260513_164613-40cb81kv/files/output.log

@@ -0,0 +1,266 @@
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1024
+  num_hidden_layers: 28
+  num_attention_heads: 16
+  num_key_value_heads: 8
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 31
+  sparse_mask_pool_size: 10
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 596,049,920
+  Trainable parameters: 596,049,920
+  Non-trainable parameters: 0
+  Model size (BF16): 1.19 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [3, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [3, 8192, 1024]           --                        True
+│    └─Embedding: 2-1                                        [3, 8192]                 [3, 8192, 1024]           155,582,464               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    └─Qwen3RMSNorm: 2-3                                     [3, 8192, 1024]           [3, 8192, 1024]           1,024                     True
+├─Linear: 1-2                                                [3, 8192, 1024]           [3, 8192, 151936]         155,582,464               True
+================================================================================================================================================================
+Total params: 751,632,384
+Trainable params: 751,632,384
+Non-trainable params: 0
+Total mult-adds (G): 2.25
+================================================================================================================================================================
+Input size (MB): 0.20
+Forward/backward pass size (MB): 63052.97
+Params size (MB): 1503.26
+Estimated Total Size (MB): 64556.43
+================================================================================================================================================================
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Loaded 5088012 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1272003)
+Dataset split: 1017602 train, 254401 validation
+Training examples (packed, train split):  1017602
+Validation examples (packed, val split):  254401
+Per-device batch size:                    3
+World size (GPUs):                        1
+Gradient accumulation steps:              8
+Effective batch size (seqs/step):         24
+Sequence length:                          8192
+Tokens per optimizer step:                196,608
+Micro-batches per epoch (per rank):       339200
+Optimizer steps per epoch:                42400
+Total optimizer steps:                    20005  (capped by max_steps; 0.47 epochs)
+Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-8000...
+Warning: no RNG state file found for rank 0; RNG will not be restored.
+Resumed from checkpoint at step 7999 (epoch 0, micro_step 0)
+Resuming training from step 7999
+Starting training...
+Warning: no loader_gen_state file found; DataLoader shuffle order on resume may differ from original run.
+Step 8000/20005 | Loss: 0.3005 | LR: 3.33e-04 | 60738.9ms/step | 0.4 samples/s
+Validation at step 8000: Loss = 3.0463, Perplexity = 21.04
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-8000
+Step 8010/20005 | Loss: 3.0209 | LR: 3.33e-04 | 43173.7ms/step | 0.6 samples/s
+Step 8020/20005 | Loss: 3.0490 | LR: 3.32e-04 | 5895.2ms/step | 4.1 samples/s
+Step 8030/20005 | Loss: 3.0187 | LR: 3.32e-04 | 5845.3ms/step | 4.1 samples/s
+Step 8040/20005 | Loss: 3.0369 | LR: 3.31e-04 | 5844.8ms/step | 4.1 samples/s
+Step 8050/20005 | Loss: 3.0430 | LR: 3.31e-04 | 5844.2ms/step | 4.1 samples/s
+Step 8060/20005 | Loss: 3.0399 | LR: 3.31e-04 | 5844.5ms/step | 4.1 samples/s
+Step 8070/20005 | Loss: 3.0496 | LR: 3.30e-04 | 5845.8ms/step | 4.1 samples/s
+Step 8080/20005 | Loss: 2.9954 | LR: 3.30e-04 | 5844.4ms/step | 4.1 samples/s
+Step 8090/20005 | Loss: 3.0354 | LR: 3.30e-04 | 5846.4ms/step | 4.1 samples/s
+Step 8100/20005 | Loss: 3.0208 | LR: 3.29e-04 | 5849.0ms/step | 4.1 samples/s
+Step 8110/20005 | Loss: 3.0160 | LR: 3.29e-04 | 5844.7ms/step | 4.1 samples/s
+Step 8120/20005 | Loss: 3.0308 | LR: 3.28e-04 | 5849.4ms/step | 4.1 samples/s
+Step 8130/20005 | Loss: 3.0105 | LR: 3.28e-04 | 5851.7ms/step | 4.1 samples/s
+Step 8140/20005 | Loss: 3.0254 | LR: 3.28e-04 | 5846.8ms/step | 4.1 samples/s
+Step 8150/20005 | Loss: 3.0212 | LR: 3.27e-04 | 5848.3ms/step | 4.1 samples/s
+Step 8160/20005 | Loss: 2.9973 | LR: 3.27e-04 | 5845.1ms/step | 4.1 samples/s
+Step 8170/20005 | Loss: 2.9866 | LR: 3.27e-04 | 5844.8ms/step | 4.1 samples/s
+Step 8180/20005 | Loss: 3.0179 | LR: 3.26e-04 | 5844.1ms/step | 4.1 samples/s
+Step 8190/20005 | Loss: 3.0313 | LR: 3.26e-04 | 5845.2ms/step | 4.1 samples/s
+Step 8200/20005 | Loss: 3.0137 | LR: 3.25e-04 | 5844.7ms/step | 4.1 samples/s
+Step 8210/20005 | Loss: 3.0414 | LR: 3.25e-04 | 5844.0ms/step | 4.1 samples/s
+Step 8220/20005 | Loss: 3.0272 | LR: 3.25e-04 | 5846.0ms/step | 4.1 samples/s
+Step 8230/20005 | Loss: 3.0355 | LR: 3.24e-04 | 5844.2ms/step | 4.1 samples/s
+Step 8240/20005 | Loss: 3.0317 | LR: 3.24e-04 | 5846.5ms/step | 4.1 samples/s
+Step 8250/20005 | Loss: 2.9886 | LR: 3.24e-04 | 5846.2ms/step | 4.1 samples/s
+Validation at step 8250: Loss = 3.0269, Perplexity = 20.63
+Step 8260/20005 | Loss: 3.0043 | LR: 3.23e-04 | 40512.8ms/step | 0.6 samples/s
+Step 8270/20005 | Loss: 3.0274 | LR: 3.23e-04 | 5844.8ms/step | 4.1 samples/s
+Step 8280/20005 | Loss: 3.0327 | LR: 3.22e-04 | 5845.8ms/step | 4.1 samples/s
+Step 8290/20005 | Loss: 3.0293 | LR: 3.22e-04 | 5848.5ms/step | 4.1 samples/s
+Step 8300/20005 | Loss: 3.0040 | LR: 3.22e-04 | 5846.7ms/step | 4.1 samples/s
+Step 8310/20005 | Loss: 3.0314 | LR: 3.21e-04 | 5845.0ms/step | 4.1 samples/s
+Step 8320/20005 | Loss: 3.0232 | LR: 3.21e-04 | 5846.6ms/step | 4.1 samples/s
+Step 8330/20005 | Loss: 3.0138 | LR: 3.20e-04 | 5849.4ms/step | 4.1 samples/s
+Step 8340/20005 | Loss: 2.9915 | LR: 3.20e-04 | 5850.4ms/step | 4.1 samples/s
+Step 8350/20005 | Loss: 3.0238 | LR: 3.20e-04 | 5873.6ms/step | 4.1 samples/s
+Step 8360/20005 | Loss: 3.0174 | LR: 3.19e-04 | 5877.6ms/step | 4.1 samples/s
+Step 8370/20005 | Loss: 3.0191 | LR: 3.19e-04 | 5881.5ms/step | 4.1 samples/s
+Step 8380/20005 | Loss: 3.0175 | LR: 3.19e-04 | 5886.7ms/step | 4.1 samples/s
+Step 8390/20005 | Loss: 2.9975 | LR: 3.18e-04 | 5889.4ms/step | 4.1 samples/s
+Step 8400/20005 | Loss: 3.0001 | LR: 3.18e-04 | 5890.0ms/step | 4.1 samples/s
+Step 8410/20005 | Loss: 3.0340 | LR: 3.17e-04 | 5891.4ms/step | 4.1 samples/s
+Step 8420/20005 | Loss: 2.9922 | LR: 3.17e-04 | 5892.5ms/step | 4.1 samples/s
+Step 8430/20005 | Loss: 2.9943 | LR: 3.17e-04 | 5896.1ms/step | 4.1 samples/s
+Step 8440/20005 | Loss: 2.9912 | LR: 3.16e-04 | 5888.7ms/step | 4.1 samples/s
+Step 8450/20005 | Loss: 3.0143 | LR: 3.16e-04 | 5892.4ms/step | 4.1 samples/s
+Step 8460/20005 | Loss: 3.0145 | LR: 3.15e-04 | 5890.0ms/step | 4.1 samples/s
+Step 8470/20005 | Loss: 3.0116 | LR: 3.15e-04 | 5887.8ms/step | 4.1 samples/s
+Step 8480/20005 | Loss: 2.9898 | LR: 3.15e-04 | 5894.9ms/step | 4.1 samples/s
+Step 8490/20005 | Loss: 3.0156 | LR: 3.14e-04 | 5895.3ms/step | 4.1 samples/s
+Step 8500/20005 | Loss: 3.0061 | LR: 3.14e-04 | 5891.2ms/step | 4.1 samples/s
+Validation at step 8500: Loss = 3.0173, Perplexity = 20.44
+Step 8510/20005 | Loss: 3.0116 | LR: 3.14e-04 | 40904.0ms/step | 0.6 samples/s
+Step 8520/20005 | Loss: 2.9885 | LR: 3.13e-04 | 5926.6ms/step | 4.0 samples/s
+Step 8530/20005 | Loss: 2.9842 | LR: 3.13e-04 | 5884.8ms/step | 4.1 samples/s
+Step 8540/20005 | Loss: 2.9798 | LR: 3.12e-04 | 5889.1ms/step | 4.1 samples/s
+Step 8550/20005 | Loss: 3.0528 | LR: 3.12e-04 | 5889.0ms/step | 4.1 samples/s
+Step 8560/20005 | Loss: 3.0133 | LR: 3.12e-04 | 5890.7ms/step | 4.1 samples/s
+Step 8570/20005 | Loss: 3.0188 | LR: 3.11e-04 | 5886.1ms/step | 4.1 samples/s
+Step 8580/20005 | Loss: 2.9773 | LR: 3.11e-04 | 5889.0ms/step | 4.1 samples/s
+Step 8590/20005 | Loss: 2.9922 | LR: 3.10e-04 | 5892.9ms/step | 4.1 samples/s
+Step 8600/20005 | Loss: 3.0110 | LR: 3.10e-04 | 5891.2ms/step | 4.1 samples/s
+Step 8610/20005 | Loss: 3.0045 | LR: 3.10e-04 | 5892.3ms/step | 4.1 samples/s
+Step 8620/20005 | Loss: 3.0010 | LR: 3.09e-04 | 5896.6ms/step | 4.1 samples/s
+Step 8630/20005 | Loss: 3.0084 | LR: 3.09e-04 | 5891.8ms/step | 4.1 samples/s
+Step 8640/20005 | Loss: 3.0234 | LR: 3.09e-04 | 5885.1ms/step | 4.1 samples/s
+Step 8650/20005 | Loss: 3.0314 | LR: 3.08e-04 | 5891.4ms/step | 4.1 samples/s
+Step 8660/20005 | Loss: 2.9966 | LR: 3.08e-04 | 5881.4ms/step | 4.1 samples/s
+Step 8670/20005 | Loss: 2.9927 | LR: 3.07e-04 | 5885.8ms/step | 4.1 samples/s
+Step 8680/20005 | Loss: 2.9941 | LR: 3.07e-04 | 5893.6ms/step | 4.1 samples/s
+Step 8690/20005 | Loss: 2.9849 | LR: 3.07e-04 | 5890.2ms/step | 4.1 samples/s
+Step 8700/20005 | Loss: 2.9973 | LR: 3.06e-04 | 5872.2ms/step | 4.1 samples/s
+Step 8710/20005 | Loss: 2.9828 | LR: 3.06e-04 | 5873.4ms/step | 4.1 samples/s
+Step 8720/20005 | Loss: 2.9969 | LR: 3.05e-04 | 5872.7ms/step | 4.1 samples/s
+Step 8730/20005 | Loss: 2.9769 | LR: 3.05e-04 | 5878.6ms/step | 4.1 samples/s
+Step 8740/20005 | Loss: 2.9891 | LR: 3.05e-04 | 5876.6ms/step | 4.1 samples/s
+Step 8750/20005 | Loss: 2.9813 | LR: 3.04e-04 | 5881.4ms/step | 4.1 samples/s
+Validation at step 8750: Loss = 3.0081, Perplexity = 20.25
+Step 8760/20005 | Loss: 2.9837 | LR: 3.04e-04 | 40797.1ms/step | 0.6 samples/s
+Step 8770/20005 | Loss: 3.0124 | LR: 3.04e-04 | 5866.5ms/step | 4.1 samples/s
+Step 8780/20005 | Loss: 3.0050 | LR: 3.03e-04 | 5867.2ms/step | 4.1 samples/s
+Step 8790/20005 | Loss: 2.9934 | LR: 3.03e-04 | 5865.9ms/step | 4.1 samples/s
+Step 8800/20005 | Loss: 2.9902 | LR: 3.02e-04 | 5868.6ms/step | 4.1 samples/s
+Step 8810/20005 | Loss: 2.9924 | LR: 3.02e-04 | 5866.2ms/step | 4.1 samples/s
+Step 8820/20005 | Loss: 2.9850 | LR: 3.02e-04 | 5866.9ms/step | 4.1 samples/s
+Step 8830/20005 | Loss: 2.9816 | LR: 3.01e-04 | 5864.8ms/step | 4.1 samples/s
+Step 8840/20005 | Loss: 2.9830 | LR: 3.01e-04 | 5866.6ms/step | 4.1 samples/s
+Step 8850/20005 | Loss: 2.9927 | LR: 3.00e-04 | 5869.9ms/step | 4.1 samples/s
+Step 8860/20005 | Loss: 2.9707 | LR: 3.00e-04 | 5868.1ms/step | 4.1 samples/s
+Step 8870/20005 | Loss: 3.0131 | LR: 3.00e-04 | 5869.0ms/step | 4.1 samples/s
+Step 8880/20005 | Loss: 2.9928 | LR: 2.99e-04 | 5867.9ms/step | 4.1 samples/s
+Step 8890/20005 | Loss: 2.9964 | LR: 2.99e-04 | 5870.2ms/step | 4.1 samples/s
+Step 8900/20005 | Loss: 2.9714 | LR: 2.98e-04 | 5865.4ms/step | 4.1 samples/s
+Step 8910/20005 | Loss: 3.0026 | LR: 2.98e-04 | 5866.9ms/step | 4.1 samples/s
+Step 8920/20005 | Loss: 2.9990 | LR: 2.98e-04 | 5865.3ms/step | 4.1 samples/s
+Step 8930/20005 | Loss: 2.9676 | LR: 2.97e-04 | 5866.0ms/step | 4.1 samples/s
+Step 8940/20005 | Loss: 2.9855 | LR: 2.97e-04 | 5864.0ms/step | 4.1 samples/s
+Step 8950/20005 | Loss: 2.9960 | LR: 2.97e-04 | 5866.3ms/step | 4.1 samples/s
+Step 8960/20005 | Loss: 3.0052 | LR: 2.96e-04 | 5866.2ms/step | 4.1 samples/s
+Step 8970/20005 | Loss: 3.0131 | LR: 2.96e-04 | 5867.7ms/step | 4.1 samples/s
+Step 8980/20005 | Loss: 3.0088 | LR: 2.95e-04 | 5862.7ms/step | 4.1 samples/s
+Step 8990/20005 | Loss: 2.9720 | LR: 2.95e-04 | 5861.5ms/step | 4.1 samples/s
+Step 9000/20005 | Loss: 2.9688 | LR: 2.95e-04 | 5865.9ms/step | 4.1 samples/s
+Validation at step 9000: Loss = 3.0002, Perplexity = 20.09
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-9000
+Step 9010/20005 | Loss: 3.0051 | LR: 2.94e-04 | 41291.0ms/step | 0.6 samples/s
+Step 9020/20005 | Loss: 2.9831 | LR: 2.94e-04 | 5909.6ms/step | 4.1 samples/s
+Step 9030/20005 | Loss: 2.9727 | LR: 2.93e-04 | 5873.1ms/step | 4.1 samples/s
+Step 9040/20005 | Loss: 2.9759 | LR: 2.93e-04 | 5877.0ms/step | 4.1 samples/s
+Step 9050/20005 | Loss: 2.9739 | LR: 2.93e-04 | 5874.8ms/step | 4.1 samples/s
+Step 9060/20005 | Loss: 3.0044 | LR: 2.92e-04 | 5868.5ms/step | 4.1 samples/s
+Step 9070/20005 | Loss: 2.9791 | LR: 2.92e-04 | 5869.0ms/step | 4.1 samples/s
+Step 9080/20005 | Loss: 3.0096 | LR: 2.91e-04 | 5868.7ms/step | 4.1 samples/s
+Step 9090/20005 | Loss: 2.9563 | LR: 2.91e-04 | 5870.9ms/step | 4.1 samples/s
+Step 9100/20005 | Loss: 3.0059 | LR: 2.91e-04 | 5871.2ms/step | 4.1 samples/s
+Step 9110/20005 | Loss: 3.0081 | LR: 2.90e-04 | 5875.1ms/step | 4.1 samples/s
+Step 9120/20005 | Loss: 2.9921 | LR: 2.90e-04 | 5876.5ms/step | 4.1 samples/s
+Step 9130/20005 | Loss: 2.9977 | LR: 2.89e-04 | 5876.8ms/step | 4.1 samples/s
+Step 9140/20005 | Loss: 2.9906 | LR: 2.89e-04 | 5872.4ms/step | 4.1 samples/s
+Step 9150/20005 | Loss: 2.9799 | LR: 2.89e-04 | 5871.3ms/step | 4.1 samples/s
+Step 9160/20005 | Loss: 2.9954 | LR: 2.88e-04 | 5874.4ms/step | 4.1 samples/s
+Step 9170/20005 | Loss: 2.9937 | LR: 2.88e-04 | 5873.1ms/step | 4.1 samples/s
+Step 9180/20005 | Loss: 2.9965 | LR: 2.87e-04 | 5873.8ms/step | 4.1 samples/s
+Step 9190/20005 | Loss: 2.9725 | LR: 2.87e-04 | 5873.2ms/step | 4.1 samples/s
+Step 9200/20005 | Loss: 2.9793 | LR: 2.87e-04 | 5871.9ms/step | 4.1 samples/s
+Step 9210/20005 | Loss: 2.9903 | LR: 2.86e-04 | 5870.8ms/step | 4.1 samples/s
+Step 9220/20005 | Loss: 2.9914 | LR: 2.86e-04 | 5874.5ms/step | 4.1 samples/s
+Step 9230/20005 | Loss: 3.0041 | LR: 2.86e-04 | 5877.4ms/step | 4.1 samples/s
+Step 9240/20005 | Loss: 2.9725 | LR: 2.85e-04 | 5876.6ms/step | 4.1 samples/s
+Step 9250/20005 | Loss: 2.9899 | LR: 2.85e-04 | 5873.9ms/step | 4.1 samples/s
+Validation at step 9250: Loss = 2.9921, Perplexity = 19.93
+Step 9260/20005 | Loss: 2.9607 | LR: 2.84e-04 | 40847.6ms/step | 0.6 samples/s
+Step 9270/20005 | Loss: 2.9754 | LR: 2.84e-04 | 5880.0ms/step | 4.1 samples/s
+Step 9280/20005 | Loss: 2.9653 | LR: 2.84e-04 | 5879.0ms/step | 4.1 samples/s
+Step 9290/20005 | Loss: 2.9909 | LR: 2.83e-04 | 5880.4ms/step | 4.1 samples/s
+Step 9300/20005 | Loss: 2.9755 | LR: 2.83e-04 | 5877.0ms/step | 4.1 samples/s
+Step 9310/20005 | Loss: 2.9887 | LR: 2.82e-04 | 5877.2ms/step | 4.1 samples/s
+Step 9320/20005 | Loss: 3.0098 | LR: 2.82e-04 | 5876.3ms/step | 4.1 samples/s
+Step 9330/20005 | Loss: 2.9954 | LR: 2.82e-04 | 5875.9ms/step | 4.1 samples/s
+Step 9340/20005 | Loss: 2.9673 | LR: 2.81e-04 | 5874.9ms/step | 4.1 samples/s
+Step 9350/20005 | Loss: 2.9928 | LR: 2.81e-04 | 5874.9ms/step | 4.1 samples/s
+Step 9360/20005 | Loss: 2.9668 | LR: 2.80e-04 | 5875.5ms/step | 4.1 samples/s
+Step 9370/20005 | Loss: 2.9698 | LR: 2.80e-04 | 5874.3ms/step | 4.1 samples/s
+Step 9380/20005 | Loss: 2.9749 | LR: 2.80e-04 | 5875.7ms/step | 4.1 samples/s
+Step 9390/20005 | Loss: 2.9713 | LR: 2.79e-04 | 5875.9ms/step | 4.1 samples/s
+Step 9400/20005 | Loss: 2.9855 | LR: 2.79e-04 | 5878.1ms/step | 4.1 samples/s
+Step 9410/20005 | Loss: 2.9873 | LR: 2.78e-04 | 5874.1ms/step | 4.1 samples/s
+Step 9420/20005 | Loss: 2.9552 | LR: 2.78e-04 | 5876.0ms/step | 4.1 samples/s
+Step 9430/20005 | Loss: 3.0031 | LR: 2.78e-04 | 5875.3ms/step | 4.1 samples/s
+Step 9440/20005 | Loss: 3.0040 | LR: 2.77e-04 | 5874.4ms/step | 4.1 samples/s
+Step 9450/20005 | Loss: 2.9816 | LR: 2.77e-04 | 5876.0ms/step | 4.1 samples/s
+Step 9460/20005 | Loss: 2.9469 | LR: 2.76e-04 | 5875.0ms/step | 4.1 samples/s
+Step 9470/20005 | Loss: 2.9661 | LR: 2.76e-04 | 5877.1ms/step | 4.1 samples/s
+Step 9480/20005 | Loss: 2.9778 | LR: 2.76e-04 | 5876.0ms/step | 4.1 samples/s
+Step 9490/20005 | Loss: 2.9720 | LR: 2.75e-04 | 5877.4ms/step | 4.1 samples/s
+Step 9500/20005 | Loss: 2.9646 | LR: 2.75e-04 | 5874.0ms/step | 4.1 samples/s
+Validation at step 9500: Loss = 2.9845, Perplexity = 19.78
+Step 9510/20005 | Loss: 2.9689 | LR: 2.74e-04 | 40859.1ms/step | 0.6 samples/s
+Step 9520/20005 | Loss: 2.9505 | LR: 2.74e-04 | 5914.9ms/step | 4.1 samples/s
+Step 9530/20005 | Loss: 2.9465 | LR: 2.74e-04 | 5875.1ms/step | 4.1 samples/s
+Step 9540/20005 | Loss: 2.9960 | LR: 2.73e-04 | 5875.3ms/step | 4.1 samples/s

epfl_final/wandb/run-20260513_164613-40cb81kv/files/requirements.txt

@@ -0,0 +1,97 @@
+pytz==2026.2
+nvidia-cusparselt-cu13==0.8.1
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.7.0
+tzdata==2026.2
+typing_extensions==4.15.0
+triton==3.7.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+safetensors==0.7.0
+regex==2026.5.9
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.5.2
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.29.7
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+numpy==2.2.6
+networkx==3.4.2
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.15
+hf-xet==1.5.0
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+async-timeout==5.0.1
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+typing-inspection==0.4.2
+sentry-sdk==2.60.0
+requests==2.34.0
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.4
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cublas==13.1.1.3
+multiprocess==0.70.19
+multidict==6.7.1
+markdown-it-py==4.2.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+exceptiongroup==1.3.1
+cuda-bindings==13.2.0
+aiosignal==1.4.0
+yarl==1.23.0
+rich==15.0.0
+pydantic==2.13.4
+pandas==2.3.3
+nvidia-cusolver==12.0.4.66
+nvidia-cudnn-cu13==9.20.0.48
+GitPython==3.1.50
+anyio==4.13.0
+wandb==0.26.1
+typer==0.25.1
+httpx==0.28.1
+aiohttp==3.13.5
+torch==2.12.0
+huggingface_hub==1.14.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.8.1
+PyGObject==3.42.1
+dbus-python==1.2.18
+distro==1.7.0
+pip==22.0.2
+setuptools==59.6.0
+ssh-import-id==5.11
+wheel==0.37.1

epfl_final/wandb/run-20260513_164613-40cb81kv/files/wandb-metadata.json

@@ -0,0 +1,36 @@
+{
+  "os":  "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.10.12",
+  "startedAt":  "2026-05-13T16:46:13.049828Z",
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "SparseAtt/train.py",
+  "codePathLocal":  "SparseAtt/train.py",
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse",
+  "host":  "h100-train-1-0-4",
+  "executable":  "/usr/bin/python3",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  1,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "7247704559616"
+    }
+  },
+  "memory":  {
+    "total":  "1623005847552"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-e8ce0e9a-801b-03d2-c898-c0cc7d746d53"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "3z35bsll7ks68msxfalm5asmg6elgp61"
+}

epfl_final/wandb/run-20260513_164613-40cb81kv/logs/debug.log

@@ -0,0 +1,19 @@
+2026-05-13 16:46:13,067 INFO    MainThread:2392 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-05-13 16:46:13,068 INFO    MainThread:2392 [wandb_setup.py:_flush():81] Configure stats pid to 2392
+2026-05-13 16:46:13,068 INFO    MainThread:2392 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-13 16:46:13,069 INFO    MainThread:2392 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260513_164613-40cb81kv/logs/debug.log
+2026-05-13 16:46:13,069 INFO    MainThread:2392 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260513_164613-40cb81kv/logs/debug-internal.log
+2026-05-13 16:46:13,070 INFO    MainThread:2392 [wandb_init.py:init():850] calling init triggers
+2026-05-13 16:46:13,071 INFO    MainThread:2392 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/run_20260511_191723/checkpoint-8000', 'num_epochs': 1, 'max_steps': 20005, 'per_device_batch_size': 3, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 250, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 200, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'dense continued 8k', 'wandb_tags': ['qwen3', '0.6b', 'dense-resume'], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 10, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'dense_mode': True, '_wandb': {}}
+2026-05-13 16:46:13,071 INFO    MainThread:2392 [wandb_init.py:init():898] starting backend
+2026-05-13 16:46:13,291 INFO    MainThread:2392 [wandb_init.py:init():913] sending inform_init request
+2026-05-13 16:46:13,636 INFO    MainThread:2392 [wandb_init.py:init():918] backend started and connected
+2026-05-13 16:46:13,638 INFO    MainThread:2392 [wandb_init.py:init():988] updated telemetry
+2026-05-13 16:46:13,639 INFO    MainThread:2392 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-05-13 16:46:14,065 INFO    MainThread:2392 [wandb_init.py:init():1056] starting run threads in backend
+2026-05-13 16:46:14,120 INFO    MainThread:2392 [wandb_run.py:_console_start():2554] atexit reg
+2026-05-13 16:46:14,120 INFO    MainThread:2392 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-05-13 16:46:14,120 INFO    MainThread:2392 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-05-13 16:46:14,121 INFO    MainThread:2392 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-05-13 16:46:14,124 INFO    MainThread:2392 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/wandb/run-20260513_164613-40cb81kv/run-40cb81kv.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a350bdba80e19bed5924c78ce29b37b4e4fea71ce5e33351ea33f016e3b36ea
+size 1671168

epfl_final/wandb/run-20260513_200715-r9mojppo/files/output.log

@@ -0,0 +1,106 @@
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1024
+  num_hidden_layers: 28
+  num_attention_heads: 16
+  num_key_value_heads: 8
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 19
+  sparse_mask_pool_size: 2000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 596,049,920
+  Trainable parameters: 596,049,920
+  Non-trainable parameters: 0
+  Model size (BF16): 1.19 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:2055: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [3, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [3, 8192, 1024]           --                        True
+│    └─Embedding: 2-1                                        [3, 8192]                 [3, 8192, 1024]           155,582,464               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    └─Qwen3RMSNorm: 2-3                                     [3, 8192, 1024]           [3, 8192, 1024]           1,024                     True
+├─Linear: 1-2                                                [3, 8192, 1024]           [3, 8192, 151936]         155,582,464               True
+================================================================================================================================================================
+Total params: 751,632,384
+Trainable params: 751,632,384
+Non-trainable params: 0
+Total mult-adds (G): 2.25
+================================================================================================================================================================
+Input size (MB): 0.20
+Forward/backward pass size (MB): 63052.97
+Params size (MB): 1503.26
+Estimated Total Size (MB): 64556.43
+================================================================================================================================================================
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Loaded 5088012 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1272003)
+Dataset split: 1017602 train, 254401 validation
+Training examples (packed, train split):  1017602
+Validation examples (packed, val split):  254401
+Per-device batch size:                    3
+World size (GPUs):                        1
+Gradient accumulation steps:              8
+Effective batch size (seqs/step):         24
+Sequence length:                          8192
+Tokens per optimizer step:                196,608
+Micro-batches per epoch (per rank):       339200
+Optimizer steps per epoch:                42400
+Total optimizer steps:                    20005  (capped by max_steps; 0.47 epochs)
+Starting training...

epfl_final/wandb/run-20260513_200715-r9mojppo/files/requirements.txt

@@ -0,0 +1,97 @@
+pytz==2026.2
+nvidia-cusparselt-cu13==0.8.1
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.7.0
+tzdata==2026.2
+typing_extensions==4.15.0
+triton==3.7.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+safetensors==0.7.0
+regex==2026.5.9
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.5.2
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.29.7
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+numpy==2.2.6
+networkx==3.4.2
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.15
+hf-xet==1.5.0
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+async-timeout==5.0.1
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+typing-inspection==0.4.2
+sentry-sdk==2.60.0
+requests==2.34.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.4
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cublas==13.1.1.3
+multiprocess==0.70.19
+multidict==6.7.1
+markdown-it-py==4.2.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+exceptiongroup==1.3.1
+cuda-bindings==13.2.0
+aiosignal==1.4.0
+yarl==1.23.0
+rich==15.0.0
+pydantic==2.13.4
+pandas==2.3.3
+nvidia-cusolver==12.0.4.66
+nvidia-cudnn-cu13==9.20.0.48
+GitPython==3.1.50
+anyio==4.13.0
+wandb==0.26.1
+typer==0.25.1
+httpx==0.28.1
+aiohttp==3.13.5
+torch==2.12.0
+huggingface_hub==1.14.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.8.1
+PyGObject==3.42.1
+dbus-python==1.2.18
+distro==1.7.0
+pip==22.0.2
+setuptools==59.6.0
+ssh-import-id==5.11
+wheel==0.37.1

epfl_final/wandb/run-20260513_200715-r9mojppo/files/wandb-metadata.json

@@ -0,0 +1,36 @@
+{
+  "os":  "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.10.12",
+  "startedAt":  "2026-05-13T20:07:15.584547Z",
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "SparseAtt/train.py",
+  "codePathLocal":  "SparseAtt/train.py",
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse",
+  "host":  "h100-train-9-0-3",
+  "executable":  "/usr/bin/python3",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  1,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "7334242979840"
+    }
+  },
+  "memory":  {
+    "total":  "1623005741056"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-3ea6c840-350f-c68c-7562-6e0affa4e153"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "5yky95jtmuzzeuh6x537xr0appx5zc37"
+}

epfl_final/wandb/run-20260513_200715-r9mojppo/logs/debug-internal.log

@@ -0,0 +1,17 @@
+{"time":"2026-05-13T20:07:15.828949269Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-05-13T20:07:15.832400522Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-05-13T20:07:16.124538454Z","level":"INFO","msg":"stream: created new stream","id":"r9mojppo"}
+{"time":"2026-05-13T20:07:16.124627607Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-05-13T20:07:16.126547263Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-05-13T20:07:16.126575454Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-05-13T20:07:16.126587833Z","level":"INFO","msg":"writer: started","stream_id":"r9mojppo"}
+{"time":"2026-05-13T20:07:16.621394759Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":4}
+{"time":"2026-05-13T20:07:16.879603939Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-13T20:07:31.622410021Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":1,"console_offset":4,"console_lines":102,"uploaded_len":2}
+{"time":"2026-05-13T20:07:32.073232242Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-13T20:07:46.621739745Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":1,"events_lines":2}
+{"time":"2026-05-13T20:07:46.912986707Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-13T20:08:01.621760912Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":3,"events_lines":2}
+{"time":"2026-05-13T20:08:01.868033211Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-13T20:08:16.621844353Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":5,"events_lines":2}
+{"time":"2026-05-13T20:08:16.865910554Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

epfl_final/wandb/run-20260513_200715-r9mojppo/logs/debug.log

@@ -0,0 +1,19 @@
+2026-05-13 20:07:15,600 INFO    MainThread:2744 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-05-13 20:07:15,600 INFO    MainThread:2744 [wandb_setup.py:_flush():81] Configure stats pid to 2744
+2026-05-13 20:07:15,601 INFO    MainThread:2744 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-13 20:07:15,601 INFO    MainThread:2744 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260513_200715-r9mojppo/logs/debug.log
+2026-05-13 20:07:15,602 INFO    MainThread:2744 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260513_200715-r9mojppo/logs/debug-internal.log
+2026-05-13 20:07:15,602 INFO    MainThread:2744 [wandb_init.py:init():850] calling init triggers
+2026-05-13 20:07:15,603 INFO    MainThread:2744 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': './checkpoints/run_20260513_200443', 'resume_from_checkpoint': None, 'num_epochs': 1, 'max_steps': 20005, 'per_device_batch_size': 3, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 250, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 200, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'sparse 30%', 'wandb_tags': ['qwen3', '0.6b'], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 19, 'sparse_mask_pool_size': 2000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache', 'dense_mode': False, '_wandb': {}}
+2026-05-13 20:07:15,604 INFO    MainThread:2744 [wandb_init.py:init():898] starting backend
+2026-05-13 20:07:15,822 INFO    MainThread:2744 [wandb_init.py:init():913] sending inform_init request
+2026-05-13 20:07:16,127 INFO    MainThread:2744 [wandb_init.py:init():918] backend started and connected
+2026-05-13 20:07:16,128 INFO    MainThread:2744 [wandb_init.py:init():988] updated telemetry
+2026-05-13 20:07:16,129 INFO    MainThread:2744 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-05-13 20:07:16,561 INFO    MainThread:2744 [wandb_init.py:init():1056] starting run threads in backend
+2026-05-13 20:07:16,617 INFO    MainThread:2744 [wandb_run.py:_console_start():2554] atexit reg
+2026-05-13 20:07:16,618 INFO    MainThread:2744 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-05-13 20:07:16,618 INFO    MainThread:2744 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-05-13 20:07:16,618 INFO    MainThread:2744 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-05-13 20:07:16,620 INFO    MainThread:2744 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/wandb/run-20260517_023932-hi2z3jh6/files/output.log

@@ -0,0 +1,245 @@
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1024
+  num_hidden_layers: 28
+  num_attention_heads: 16
+  num_key_value_heads: 8
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 18
+  sparse_mask_pool_size: 2000
+  sparse_max_seq_length: 8192
+
+Parameter count:
+  Total parameters: 596,049,920
+  Trainable parameters: 596,049,920
+  Non-trainable parameters: 0
+  Model size (BF16): 1.19 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:2055: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [3, 8192]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [3, 8192, 1024]           --                        True
+│    └─Embedding: 2-1                                        [3, 8192]                 [3, 8192, 1024]           155,582,464               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [3, 8192, 1024]           [3, 8192, 1024]           15,730,944                True
+│    └─Qwen3RMSNorm: 2-3                                     [3, 8192, 1024]           [3, 8192, 1024]           1,024                     True
+├─Linear: 1-2                                                [3, 8192, 1024]           [3, 8192, 151936]         155,582,464               True
+================================================================================================================================================================
+Total params: 751,632,384
+Trainable params: 751,632,384
+Non-trainable params: 0
+Total mult-adds (G): 2.25
+================================================================================================================================================================
+Input size (MB): 0.20
+Forward/backward pass size (MB): 63052.97
+Params size (MB): 1503.26
+Estimated Total Size (MB): 64556.43
+================================================================================================================================================================
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
+Loaded 5088012 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1272003)
+Dataset split: 1017602 train, 254401 validation
+Training examples (packed, train split):  1017602
+Validation examples (packed, val split):  254401
+Per-device batch size:                    3
+World size (GPUs):                        1
+Gradient accumulation steps:              8
+Effective batch size (seqs/step):         24
+Sequence length:                          8192
+Tokens per optimizer step:                196,608
+Micro-batches per epoch (per rank):       339200
+Optimizer steps per epoch:                42400
+Total optimizer steps:                    20005  (capped by max_steps; 0.47 epochs)
+Starting training...
+Step 10/20005 | Loss: 12.0480 | LR: 2.00e-05 | 8374.8ms/step | 2.9 samples/s
+Step 20/20005 | Loss: 11.1479 | LR: 4.00e-05 | 2293.1ms/step | 10.5 samples/s
+Step 30/20005 | Loss: 10.3876 | LR: 6.00e-05 | 2291.7ms/step | 10.5 samples/s
+Step 40/20005 | Loss: 9.8708 | LR: 8.00e-05 | 2286.8ms/step | 10.5 samples/s
+Step 50/20005 | Loss: 9.0176 | LR: 1.00e-04 | 2281.1ms/step | 10.5 samples/s
+Step 60/20005 | Loss: 8.2251 | LR: 1.20e-04 | 2275.5ms/step | 10.5 samples/s
+Step 70/20005 | Loss: 7.6717 | LR: 1.40e-04 | 2289.0ms/step | 10.5 samples/s
+Step 80/20005 | Loss: 7.4142 | LR: 1.60e-04 | 2300.3ms/step | 10.4 samples/s
+Step 90/20005 | Loss: 7.2477 | LR: 1.80e-04 | 2301.5ms/step | 10.4 samples/s
+Step 100/20005 | Loss: 7.0899 | LR: 2.00e-04 | 2303.8ms/step | 10.4 samples/s
+Step 110/20005 | Loss: 6.9225 | LR: 2.20e-04 | 2305.5ms/step | 10.4 samples/s
+Step 120/20005 | Loss: 6.7848 | LR: 2.40e-04 | 2306.7ms/step | 10.4 samples/s
+Step 130/20005 | Loss: 6.6476 | LR: 2.60e-04 | 2307.9ms/step | 10.4 samples/s
+Step 140/20005 | Loss: 6.5474 | LR: 2.80e-04 | 2305.2ms/step | 10.4 samples/s
+Step 150/20005 | Loss: 6.4523 | LR: 3.00e-04 | 2307.4ms/step | 10.4 samples/s
+Step 160/20005 | Loss: 6.3559 | LR: 3.20e-04 | 2307.6ms/step | 10.4 samples/s
+Step 170/20005 | Loss: 6.2772 | LR: 3.40e-04 | 2306.1ms/step | 10.4 samples/s
+Step 180/20005 | Loss: 6.2081 | LR: 3.60e-04 | 2306.9ms/step | 10.4 samples/s
+Step 190/20005 | Loss: 6.1666 | LR: 3.80e-04 | 2309.4ms/step | 10.4 samples/s
+Step 200/20005 | Loss: 6.1099 | LR: 4.00e-04 | 2309.5ms/step | 10.4 samples/s
+Step 210/20005 | Loss: 6.0363 | LR: 4.20e-04 | 2307.6ms/step | 10.4 samples/s
+Step 220/20005 | Loss: 6.0059 | LR: 4.40e-04 | 2308.5ms/step | 10.4 samples/s
+Step 230/20005 | Loss: 5.9565 | LR: 4.60e-04 | 2306.4ms/step | 10.4 samples/s
+Step 240/20005 | Loss: 5.8671 | LR: 4.80e-04 | 2306.9ms/step | 10.4 samples/s
+Step 250/20005 | Loss: 5.7959 | LR: 5.00e-04 | 2307.3ms/step | 10.4 samples/s
+Validation at step 250: Loss = 5.7678, Perplexity = 319.84
+Step 260/20005 | Loss: 5.7540 | LR: 5.00e-04 | 19802.1ms/step | 1.2 samples/s
+Step 270/20005 | Loss: 5.6804 | LR: 5.00e-04 | 2309.1ms/step | 10.4 samples/s
+Step 280/20005 | Loss: 5.6332 | LR: 5.00e-04 | 2312.7ms/step | 10.4 samples/s
+Step 290/20005 | Loss: 5.5953 | LR: 5.00e-04 | 2308.8ms/step | 10.4 samples/s
+Step 300/20005 | Loss: 5.4834 | LR: 5.00e-04 | 2307.3ms/step | 10.4 samples/s
+Step 310/20005 | Loss: 5.4939 | LR: 5.00e-04 | 2306.5ms/step | 10.4 samples/s
+Step 320/20005 | Loss: 5.4410 | LR: 5.00e-04 | 2307.0ms/step | 10.4 samples/s
+Step 330/20005 | Loss: 5.3622 | LR: 5.00e-04 | 2308.5ms/step | 10.4 samples/s
+Step 340/20005 | Loss: 5.3009 | LR: 5.00e-04 | 2308.1ms/step | 10.4 samples/s
+Step 350/20005 | Loss: 5.2969 | LR: 5.00e-04 | 2307.7ms/step | 10.4 samples/s
+Step 360/20005 | Loss: 5.2176 | LR: 5.00e-04 | 2312.4ms/step | 10.4 samples/s
+Step 370/20005 | Loss: 5.2223 | LR: 5.00e-04 | 2310.7ms/step | 10.4 samples/s
+Step 380/20005 | Loss: 5.1195 | LR: 5.00e-04 | 2308.7ms/step | 10.4 samples/s
+Step 390/20005 | Loss: 5.0926 | LR: 5.00e-04 | 2309.7ms/step | 10.4 samples/s
+Step 400/20005 | Loss: 5.0550 | LR: 5.00e-04 | 2312.8ms/step | 10.4 samples/s
+Step 410/20005 | Loss: 5.0757 | LR: 5.00e-04 | 2311.4ms/step | 10.4 samples/s
+Step 420/20005 | Loss: 5.0367 | LR: 5.00e-04 | 2309.7ms/step | 10.4 samples/s
+Step 430/20005 | Loss: 4.9661 | LR: 5.00e-04 | 2307.4ms/step | 10.4 samples/s
+Step 440/20005 | Loss: 4.9184 | LR: 5.00e-04 | 2307.5ms/step | 10.4 samples/s
+Step 450/20005 | Loss: 4.9099 | LR: 5.00e-04 | 2308.7ms/step | 10.4 samples/s
+Step 460/20005 | Loss: 4.8951 | LR: 5.00e-04 | 2305.6ms/step | 10.4 samples/s
+Step 470/20005 | Loss: 4.8453 | LR: 5.00e-04 | 2309.0ms/step | 10.4 samples/s
+Step 480/20005 | Loss: 4.8152 | LR: 5.00e-04 | 2309.6ms/step | 10.4 samples/s
+Step 490/20005 | Loss: 4.7827 | LR: 5.00e-04 | 2464.6ms/step | 9.7 samples/s
+Step 500/20005 | Loss: 4.7654 | LR: 5.00e-04 | 2305.6ms/step | 10.4 samples/s
+Validation at step 500: Loss = 4.7523, Perplexity = 115.84
+Step 510/20005 | Loss: 4.7384 | LR: 5.00e-04 | 17401.1ms/step | 1.4 samples/s
+Step 520/20005 | Loss: 4.6919 | LR: 5.00e-04 | 2312.1ms/step | 10.4 samples/s
+Step 530/20005 | Loss: 4.6701 | LR: 5.00e-04 | 2317.8ms/step | 10.4 samples/s
+Step 540/20005 | Loss: 4.6117 | LR: 5.00e-04 | 2315.3ms/step | 10.4 samples/s
+Step 550/20005 | Loss: 4.6440 | LR: 5.00e-04 | 2318.6ms/step | 10.4 samples/s
+Step 560/20005 | Loss: 4.5925 | LR: 5.00e-04 | 2315.9ms/step | 10.4 samples/s
+Step 570/20005 | Loss: 4.5736 | LR: 5.00e-04 | 2316.9ms/step | 10.4 samples/s
+Step 580/20005 | Loss: 4.5060 | LR: 5.00e-04 | 2319.4ms/step | 10.3 samples/s
+Step 590/20005 | Loss: 4.4832 | LR: 5.00e-04 | 2317.8ms/step | 10.4 samples/s
+Step 600/20005 | Loss: 4.4877 | LR: 5.00e-04 | 2319.8ms/step | 10.3 samples/s
+Step 610/20005 | Loss: 4.4661 | LR: 5.00e-04 | 2323.7ms/step | 10.3 samples/s
+Step 620/20005 | Loss: 4.4391 | LR: 5.00e-04 | 2321.2ms/step | 10.3 samples/s
+Step 630/20005 | Loss: 4.4255 | LR: 5.00e-04 | 2321.0ms/step | 10.3 samples/s
+Step 640/20005 | Loss: 4.4375 | LR: 5.00e-04 | 2320.3ms/step | 10.3 samples/s
+Step 650/20005 | Loss: 4.4180 | LR: 4.99e-04 | 2319.8ms/step | 10.3 samples/s
+Step 660/20005 | Loss: 4.3344 | LR: 4.99e-04 | 2317.9ms/step | 10.4 samples/s
+Step 670/20005 | Loss: 4.3440 | LR: 4.99e-04 | 2314.6ms/step | 10.4 samples/s
+Step 680/20005 | Loss: 4.3308 | LR: 4.99e-04 | 2319.8ms/step | 10.3 samples/s
+Step 690/20005 | Loss: 4.2861 | LR: 4.99e-04 | 2322.0ms/step | 10.3 samples/s
+Step 700/20005 | Loss: 4.2852 | LR: 4.99e-04 | 2318.6ms/step | 10.4 samples/s
+Step 710/20005 | Loss: 4.2833 | LR: 4.99e-04 | 2315.7ms/step | 10.4 samples/s
+Step 720/20005 | Loss: 4.2582 | LR: 4.99e-04 | 2317.4ms/step | 10.4 samples/s
+Step 730/20005 | Loss: 4.2297 | LR: 4.99e-04 | 2319.9ms/step | 10.3 samples/s
+Step 740/20005 | Loss: 4.2219 | LR: 4.99e-04 | 2321.4ms/step | 10.3 samples/s
+Step 750/20005 | Loss: 4.1901 | LR: 4.99e-04 | 2318.6ms/step | 10.4 samples/s
+Validation at step 750: Loss = 4.2170, Perplexity = 67.83
+Step 760/20005 | Loss: 4.2028 | LR: 4.99e-04 | 17478.8ms/step | 1.4 samples/s
+Step 770/20005 | Loss: 4.2164 | LR: 4.99e-04 | 2322.4ms/step | 10.3 samples/s
+Step 780/20005 | Loss: 4.1790 | LR: 4.99e-04 | 2320.1ms/step | 10.3 samples/s
+Step 790/20005 | Loss: 4.1733 | LR: 4.99e-04 | 2319.4ms/step | 10.3 samples/s
+Step 800/20005 | Loss: 4.1427 | LR: 4.99e-04 | 2319.0ms/step | 10.3 samples/s
+Step 810/20005 | Loss: 4.1342 | LR: 4.99e-04 | 2318.7ms/step | 10.4 samples/s
+Step 820/20005 | Loss: 4.1324 | LR: 4.99e-04 | 2319.6ms/step | 10.3 samples/s
+Step 830/20005 | Loss: 4.1026 | LR: 4.99e-04 | 2318.2ms/step | 10.4 samples/s
+Step 840/20005 | Loss: 4.0817 | LR: 4.99e-04 | 2318.0ms/step | 10.4 samples/s
+Step 850/20005 | Loss: 4.0863 | LR: 4.99e-04 | 2317.7ms/step | 10.4 samples/s
+Step 860/20005 | Loss: 4.0704 | LR: 4.99e-04 | 2318.6ms/step | 10.4 samples/s
+Step 870/20005 | Loss: 4.0846 | LR: 4.99e-04 | 2322.0ms/step | 10.3 samples/s
+Step 880/20005 | Loss: 4.0708 | LR: 4.99e-04 | 2319.7ms/step | 10.3 samples/s
+Step 890/20005 | Loss: 4.0503 | LR: 4.99e-04 | 2316.6ms/step | 10.4 samples/s
+Step 900/20005 | Loss: 4.0287 | LR: 4.99e-04 | 2317.0ms/step | 10.4 samples/s
+Step 910/20005 | Loss: 4.0380 | LR: 4.99e-04 | 2317.0ms/step | 10.4 samples/s
+Step 920/20005 | Loss: 4.0394 | LR: 4.99e-04 | 2320.7ms/step | 10.3 samples/s
+Step 930/20005 | Loss: 3.9820 | LR: 4.99e-04 | 2319.2ms/step | 10.3 samples/s
+Step 940/20005 | Loss: 4.0125 | LR: 4.98e-04 | 2318.8ms/step | 10.4 samples/s
+Step 950/20005 | Loss: 3.9945 | LR: 4.98e-04 | 2317.5ms/step | 10.4 samples/s
+Step 960/20005 | Loss: 4.0175 | LR: 4.98e-04 | 2318.9ms/step | 10.3 samples/s
+Step 970/20005 | Loss: 4.0122 | LR: 4.98e-04 | 2317.8ms/step | 10.4 samples/s
+Step 980/20005 | Loss: 3.9866 | LR: 4.98e-04 | 2319.2ms/step | 10.3 samples/s
+Step 990/20005 | Loss: 3.9478 | LR: 4.98e-04 | 2322.1ms/step | 10.3 samples/s
+Step 1000/20005 | Loss: 3.9490 | LR: 4.98e-04 | 2320.6ms/step | 10.3 samples/s
+Validation at step 1000: Loss = 3.9642, Perplexity = 52.68
+Saved checkpoint to checkpoints/run_20260517_023855/checkpoint-1000
+Step 1010/20005 | Loss: 3.9754 | LR: 4.98e-04 | 17894.0ms/step | 1.3 samples/s
+Step 1020/20005 | Loss: 3.9452 | LR: 4.98e-04 | 2316.2ms/step | 10.4 samples/s
+Step 1030/20005 | Loss: 3.9162 | LR: 4.98e-04 | 2319.5ms/step | 10.3 samples/s
+Step 1040/20005 | Loss: 3.9190 | LR: 4.98e-04 | 2319.3ms/step | 10.3 samples/s
+Step 1050/20005 | Loss: 3.9139 | LR: 4.98e-04 | 2316.4ms/step | 10.4 samples/s
+Step 1060/20005 | Loss: 3.9369 | LR: 4.98e-04 | 2315.1ms/step | 10.4 samples/s
+Step 1070/20005 | Loss: 3.9042 | LR: 4.98e-04 | 2317.9ms/step | 10.4 samples/s
+Step 1080/20005 | Loss: 3.9171 | LR: 4.98e-04 | 2317.7ms/step | 10.4 samples/s
+Step 1090/20005 | Loss: 3.8790 | LR: 4.98e-04 | 2316.5ms/step | 10.4 samples/s
+Step 1100/20005 | Loss: 3.8872 | LR: 4.98e-04 | 2315.5ms/step | 10.4 samples/s
+Step 1110/20005 | Loss: 3.9088 | LR: 4.98e-04 | 2318.1ms/step | 10.4 samples/s
+Step 1120/20005 | Loss: 3.8880 | LR: 4.98e-04 | 2316.4ms/step | 10.4 samples/s
+Step 1130/20005 | Loss: 3.8920 | LR: 4.98e-04 | 2316.9ms/step | 10.4 samples/s
+Step 1140/20005 | Loss: 3.8729 | LR: 4.98e-04 | 2315.2ms/step | 10.4 samples/s
+Step 1150/20005 | Loss: 3.8558 | LR: 4.97e-04 | 2314.9ms/step | 10.4 samples/s
+Step 1160/20005 | Loss: 3.8576 | LR: 4.97e-04 | 2316.4ms/step | 10.4 samples/s
+Step 1170/20005 | Loss: 3.8610 | LR: 4.97e-04 | 2316.8ms/step | 10.4 samples/s
+Step 1180/20005 | Loss: 3.8576 | LR: 4.97e-04 | 2318.6ms/step | 10.4 samples/s
+Step 1190/20005 | Loss: 3.8300 | LR: 4.97e-04 | 2310.9ms/step | 10.4 samples/s
+Step 1200/20005 | Loss: 3.8220 | LR: 4.97e-04 | 2315.8ms/step | 10.4 samples/s
+Step 1210/20005 | Loss: 3.8281 | LR: 4.97e-04 | 2315.0ms/step | 10.4 samples/s
+Step 1220/20005 | Loss: 3.8404 | LR: 4.97e-04 | 2317.0ms/step | 10.4 samples/s
+Step 1230/20005 | Loss: 3.8407 | LR: 4.97e-04 | 2313.9ms/step | 10.4 samples/s
+Step 1240/20005 | Loss: 3.7954 | LR: 4.97e-04 | 2316.0ms/step | 10.4 samples/s
+Step 1250/20005 | Loss: 3.8445 | LR: 4.97e-04 | 2320.9ms/step | 10.3 samples/s
+Validation at step 1250: Loss = 3.8064, Perplexity = 44.99
+Step 1260/20005 | Loss: 3.7741 | LR: 4.97e-04 | 17457.6ms/step | 1.4 samples/s
+Step 1270/20005 | Loss: 3.7992 | LR: 4.97e-04 | 2318.0ms/step | 10.4 samples/s
+Step 1280/20005 | Loss: 3.7852 | LR: 4.97e-04 | 2315.2ms/step | 10.4 samples/s
+Step 1290/20005 | Loss: 3.7934 | LR: 4.97e-04 | 2319.6ms/step | 10.3 samples/s
+Step 1300/20005 | Loss: 3.7663 | LR: 4.97e-04 | 2316.4ms/step | 10.4 samples/s
+Step 1310/20005 | Loss: 3.7880 | LR: 4.96e-04 | 2317.9ms/step | 10.4 samples/s
+Step 1320/20005 | Loss: 3.7973 | LR: 4.96e-04 | 2317.8ms/step | 10.4 samples/s
+Step 1330/20005 | Loss: 3.7853 | LR: 4.96e-04 | 2314.5ms/step | 10.4 samples/s

epfl_final/wandb/run-20260517_023932-hi2z3jh6/files/requirements.txt

@@ -0,0 +1,97 @@
+pytz==2026.2
+nvidia-cusparselt-cu13==0.8.1
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.7.0
+tzdata==2026.2
+typing_extensions==4.15.0
+triton==3.7.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+safetensors==0.7.0
+regex==2026.5.9
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.5.2
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.29.7
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+numpy==2.2.6
+networkx==3.4.2
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.15
+hf-xet==1.5.0
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.4.0
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+async-timeout==5.0.1
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+typing-inspection==0.4.2
+sentry-sdk==2.60.0
+requests==2.34.2
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.4
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cublas==13.1.1.3
+multiprocess==0.70.19
+multidict==6.7.1
+markdown-it-py==4.2.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+exceptiongroup==1.3.1
+cuda-bindings==13.2.0
+aiosignal==1.4.0
+yarl==1.23.0
+rich==15.0.0
+pydantic==2.13.4
+pandas==2.3.3
+nvidia-cusolver==12.0.4.66
+nvidia-cudnn-cu13==9.20.0.48
+GitPython==3.1.50
+anyio==4.13.0
+wandb==0.27.0
+typer==0.25.1
+httpx==0.28.1
+aiohttp==3.13.5
+torch==2.12.0
+huggingface_hub==1.15.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.8.1
+PyGObject==3.42.1
+dbus-python==1.2.18
+distro==1.7.0
+pip==22.0.2
+setuptools==59.6.0
+ssh-import-id==5.11
+wheel==0.37.1

epfl_final/wandb/run-20260517_023932-hi2z3jh6/files/wandb-metadata.json

@@ -0,0 +1,36 @@
+{
+  "os": "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.12",
+  "startedAt": "2026-05-17T02:39:32.492332Z",
+  "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath": "SparseAtt/train.py",
+  "codePathLocal": "SparseAtt/train.py",
+  "email": "ruben.navasardyan@mail.utoronto.ca",
+  "root": "/mloscratch/homes/navasard/sparse",
+  "host": "h100-train-14-0-3",
+  "executable": "/usr/bin/python3",
+  "cpu_count": 96,
+  "cpu_count_logical": 192,
+  "gpu": "NVIDIA H100 80GB HBM3",
+  "gpu_count": 1,
+  "disk": {
+    "/": {
+      "total": "9374931091456",
+      "used": "6279380570112"
+    }
+  },
+  "memory": {
+    "total": "1623005847552"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H100 80GB HBM3",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-581c6a63-5bcc-c76f-16dd-48cb7b247128"
+    }
+  ],
+  "cudaVersion": "13.0",
+  "writerId": "of3l55g0mwmpwkjdd292y1wm2xp68kak"
+}

epfl_final/wandb/run-20260517_023932-hi2z3jh6/logs/debug-internal.log

@@ -0,0 +1,533 @@
+{"time":"2026-05-17T02:39:32.758711377Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-05-17T02:39:32.761709088Z","level":"INFO","msg":"stream: starting","core version":"0.27.0"}
+{"time":"2026-05-17T02:39:33.055914603Z","level":"INFO","msg":"stream: created new stream","id":"hi2z3jh6"}
+{"time":"2026-05-17T02:39:33.056081794Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-05-17T02:39:33.058228047Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-05-17T02:39:33.058302419Z","level":"INFO","msg":"writer: started","stream_id":"hi2z3jh6"}
+{"time":"2026-05-17T02:39:33.058340676Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-05-17T02:39:33.492158196Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":3}
+{"time":"2026-05-17T02:39:33.759034362Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:39:48.493157075Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":1,"console_offset":3,"console_lines":103,"uploaded_len":2}
+{"time":"2026-05-17T02:39:48.7405926Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:40:03.492632137Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":1,"events_lines":2}
+{"time":"2026-05-17T02:40:03.701998586Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:40:18.492980901Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":3,"events_lines":2}
+{"time":"2026-05-17T02:40:18.694150722Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:40:33.492406909Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":5,"events_lines":2}
+{"time":"2026-05-17T02:40:33.736412533Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:40:48.492266388Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":7,"events_lines":2}
+{"time":"2026-05-17T02:40:48.738039005Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:41:03.49244106Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":9,"events_lines":2,"console_offset":106,"console_lines":1}
+{"time":"2026-05-17T02:41:03.707305695Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:41:18.493069624Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":11,"events_lines":2}
+{"time":"2026-05-17T02:41:18.704252571Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:41:33.492465847Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":13,"events_lines":2,"console_offset":107,"console_lines":1}
+{"time":"2026-05-17T02:41:33.734144096Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:41:48.492626779Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":15,"events_lines":2}
+{"time":"2026-05-17T02:41:48.706376981Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:42:03.492952909Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":17,"events_lines":2,"console_offset":108,"console_lines":1}
+{"time":"2026-05-17T02:42:03.706513716Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:42:18.492738587Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":1,"events_offset":19,"events_lines":2,"console_offset":109,"console_lines":1}
+{"time":"2026-05-17T02:42:18.715702935Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:42:33.493072378Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":21,"events_lines":2}
+{"time":"2026-05-17T02:42:33.704516407Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:42:48.492977365Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":3,"history_lines":1,"events_offset":23,"events_lines":2,"console_offset":110,"console_lines":1}
+{"time":"2026-05-17T02:42:48.712169068Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:43:03.492333238Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":4,"history_lines":1,"events_offset":25,"events_lines":2,"console_offset":111,"console_lines":1}
+{"time":"2026-05-17T02:43:03.727131857Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:43:18.492603965Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":27,"events_lines":2}
+{"time":"2026-05-17T02:43:18.746684577Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:43:33.493078587Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":1,"events_offset":29,"events_lines":2,"console_offset":112,"console_lines":1}
+{"time":"2026-05-17T02:43:33.743746255Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:43:48.49244764Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":1,"events_offset":31,"events_lines":2,"console_offset":113,"console_lines":1}
+{"time":"2026-05-17T02:43:48.741471159Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:44:03.492995022Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":33,"events_lines":2}
+{"time":"2026-05-17T02:44:03.802671198Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:44:18.492891367Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":7,"history_lines":1,"events_offset":35,"events_lines":2,"console_offset":114,"console_lines":1}
+{"time":"2026-05-17T02:44:18.713024523Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:44:33.492375145Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":8,"history_lines":1,"events_offset":37,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-05-17T02:44:33.750374173Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:44:48.492869046Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":39,"events_lines":2}
+{"time":"2026-05-17T02:44:48.816534503Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:45:03.493148486Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":1,"events_offset":41,"events_lines":2,"console_offset":116,"console_lines":1}
+{"time":"2026-05-17T02:45:03.772797575Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:45:18.493023137Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":10,"history_lines":1,"events_offset":43,"events_lines":2,"console_offset":117,"console_lines":1}
+{"time":"2026-05-17T02:45:18.731450375Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:45:33.492974604Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":45,"events_lines":2}
+{"time":"2026-05-17T02:45:33.769223397Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:45:48.492914584Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":11,"history_lines":1,"events_offset":47,"events_lines":2,"console_offset":118,"console_lines":1}
+{"time":"2026-05-17T02:45:48.703159531Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:46:03.492408834Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":12,"history_lines":1,"events_offset":49,"events_lines":2,"console_offset":119,"console_lines":1}
+{"time":"2026-05-17T02:46:03.824262377Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:46:18.493082155Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":51,"events_lines":2}
+{"time":"2026-05-17T02:46:19.078585485Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:46:33.493063447Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":13,"history_lines":1,"events_offset":53,"events_lines":2,"console_offset":120,"console_lines":1}
+{"time":"2026-05-17T02:46:33.703324779Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:46:48.492929546Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":14,"history_lines":1,"events_offset":55,"events_lines":2,"console_offset":121,"console_lines":1}
+{"time":"2026-05-17T02:46:48.715577618Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:47:03.492384006Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":57,"events_lines":2}
+{"time":"2026-05-17T02:47:03.706498013Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:47:18.492516765Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":15,"history_lines":1,"events_offset":59,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-05-17T02:47:18.701200829Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:47:33.492941192Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":61,"events_lines":2}
+{"time":"2026-05-17T02:47:33.700508779Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:47:48.492841422Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":16,"history_lines":1,"events_offset":63,"events_lines":2,"console_offset":123,"console_lines":1}
+{"time":"2026-05-17T02:47:48.700299415Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:48:03.493185038Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":17,"history_lines":1,"events_offset":65,"events_lines":2,"console_offset":124,"console_lines":1}
+{"time":"2026-05-17T02:48:03.724098152Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:48:18.49298018Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":67,"events_lines":2}
+{"time":"2026-05-17T02:48:18.763140279Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:48:33.492538396Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":18,"history_lines":1,"events_offset":69,"events_lines":2,"console_offset":125,"console_lines":1}
+{"time":"2026-05-17T02:48:33.725386546Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:48:48.493219939Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":19,"history_lines":1,"events_offset":71,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-05-17T02:48:48.743684794Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:49:03.492204835Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":73,"events_lines":2}
+{"time":"2026-05-17T02:49:03.708470533Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:49:18.492772687Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":20,"history_lines":1,"events_offset":75,"events_lines":2,"console_offset":127,"console_lines":1}
+{"time":"2026-05-17T02:49:18.723043386Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:49:33.492937324Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":21,"history_lines":1,"events_offset":77,"events_lines":2,"console_offset":128,"console_lines":1}
+{"time":"2026-05-17T02:49:33.782298858Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:49:48.492479166Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":79,"events_lines":2}
+{"time":"2026-05-17T02:49:48.785782038Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:50:03.492879277Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":22,"history_lines":1,"events_offset":81,"events_lines":2,"console_offset":129,"console_lines":1}
+{"time":"2026-05-17T02:50:03.864587618Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:50:18.492601359Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":23,"history_lines":1,"events_offset":83,"events_lines":2,"console_offset":130,"console_lines":1}
+{"time":"2026-05-17T02:50:18.908408542Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:50:33.492217262Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":85,"events_lines":2}
+{"time":"2026-05-17T02:50:33.736779783Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:50:48.492411283Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":87,"events_lines":2}
+{"time":"2026-05-17T02:50:48.802350257Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:51:03.49260289Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":89,"events_lines":2}
+{"time":"2026-05-17T02:51:03.713791441Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:51:18.492686957Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":91,"events_lines":2}
+{"time":"2026-05-17T02:51:18.725648627Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:51:33.492144171Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":93,"events_lines":2}
+{"time":"2026-05-17T02:51:33.720073665Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:51:48.492999906Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":95,"events_lines":2}
+{"time":"2026-05-17T02:51:48.74548231Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:52:03.492434276Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":97,"events_lines":2}
+{"time":"2026-05-17T02:52:03.767581606Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:52:18.493055109Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":99,"events_lines":2}
+{"time":"2026-05-17T02:52:18.738934527Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:52:33.492330991Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":101,"events_lines":2}
+{"time":"2026-05-17T02:52:33.763659225Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:52:48.492646555Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":103,"events_lines":2}
+{"time":"2026-05-17T02:52:48.706095704Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:53:03.492405332Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":105,"events_lines":2}
+{"time":"2026-05-17T02:53:03.841553275Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:53:18.492360236Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":107,"events_lines":2,"console_offset":131,"console_lines":1}
+{"time":"2026-05-17T02:53:18.775191028Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:53:33.492390202Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":24,"history_lines":1,"events_offset":109,"events_lines":2,"console_offset":132,"console_lines":1}
+{"time":"2026-05-17T02:53:33.769582406Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:53:48.493098988Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":111,"events_lines":2}
+{"time":"2026-05-17T02:53:48.751133713Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:54:03.492267051Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":25,"history_lines":1,"events_offset":113,"events_lines":2,"console_offset":133,"console_lines":1}
+{"time":"2026-05-17T02:54:03.786807036Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:54:18.492920974Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":115,"events_lines":2}
+{"time":"2026-05-17T02:54:18.809651193Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:54:33.492638993Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":26,"history_lines":1,"events_offset":117,"events_lines":2,"console_offset":134,"console_lines":1}
+{"time":"2026-05-17T02:54:34.162149047Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:54:48.49258941Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":27,"history_lines":1,"events_offset":119,"events_lines":2,"console_offset":135,"console_lines":1}
+{"time":"2026-05-17T02:54:48.747993744Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:55:03.492166937Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":121,"events_lines":2}
+{"time":"2026-05-17T02:55:03.880144429Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:55:18.492593669Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":28,"history_lines":1,"events_offset":123,"events_lines":2,"console_offset":136,"console_lines":1}
+{"time":"2026-05-17T02:55:18.758845524Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:55:33.492383806Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":29,"history_lines":1,"events_offset":125,"events_lines":2,"console_offset":137,"console_lines":1}
+{"time":"2026-05-17T02:55:33.791458691Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:55:48.493067083Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":127,"events_lines":2}
+{"time":"2026-05-17T02:55:48.786821149Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:56:03.49259103Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":30,"history_lines":1,"events_offset":129,"events_lines":2,"console_offset":138,"console_lines":1}
+{"time":"2026-05-17T02:56:03.736954134Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:56:18.493021598Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":31,"history_lines":1,"events_offset":131,"events_lines":2,"console_offset":139,"console_lines":1}
+{"time":"2026-05-17T02:56:19.036310124Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:56:33.492374298Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":133,"events_lines":2}
+{"time":"2026-05-17T02:56:33.763446662Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:56:48.492276883Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":32,"history_lines":1,"events_offset":135,"events_lines":2,"console_offset":140,"console_lines":1}
+{"time":"2026-05-17T02:56:48.919460142Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:57:03.492538908Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":33,"history_lines":1,"events_offset":137,"events_lines":2,"console_offset":141,"console_lines":1}
+{"time":"2026-05-17T02:57:03.739403028Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:57:18.49317828Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":139,"events_lines":2}
+{"time":"2026-05-17T02:57:18.76584637Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:57:33.49301082Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":34,"history_lines":1,"events_offset":141,"events_lines":2,"console_offset":142,"console_lines":1}
+{"time":"2026-05-17T02:57:33.709371022Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:57:48.492538863Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":35,"history_lines":1,"events_offset":143,"events_lines":2,"console_offset":143,"console_lines":1}
+{"time":"2026-05-17T02:57:48.722686009Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:58:03.492608159Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":145,"events_lines":2}
+{"time":"2026-05-17T02:58:03.69108178Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:58:18.493120059Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":36,"history_lines":1,"events_offset":147,"events_lines":2,"console_offset":144,"console_lines":1}
+{"time":"2026-05-17T02:58:18.701982632Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:58:33.492668793Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":37,"history_lines":1,"events_offset":149,"events_lines":2,"console_offset":145,"console_lines":1}
+{"time":"2026-05-17T02:58:33.795654372Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:58:48.493005601Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":151,"events_lines":2}
+{"time":"2026-05-17T02:58:48.705085675Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:59:03.492581485Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":38,"history_lines":1,"events_offset":153,"events_lines":2,"console_offset":146,"console_lines":1}
+{"time":"2026-05-17T02:59:03.727280507Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:59:18.492869499Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":155,"events_lines":2}
+{"time":"2026-05-17T02:59:18.72192811Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:59:33.492270852Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":39,"history_lines":1,"events_offset":157,"events_lines":2,"console_offset":147,"console_lines":1}
+{"time":"2026-05-17T02:59:33.698840231Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T02:59:48.49322061Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":40,"history_lines":1,"events_offset":159,"events_lines":2,"console_offset":148,"console_lines":1}
+{"time":"2026-05-17T02:59:48.739096866Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:00:03.492578698Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":161,"events_lines":2}
+{"time":"2026-05-17T03:00:03.764557232Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:00:18.492929065Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":41,"history_lines":1,"events_offset":163,"events_lines":2,"console_offset":149,"console_lines":1}
+{"time":"2026-05-17T03:00:18.895581205Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:00:33.492831861Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":42,"history_lines":1,"events_offset":165,"events_lines":2,"console_offset":150,"console_lines":1}
+{"time":"2026-05-17T03:00:33.968792639Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:00:48.493101968Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":167,"events_lines":2}
+{"time":"2026-05-17T03:00:48.853008296Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:01:03.493160027Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":43,"history_lines":1,"events_offset":169,"events_lines":2,"console_offset":151,"console_lines":1}
+{"time":"2026-05-17T03:01:03.766976131Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:01:18.492394479Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":44,"history_lines":1,"events_offset":171,"events_lines":2,"console_offset":152,"console_lines":1}
+{"time":"2026-05-17T03:01:18.846436069Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:01:33.492511196Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":173,"events_lines":2}
+{"time":"2026-05-17T03:01:33.736415261Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:01:48.4931635Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":45,"history_lines":1,"events_offset":175,"events_lines":2,"console_offset":153,"console_lines":1}
+{"time":"2026-05-17T03:01:48.888890776Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:02:03.492770515Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":46,"history_lines":1,"events_offset":177,"events_lines":2,"console_offset":154,"console_lines":1}
+{"time":"2026-05-17T03:02:03.743260921Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:02:18.492775247Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":179,"events_lines":2}
+{"time":"2026-05-17T03:02:18.690294473Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:02:33.493053069Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":47,"history_lines":1,"events_offset":181,"events_lines":2,"console_offset":155,"console_lines":1}
+{"time":"2026-05-17T03:02:33.821121999Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:02:48.492936349Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":183,"events_lines":2}
+{"time":"2026-05-17T03:02:48.718536165Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:03:03.493267722Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":48,"history_lines":1,"events_offset":185,"events_lines":2,"console_offset":156,"console_lines":1}
+{"time":"2026-05-17T03:03:03.711255076Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:03:18.493192033Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":187,"events_lines":2}
+{"time":"2026-05-17T03:03:19.207182172Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:03:33.492544545Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":189,"events_lines":2}
+{"time":"2026-05-17T03:03:33.721013019Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:03:48.49290394Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":191,"events_lines":2}
+{"time":"2026-05-17T03:03:48.700035762Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:04:03.492605537Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":193,"events_lines":2}
+{"time":"2026-05-17T03:04:03.716428875Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:04:18.492900054Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":195,"events_lines":2}
+{"time":"2026-05-17T03:04:18.75079549Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:04:33.492302812Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":197,"events_lines":2}
+{"time":"2026-05-17T03:04:33.715704537Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:04:48.492312982Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":199,"events_lines":2}
+{"time":"2026-05-17T03:04:48.737401686Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:05:03.492337644Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":201,"events_lines":2}
+{"time":"2026-05-17T03:05:03.79213055Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:05:18.49238557Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":203,"events_lines":2}
+{"time":"2026-05-17T03:05:18.77594714Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:05:33.492570172Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":205,"events_lines":2,"console_offset":157,"console_lines":1}
+{"time":"2026-05-17T03:05:33.863401169Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:05:48.492433451Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":49,"history_lines":1,"events_offset":207,"events_lines":2,"console_offset":158,"console_lines":1}
+{"time":"2026-05-17T03:05:48.939923021Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:06:03.49274894Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":209,"events_lines":2}
+{"time":"2026-05-17T03:06:03.728492349Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:06:18.492971819Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":50,"history_lines":1,"events_offset":211,"events_lines":2,"console_offset":159,"console_lines":1}
+{"time":"2026-05-17T03:06:18.711146605Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:06:33.492438694Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":51,"history_lines":1,"events_offset":213,"events_lines":2,"console_offset":160,"console_lines":1}
+{"time":"2026-05-17T03:06:33.694369084Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:06:48.492323966Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":215,"events_lines":2}
+{"time":"2026-05-17T03:06:48.716729068Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:07:03.493115029Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":52,"history_lines":1,"events_offset":217,"events_lines":2,"console_offset":161,"console_lines":1}
+{"time":"2026-05-17T03:07:03.862136299Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:07:18.492660021Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":53,"history_lines":1,"events_offset":219,"events_lines":2,"console_offset":162,"console_lines":1}
+{"time":"2026-05-17T03:07:18.867756765Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:07:33.492495118Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":221,"events_lines":2}
+{"time":"2026-05-17T03:07:34.070780171Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:07:48.493246732Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":54,"history_lines":1,"events_offset":223,"events_lines":2,"console_offset":163,"console_lines":1}
+{"time":"2026-05-17T03:07:49.097545812Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:08:03.493259987Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":55,"history_lines":1,"events_offset":225,"events_lines":2,"console_offset":164,"console_lines":1}
+{"time":"2026-05-17T03:08:03.71529548Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:08:18.492688854Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":227,"events_lines":2}
+{"time":"2026-05-17T03:08:18.691452629Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:08:33.492814127Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":56,"history_lines":1,"events_offset":229,"events_lines":2,"console_offset":165,"console_lines":1}
+{"time":"2026-05-17T03:08:34.081051807Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:08:48.492577747Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":57,"history_lines":1,"events_offset":231,"events_lines":2,"console_offset":166,"console_lines":1}
+{"time":"2026-05-17T03:08:48.720112168Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:09:03.492767917Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":233,"events_lines":2}
+{"time":"2026-05-17T03:09:03.711452428Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:09:18.492968403Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":58,"history_lines":1,"events_offset":235,"events_lines":2,"console_offset":167,"console_lines":1}
+{"time":"2026-05-17T03:09:18.690682452Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:09:33.492678633Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":237,"events_lines":2}
+{"time":"2026-05-17T03:09:33.700261322Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:09:48.492678498Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":59,"history_lines":1,"events_offset":239,"events_lines":2,"console_offset":168,"console_lines":1}
+{"time":"2026-05-17T03:09:48.724984825Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:10:03.492994587Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":60,"history_lines":1,"events_offset":241,"events_lines":2,"console_offset":169,"console_lines":1}
+{"time":"2026-05-17T03:10:03.737531502Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:10:18.492097004Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":243,"events_lines":2}
+{"time":"2026-05-17T03:10:18.744004368Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:10:33.492567257Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":61,"history_lines":1,"events_offset":245,"events_lines":2,"console_offset":170,"console_lines":1}
+{"time":"2026-05-17T03:10:33.708933525Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:10:48.493102186Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":62,"history_lines":1,"events_offset":247,"events_lines":2,"console_offset":171,"console_lines":1}
+{"time":"2026-05-17T03:10:48.801041407Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:11:03.492961401Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":249,"events_lines":2}
+{"time":"2026-05-17T03:11:03.780706643Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:11:18.49251075Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":63,"history_lines":1,"events_offset":251,"events_lines":2,"console_offset":172,"console_lines":1}
+{"time":"2026-05-17T03:11:18.744491597Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:11:33.492798278Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":64,"history_lines":1,"events_offset":253,"events_lines":2,"console_offset":173,"console_lines":1}
+{"time":"2026-05-17T03:11:33.861735791Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:11:48.49300856Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":255,"events_lines":2}
+{"time":"2026-05-17T03:11:48.902881634Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:12:03.492324548Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":65,"history_lines":1,"events_offset":257,"events_lines":2,"console_offset":174,"console_lines":1}
+{"time":"2026-05-17T03:12:03.790228227Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:12:18.4929472Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":66,"history_lines":1,"events_offset":259,"events_lines":2,"console_offset":175,"console_lines":1}
+{"time":"2026-05-17T03:12:18.821321061Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:12:33.492846044Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":261,"events_lines":2}
+{"time":"2026-05-17T03:12:33.714490048Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:12:48.493217427Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":67,"history_lines":1,"events_offset":263,"events_lines":2,"console_offset":176,"console_lines":1}
+{"time":"2026-05-17T03:12:48.892246996Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:13:03.492408088Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":68,"history_lines":1,"events_offset":265,"events_lines":2,"console_offset":177,"console_lines":1}
+{"time":"2026-05-17T03:13:03.754036375Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:13:18.493130649Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":267,"events_lines":2}
+{"time":"2026-05-17T03:13:18.722532225Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:13:33.492354209Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":69,"history_lines":1,"events_offset":269,"events_lines":2,"console_offset":178,"console_lines":1}
+{"time":"2026-05-17T03:13:33.880728014Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:13:48.492933335Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":271,"events_lines":2}
+{"time":"2026-05-17T03:13:48.702165699Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:14:03.492802895Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":70,"history_lines":1,"events_offset":273,"events_lines":2,"console_offset":179,"console_lines":1}
+{"time":"2026-05-17T03:14:03.705254864Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:14:18.492471463Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":71,"history_lines":1,"events_offset":275,"events_lines":2,"console_offset":180,"console_lines":1}
+{"time":"2026-05-17T03:14:18.716336093Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:14:33.49302467Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":277,"events_lines":2}
+{"time":"2026-05-17T03:14:33.697495813Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:14:48.492276953Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":72,"history_lines":1,"events_offset":279,"events_lines":2,"console_offset":181,"console_lines":1}
+{"time":"2026-05-17T03:14:48.711393972Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:15:03.492778692Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":73,"history_lines":1,"events_offset":281,"events_lines":2,"console_offset":182,"console_lines":1}
+{"time":"2026-05-17T03:15:03.736107085Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:15:18.492219969Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":283,"events_lines":2}
+{"time":"2026-05-17T03:15:18.714923275Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:15:33.492412675Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":285,"events_lines":2}
+{"time":"2026-05-17T03:15:33.704003Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:15:48.49225263Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":287,"events_lines":2}
+{"time":"2026-05-17T03:15:48.692747351Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:16:03.492809152Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":289,"events_lines":2}
+{"time":"2026-05-17T03:16:03.796944239Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:16:18.493059133Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":291,"events_lines":2}
+{"time":"2026-05-17T03:16:18.708827464Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:16:33.492184446Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":293,"events_lines":2}
+{"time":"2026-05-17T03:16:33.692797304Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:16:48.492117601Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":295,"events_lines":2}
+{"time":"2026-05-17T03:16:48.746132043Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:17:03.493206653Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":297,"events_lines":2}
+{"time":"2026-05-17T03:17:03.714949156Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:17:18.492355671Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":299,"events_lines":2}
+{"time":"2026-05-17T03:17:18.775638931Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:17:33.492928146Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":301,"events_lines":2,"console_offset":183,"console_lines":1}
+{"time":"2026-05-17T03:17:33.730352802Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:17:48.493201964Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":303,"events_lines":2}
+{"time":"2026-05-17T03:17:48.715309495Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:18:03.493138324Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":74,"history_lines":1,"events_offset":305,"events_lines":2,"console_offset":184,"console_lines":1}
+{"time":"2026-05-17T03:18:03.945193992Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:18:18.492871409Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":75,"history_lines":1,"events_offset":307,"events_lines":2,"console_offset":185,"console_lines":1}
+{"time":"2026-05-17T03:18:18.777197275Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:18:33.4926415Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":309,"events_lines":2}
+{"time":"2026-05-17T03:18:33.717738481Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:18:48.49261738Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":76,"history_lines":1,"events_offset":311,"events_lines":2,"console_offset":186,"console_lines":1}
+{"time":"2026-05-17T03:18:48.87833199Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:19:03.492387986Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":313,"events_lines":2}
+{"time":"2026-05-17T03:19:03.690562778Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:19:18.492905831Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":77,"history_lines":1,"events_offset":315,"events_lines":2,"console_offset":187,"console_lines":1}
+{"time":"2026-05-17T03:19:18.762063344Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:19:33.492643403Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":78,"history_lines":1,"events_offset":317,"events_lines":2,"console_offset":188,"console_lines":1}
+{"time":"2026-05-17T03:19:33.703101225Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:19:48.49291683Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":319,"events_lines":2}
+{"time":"2026-05-17T03:19:48.703374703Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:20:03.492542064Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":79,"history_lines":1,"events_offset":321,"events_lines":2,"console_offset":189,"console_lines":1}
+{"time":"2026-05-17T03:20:03.740276228Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:20:18.492257323Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":80,"history_lines":1,"events_offset":323,"events_lines":2,"console_offset":190,"console_lines":1}
+{"time":"2026-05-17T03:20:18.704410582Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:20:33.492252483Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":325,"events_lines":2}
+{"time":"2026-05-17T03:20:33.70457132Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:20:48.492508253Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":81,"history_lines":1,"events_offset":327,"events_lines":2,"console_offset":191,"console_lines":1}
+{"time":"2026-05-17T03:20:48.718557685Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:21:03.49301397Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":82,"history_lines":1,"events_offset":329,"events_lines":2,"console_offset":192,"console_lines":1}
+{"time":"2026-05-17T03:21:03.707162596Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:21:18.492997381Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":331,"events_lines":2}
+{"time":"2026-05-17T03:21:18.809515082Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:21:33.492525919Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":83,"history_lines":1,"events_offset":333,"events_lines":2,"console_offset":193,"console_lines":1}
+{"time":"2026-05-17T03:21:33.699070744Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:21:48.493095912Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":84,"history_lines":1,"events_offset":335,"events_lines":2,"console_offset":194,"console_lines":1}
+{"time":"2026-05-17T03:21:48.709520797Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:22:03.492814456Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":337,"events_lines":2}
+{"time":"2026-05-17T03:22:03.699448404Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:22:18.492481032Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":85,"history_lines":1,"events_offset":339,"events_lines":2,"console_offset":195,"console_lines":1}
+{"time":"2026-05-17T03:22:18.705917198Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:22:33.492311594Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":86,"history_lines":1,"events_offset":341,"events_lines":2,"console_offset":196,"console_lines":1}
+{"time":"2026-05-17T03:22:33.714147957Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:22:48.492890019Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":343,"events_lines":2}
+{"time":"2026-05-17T03:22:48.792103418Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:23:03.492455944Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":87,"history_lines":1,"events_offset":345,"events_lines":2,"console_offset":197,"console_lines":1}
+{"time":"2026-05-17T03:23:03.779653985Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:23:18.492495049Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":347,"events_lines":2}
+{"time":"2026-05-17T03:23:18.717921776Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:23:33.493162348Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":88,"history_lines":1,"events_offset":349,"events_lines":2,"console_offset":198,"console_lines":1}
+{"time":"2026-05-17T03:23:33.734727345Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:23:48.49265307Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":89,"history_lines":1,"events_offset":351,"events_lines":2,"console_offset":199,"console_lines":1}
+{"time":"2026-05-17T03:23:48.732333356Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:24:03.492299014Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":353,"events_lines":2}
+{"time":"2026-05-17T03:24:03.757614847Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:24:18.492532972Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":90,"history_lines":1,"events_offset":355,"events_lines":2,"console_offset":200,"console_lines":1}
+{"time":"2026-05-17T03:24:18.765557504Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:24:33.492764116Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":91,"history_lines":1,"events_offset":357,"events_lines":2,"console_offset":201,"console_lines":1}
+{"time":"2026-05-17T03:24:33.697834045Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:24:48.492231142Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":359,"events_lines":2}
+{"time":"2026-05-17T03:24:48.741813241Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:25:03.492719562Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":92,"history_lines":1,"events_offset":361,"events_lines":2,"console_offset":202,"console_lines":1}
+{"time":"2026-05-17T03:25:03.725147184Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:25:18.492530445Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":93,"history_lines":1,"events_offset":363,"events_lines":2,"console_offset":203,"console_lines":1}
+{"time":"2026-05-17T03:25:18.72852164Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:25:33.492244341Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":365,"events_lines":2}
+{"time":"2026-05-17T03:25:33.897290038Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:25:48.492429406Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":94,"history_lines":1,"events_offset":367,"events_lines":2,"console_offset":204,"console_lines":1}
+{"time":"2026-05-17T03:25:48.794740639Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:26:03.492285576Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":95,"history_lines":1,"events_offset":369,"events_lines":2,"console_offset":205,"console_lines":1}
+{"time":"2026-05-17T03:26:03.792775894Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:26:18.492892495Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":371,"events_lines":2}
+{"time":"2026-05-17T03:26:18.752923459Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:26:33.492252129Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":96,"history_lines":1,"events_offset":373,"events_lines":2,"console_offset":206,"console_lines":1}
+{"time":"2026-05-17T03:26:33.780576362Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:26:48.493126259Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":97,"history_lines":1,"events_offset":375,"events_lines":2,"console_offset":207,"console_lines":1}
+{"time":"2026-05-17T03:26:49.1886677Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:27:03.492924953Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":377,"events_lines":2}
+{"time":"2026-05-17T03:27:03.727741864Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:27:18.493150719Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":98,"history_lines":1,"events_offset":379,"events_lines":2,"console_offset":208,"console_lines":1}
+{"time":"2026-05-17T03:27:18.701279422Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:27:33.492743634Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":381,"events_lines":2}
+{"time":"2026-05-17T03:27:33.706065338Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:27:48.492434476Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":383,"events_lines":2}
+{"time":"2026-05-17T03:27:48.718170246Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:28:03.493051089Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":385,"events_lines":2}
+{"time":"2026-05-17T03:28:03.764534778Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:28:18.49312684Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":387,"events_lines":2}
+{"time":"2026-05-17T03:28:18.76369229Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:28:33.493123943Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":389,"events_lines":2}
+{"time":"2026-05-17T03:28:33.850044658Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:28:48.49263752Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":391,"events_lines":2}
+{"time":"2026-05-17T03:28:48.913257699Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:29:03.492300861Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":393,"events_lines":2}
+{"time":"2026-05-17T03:29:03.797276038Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:29:18.493049693Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":395,"events_lines":2}
+{"time":"2026-05-17T03:29:18.736001995Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:29:33.492984182Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":397,"events_lines":2}
+{"time":"2026-05-17T03:29:33.905165665Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:29:48.493108125Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":399,"events_lines":2,"console_offset":209,"console_lines":2}
+{"time":"2026-05-17T03:29:48.913185379Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:30:03.492272127Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":401,"events_lines":2}
+{"time":"2026-05-17T03:30:03.708439595Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:30:18.492779336Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":99,"history_lines":1,"events_offset":403,"events_lines":2,"console_offset":211,"console_lines":1}
+{"time":"2026-05-17T03:30:18.705025133Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:30:33.492960305Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":100,"history_lines":1,"events_offset":405,"events_lines":2,"console_offset":212,"console_lines":1}
+{"time":"2026-05-17T03:30:33.693817575Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:30:48.493131549Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":407,"events_lines":2}
+{"time":"2026-05-17T03:30:48.723917235Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:31:03.492968362Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":101,"history_lines":1,"events_offset":409,"events_lines":2,"console_offset":213,"console_lines":1}
+{"time":"2026-05-17T03:31:03.713382729Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:31:18.492188658Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":411,"events_lines":2}
+{"time":"2026-05-17T03:31:18.699645651Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:31:33.493249078Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":102,"history_lines":1,"events_offset":413,"events_lines":2,"console_offset":214,"console_lines":1}
+{"time":"2026-05-17T03:31:33.693234658Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:31:48.492552038Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":103,"history_lines":1,"events_offset":415,"events_lines":2,"console_offset":215,"console_lines":1}
+{"time":"2026-05-17T03:31:48.752372598Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:32:03.492475854Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":417,"events_lines":2}
+{"time":"2026-05-17T03:32:03.699289786Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:32:18.493236722Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":104,"history_lines":1,"events_offset":419,"events_lines":2,"console_offset":216,"console_lines":1}
+{"time":"2026-05-17T03:32:18.729305346Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:32:33.492774492Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":105,"history_lines":1,"events_offset":421,"events_lines":2,"console_offset":217,"console_lines":1}
+{"time":"2026-05-17T03:32:33.719364787Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:32:48.492425083Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":423,"events_lines":2}
+{"time":"2026-05-17T03:32:48.726429156Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:33:03.493049126Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":106,"history_lines":1,"events_offset":425,"events_lines":2,"console_offset":218,"console_lines":1}
+{"time":"2026-05-17T03:33:03.698249484Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:33:18.492537342Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":107,"history_lines":1,"events_offset":427,"events_lines":2,"console_offset":219,"console_lines":1}
+{"time":"2026-05-17T03:33:18.733390105Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:33:33.492240802Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":429,"events_lines":2}
+{"time":"2026-05-17T03:33:33.714359428Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:33:48.492445194Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":108,"history_lines":1,"events_offset":431,"events_lines":2,"console_offset":220,"console_lines":1}
+{"time":"2026-05-17T03:33:48.70422663Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:34:03.49230559Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":109,"history_lines":1,"events_offset":433,"events_lines":2,"console_offset":221,"console_lines":1}
+{"time":"2026-05-17T03:34:03.737104582Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:34:18.493038346Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":435,"events_lines":2}
+{"time":"2026-05-17T03:34:18.717930319Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:34:33.493255107Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":110,"history_lines":1,"events_offset":437,"events_lines":2,"console_offset":222,"console_lines":1}
+{"time":"2026-05-17T03:34:33.724740386Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:34:48.493002583Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":111,"history_lines":1,"events_offset":439,"events_lines":2,"console_offset":223,"console_lines":1}
+{"time":"2026-05-17T03:34:48.724054371Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:35:03.493025713Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":441,"events_lines":2}
+{"time":"2026-05-17T03:35:03.732502879Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:35:18.493235063Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":112,"history_lines":1,"events_offset":443,"events_lines":2,"console_offset":224,"console_lines":1}
+{"time":"2026-05-17T03:35:18.751090541Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:35:33.492954907Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":445,"events_lines":2}
+{"time":"2026-05-17T03:35:33.872114385Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:35:48.492585347Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":113,"history_lines":1,"events_offset":447,"events_lines":2,"console_offset":225,"console_lines":1}
+{"time":"2026-05-17T03:35:48.728942606Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:36:03.493107057Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":114,"history_lines":1,"events_offset":449,"events_lines":2,"console_offset":226,"console_lines":1}
+{"time":"2026-05-17T03:36:03.743457198Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:36:18.493167771Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":451,"events_lines":2}
+{"time":"2026-05-17T03:36:18.721802402Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:36:33.492425912Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":115,"history_lines":1,"events_offset":453,"events_lines":2,"console_offset":227,"console_lines":1}
+{"time":"2026-05-17T03:36:33.75811182Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:36:48.492489733Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":116,"history_lines":1,"events_offset":455,"events_lines":2,"console_offset":228,"console_lines":1}
+{"time":"2026-05-17T03:36:48.734250522Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:37:03.49293721Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":457,"events_lines":2}
+{"time":"2026-05-17T03:37:03.748613612Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:37:18.492370144Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":117,"history_lines":1,"events_offset":459,"events_lines":2,"console_offset":229,"console_lines":1}
+{"time":"2026-05-17T03:37:18.777467511Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:37:33.492236669Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":118,"history_lines":1,"events_offset":461,"events_lines":2,"console_offset":230,"console_lines":1}
+{"time":"2026-05-17T03:37:33.89818454Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:37:48.492907442Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":463,"events_lines":2}
+{"time":"2026-05-17T03:37:48.747517882Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:38:03.493116731Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":119,"history_lines":1,"events_offset":465,"events_lines":2,"console_offset":231,"console_lines":1}
+{"time":"2026-05-17T03:38:03.780294283Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:38:18.4927265Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":120,"history_lines":1,"events_offset":467,"events_lines":2,"console_offset":232,"console_lines":1}
+{"time":"2026-05-17T03:38:18.74835438Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:38:33.492668248Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":469,"events_lines":2}
+{"time":"2026-05-17T03:38:33.720208366Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:38:48.492634152Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":121,"history_lines":1,"events_offset":471,"events_lines":2,"console_offset":233,"console_lines":1}
+{"time":"2026-05-17T03:38:48.988176978Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:39:03.492397943Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":122,"history_lines":1,"events_offset":473,"events_lines":2,"console_offset":234,"console_lines":1}
+{"time":"2026-05-17T03:39:03.960546368Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:39:18.49277208Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":475,"events_lines":2}
+{"time":"2026-05-17T03:39:18.719547369Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:39:33.493167189Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":123,"history_lines":1,"events_offset":477,"events_lines":2,"console_offset":235,"console_lines":1}
+{"time":"2026-05-17T03:39:33.688793798Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:39:48.492485961Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":479,"events_lines":2}
+{"time":"2026-05-17T03:39:48.772109525Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:40:03.492622612Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":481,"events_lines":2}
+{"time":"2026-05-17T03:40:03.688784254Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:40:18.492893384Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":483,"events_lines":2}
+{"time":"2026-05-17T03:40:18.724225958Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:40:33.492344895Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":485,"events_lines":2}
+{"time":"2026-05-17T03:40:33.70538528Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:40:48.493097189Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":487,"events_lines":2}
+{"time":"2026-05-17T03:40:48.734683254Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:41:03.493125206Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":489,"events_lines":2}
+{"time":"2026-05-17T03:41:03.904069659Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:41:18.492377346Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":491,"events_lines":2}
+{"time":"2026-05-17T03:41:18.713448004Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:41:33.492476429Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":493,"events_lines":2}
+{"time":"2026-05-17T03:41:33.769298372Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:41:48.49276688Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":495,"events_lines":2}
+{"time":"2026-05-17T03:41:48.76569935Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:42:03.49310432Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":497,"events_lines":2,"console_offset":236,"console_lines":1}
+{"time":"2026-05-17T03:42:03.740430267Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:42:18.492919797Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":499,"events_lines":2}
+{"time":"2026-05-17T03:42:18.761943476Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:42:33.49303868Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":124,"history_lines":1,"events_offset":501,"events_lines":2,"console_offset":237,"console_lines":1}
+{"time":"2026-05-17T03:42:33.713338049Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:42:48.492964423Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":125,"history_lines":1,"events_offset":503,"events_lines":2,"console_offset":238,"console_lines":1}
+{"time":"2026-05-17T03:42:48.702195125Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:43:03.493186059Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":505,"events_lines":2}
+{"time":"2026-05-17T03:43:03.707937615Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:43:18.493068054Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":126,"history_lines":1,"events_offset":507,"events_lines":2,"console_offset":239,"console_lines":1}
+{"time":"2026-05-17T03:43:18.696200025Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:43:33.492629018Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":127,"history_lines":1,"events_offset":509,"events_lines":2,"console_offset":240,"console_lines":1}
+{"time":"2026-05-17T03:43:33.716914735Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:43:48.492460679Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":511,"events_lines":2}
+{"time":"2026-05-17T03:43:48.697280776Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:44:03.492638741Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":128,"history_lines":1,"events_offset":513,"events_lines":2,"console_offset":241,"console_lines":1}
+{"time":"2026-05-17T03:44:03.69349291Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:44:18.493193188Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":129,"history_lines":1,"events_offset":515,"events_lines":2,"console_offset":242,"console_lines":1}
+{"time":"2026-05-17T03:44:18.707252844Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:44:33.492971148Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":517,"events_lines":2}
+{"time":"2026-05-17T03:44:33.720953508Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:44:48.492826796Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":130,"history_lines":1,"events_offset":519,"events_lines":2,"console_offset":243,"console_lines":1}
+{"time":"2026-05-17T03:44:48.777520434Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-05-17T03:45:03.492934181Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":131,"history_lines":1,"events_offset":521,"events_lines":2,"console_offset":244,"console_lines":1}
+{"time":"2026-05-17T03:45:03.719129327Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

epfl_final/wandb/run-20260517_023932-hi2z3jh6/logs/debug.log

@@ -0,0 +1,19 @@
+2026-05-17 02:39:32,510 INFO    MainThread:4622 [wandb_setup.py:_flush():81] Current SDK version is 0.27.0
+2026-05-17 02:39:32,510 INFO    MainThread:4622 [wandb_setup.py:_flush():81] Configure stats pid to 4622
+2026-05-17 02:39:32,511 INFO    MainThread:4622 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-17 02:39:32,512 INFO    MainThread:4622 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260517_023932-hi2z3jh6/logs/debug.log
+2026-05-17 02:39:32,512 INFO    MainThread:4622 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260517_023932-hi2z3jh6/logs/debug-internal.log
+2026-05-17 02:39:32,513 INFO    MainThread:4622 [wandb_init.py:init():851] calling init triggers
+2026-05-17 02:39:32,514 INFO    MainThread:4622 [wandb_init.py:init():856] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': './checkpoints/run_20260517_023855', 'resume_from_checkpoint': None, 'num_epochs': 1, 'max_steps': 20005, 'per_device_batch_size': 3, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 250, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 200, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'sparse 30%', 'wandb_tags': ['qwen3', '0.6b'], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 18, 'sparse_mask_pool_size': 2000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_19', 'dense_mode': False, '_wandb': {}}
+2026-05-17 02:39:32,514 INFO    MainThread:4622 [wandb_init.py:init():899] starting backend
+2026-05-17 02:39:32,749 INFO    MainThread:4622 [wandb_init.py:init():914] sending inform_init request
+2026-05-17 02:39:33,059 INFO    MainThread:4622 [wandb_init.py:init():919] backend started and connected
+2026-05-17 02:39:33,062 INFO    MainThread:4622 [wandb_init.py:init():989] updated telemetry
+2026-05-17 02:39:33,064 INFO    MainThread:4622 [wandb_init.py:init():1012] communicating run to backend with 90.0 second timeout
+2026-05-17 02:39:33,402 INFO    MainThread:4622 [wandb_init.py:init():1057] starting run threads in backend
+2026-05-17 02:39:33,488 INFO    MainThread:4622 [wandb_run.py:_console_start():2509] atexit reg
+2026-05-17 02:39:33,488 INFO    MainThread:4622 [wandb_run.py:_redirect():2359] redirect: wrap_raw
+2026-05-17 02:39:33,488 INFO    MainThread:4622 [wandb_run.py:_redirect():2428] Wrapping output streams.
+2026-05-17 02:39:33,488 INFO    MainThread:4622 [wandb_run.py:_redirect():2451] Redirects installed.
+2026-05-17 02:39:33,491 INFO    MainThread:4622 [wandb_init.py:init():1095] run started, returning control to user process

epfl_final/wandb/run-20260517_023932-hi2z3jh6/run-hi2z3jh6.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e40a65bcb649da5fdf39e3e9cb70bc3d89af92bfeeac79ffd5a0f5393f596b0
+size 622592