EPFL final checkpoints and W&B history 401-450 of 1463

.gitattributes

@@ -50,3 +50,8 @@ epfl_final/SparseAtt/wandb/run-20260707_171131-304r0ilz/run-304r0ilz.wandb filte
 epfl_final/SparseAtt/wandb/run-20260708_045920-qlqhc3gy/run-qlqhc3gy.wandb filter=lfs diff=lfs merge=lfs -text
 epfl_final/SparseAtt/wandb/run-20260708_054405-43sa4ouk/run-43sa4ouk.wandb filter=lfs diff=lfs merge=lfs -text
 epfl_final/SparseAtt/wandb/run-20260708_060003-qlqhc3gy/run-qlqhc3gy.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260708_065216-cd66e13p/run-cd66e13p.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260708_112402-hn868f80/run-hn868f80.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260708_173250-cd66e13p/run-cd66e13p.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260708_205109-ti3a1x4c/run-ti3a1x4c.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260708_205109-82re6m3i/run-82re6m3i.wandb filter=lfs diff=lfs merge=lfs -text

epfl_final/SparseAtt/wandb/run-20260708_063331-i61v7k8r/files/output.log

@@ -0,0 +1,2 @@
+Starting training...
+Switching attention mode for step 1: dense (dense step interval)

epfl_final/SparseAtt/wandb/run-20260708_063331-i61v7k8r/files/requirements.txt

@@ -0,0 +1,141 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5

epfl_final/SparseAtt/wandb/run-20260708_063704-t6a1jz9b/files/output.log

@@ -0,0 +1,7 @@
+Starting training...
+Switching attention mode for step 1: dense (dense step interval)
+Step 10/37465 | Loss: 12.2534 | LR: 6.67e-06 | 13041.4ms/step | 1.5 samples/s
+Step 20/37465 | Loss: 12.1201 | LR: 1.33e-05 | 6349.5ms/step | 3.1 samples/s
+Step 30/37465 | Loss: 11.6859 | LR: 2.00e-05 | 6356.2ms/step | 3.1 samples/s
+Step 40/37465 | Loss: 10.6683 | LR: 2.67e-05 | 6340.9ms/step | 3.2 samples/s
+Step 50/37465 | Loss: 10.0551 | LR: 3.33e-05 | 6325.8ms/step | 3.2 samples/s

epfl_final/SparseAtt/wandb/run-20260708_063704-t6a1jz9b/files/requirements.txt

@@ -0,0 +1,141 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5

epfl_final/SparseAtt/wandb/run-20260708_063704-t6a1jz9b/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-07-08T06:37:04.570248Z",
+  "args":  [
+    "--config",
+    "configs/train_config_30k_dense_b128.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "88298c7a418622530ca5426b22b1e5ebd10ca0cb"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-train-4-gpu-2-0-2",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "5564801204224"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-51f13867-a72c-7b65-ff60-661787f0c18a"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bab1e315-c967-43cf-bdfb-cac1a8dfdfac"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-069bdfaf-ef1c-f7f2-f9e3-2249bb9dfc41"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bdb5e23e-0700-20a0-0b21-1ad0901f08b7"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "s2qr98e2llij904fcrfpuos4dxgukzau"
+}

epfl_final/SparseAtt/wandb/run-20260708_063704-t6a1jz9b/logs/debug-internal.log

@@ -0,0 +1,61 @@
+{"time":"2026-07-08T06:37:04.871067196Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-07-08T06:37:04.871620763Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-07-08T06:37:05.189790598Z","level":"INFO","msg":"stream: created new stream","id":"t6a1jz9b"}
+{"time":"2026-07-08T06:37:05.189935394Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-07-08T06:37:05.191848901Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-07-08T06:37:05.191926968Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-07-08T06:37:05.191927238Z","level":"INFO","msg":"writer: started","stream_id":"t6a1jz9b"}
+{"time":"2026-07-08T06:37:05.836178412Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-07-08T06:37:06.100904402Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T06:37:20.836885871Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"console_offset":1,"console_lines":1,"uploaded_len":2}
+{"time":"2026-07-08T06:37:21.153168007Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T06:37:35.836764302Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
+{"time":"2026-07-08T06:37:36.062615403Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T06:37:50.836354384Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":4,"events_lines":2}
+{"time":"2026-07-08T06:37:51.175301544Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T06:38:05.836588606Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":6,"events_lines":2}
+{"time":"2026-07-08T06:38:06.229906886Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T06:38:20.83684332Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":8,"events_lines":2}
+{"time":"2026-07-08T06:38:21.171410079Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T06:38:35.837227056Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":10,"events_lines":2}
+{"time":"2026-07-08T06:38:36.119688649Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T06:38:50.837181611Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":12,"events_lines":2}
+{"time":"2026-07-08T06:38:51.165906877Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T06:39:05.837227041Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":14,"events_lines":2}
+{"time":"2026-07-08T06:39:06.116414576Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T06:39:20.8372908Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":16,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-07-08T06:39:21.120251499Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T06:39:35.836790305Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":18,"events_lines":2}
+{"time":"2026-07-08T06:39:36.08150913Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T06:39:50.836735286Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":20,"events_lines":2}
+{"time":"2026-07-08T06:39:51.171624126Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T06:40:05.83677591Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":22,"events_lines":2}
+{"time":"2026-07-08T06:40:06.213301133Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T06:40:20.836787991Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":24,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-07-08T06:40:21.141459199Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T06:40:35.836299995Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":26,"events_lines":2}
+{"time":"2026-07-08T06:40:36.118140762Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T06:40:50.836441901Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":28,"events_lines":2}
+{"time":"2026-07-08T06:40:51.141236253Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T06:41:05.837067214Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":30,"events_lines":2}
+{"time":"2026-07-08T06:41:06.153035022Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T06:41:20.837331402Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":32,"events_lines":2}
+{"time":"2026-07-08T06:41:21.169275205Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T06:41:35.837066561Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":34,"events_lines":2,"console_offset":4,"console_lines":1}
+{"time":"2026-07-08T06:41:36.233759286Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T06:41:50.837048918Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":36,"events_lines":2}
+{"time":"2026-07-08T06:41:51.238988558Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T06:42:05.836798097Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":38,"events_lines":2}
+{"time":"2026-07-08T06:42:06.126843238Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T06:42:20.837063558Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":40,"events_lines":2}
+{"time":"2026-07-08T06:42:21.178339596Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T06:42:35.836973587Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":1,"events_offset":42,"events_lines":2,"console_offset":5,"console_lines":1}
+{"time":"2026-07-08T06:42:36.145508907Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T06:42:50.836987632Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":44,"events_lines":2}
+{"time":"2026-07-08T06:42:51.152559169Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T06:43:05.836569592Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":46,"events_lines":2}
+{"time":"2026-07-08T06:43:06.170965066Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T06:43:20.837156922Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":48,"events_lines":2}
+{"time":"2026-07-08T06:43:21.328844393Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T06:43:35.836608798Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":3,"history_lines":1,"events_offset":50,"events_lines":2,"console_offset":6,"console_lines":1}
+{"time":"2026-07-08T06:43:36.15991944Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

epfl_final/SparseAtt/wandb/run-20260708_063704-t6a1jz9b/logs/debug.log

@@ -0,0 +1,19 @@
+2026-07-08 06:37:04,590 INFO    MainThread:2544 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-07-08 06:37:04,591 INFO    MainThread:2544 [wandb_setup.py:_flush():81] Configure stats pid to 2544
+2026-07-08 06:37:04,591 INFO    MainThread:2544 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-08 06:37:04,591 INFO    MainThread:2544 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_063704-t6a1jz9b/logs/debug.log
+2026-07-08 06:37:04,591 INFO    MainThread:2544 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_063704-t6a1jz9b/logs/debug-internal.log
+2026-07-08 06:37:04,591 INFO    MainThread:2544 [wandb_init.py:init():850] calling init triggers
+2026-07-08 06:37:04,591 INFO    MainThread:2544 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/longctx30k/tokenized', 'val_path': None, 'max_seq_length': 30720, 'num_workers': 4, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_063617', 'resume_from_checkpoint': None, 'force_new_run': False, 'num_epochs': 1, 'max_steps': None, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 5, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 300, 'eval_steps': 500, 'eval_batches': 20, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b dense 30k longctx', 'wandb_tags': ['qwen3', '1.2b', 'dense', '30k', 'longctx', 'b128', 'scaleladder'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 120, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
+2026-07-08 06:37:04,591 INFO    MainThread:2544 [wandb_init.py:init():898] starting backend
+2026-07-08 06:37:04,856 INFO    MainThread:2544 [wandb_init.py:init():913] sending inform_init request
+2026-07-08 06:37:05,192 INFO    MainThread:2544 [wandb_init.py:init():918] backend started and connected
+2026-07-08 06:37:05,195 INFO    MainThread:2544 [wandb_init.py:init():988] updated telemetry
+2026-07-08 06:37:05,211 INFO    MainThread:2544 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-07-08 06:37:05,613 INFO    MainThread:2544 [wandb_init.py:init():1056] starting run threads in backend
+2026-07-08 06:37:05,826 INFO    MainThread:2544 [wandb_run.py:_console_start():2554] atexit reg
+2026-07-08 06:37:05,826 INFO    MainThread:2544 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-07-08 06:37:05,826 INFO    MainThread:2544 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-07-08 06:37:05,826 INFO    MainThread:2544 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-07-08 06:37:05,831 INFO    MainThread:2544 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/run-20260708_065216-cd66e13p/files/output.log

@@ -0,0 +1,363 @@
+Starting training...
+Switching attention mode for step 1: dense (dense step interval)
+Step 10/37465 | Loss: 12.2534 | LR: 6.67e-06 | 16466.7ms/step | 1.2 samples/s
+Step 20/37465 | Loss: 12.1201 | LR: 1.33e-05 | 6357.6ms/step | 3.1 samples/s
+Step 30/37465 | Loss: 11.6858 | LR: 2.00e-05 | 6363.5ms/step | 3.1 samples/s
+Step 40/37465 | Loss: 10.6683 | LR: 2.67e-05 | 6353.9ms/step | 3.1 samples/s
+Step 50/37465 | Loss: 10.0551 | LR: 3.33e-05 | 6330.1ms/step | 3.2 samples/s
+Step 60/37465 | Loss: 9.5722 | LR: 4.00e-05 | 6319.8ms/step | 3.2 samples/s
+Step 70/37465 | Loss: 9.1242 | LR: 4.67e-05 | 6319.8ms/step | 3.2 samples/s
+Step 80/37465 | Loss: 8.6455 | LR: 5.33e-05 | 6326.9ms/step | 3.2 samples/s
+Step 90/37465 | Loss: 8.0658 | LR: 6.00e-05 | 6329.3ms/step | 3.2 samples/s
+Step 100/37465 | Loss: 7.7199 | LR: 6.67e-05 | 6327.5ms/step | 3.2 samples/s
+Step 110/37465 | Loss: 7.3702 | LR: 7.33e-05 | 6331.3ms/step | 3.2 samples/s
+Step 120/37465 | Loss: 7.1084 | LR: 8.00e-05 | 6339.4ms/step | 3.2 samples/s
+Step 130/37465 | Loss: 6.9197 | LR: 8.67e-05 | 6338.2ms/step | 3.2 samples/s
+Step 140/37465 | Loss: 6.8033 | LR: 9.33e-05 | 6347.0ms/step | 3.2 samples/s
+Step 150/37465 | Loss: 6.8035 | LR: 1.00e-04 | 6351.2ms/step | 3.1 samples/s
+Step 160/37465 | Loss: 6.6536 | LR: 1.07e-04 | 6354.2ms/step | 3.1 samples/s
+Step 170/37465 | Loss: 6.3240 | LR: 1.13e-04 | 6361.6ms/step | 3.1 samples/s
+Step 180/37465 | Loss: 6.1787 | LR: 1.20e-04 | 6363.3ms/step | 3.1 samples/s
+Step 190/37465 | Loss: 6.2520 | LR: 1.27e-04 | 6359.6ms/step | 3.1 samples/s
+Step 200/37465 | Loss: 6.0200 | LR: 1.33e-04 | 6364.1ms/step | 3.1 samples/s
+Step 210/37465 | Loss: 6.0067 | LR: 1.40e-04 | 6360.8ms/step | 3.1 samples/s
+Step 220/37465 | Loss: 5.9344 | LR: 1.47e-04 | 6362.6ms/step | 3.1 samples/s
+Step 230/37465 | Loss: 5.9027 | LR: 1.53e-04 | 6366.6ms/step | 3.1 samples/s
+Step 240/37465 | Loss: 5.8670 | LR: 1.60e-04 | 6356.6ms/step | 3.1 samples/s
+Step 250/37465 | Loss: 5.8162 | LR: 1.67e-04 | 6368.7ms/step | 3.1 samples/s
+Step 260/37465 | Loss: 5.6788 | LR: 1.73e-04 | 6369.0ms/step | 3.1 samples/s
+Step 270/37465 | Loss: 5.7045 | LR: 1.80e-04 | 6368.3ms/step | 3.1 samples/s
+Step 280/37465 | Loss: 5.7118 | LR: 1.87e-04 | 6360.1ms/step | 3.1 samples/s
+Step 290/37465 | Loss: 5.5616 | LR: 1.93e-04 | 6365.8ms/step | 3.1 samples/s
+Step 300/37465 | Loss: 5.3729 | LR: 2.00e-04 | 6371.3ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-300
+Step 310/37465 | Loss: 5.2669 | LR: 2.07e-04 | 7332.6ms/step | 2.7 samples/s
+Step 320/37465 | Loss: 5.3585 | LR: 2.13e-04 | 6363.0ms/step | 3.1 samples/s
+Step 330/37465 | Loss: 5.5393 | LR: 2.20e-04 | 6364.3ms/step | 3.1 samples/s
+Step 340/37465 | Loss: 5.3772 | LR: 2.27e-04 | 6362.6ms/step | 3.1 samples/s
+Step 350/37465 | Loss: 5.2125 | LR: 2.33e-04 | 6367.0ms/step | 3.1 samples/s
+Step 360/37465 | Loss: 5.2798 | LR: 2.40e-04 | 6363.3ms/step | 3.1 samples/s
+Step 370/37465 | Loss: 5.1172 | LR: 2.47e-04 | 6363.6ms/step | 3.1 samples/s
+Step 380/37465 | Loss: 5.3529 | LR: 2.53e-04 | 6367.7ms/step | 3.1 samples/s
+Step 390/37465 | Loss: 5.2645 | LR: 2.60e-04 | 6367.3ms/step | 3.1 samples/s
+Step 400/37465 | Loss: 5.2519 | LR: 2.67e-04 | 6365.2ms/step | 3.1 samples/s
+Step 410/37465 | Loss: 5.0620 | LR: 2.73e-04 | 6366.8ms/step | 3.1 samples/s
+Step 420/37465 | Loss: 4.8713 | LR: 2.80e-04 | 6364.6ms/step | 3.1 samples/s
+Step 430/37465 | Loss: 4.9729 | LR: 2.87e-04 | 6365.5ms/step | 3.1 samples/s
+Step 440/37465 | Loss: 4.8967 | LR: 2.93e-04 | 6369.5ms/step | 3.1 samples/s
+Step 450/37465 | Loss: 4.8868 | LR: 3.00e-04 | 6367.4ms/step | 3.1 samples/s
+Step 460/37465 | Loss: 4.6993 | LR: 3.07e-04 | 6368.1ms/step | 3.1 samples/s
+Step 470/37465 | Loss: 4.7116 | LR: 3.13e-04 | 6370.6ms/step | 3.1 samples/s
+Step 480/37465 | Loss: 4.8928 | LR: 3.20e-04 | 6368.0ms/step | 3.1 samples/s
+Step 490/37465 | Loss: 4.6241 | LR: 3.27e-04 | 6368.7ms/step | 3.1 samples/s
+Step 500/37465 | Loss: 4.8609 | LR: 3.33e-04 | 6369.7ms/step | 3.1 samples/s
+Validation at step 500: Loss = 4.6498, Perplexity = 104.56
+Step 510/37465 | Loss: 4.4969 | LR: 3.40e-04 | 10432.4ms/step | 1.9 samples/s
+Step 520/37465 | Loss: 4.6458 | LR: 3.47e-04 | 6360.4ms/step | 3.1 samples/s
+Step 530/37465 | Loss: 4.8233 | LR: 3.53e-04 | 6367.5ms/step | 3.1 samples/s
+Step 540/37465 | Loss: 4.9458 | LR: 3.60e-04 | 6369.9ms/step | 3.1 samples/s
+Step 550/37465 | Loss: 4.4279 | LR: 3.67e-04 | 6367.6ms/step | 3.1 samples/s
+Step 560/37465 | Loss: 4.4328 | LR: 3.73e-04 | 6368.9ms/step | 3.1 samples/s
+Step 570/37465 | Loss: 4.4073 | LR: 3.80e-04 | 6368.9ms/step | 3.1 samples/s
+Step 580/37465 | Loss: 4.0842 | LR: 3.87e-04 | 6369.7ms/step | 3.1 samples/s
+Step 590/37465 | Loss: 4.4378 | LR: 3.93e-04 | 6371.7ms/step | 3.1 samples/s
+Step 600/37465 | Loss: 4.5479 | LR: 4.00e-04 | 6371.6ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-600
+Step 610/37465 | Loss: 4.2887 | LR: 4.07e-04 | 7304.6ms/step | 2.7 samples/s
+Step 620/37465 | Loss: 4.1438 | LR: 4.13e-04 | 6380.9ms/step | 3.1 samples/s
+Step 630/37465 | Loss: 4.2081 | LR: 4.20e-04 | 6371.7ms/step | 3.1 samples/s
+Step 640/37465 | Loss: 4.2991 | LR: 4.27e-04 | 6372.2ms/step | 3.1 samples/s
+Step 650/37465 | Loss: 4.2150 | LR: 4.33e-04 | 6368.2ms/step | 3.1 samples/s
+Step 660/37465 | Loss: 4.3003 | LR: 4.40e-04 | 6376.6ms/step | 3.1 samples/s
+Step 670/37465 | Loss: 4.4093 | LR: 4.47e-04 | 6375.2ms/step | 3.1 samples/s
+Step 680/37465 | Loss: 4.1509 | LR: 4.53e-04 | 6370.9ms/step | 3.1 samples/s
+Step 690/37465 | Loss: 3.9877 | LR: 4.60e-04 | 6374.5ms/step | 3.1 samples/s
+Step 700/37465 | Loss: 4.1206 | LR: 4.67e-04 | 6372.4ms/step | 3.1 samples/s
+Step 710/37465 | Loss: 4.0514 | LR: 4.73e-04 | 6374.8ms/step | 3.1 samples/s
+Step 720/37465 | Loss: 3.9697 | LR: 4.80e-04 | 6372.0ms/step | 3.1 samples/s
+Step 730/37465 | Loss: 3.5586 | LR: 4.87e-04 | 6372.2ms/step | 3.1 samples/s
+Step 740/37465 | Loss: 4.0557 | LR: 4.93e-04 | 6369.6ms/step | 3.1 samples/s
+Step 750/37465 | Loss: 3.7544 | LR: 5.00e-04 | 6375.6ms/step | 3.1 samples/s
+Step 760/37465 | Loss: 3.6661 | LR: 5.00e-04 | 6374.7ms/step | 3.1 samples/s
+Step 770/37465 | Loss: 3.6655 | LR: 5.00e-04 | 6379.4ms/step | 3.1 samples/s
+Step 780/37465 | Loss: 3.7366 | LR: 5.00e-04 | 6373.3ms/step | 3.1 samples/s
+Step 790/37465 | Loss: 3.3627 | LR: 5.00e-04 | 6377.5ms/step | 3.1 samples/s
+Step 800/37465 | Loss: 3.7698 | LR: 5.00e-04 | 6377.4ms/step | 3.1 samples/s
+Step 810/37465 | Loss: 3.8066 | LR: 5.00e-04 | 6371.7ms/step | 3.1 samples/s
+Step 820/37465 | Loss: 3.6056 | LR: 5.00e-04 | 6376.1ms/step | 3.1 samples/s
+Step 830/37465 | Loss: 3.6018 | LR: 5.00e-04 | 6376.6ms/step | 3.1 samples/s
+Step 840/37465 | Loss: 3.7742 | LR: 5.00e-04 | 6375.5ms/step | 3.1 samples/s
+Step 850/37465 | Loss: 3.4180 | LR: 5.00e-04 | 6378.2ms/step | 3.1 samples/s
+Step 860/37465 | Loss: 3.6353 | LR: 5.00e-04 | 6374.2ms/step | 3.1 samples/s
+Step 870/37465 | Loss: 3.5633 | LR: 5.00e-04 | 6373.2ms/step | 3.1 samples/s
+Step 880/37465 | Loss: 3.4691 | LR: 5.00e-04 | 6374.4ms/step | 3.1 samples/s
+Step 890/37465 | Loss: 3.5110 | LR: 5.00e-04 | 6370.8ms/step | 3.1 samples/s
+Step 900/37465 | Loss: 3.1987 | LR: 5.00e-04 | 6375.8ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-900
+Step 910/37465 | Loss: 3.2169 | LR: 5.00e-04 | 7315.5ms/step | 2.7 samples/s
+Step 920/37465 | Loss: 3.3605 | LR: 5.00e-04 | 6378.5ms/step | 3.1 samples/s
+Step 930/37465 | Loss: 3.2232 | LR: 5.00e-04 | 6380.6ms/step | 3.1 samples/s
+Step 940/37465 | Loss: 3.4859 | LR: 5.00e-04 | 6380.2ms/step | 3.1 samples/s
+Step 950/37465 | Loss: 3.7345 | LR: 5.00e-04 | 6381.3ms/step | 3.1 samples/s
+Step 960/37465 | Loss: 3.0550 | LR: 5.00e-04 | 6386.2ms/step | 3.1 samples/s
+Step 970/37465 | Loss: 3.4433 | LR: 5.00e-04 | 6389.7ms/step | 3.1 samples/s
+Step 980/37465 | Loss: 3.2395 | LR: 5.00e-04 | 6383.5ms/step | 3.1 samples/s
+Step 990/37465 | Loss: 3.2277 | LR: 5.00e-04 | 6383.7ms/step | 3.1 samples/s
+Step 1000/37465 | Loss: 3.0568 | LR: 5.00e-04 | 6387.2ms/step | 3.1 samples/s
+Validation at step 1000: Loss = 3.2705, Perplexity = 26.32
+Step 1010/37465 | Loss: 3.1221 | LR: 5.00e-04 | 7136.6ms/step | 2.8 samples/s
+Step 1020/37465 | Loss: 3.3939 | LR: 5.00e-04 | 6387.9ms/step | 3.1 samples/s
+Step 1030/37465 | Loss: 2.9741 | LR: 5.00e-04 | 6392.5ms/step | 3.1 samples/s
+Step 1040/37465 | Loss: 3.3890 | LR: 5.00e-04 | 6384.8ms/step | 3.1 samples/s
+Step 1050/37465 | Loss: 3.1908 | LR: 5.00e-04 | 6385.4ms/step | 3.1 samples/s
+Step 1060/37465 | Loss: 3.2037 | LR: 5.00e-04 | 6391.9ms/step | 3.1 samples/s
+Step 1070/37465 | Loss: 3.1875 | LR: 5.00e-04 | 6389.5ms/step | 3.1 samples/s
+Step 1080/37465 | Loss: 3.1991 | LR: 5.00e-04 | 6397.2ms/step | 3.1 samples/s
+Step 1090/37465 | Loss: 3.1953 | LR: 5.00e-04 | 6388.9ms/step | 3.1 samples/s
+Step 1100/37465 | Loss: 3.0628 | LR: 5.00e-04 | 6385.3ms/step | 3.1 samples/s
+Step 1110/37465 | Loss: 2.9822 | LR: 5.00e-04 | 6388.1ms/step | 3.1 samples/s
+Step 1120/37465 | Loss: 2.9275 | LR: 5.00e-04 | 6390.9ms/step | 3.1 samples/s
+Step 1130/37465 | Loss: 3.1709 | LR: 5.00e-04 | 6384.2ms/step | 3.1 samples/s
+Step 1140/37465 | Loss: 3.1406 | LR: 5.00e-04 | 6396.0ms/step | 3.1 samples/s
+Step 1150/37465 | Loss: 2.9701 | LR: 5.00e-04 | 6386.2ms/step | 3.1 samples/s
+Step 1160/37465 | Loss: 3.2755 | LR: 5.00e-04 | 6389.9ms/step | 3.1 samples/s
+Step 1170/37465 | Loss: 2.8357 | LR: 5.00e-04 | 6391.5ms/step | 3.1 samples/s
+Step 1180/37465 | Loss: 3.2176 | LR: 5.00e-04 | 6391.0ms/step | 3.1 samples/s
+Step 1190/37465 | Loss: 3.0387 | LR: 5.00e-04 | 6388.0ms/step | 3.1 samples/s
+Step 1200/37465 | Loss: 2.7643 | LR: 5.00e-04 | 6386.4ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-1200
+Step 1210/37465 | Loss: 3.1515 | LR: 5.00e-04 | 7315.0ms/step | 2.7 samples/s
+Step 1220/37465 | Loss: 3.2124 | LR: 5.00e-04 | 6384.4ms/step | 3.1 samples/s
+Step 1230/37465 | Loss: 2.8344 | LR: 5.00e-04 | 6379.6ms/step | 3.1 samples/s
+Step 1240/37465 | Loss: 3.0946 | LR: 5.00e-04 | 6375.6ms/step | 3.1 samples/s
+Step 1250/37465 | Loss: 2.9099 | LR: 5.00e-04 | 6378.2ms/step | 3.1 samples/s
+Step 1260/37465 | Loss: 2.9297 | LR: 5.00e-04 | 6374.3ms/step | 3.1 samples/s
+Step 1270/37465 | Loss: 3.0279 | LR: 5.00e-04 | 6376.6ms/step | 3.1 samples/s
+Step 1280/37465 | Loss: 2.8854 | LR: 5.00e-04 | 6377.8ms/step | 3.1 samples/s
+Step 1290/37465 | Loss: 2.9985 | LR: 5.00e-04 | 6378.2ms/step | 3.1 samples/s
+Step 1300/37465 | Loss: 3.3021 | LR: 5.00e-04 | 6378.0ms/step | 3.1 samples/s
+Step 1310/37465 | Loss: 3.0772 | LR: 5.00e-04 | 6379.3ms/step | 3.1 samples/s
+Step 1320/37465 | Loss: 2.9217 | LR: 5.00e-04 | 6377.9ms/step | 3.1 samples/s
+Step 1330/37465 | Loss: 3.1318 | LR: 5.00e-04 | 6378.1ms/step | 3.1 samples/s
+Step 1340/37465 | Loss: 3.0401 | LR: 5.00e-04 | 6380.0ms/step | 3.1 samples/s
+Step 1350/37465 | Loss: 2.9640 | LR: 5.00e-04 | 6379.0ms/step | 3.1 samples/s
+Step 1360/37465 | Loss: 2.8963 | LR: 5.00e-04 | 6376.5ms/step | 3.1 samples/s
+Step 1370/37465 | Loss: 3.2083 | LR: 5.00e-04 | 6380.2ms/step | 3.1 samples/s
+Step 1380/37465 | Loss: 2.9132 | LR: 5.00e-04 | 6379.1ms/step | 3.1 samples/s
+Step 1390/37465 | Loss: 2.8943 | LR: 5.00e-04 | 6374.9ms/step | 3.1 samples/s
+Step 1400/37465 | Loss: 2.9270 | LR: 5.00e-04 | 6377.8ms/step | 3.1 samples/s
+Step 1410/37465 | Loss: 3.1229 | LR: 5.00e-04 | 6381.8ms/step | 3.1 samples/s
+Step 1420/37465 | Loss: 2.7350 | LR: 5.00e-04 | 6381.4ms/step | 3.1 samples/s
+Step 1430/37465 | Loss: 3.2892 | LR: 5.00e-04 | 6377.8ms/step | 3.1 samples/s
+Step 1440/37465 | Loss: 2.9455 | LR: 5.00e-04 | 6382.3ms/step | 3.1 samples/s
+Step 1450/37465 | Loss: 3.0110 | LR: 5.00e-04 | 6375.8ms/step | 3.1 samples/s
+Step 1460/37465 | Loss: 3.0622 | LR: 5.00e-04 | 6377.9ms/step | 3.1 samples/s
+Step 1470/37465 | Loss: 2.9322 | LR: 5.00e-04 | 6379.8ms/step | 3.1 samples/s
+Step 1480/37465 | Loss: 2.7953 | LR: 5.00e-04 | 6377.0ms/step | 3.1 samples/s
+Step 1490/37465 | Loss: 2.9015 | LR: 4.99e-04 | 6379.9ms/step | 3.1 samples/s
+Step 1500/37465 | Loss: 2.8845 | LR: 4.99e-04 | 6381.4ms/step | 3.1 samples/s
+Validation at step 1500: Loss = 2.9509, Perplexity = 19.12
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-1500
+Step 1510/37465 | Loss: 2.9572 | LR: 4.99e-04 | 8029.5ms/step | 2.5 samples/s
+Step 1520/37465 | Loss: 2.8701 | LR: 4.99e-04 | 6387.5ms/step | 3.1 samples/s
+Step 1530/37465 | Loss: 2.8017 | LR: 4.99e-04 | 6387.6ms/step | 3.1 samples/s
+Step 1540/37465 | Loss: 2.7911 | LR: 4.99e-04 | 6386.7ms/step | 3.1 samples/s
+Step 1550/37465 | Loss: 3.0827 | LR: 4.99e-04 | 6385.9ms/step | 3.1 samples/s
+Step 1560/37465 | Loss: 2.9479 | LR: 4.99e-04 | 6387.4ms/step | 3.1 samples/s
+Step 1570/37465 | Loss: 2.7598 | LR: 4.99e-04 | 6390.2ms/step | 3.1 samples/s
+Step 1580/37465 | Loss: 3.1258 | LR: 4.99e-04 | 6387.3ms/step | 3.1 samples/s
+Step 1590/37465 | Loss: 2.9865 | LR: 4.99e-04 | 6389.2ms/step | 3.1 samples/s
+Step 1600/37465 | Loss: 3.2862 | LR: 4.99e-04 | 6383.2ms/step | 3.1 samples/s
+Step 1610/37465 | Loss: 2.7886 | LR: 4.99e-04 | 6386.6ms/step | 3.1 samples/s
+Step 1620/37465 | Loss: 3.0593 | LR: 4.99e-04 | 6384.5ms/step | 3.1 samples/s
+Step 1630/37465 | Loss: 2.9216 | LR: 4.99e-04 | 6390.2ms/step | 3.1 samples/s
+Step 1640/37465 | Loss: 2.8619 | LR: 4.99e-04 | 6391.2ms/step | 3.1 samples/s
+Step 1650/37465 | Loss: 2.6076 | LR: 4.99e-04 | 6387.9ms/step | 3.1 samples/s
+Step 1660/37465 | Loss: 2.7470 | LR: 4.99e-04 | 6386.2ms/step | 3.1 samples/s
+Step 1670/37465 | Loss: 2.7488 | LR: 4.99e-04 | 6394.0ms/step | 3.1 samples/s
+Step 1680/37465 | Loss: 3.0364 | LR: 4.99e-04 | 6389.6ms/step | 3.1 samples/s
+Step 1690/37465 | Loss: 2.9046 | LR: 4.99e-04 | 6398.3ms/step | 3.1 samples/s
+Step 1700/37465 | Loss: 2.9512 | LR: 4.99e-04 | 6388.5ms/step | 3.1 samples/s
+Step 1710/37465 | Loss: 2.9809 | LR: 4.99e-04 | 6382.4ms/step | 3.1 samples/s
+Step 1720/37465 | Loss: 3.0311 | LR: 4.99e-04 | 6392.7ms/step | 3.1 samples/s
+Step 1730/37465 | Loss: 2.7612 | LR: 4.99e-04 | 6390.9ms/step | 3.1 samples/s
+Step 1740/37465 | Loss: 2.9211 | LR: 4.99e-04 | 6379.4ms/step | 3.1 samples/s
+Step 1750/37465 | Loss: 2.9566 | LR: 4.99e-04 | 6387.7ms/step | 3.1 samples/s
+Step 1760/37465 | Loss: 2.8662 | LR: 4.99e-04 | 6387.3ms/step | 3.1 samples/s
+Step 1770/37465 | Loss: 2.8642 | LR: 4.99e-04 | 6385.7ms/step | 3.1 samples/s
+Step 1780/37465 | Loss: 2.7108 | LR: 4.99e-04 | 6390.8ms/step | 3.1 samples/s
+Step 1790/37465 | Loss: 2.9409 | LR: 4.99e-04 | 6388.7ms/step | 3.1 samples/s
+Step 1800/37465 | Loss: 2.6069 | LR: 4.99e-04 | 6392.8ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-1800
+Step 1810/37465 | Loss: 2.9186 | LR: 4.99e-04 | 7290.9ms/step | 2.7 samples/s
+Step 1820/37465 | Loss: 2.9307 | LR: 4.99e-04 | 6389.4ms/step | 3.1 samples/s
+Step 1830/37465 | Loss: 2.8242 | LR: 4.99e-04 | 6394.3ms/step | 3.1 samples/s
+Step 1840/37465 | Loss: 2.9409 | LR: 4.99e-04 | 6387.8ms/step | 3.1 samples/s
+Step 1850/37465 | Loss: 2.7138 | LR: 4.99e-04 | 6391.3ms/step | 3.1 samples/s
+Step 1860/37465 | Loss: 2.7519 | LR: 4.99e-04 | 6391.2ms/step | 3.1 samples/s
+Step 1870/37465 | Loss: 3.1458 | LR: 4.99e-04 | 6386.0ms/step | 3.1 samples/s
+Step 1880/37465 | Loss: 2.8918 | LR: 4.99e-04 | 6383.1ms/step | 3.1 samples/s
+Step 1890/37465 | Loss: 2.8751 | LR: 4.99e-04 | 6379.4ms/step | 3.1 samples/s
+Step 1900/37465 | Loss: 2.7494 | LR: 4.99e-04 | 6380.6ms/step | 3.1 samples/s
+Step 1910/37465 | Loss: 2.7345 | LR: 4.99e-04 | 6379.9ms/step | 3.1 samples/s
+Step 1920/37465 | Loss: 2.9475 | LR: 4.99e-04 | 6382.4ms/step | 3.1 samples/s
+Step 1930/37465 | Loss: 2.8216 | LR: 4.99e-04 | 6382.3ms/step | 3.1 samples/s
+Step 1940/37465 | Loss: 2.8334 | LR: 4.99e-04 | 6382.6ms/step | 3.1 samples/s
+Step 1950/37465 | Loss: 2.7589 | LR: 4.99e-04 | 6384.2ms/step | 3.1 samples/s
+Step 1960/37465 | Loss: 2.7681 | LR: 4.99e-04 | 6381.6ms/step | 3.1 samples/s
+Step 1970/37465 | Loss: 2.9729 | LR: 4.99e-04 | 6382.0ms/step | 3.1 samples/s
+Step 1980/37465 | Loss: 2.7409 | LR: 4.99e-04 | 6385.0ms/step | 3.1 samples/s
+Step 1990/37465 | Loss: 2.6951 | LR: 4.99e-04 | 6389.5ms/step | 3.1 samples/s
+Step 2000/37465 | Loss: 2.7914 | LR: 4.99e-04 | 6383.1ms/step | 3.1 samples/s
+Validation at step 2000: Loss = 2.7777, Perplexity = 16.08
+Step 2010/37465 | Loss: 2.7050 | LR: 4.99e-04 | 7128.8ms/step | 2.8 samples/s
+Step 2020/37465 | Loss: 2.7686 | LR: 4.99e-04 | 6384.7ms/step | 3.1 samples/s
+Step 2030/37465 | Loss: 2.6295 | LR: 4.99e-04 | 6385.0ms/step | 3.1 samples/s
+Step 2040/37465 | Loss: 2.6745 | LR: 4.98e-04 | 6379.5ms/step | 3.1 samples/s
+Step 2050/37465 | Loss: 2.7631 | LR: 4.98e-04 | 6382.7ms/step | 3.1 samples/s
+Step 2060/37465 | Loss: 2.6218 | LR: 4.98e-04 | 6384.9ms/step | 3.1 samples/s
+Step 2070/37465 | Loss: 2.6614 | LR: 4.98e-04 | 6386.5ms/step | 3.1 samples/s
+Step 2080/37465 | Loss: 2.8349 | LR: 4.98e-04 | 6382.5ms/step | 3.1 samples/s
+Step 2090/37465 | Loss: 2.8247 | LR: 4.98e-04 | 6382.2ms/step | 3.1 samples/s
+Step 2100/37465 | Loss: 2.9305 | LR: 4.98e-04 | 6379.0ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-2100
+Step 2110/37465 | Loss: 2.8710 | LR: 4.98e-04 | 7287.2ms/step | 2.7 samples/s
+Step 2120/37465 | Loss: 2.7928 | LR: 4.98e-04 | 6384.3ms/step | 3.1 samples/s
+Step 2130/37465 | Loss: 2.5135 | LR: 4.98e-04 | 6387.6ms/step | 3.1 samples/s
+Step 2140/37465 | Loss: 2.8393 | LR: 4.98e-04 | 6384.1ms/step | 3.1 samples/s
+Step 2150/37465 | Loss: 2.7712 | LR: 4.98e-04 | 6387.7ms/step | 3.1 samples/s
+Step 2160/37465 | Loss: 2.6872 | LR: 4.98e-04 | 6382.7ms/step | 3.1 samples/s
+Step 2170/37465 | Loss: 2.7883 | LR: 4.98e-04 | 6379.3ms/step | 3.1 samples/s
+Step 2180/37465 | Loss: 2.8064 | LR: 4.98e-04 | 6374.1ms/step | 3.1 samples/s
+Step 2190/37465 | Loss: 2.7624 | LR: 4.98e-04 | 6382.0ms/step | 3.1 samples/s
+Step 2200/37465 | Loss: 2.6835 | LR: 4.98e-04 | 6383.3ms/step | 3.1 samples/s
+Step 2210/37465 | Loss: 2.6176 | LR: 4.98e-04 | 6382.6ms/step | 3.1 samples/s
+Step 2220/37465 | Loss: 2.8132 | LR: 4.98e-04 | 6382.2ms/step | 3.1 samples/s
+Step 2230/37465 | Loss: 2.8084 | LR: 4.98e-04 | 6378.5ms/step | 3.1 samples/s
+Step 2240/37465 | Loss: 2.8041 | LR: 4.98e-04 | 6377.2ms/step | 3.1 samples/s
+Step 2250/37465 | Loss: 2.6921 | LR: 4.98e-04 | 6380.4ms/step | 3.1 samples/s
+Step 2260/37465 | Loss: 2.8456 | LR: 4.98e-04 | 6376.4ms/step | 3.1 samples/s
+Step 2270/37465 | Loss: 2.7451 | LR: 4.98e-04 | 6379.1ms/step | 3.1 samples/s
+Step 2280/37465 | Loss: 2.5784 | LR: 4.98e-04 | 6382.5ms/step | 3.1 samples/s
+Step 2290/37465 | Loss: 2.7031 | LR: 4.98e-04 | 6381.7ms/step | 3.1 samples/s
+Step 2300/37465 | Loss: 2.8041 | LR: 4.98e-04 | 6377.0ms/step | 3.1 samples/s
+Step 2310/37465 | Loss: 2.8593 | LR: 4.98e-04 | 6379.1ms/step | 3.1 samples/s
+Step 2320/37465 | Loss: 2.7207 | LR: 4.98e-04 | 6382.5ms/step | 3.1 samples/s
+Step 2330/37465 | Loss: 2.9899 | LR: 4.98e-04 | 6372.6ms/step | 3.1 samples/s
+Step 2340/37465 | Loss: 2.5633 | LR: 4.98e-04 | 6377.4ms/step | 3.1 samples/s
+Step 2350/37465 | Loss: 2.9726 | LR: 4.98e-04 | 6379.6ms/step | 3.1 samples/s
+Step 2360/37465 | Loss: 2.5830 | LR: 4.98e-04 | 6378.0ms/step | 3.1 samples/s
+Step 2370/37465 | Loss: 2.8398 | LR: 4.98e-04 | 6378.2ms/step | 3.1 samples/s
+Step 2380/37465 | Loss: 2.6590 | LR: 4.98e-04 | 6377.3ms/step | 3.1 samples/s
+Step 2390/37465 | Loss: 2.5337 | LR: 4.98e-04 | 6380.3ms/step | 3.1 samples/s
+Step 2400/37465 | Loss: 2.5193 | LR: 4.98e-04 | 6379.5ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-2400
+Step 2410/37465 | Loss: 2.6855 | LR: 4.97e-04 | 7312.0ms/step | 2.7 samples/s
+Step 2420/37465 | Loss: 2.8776 | LR: 4.97e-04 | 6381.3ms/step | 3.1 samples/s
+Step 2430/37465 | Loss: 2.8649 | LR: 4.97e-04 | 6384.6ms/step | 3.1 samples/s
+Step 2440/37465 | Loss: 2.8029 | LR: 4.97e-04 | 6385.3ms/step | 3.1 samples/s
+Step 2450/37465 | Loss: 2.7996 | LR: 4.97e-04 | 6390.0ms/step | 3.1 samples/s
+Step 2460/37465 | Loss: 2.7896 | LR: 4.97e-04 | 6391.3ms/step | 3.1 samples/s
+Step 2470/37465 | Loss: 2.8103 | LR: 4.97e-04 | 6381.5ms/step | 3.1 samples/s
+Step 2480/37465 | Loss: 2.6114 | LR: 4.97e-04 | 6383.0ms/step | 3.1 samples/s
+Step 2490/37465 | Loss: 2.6087 | LR: 4.97e-04 | 6371.4ms/step | 3.1 samples/s
+Step 2500/37465 | Loss: 2.7416 | LR: 4.97e-04 | 6382.6ms/step | 3.1 samples/s
+Validation at step 2500: Loss = 2.6779, Perplexity = 14.55
+Step 2510/37465 | Loss: 2.8022 | LR: 4.97e-04 | 7132.4ms/step | 2.8 samples/s
+Step 2520/37465 | Loss: 2.4969 | LR: 4.97e-04 | 6380.0ms/step | 3.1 samples/s
+Step 2530/37465 | Loss: 2.6487 | LR: 4.97e-04 | 6378.3ms/step | 3.1 samples/s
+Step 2540/37465 | Loss: 2.9364 | LR: 4.97e-04 | 6381.9ms/step | 3.1 samples/s
+Step 2550/37465 | Loss: 2.4333 | LR: 4.97e-04 | 6374.7ms/step | 3.1 samples/s
+Step 2560/37465 | Loss: 2.7193 | LR: 4.97e-04 | 6375.6ms/step | 3.1 samples/s
+Step 2570/37465 | Loss: 2.6816 | LR: 4.97e-04 | 6378.3ms/step | 3.1 samples/s
+Step 2580/37465 | Loss: 2.4674 | LR: 4.97e-04 | 6378.2ms/step | 3.1 samples/s
+Step 2590/37465 | Loss: 2.7053 | LR: 4.97e-04 | 6378.6ms/step | 3.1 samples/s
+Step 2600/37465 | Loss: 2.8359 | LR: 4.97e-04 | 6378.8ms/step | 3.1 samples/s
+Step 2610/37465 | Loss: 2.5548 | LR: 4.97e-04 | 6379.4ms/step | 3.1 samples/s
+Step 2620/37465 | Loss: 2.8504 | LR: 4.97e-04 | 6382.6ms/step | 3.1 samples/s
+Step 2630/37465 | Loss: 2.6607 | LR: 4.97e-04 | 6385.9ms/step | 3.1 samples/s
+Step 2640/37465 | Loss: 2.5939 | LR: 4.97e-04 | 6377.5ms/step | 3.1 samples/s
+Step 2650/37465 | Loss: 2.5936 | LR: 4.97e-04 | 6378.4ms/step | 3.1 samples/s
+Step 2660/37465 | Loss: 2.4835 | LR: 4.97e-04 | 6379.9ms/step | 3.1 samples/s
+Step 2670/37465 | Loss: 2.5658 | LR: 4.97e-04 | 6374.7ms/step | 3.1 samples/s
+Step 2680/37465 | Loss: 2.8609 | LR: 4.97e-04 | 6379.0ms/step | 3.1 samples/s
+Step 2690/37465 | Loss: 2.5883 | LR: 4.97e-04 | 6382.0ms/step | 3.1 samples/s
+Step 2700/37465 | Loss: 2.7509 | LR: 4.97e-04 | 6381.7ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-2700
+Step 2710/37465 | Loss: 2.5879 | LR: 4.96e-04 | 7293.1ms/step | 2.7 samples/s
+Step 2720/37465 | Loss: 2.5680 | LR: 4.96e-04 | 6377.8ms/step | 3.1 samples/s
+Step 2730/37465 | Loss: 2.6055 | LR: 4.96e-04 | 6377.1ms/step | 3.1 samples/s
+Step 2740/37465 | Loss: 2.6871 | LR: 4.96e-04 | 6382.4ms/step | 3.1 samples/s
+Step 2750/37465 | Loss: 2.5974 | LR: 4.96e-04 | 6387.7ms/step | 3.1 samples/s
+Step 2760/37465 | Loss: 2.8237 | LR: 4.96e-04 | 6378.7ms/step | 3.1 samples/s
+Step 2770/37465 | Loss: 2.6716 | LR: 4.96e-04 | 6382.1ms/step | 3.1 samples/s
+Step 2780/37465 | Loss: 2.7498 | LR: 4.96e-04 | 6380.9ms/step | 3.1 samples/s
+Step 2790/37465 | Loss: 2.4863 | LR: 4.96e-04 | 6380.2ms/step | 3.1 samples/s
+Step 2800/37465 | Loss: 2.5453 | LR: 4.96e-04 | 6383.2ms/step | 3.1 samples/s
+Step 2810/37465 | Loss: 2.5213 | LR: 4.96e-04 | 6386.4ms/step | 3.1 samples/s
+Step 2820/37465 | Loss: 2.3568 | LR: 4.96e-04 | 6391.6ms/step | 3.1 samples/s
+Step 2830/37465 | Loss: 2.4480 | LR: 4.96e-04 | 6387.9ms/step | 3.1 samples/s
+Step 2840/37465 | Loss: 2.8941 | LR: 4.96e-04 | 6383.9ms/step | 3.1 samples/s
+Step 2850/37465 | Loss: 2.5534 | LR: 4.96e-04 | 6384.4ms/step | 3.1 samples/s
+Step 2860/37465 | Loss: 2.5181 | LR: 4.96e-04 | 6387.4ms/step | 3.1 samples/s
+Step 2870/37465 | Loss: 2.7222 | LR: 4.96e-04 | 6384.2ms/step | 3.1 samples/s
+Step 2880/37465 | Loss: 2.6761 | LR: 4.96e-04 | 6383.9ms/step | 3.1 samples/s
+Step 2890/37465 | Loss: 2.6068 | LR: 4.96e-04 | 6388.8ms/step | 3.1 samples/s
+Step 2900/37465 | Loss: 2.5596 | LR: 4.96e-04 | 6387.7ms/step | 3.1 samples/s
+Step 2910/37465 | Loss: 2.6637 | LR: 4.96e-04 | 6385.2ms/step | 3.1 samples/s
+Step 2920/37465 | Loss: 2.4493 | LR: 4.96e-04 | 6381.5ms/step | 3.1 samples/s
+Step 2930/37465 | Loss: 2.5966 | LR: 4.96e-04 | 6384.4ms/step | 3.1 samples/s
+Step 2940/37465 | Loss: 2.5092 | LR: 4.96e-04 | 6381.1ms/step | 3.1 samples/s
+Step 2950/37465 | Loss: 2.5356 | LR: 4.96e-04 | 6383.8ms/step | 3.1 samples/s
+Step 2960/37465 | Loss: 2.5739 | LR: 4.96e-04 | 6386.1ms/step | 3.1 samples/s
+Step 2970/37465 | Loss: 2.6963 | LR: 4.96e-04 | 6387.8ms/step | 3.1 samples/s
+Step 2980/37465 | Loss: 2.3348 | LR: 4.95e-04 | 6385.2ms/step | 3.1 samples/s
+Step 2990/37465 | Loss: 2.7376 | LR: 4.95e-04 | 6388.7ms/step | 3.1 samples/s
+Step 3000/37465 | Loss: 2.6397 | LR: 4.95e-04 | 6387.3ms/step | 3.1 samples/s
+Validation at step 3000: Loss = 2.6081, Perplexity = 13.57
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-3000
+Step 3010/37465 | Loss: 2.6149 | LR: 4.95e-04 | 8062.9ms/step | 2.5 samples/s
+Step 3020/37465 | Loss: 2.6029 | LR: 4.95e-04 | 6386.1ms/step | 3.1 samples/s
+Step 3030/37465 | Loss: 2.8015 | LR: 4.95e-04 | 6384.6ms/step | 3.1 samples/s
+Step 3040/37465 | Loss: 2.7272 | LR: 4.95e-04 | 6380.3ms/step | 3.1 samples/s
+Step 3050/37465 | Loss: 2.6205 | LR: 4.95e-04 | 6380.2ms/step | 3.1 samples/s
+Step 3060/37465 | Loss: 2.6117 | LR: 4.95e-04 | 6386.5ms/step | 3.1 samples/s
+Step 3070/37465 | Loss: 2.4390 | LR: 4.95e-04 | 6384.2ms/step | 3.1 samples/s
+Step 3080/37465 | Loss: 2.3941 | LR: 4.95e-04 | 6385.5ms/step | 3.1 samples/s
+Step 3090/37465 | Loss: 2.5643 | LR: 4.95e-04 | 6385.4ms/step | 3.1 samples/s
+Step 3100/37465 | Loss: 2.6672 | LR: 4.95e-04 | 6382.4ms/step | 3.1 samples/s
+Step 3110/37465 | Loss: 2.6058 | LR: 4.95e-04 | 6380.2ms/step | 3.1 samples/s
+Step 3120/37465 | Loss: 2.4922 | LR: 4.95e-04 | 6381.2ms/step | 3.1 samples/s
+Step 3130/37465 | Loss: 2.5960 | LR: 4.95e-04 | 6383.0ms/step | 3.1 samples/s
+Step 3140/37465 | Loss: 2.4059 | LR: 4.95e-04 | 6378.9ms/step | 3.1 samples/s
+Step 3150/37465 | Loss: 2.3778 | LR: 4.95e-04 | 6382.1ms/step | 3.1 samples/s
+Step 3160/37465 | Loss: 2.5527 | LR: 4.95e-04 | 6374.2ms/step | 3.1 samples/s
+Step 3170/37465 | Loss: 2.8536 | LR: 4.95e-04 | 6381.6ms/step | 3.1 samples/s
+Step 3180/37465 | Loss: 2.7932 | LR: 4.95e-04 | 6378.5ms/step | 3.1 samples/s
+Step 3190/37465 | Loss: 2.2729 | LR: 4.95e-04 | 6379.2ms/step | 3.1 samples/s
+Step 3200/37465 | Loss: 2.4523 | LR: 4.95e-04 | 6376.7ms/step | 3.1 samples/s
+Step 3210/37465 | Loss: 2.3870 | LR: 4.94e-04 | 6389.6ms/step | 3.1 samples/s
+Step 3220/37465 | Loss: 2.6399 | LR: 4.94e-04 | 6385.6ms/step | 3.1 samples/s
+Step 3230/37465 | Loss: 2.6069 | LR: 4.94e-04 | 6382.8ms/step | 3.1 samples/s
+Step 3240/37465 | Loss: 2.4823 | LR: 4.94e-04 | 6378.3ms/step | 3.1 samples/s
+Step 3250/37465 | Loss: 2.4919 | LR: 4.94e-04 | 6378.8ms/step | 3.1 samples/s
+Step 3260/37465 | Loss: 2.6710 | LR: 4.94e-04 | 6378.5ms/step | 3.1 samples/s
+Step 3270/37465 | Loss: 2.6290 | LR: 4.94e-04 | 6377.1ms/step | 3.1 samples/s
+Step 3280/37465 | Loss: 2.5979 | LR: 4.94e-04 | 6382.3ms/step | 3.1 samples/s
+Step 3290/37465 | Loss: 2.4953 | LR: 4.94e-04 | 6383.3ms/step | 3.1 samples/s
+Step 3300/37465 | Loss: 2.4728 | LR: 4.94e-04 | 6378.9ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-3300
+Step 3310/37465 | Loss: 2.6605 | LR: 4.94e-04 | 7302.7ms/step | 2.7 samples/s
+Step 3320/37465 | Loss: 2.3535 | LR: 4.94e-04 | 6384.4ms/step | 3.1 samples/s
+Step 3330/37465 | Loss: 2.6802 | LR: 4.94e-04 | 6381.3ms/step | 3.1 samples/s
+Step 3340/37465 | Loss: 2.6431 | LR: 4.94e-04 | 6383.2ms/step | 3.1 samples/s
+Step 3350/37465 | Loss: 2.9169 | LR: 4.94e-04 | 6392.3ms/step | 3.1 samples/s
+Step 3360/37465 | Loss: 2.5130 | LR: 4.94e-04 | 6380.6ms/step | 3.1 samples/s
+Step 3370/37465 | Loss: 2.5432 | LR: 4.94e-04 | 6380.1ms/step | 3.1 samples/s
+Step 3380/37465 | Loss: 2.5796 | LR: 4.94e-04 | 6382.4ms/step | 3.1 samples/s
+Step 3390/37465 | Loss: 2.4650 | LR: 4.94e-04 | 6383.2ms/step | 3.1 samples/s
+Step 3400/37465 | Loss: 2.3815 | LR: 4.94e-04 | 6386.5ms/step | 3.1 samples/s
+Step 3410/37465 | Loss: 2.4313 | LR: 4.94e-04 | 6382.9ms/step | 3.1 samples/s
+Step 3420/37465 | Loss: 2.5677 | LR: 4.94e-04 | 6383.0ms/step | 3.1 samples/s
+Step 3430/37465 | Loss: 2.7321 | LR: 4.93e-04 | 6379.4ms/step | 3.1 samples/s
+Step 3440/37465 | Loss: 2.6157 | LR: 4.93e-04 | 6381.4ms/step | 3.1 samples/s

epfl_final/SparseAtt/wandb/run-20260708_065216-cd66e13p/files/requirements.txt

@@ -0,0 +1,141 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5

epfl_final/SparseAtt/wandb/run-20260708_065216-cd66e13p/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-07-08T06:52:16.387264Z",
+  "args":  [
+    "--config",
+    "configs/train_config_30k_dense_b128.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "88298c7a418622530ca5426b22b1e5ebd10ca0cb"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-train-4-gpu-2-0-0",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "5564549279744"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-51f13867-a72c-7b65-ff60-661787f0c18a"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bab1e315-c967-43cf-bdfb-cac1a8dfdfac"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-069bdfaf-ef1c-f7f2-f9e3-2249bb9dfc41"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bdb5e23e-0700-20a0-0b21-1ad0901f08b7"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "0nr1hjiblwp2cyi1qboitgxvq64lfuzy"
+}

epfl_final/SparseAtt/wandb/run-20260708_065216-cd66e13p/logs/debug.log

@@ -0,0 +1,19 @@
+2026-07-08 06:52:16,409 INFO    MainThread:208 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-07-08 06:52:16,410 INFO    MainThread:208 [wandb_setup.py:_flush():81] Configure stats pid to 208
+2026-07-08 06:52:16,410 INFO    MainThread:208 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-08 06:52:16,410 INFO    MainThread:208 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_065216-cd66e13p/logs/debug.log
+2026-07-08 06:52:16,410 INFO    MainThread:208 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_065216-cd66e13p/logs/debug-internal.log
+2026-07-08 06:52:16,410 INFO    MainThread:208 [wandb_init.py:init():850] calling init triggers
+2026-07-08 06:52:16,410 INFO    MainThread:208 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/longctx30k/tokenized', 'val_path': None, 'max_seq_length': 30720, 'num_workers': 4, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119', 'resume_from_checkpoint': None, 'force_new_run': False, 'num_epochs': 1, 'max_steps': None, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 5, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 300, 'eval_steps': 500, 'eval_batches': 20, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b dense 30k longctx', 'wandb_tags': ['qwen3', '1.2b', 'dense', '30k', 'longctx', 'b128', 'scaleladder'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 120, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
+2026-07-08 06:52:16,410 INFO    MainThread:208 [wandb_init.py:init():898] starting backend
+2026-07-08 06:52:16,968 INFO    MainThread:208 [wandb_init.py:init():913] sending inform_init request
+2026-07-08 06:52:17,484 INFO    MainThread:208 [wandb_init.py:init():918] backend started and connected
+2026-07-08 06:52:17,494 INFO    MainThread:208 [wandb_init.py:init():988] updated telemetry
+2026-07-08 06:52:17,520 INFO    MainThread:208 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-07-08 06:52:18,346 INFO    MainThread:208 [wandb_init.py:init():1056] starting run threads in backend
+2026-07-08 06:52:18,574 INFO    MainThread:208 [wandb_run.py:_console_start():2554] atexit reg
+2026-07-08 06:52:18,574 INFO    MainThread:208 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-07-08 06:52:18,574 INFO    MainThread:208 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-07-08 06:52:18,574 INFO    MainThread:208 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-07-08 06:52:18,582 INFO    MainThread:208 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/run-20260708_065216-cd66e13p/run-cd66e13p.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7d3f2568499a2e9dd531478ba7e3639311b04223699b2e6ea720e6476161437
+size 7176192

epfl_final/SparseAtt/wandb/run-20260708_112402-hn868f80/files/output.log

@@ -0,0 +1,467 @@
+Starting training...
+Switching attention mode for step 1: sparse_stochastic (configured sparse attention)
+Step 10/37465 | Loss: 12.2529 | LR: 6.67e-06 | 16108.6ms/step | 1.2 samples/s
+Step 20/37465 | Loss: 12.1155 | LR: 1.33e-05 | 8131.3ms/step | 2.5 samples/s
+Step 30/37465 | Loss: 11.6636 | LR: 2.00e-05 | 8123.6ms/step | 2.5 samples/s
+Step 40/37465 | Loss: 10.6239 | LR: 2.67e-05 | 8092.5ms/step | 2.5 samples/s
+Step 50/37465 | Loss: 10.0364 | LR: 3.33e-05 | 8084.0ms/step | 2.5 samples/s
+Step 60/37465 | Loss: 9.5529 | LR: 4.00e-05 | 8086.0ms/step | 2.5 samples/s
+Step 70/37465 | Loss: 9.0991 | LR: 4.67e-05 | 8086.6ms/step | 2.5 samples/s
+Step 80/37465 | Loss: 8.6253 | LR: 5.33e-05 | 8098.9ms/step | 2.5 samples/s
+Step 90/37465 | Loss: 8.0489 | LR: 6.00e-05 | 8077.6ms/step | 2.5 samples/s
+Step 100/37465 | Loss: 7.7035 | LR: 6.67e-05 | 8081.7ms/step | 2.5 samples/s
+Step 110/37465 | Loss: 7.3558 | LR: 7.33e-05 | 8080.7ms/step | 2.5 samples/s
+Step 120/37465 | Loss: 7.0939 | LR: 8.00e-05 | 8095.0ms/step | 2.5 samples/s
+Step 130/37465 | Loss: 6.9100 | LR: 8.67e-05 | 8109.2ms/step | 2.5 samples/s
+Step 140/37465 | Loss: 6.8013 | LR: 9.33e-05 | 8203.9ms/step | 2.4 samples/s
+Step 150/37465 | Loss: 6.8181 | LR: 1.00e-04 | 8287.3ms/step | 2.4 samples/s
+Step 160/37465 | Loss: 6.6680 | LR: 1.07e-04 | 8388.0ms/step | 2.4 samples/s
+Step 170/37465 | Loss: 6.3391 | LR: 1.13e-04 | 8137.4ms/step | 2.5 samples/s
+Step 180/37465 | Loss: 6.1800 | LR: 1.20e-04 | 8334.3ms/step | 2.4 samples/s
+Step 190/37465 | Loss: 6.2515 | LR: 1.27e-04 | 8359.5ms/step | 2.4 samples/s
+Step 200/37465 | Loss: 6.0149 | LR: 1.33e-04 | 8217.3ms/step | 2.4 samples/s
+Step 210/37465 | Loss: 5.9924 | LR: 1.40e-04 | 8219.1ms/step | 2.4 samples/s
+Step 220/37465 | Loss: 5.9197 | LR: 1.47e-04 | 8339.7ms/step | 2.4 samples/s
+Step 230/37465 | Loss: 5.8678 | LR: 1.53e-04 | 8380.4ms/step | 2.4 samples/s
+Step 240/37465 | Loss: 5.8344 | LR: 1.60e-04 | 8227.7ms/step | 2.4 samples/s
+Step 250/37465 | Loss: 5.7789 | LR: 1.67e-04 | 8311.3ms/step | 2.4 samples/s
+Step 260/37465 | Loss: 5.6418 | LR: 1.73e-04 | 8387.2ms/step | 2.4 samples/s
+Step 270/37465 | Loss: 5.6752 | LR: 1.80e-04 | 8305.4ms/step | 2.4 samples/s
+Step 280/37465 | Loss: 5.6590 | LR: 1.87e-04 | 8264.8ms/step | 2.4 samples/s
+Step 290/37465 | Loss: 5.4748 | LR: 1.93e-04 | 8364.3ms/step | 2.4 samples/s
+Step 300/37465 | Loss: 5.3536 | LR: 2.00e-04 | 8401.3ms/step | 2.4 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-300
+Step 310/37465 | Loss: 5.2326 | LR: 2.07e-04 | 9109.2ms/step | 2.2 samples/s
+Step 320/37465 | Loss: 5.3250 | LR: 2.13e-04 | 8331.6ms/step | 2.4 samples/s
+Step 330/37465 | Loss: 5.5214 | LR: 2.20e-04 | 8361.3ms/step | 2.4 samples/s
+Step 340/37465 | Loss: 5.3371 | LR: 2.27e-04 | 8293.6ms/step | 2.4 samples/s
+Step 350/37465 | Loss: 5.1644 | LR: 2.33e-04 | 8315.2ms/step | 2.4 samples/s
+Step 360/37465 | Loss: 5.2510 | LR: 2.40e-04 | 8461.1ms/step | 2.4 samples/s
+Step 370/37465 | Loss: 5.0816 | LR: 2.47e-04 | 8194.5ms/step | 2.4 samples/s
+Step 380/37465 | Loss: 5.3100 | LR: 2.53e-04 | 8402.5ms/step | 2.4 samples/s
+Step 390/37465 | Loss: 5.2247 | LR: 2.60e-04 | 8362.2ms/step | 2.4 samples/s
+Step 400/37465 | Loss: 5.2271 | LR: 2.67e-04 | 8184.1ms/step | 2.4 samples/s
+Step 410/37465 | Loss: 5.0511 | LR: 2.73e-04 | 8313.9ms/step | 2.4 samples/s
+Step 420/37465 | Loss: 4.8262 | LR: 2.80e-04 | 8418.4ms/step | 2.4 samples/s
+Step 430/37465 | Loss: 4.9368 | LR: 2.87e-04 | 8264.9ms/step | 2.4 samples/s
+Step 440/37465 | Loss: 4.8697 | LR: 2.93e-04 | 8337.8ms/step | 2.4 samples/s
+Step 450/37465 | Loss: 4.8496 | LR: 3.00e-04 | 8381.9ms/step | 2.4 samples/s
+Step 460/37465 | Loss: 4.6666 | LR: 3.07e-04 | 8243.3ms/step | 2.4 samples/s
+Step 470/37465 | Loss: 4.6707 | LR: 3.13e-04 | 8268.4ms/step | 2.4 samples/s
+Step 480/37465 | Loss: 4.8633 | LR: 3.20e-04 | 8348.5ms/step | 2.4 samples/s
+Step 490/37465 | Loss: 4.5865 | LR: 3.27e-04 | 8334.4ms/step | 2.4 samples/s
+Step 500/37465 | Loss: 4.8181 | LR: 3.33e-04 | 8301.3ms/step | 2.4 samples/s
+Validation at step 500: Loss = 4.6234, Perplexity = 101.84
+Step 510/37465 | Loss: 4.4567 | LR: 3.40e-04 | 12530.7ms/step | 1.6 samples/s
+Step 520/37465 | Loss: 4.6110 | LR: 3.47e-04 | 8414.2ms/step | 2.4 samples/s
+Step 530/37465 | Loss: 4.7611 | LR: 3.53e-04 | 8301.6ms/step | 2.4 samples/s
+Step 540/37465 | Loss: 4.8849 | LR: 3.60e-04 | 8380.6ms/step | 2.4 samples/s
+Step 550/37465 | Loss: 4.3841 | LR: 3.67e-04 | 8311.8ms/step | 2.4 samples/s
+Step 560/37465 | Loss: 4.3966 | LR: 3.73e-04 | 8342.6ms/step | 2.4 samples/s
+Step 570/37465 | Loss: 4.3736 | LR: 3.80e-04 | 8409.2ms/step | 2.4 samples/s
+Step 580/37465 | Loss: 4.0383 | LR: 3.87e-04 | 8353.1ms/step | 2.4 samples/s
+Step 590/37465 | Loss: 4.4026 | LR: 3.93e-04 | 8452.4ms/step | 2.4 samples/s
+Step 600/37465 | Loss: 4.4882 | LR: 4.00e-04 | 8403.1ms/step | 2.4 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-600
+Step 610/37465 | Loss: 4.2267 | LR: 4.07e-04 | 9073.7ms/step | 2.2 samples/s
+Step 620/37465 | Loss: 4.0985 | LR: 4.13e-04 | 8221.0ms/step | 2.4 samples/s
+Step 630/37465 | Loss: 4.1605 | LR: 4.20e-04 | 8221.4ms/step | 2.4 samples/s
+Step 640/37465 | Loss: 4.2581 | LR: 4.27e-04 | 8149.4ms/step | 2.5 samples/s
+Step 650/37465 | Loss: 4.1771 | LR: 4.33e-04 | 8182.6ms/step | 2.4 samples/s
+Step 660/37465 | Loss: 4.2448 | LR: 4.40e-04 | 8229.3ms/step | 2.4 samples/s
+Step 670/37465 | Loss: 4.3723 | LR: 4.47e-04 | 8152.3ms/step | 2.5 samples/s
+Step 680/37465 | Loss: 4.1398 | LR: 4.53e-04 | 8190.6ms/step | 2.4 samples/s
+Step 690/37465 | Loss: 3.9610 | LR: 4.60e-04 | 8376.1ms/step | 2.4 samples/s
+Step 700/37465 | Loss: 4.0882 | LR: 4.67e-04 | 8297.2ms/step | 2.4 samples/s
+Step 710/37465 | Loss: 4.0269 | LR: 4.73e-04 | 8247.7ms/step | 2.4 samples/s
+Step 720/37465 | Loss: 3.9731 | LR: 4.80e-04 | 8152.7ms/step | 2.5 samples/s
+Step 730/37465 | Loss: 3.5717 | LR: 4.87e-04 | 8128.0ms/step | 2.5 samples/s
+Step 740/37465 | Loss: 4.0327 | LR: 4.93e-04 | 8136.5ms/step | 2.5 samples/s
+Step 750/37465 | Loss: 3.7517 | LR: 5.00e-04 | 8156.7ms/step | 2.5 samples/s
+Step 760/37465 | Loss: 3.6823 | LR: 5.00e-04 | 8179.4ms/step | 2.4 samples/s
+Step 770/37465 | Loss: 3.6981 | LR: 5.00e-04 | 8187.6ms/step | 2.4 samples/s
+Step 780/37465 | Loss: 3.7687 | LR: 5.00e-04 | 8195.8ms/step | 2.4 samples/s
+Step 790/37465 | Loss: 3.4020 | LR: 5.00e-04 | 8201.2ms/step | 2.4 samples/s
+Step 800/37465 | Loss: 3.7875 | LR: 5.00e-04 | 8244.4ms/step | 2.4 samples/s
+Step 810/37465 | Loss: 3.8170 | LR: 5.00e-04 | 8204.8ms/step | 2.4 samples/s
+Step 820/37465 | Loss: 3.6359 | LR: 5.00e-04 | 8149.9ms/step | 2.5 samples/s
+Step 830/37465 | Loss: 3.6302 | LR: 5.00e-04 | 8159.7ms/step | 2.5 samples/s
+Step 840/37465 | Loss: 3.8067 | LR: 5.00e-04 | 8152.7ms/step | 2.5 samples/s
+Step 850/37465 | Loss: 3.4456 | LR: 5.00e-04 | 8219.3ms/step | 2.4 samples/s
+Step 860/37465 | Loss: 3.6688 | LR: 5.00e-04 | 8306.0ms/step | 2.4 samples/s
+Step 870/37465 | Loss: 3.6052 | LR: 5.00e-04 | 8273.8ms/step | 2.4 samples/s
+Step 880/37465 | Loss: 3.4955 | LR: 5.00e-04 | 8288.3ms/step | 2.4 samples/s
+Step 890/37465 | Loss: 3.5420 | LR: 5.00e-04 | 8239.0ms/step | 2.4 samples/s
+Step 900/37465 | Loss: 3.2254 | LR: 5.00e-04 | 8169.4ms/step | 2.4 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-900
+Step 910/37465 | Loss: 3.2560 | LR: 5.00e-04 | 9184.3ms/step | 2.2 samples/s
+Step 920/37465 | Loss: 3.4002 | LR: 5.00e-04 | 8214.5ms/step | 2.4 samples/s
+Step 930/37465 | Loss: 3.2559 | LR: 5.00e-04 | 8258.3ms/step | 2.4 samples/s
+Step 940/37465 | Loss: 3.5204 | LR: 5.00e-04 | 8186.9ms/step | 2.4 samples/s
+Step 950/37465 | Loss: 3.7731 | LR: 5.00e-04 | 8245.3ms/step | 2.4 samples/s
+Step 960/37465 | Loss: 3.1069 | LR: 5.00e-04 | 8158.7ms/step | 2.5 samples/s
+Step 970/37465 | Loss: 3.4975 | LR: 5.00e-04 | 8265.2ms/step | 2.4 samples/s
+Step 980/37465 | Loss: 3.3028 | LR: 5.00e-04 | 8266.4ms/step | 2.4 samples/s
+Step 990/37465 | Loss: 3.2725 | LR: 5.00e-04 | 8200.5ms/step | 2.4 samples/s
+Step 1000/37465 | Loss: 3.1093 | LR: 5.00e-04 | 8170.5ms/step | 2.4 samples/s
+Validation at step 1000: Loss = 3.3107, Perplexity = 27.40
+Step 1010/37465 | Loss: 3.1640 | LR: 5.00e-04 | 8912.6ms/step | 2.2 samples/s
+Step 1020/37465 | Loss: 3.4299 | LR: 5.00e-04 | 8212.7ms/step | 2.4 samples/s
+Step 1030/37465 | Loss: 3.0263 | LR: 5.00e-04 | 8212.9ms/step | 2.4 samples/s
+Step 1040/37465 | Loss: 3.4330 | LR: 5.00e-04 | 8177.0ms/step | 2.4 samples/s
+Step 1050/37465 | Loss: 3.2475 | LR: 5.00e-04 | 8203.6ms/step | 2.4 samples/s
+Step 1060/37465 | Loss: 3.2549 | LR: 5.00e-04 | 8239.7ms/step | 2.4 samples/s
+Step 1070/37465 | Loss: 3.2350 | LR: 5.00e-04 | 8174.6ms/step | 2.4 samples/s
+Step 1080/37465 | Loss: 3.2388 | LR: 5.00e-04 | 8144.2ms/step | 2.5 samples/s
+Step 1090/37465 | Loss: 3.2410 | LR: 5.00e-04 | 8155.1ms/step | 2.5 samples/s
+Step 1100/37465 | Loss: 3.0985 | LR: 5.00e-04 | 8152.8ms/step | 2.5 samples/s
+Step 1110/37465 | Loss: 3.0416 | LR: 5.00e-04 | 8235.7ms/step | 2.4 samples/s
+Step 1120/37465 | Loss: 2.9873 | LR: 5.00e-04 | 8241.8ms/step | 2.4 samples/s
+Step 1130/37465 | Loss: 3.2195 | LR: 5.00e-04 | 8187.2ms/step | 2.4 samples/s
+Step 1140/37465 | Loss: 3.2064 | LR: 5.00e-04 | 8172.4ms/step | 2.4 samples/s
+Step 1150/37465 | Loss: 3.0180 | LR: 5.00e-04 | 8228.4ms/step | 2.4 samples/s
+Step 1160/37465 | Loss: 3.3100 | LR: 5.00e-04 | 8210.0ms/step | 2.4 samples/s
+Step 1170/37465 | Loss: 2.8792 | LR: 5.00e-04 | 8227.6ms/step | 2.4 samples/s
+Step 1180/37465 | Loss: 3.2582 | LR: 5.00e-04 | 8248.9ms/step | 2.4 samples/s
+Step 1190/37465 | Loss: 3.0823 | LR: 5.00e-04 | 8269.8ms/step | 2.4 samples/s
+Step 1200/37465 | Loss: 2.8300 | LR: 5.00e-04 | 8179.5ms/step | 2.4 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-1200
+Step 1210/37465 | Loss: 3.1854 | LR: 5.00e-04 | 9095.3ms/step | 2.2 samples/s
+Step 1220/37465 | Loss: 3.2507 | LR: 5.00e-04 | 8178.1ms/step | 2.4 samples/s
+Step 1230/37465 | Loss: 2.8841 | LR: 5.00e-04 | 8179.4ms/step | 2.4 samples/s
+Step 1240/37465 | Loss: 3.1349 | LR: 5.00e-04 | 8155.6ms/step | 2.5 samples/s
+Step 1250/37465 | Loss: 2.9573 | LR: 5.00e-04 | 8202.6ms/step | 2.4 samples/s
+Step 1260/37465 | Loss: 2.9792 | LR: 5.00e-04 | 8185.9ms/step | 2.4 samples/s
+Step 1270/37465 | Loss: 3.0829 | LR: 5.00e-04 | 8252.0ms/step | 2.4 samples/s
+Step 1280/37465 | Loss: 2.9335 | LR: 5.00e-04 | 8206.7ms/step | 2.4 samples/s
+Step 1290/37465 | Loss: 3.0606 | LR: 5.00e-04 | 8187.6ms/step | 2.4 samples/s
+Step 1300/37465 | Loss: 3.3668 | LR: 5.00e-04 | 8204.1ms/step | 2.4 samples/s
+Step 1310/37465 | Loss: 3.1303 | LR: 5.00e-04 | 8189.0ms/step | 2.4 samples/s
+Step 1320/37465 | Loss: 2.9707 | LR: 5.00e-04 | 8211.9ms/step | 2.4 samples/s
+Step 1330/37465 | Loss: 3.1654 | LR: 5.00e-04 | 8237.1ms/step | 2.4 samples/s
+Step 1340/37465 | Loss: 3.0821 | LR: 5.00e-04 | 8188.5ms/step | 2.4 samples/s
+Step 1350/37465 | Loss: 3.0063 | LR: 5.00e-04 | 8214.2ms/step | 2.4 samples/s
+Step 1360/37465 | Loss: 2.9233 | LR: 5.00e-04 | 8179.1ms/step | 2.4 samples/s
+Step 1370/37465 | Loss: 3.2369 | LR: 5.00e-04 | 8193.5ms/step | 2.4 samples/s
+Step 1380/37465 | Loss: 2.9580 | LR: 5.00e-04 | 8154.0ms/step | 2.5 samples/s
+Step 1390/37465 | Loss: 2.9403 | LR: 5.00e-04 | 8206.3ms/step | 2.4 samples/s
+Step 1400/37465 | Loss: 2.9784 | LR: 5.00e-04 | 8174.0ms/step | 2.4 samples/s
+Step 1410/37465 | Loss: 3.1715 | LR: 5.00e-04 | 8226.6ms/step | 2.4 samples/s
+Step 1420/37465 | Loss: 2.7800 | LR: 5.00e-04 | 8163.5ms/step | 2.4 samples/s
+Step 1430/37465 | Loss: 3.3297 | LR: 5.00e-04 | 8170.5ms/step | 2.4 samples/s
+Step 1440/37465 | Loss: 2.9857 | LR: 5.00e-04 | 8143.5ms/step | 2.5 samples/s
+Step 1450/37465 | Loss: 3.0472 | LR: 5.00e-04 | 8186.8ms/step | 2.4 samples/s
+Step 1460/37465 | Loss: 3.1017 | LR: 5.00e-04 | 8201.9ms/step | 2.4 samples/s
+Step 1470/37465 | Loss: 2.9672 | LR: 5.00e-04 | 8281.7ms/step | 2.4 samples/s
+Step 1480/37465 | Loss: 2.8356 | LR: 5.00e-04 | 8175.8ms/step | 2.4 samples/s
+Step 1490/37465 | Loss: 2.9505 | LR: 4.99e-04 | 8182.0ms/step | 2.4 samples/s
+Step 1500/37465 | Loss: 2.9152 | LR: 4.99e-04 | 8204.8ms/step | 2.4 samples/s
+Validation at step 1500: Loss = 2.9929, Perplexity = 19.94
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-1500
+Step 1510/37465 | Loss: 2.9940 | LR: 4.99e-04 | 9734.2ms/step | 2.1 samples/s
+Step 1520/37465 | Loss: 2.9082 | LR: 4.99e-04 | 8144.7ms/step | 2.5 samples/s
+Step 1530/37465 | Loss: 2.8439 | LR: 4.99e-04 | 8170.1ms/step | 2.4 samples/s
+Step 1540/37465 | Loss: 2.8328 | LR: 4.99e-04 | 8234.3ms/step | 2.4 samples/s
+Step 1550/37465 | Loss: 3.1183 | LR: 4.99e-04 | 8230.8ms/step | 2.4 samples/s
+Step 1560/37465 | Loss: 2.9935 | LR: 4.99e-04 | 8171.8ms/step | 2.4 samples/s
+Step 1570/37465 | Loss: 2.7985 | LR: 4.99e-04 | 8197.9ms/step | 2.4 samples/s
+Step 1580/37465 | Loss: 3.1618 | LR: 4.99e-04 | 8187.2ms/step | 2.4 samples/s
+Step 1590/37465 | Loss: 3.0263 | LR: 4.99e-04 | 8168.4ms/step | 2.4 samples/s
+Step 1600/37465 | Loss: 3.3189 | LR: 4.99e-04 | 8201.7ms/step | 2.4 samples/s
+Step 1610/37465 | Loss: 2.8287 | LR: 4.99e-04 | 8183.2ms/step | 2.4 samples/s
+Step 1620/37465 | Loss: 3.0913 | LR: 4.99e-04 | 8221.1ms/step | 2.4 samples/s
+Step 1630/37465 | Loss: 2.9591 | LR: 4.99e-04 | 8224.3ms/step | 2.4 samples/s
+Step 1640/37465 | Loss: 2.8990 | LR: 4.99e-04 | 8187.6ms/step | 2.4 samples/s
+Step 1650/37465 | Loss: 2.6485 | LR: 4.99e-04 | 8256.0ms/step | 2.4 samples/s
+Step 1660/37465 | Loss: 2.7825 | LR: 4.99e-04 | 8186.0ms/step | 2.4 samples/s
+Step 1670/37465 | Loss: 2.7878 | LR: 4.99e-04 | 8184.7ms/step | 2.4 samples/s
+Step 1680/37465 | Loss: 3.0618 | LR: 4.99e-04 | 8154.6ms/step | 2.5 samples/s
+Step 1690/37465 | Loss: 2.9331 | LR: 4.99e-04 | 8154.0ms/step | 2.5 samples/s
+Step 1700/37465 | Loss: 2.9924 | LR: 4.99e-04 | 8170.4ms/step | 2.4 samples/s
+Step 1710/37465 | Loss: 3.0073 | LR: 4.99e-04 | 8154.1ms/step | 2.5 samples/s
+Step 1720/37465 | Loss: 3.0590 | LR: 4.99e-04 | 8182.2ms/step | 2.4 samples/s
+Step 1730/37465 | Loss: 2.8025 | LR: 4.99e-04 | 8192.3ms/step | 2.4 samples/s
+Step 1740/37465 | Loss: 2.9592 | LR: 4.99e-04 | 8182.5ms/step | 2.4 samples/s
+Step 1750/37465 | Loss: 2.9930 | LR: 4.99e-04 | 8149.3ms/step | 2.5 samples/s
+Step 1760/37465 | Loss: 2.9122 | LR: 4.99e-04 | 8202.2ms/step | 2.4 samples/s
+Step 1770/37465 | Loss: 2.9019 | LR: 4.99e-04 | 8199.3ms/step | 2.4 samples/s
+Step 1780/37465 | Loss: 2.7550 | LR: 4.99e-04 | 8217.9ms/step | 2.4 samples/s
+Step 1790/37465 | Loss: 2.9807 | LR: 4.99e-04 | 8218.4ms/step | 2.4 samples/s
+Step 1800/37465 | Loss: 2.6345 | LR: 4.99e-04 | 8250.8ms/step | 2.4 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-1800
+Step 1810/37465 | Loss: 2.9508 | LR: 4.99e-04 | 9148.1ms/step | 2.2 samples/s
+Step 1820/37465 | Loss: 2.9640 | LR: 4.99e-04 | 8218.2ms/step | 2.4 samples/s
+Step 1830/37465 | Loss: 2.8576 | LR: 4.99e-04 | 8240.7ms/step | 2.4 samples/s
+Step 1840/37465 | Loss: 2.9748 | LR: 4.99e-04 | 8187.0ms/step | 2.4 samples/s
+Step 1850/37465 | Loss: 2.7509 | LR: 4.99e-04 | 8185.5ms/step | 2.4 samples/s
+Step 1860/37465 | Loss: 2.7869 | LR: 4.99e-04 | 8164.2ms/step | 2.4 samples/s
+Step 1870/37465 | Loss: 3.1765 | LR: 4.99e-04 | 8173.6ms/step | 2.4 samples/s
+Step 1880/37465 | Loss: 2.9232 | LR: 4.99e-04 | 8160.3ms/step | 2.5 samples/s
+Step 1890/37465 | Loss: 2.9119 | LR: 4.99e-04 | 8150.2ms/step | 2.5 samples/s
+Step 1900/37465 | Loss: 2.7851 | LR: 4.99e-04 | 8160.2ms/step | 2.5 samples/s
+Step 1910/37465 | Loss: 2.7634 | LR: 4.99e-04 | 8163.8ms/step | 2.4 samples/s
+Step 1920/37465 | Loss: 2.9797 | LR: 4.99e-04 | 8163.2ms/step | 2.5 samples/s
+Step 1930/37465 | Loss: 2.8600 | LR: 4.99e-04 | 8142.1ms/step | 2.5 samples/s
+Step 1940/37465 | Loss: 2.8724 | LR: 4.99e-04 | 8167.8ms/step | 2.4 samples/s
+Step 1950/37465 | Loss: 2.7951 | LR: 4.99e-04 | 8183.9ms/step | 2.4 samples/s
+Step 1960/37465 | Loss: 2.7944 | LR: 4.99e-04 | 8172.5ms/step | 2.4 samples/s
+Step 1970/37465 | Loss: 3.0025 | LR: 4.99e-04 | 8171.6ms/step | 2.4 samples/s
+Step 1980/37465 | Loss: 2.7713 | LR: 4.99e-04 | 8160.0ms/step | 2.5 samples/s
+Step 1990/37465 | Loss: 2.7367 | LR: 4.99e-04 | 8149.8ms/step | 2.5 samples/s
+Step 2000/37465 | Loss: 2.8296 | LR: 4.99e-04 | 8178.1ms/step | 2.4 samples/s
+Validation at step 2000: Loss = 2.8157, Perplexity = 16.71
+Step 2010/37465 | Loss: 2.7407 | LR: 4.99e-04 | 8857.0ms/step | 2.3 samples/s
+Step 2020/37465 | Loss: 2.8025 | LR: 4.99e-04 | 8144.6ms/step | 2.5 samples/s
+Step 2030/37465 | Loss: 2.6655 | LR: 4.99e-04 | 8163.9ms/step | 2.4 samples/s
+Step 2040/37465 | Loss: 2.7126 | LR: 4.98e-04 | 8212.2ms/step | 2.4 samples/s
+Step 2050/37465 | Loss: 2.8032 | LR: 4.98e-04 | 8265.4ms/step | 2.4 samples/s
+Step 2060/37465 | Loss: 2.6603 | LR: 4.98e-04 | 8265.1ms/step | 2.4 samples/s
+Step 2070/37465 | Loss: 2.6939 | LR: 4.98e-04 | 8219.6ms/step | 2.4 samples/s
+Step 2080/37465 | Loss: 2.8709 | LR: 4.98e-04 | 8153.7ms/step | 2.5 samples/s
+Step 2090/37465 | Loss: 2.8547 | LR: 4.98e-04 | 8164.6ms/step | 2.4 samples/s
+Step 2100/37465 | Loss: 2.9556 | LR: 4.98e-04 | 8154.4ms/step | 2.5 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-2100
+Step 2110/37465 | Loss: 2.8995 | LR: 4.98e-04 | 9140.8ms/step | 2.2 samples/s
+Step 2120/37465 | Loss: 2.8264 | LR: 4.98e-04 | 8196.8ms/step | 2.4 samples/s
+Step 2130/37465 | Loss: 2.5502 | LR: 4.98e-04 | 8167.0ms/step | 2.4 samples/s
+Step 2140/37465 | Loss: 2.8676 | LR: 4.98e-04 | 8158.7ms/step | 2.5 samples/s
+Step 2150/37465 | Loss: 2.8053 | LR: 4.98e-04 | 8175.0ms/step | 2.4 samples/s
+Step 2160/37465 | Loss: 2.7136 | LR: 4.98e-04 | 8182.5ms/step | 2.4 samples/s
+Step 2170/37465 | Loss: 2.8158 | LR: 4.98e-04 | 8178.2ms/step | 2.4 samples/s
+Step 2180/37465 | Loss: 2.8378 | LR: 4.98e-04 | 8164.8ms/step | 2.4 samples/s
+Step 2190/37465 | Loss: 2.7915 | LR: 4.98e-04 | 8185.7ms/step | 2.4 samples/s
+Step 2200/37465 | Loss: 2.7099 | LR: 4.98e-04 | 8155.4ms/step | 2.5 samples/s
+Step 2210/37465 | Loss: 2.6477 | LR: 4.98e-04 | 8213.5ms/step | 2.4 samples/s
+Step 2220/37465 | Loss: 2.8399 | LR: 4.98e-04 | 8167.4ms/step | 2.4 samples/s
+Step 2230/37465 | Loss: 2.8308 | LR: 4.98e-04 | 8169.0ms/step | 2.4 samples/s
+Step 2240/37465 | Loss: 2.8296 | LR: 4.98e-04 | 8170.1ms/step | 2.4 samples/s
+Step 2250/37465 | Loss: 2.7188 | LR: 4.98e-04 | 8209.2ms/step | 2.4 samples/s
+Step 2260/37465 | Loss: 2.8725 | LR: 4.98e-04 | 8153.3ms/step | 2.5 samples/s
+Step 2270/37465 | Loss: 2.7671 | LR: 4.98e-04 | 8187.0ms/step | 2.4 samples/s
+Step 2280/37465 | Loss: 2.5925 | LR: 4.98e-04 | 8195.9ms/step | 2.4 samples/s
+Step 2290/37465 | Loss: 2.7323 | LR: 4.98e-04 | 8184.2ms/step | 2.4 samples/s
+Step 2300/37465 | Loss: 2.8295 | LR: 4.98e-04 | 8156.4ms/step | 2.5 samples/s
+Step 2310/37465 | Loss: 2.8868 | LR: 4.98e-04 | 8166.3ms/step | 2.4 samples/s
+Step 2320/37465 | Loss: 2.7510 | LR: 4.98e-04 | 8204.4ms/step | 2.4 samples/s
+Step 2330/37465 | Loss: 3.0182 | LR: 4.98e-04 | 8176.3ms/step | 2.4 samples/s
+Step 2340/37465 | Loss: 2.6166 | LR: 4.98e-04 | 8154.8ms/step | 2.5 samples/s
+Step 2350/37465 | Loss: 3.0000 | LR: 4.98e-04 | 8207.7ms/step | 2.4 samples/s
+Step 2360/37465 | Loss: 2.6088 | LR: 4.98e-04 | 8220.1ms/step | 2.4 samples/s
+Step 2370/37465 | Loss: 2.8639 | LR: 4.98e-04 | 8221.8ms/step | 2.4 samples/s
+Step 2380/37465 | Loss: 2.6826 | LR: 4.98e-04 | 8219.5ms/step | 2.4 samples/s
+Step 2390/37465 | Loss: 2.5604 | LR: 4.98e-04 | 8159.0ms/step | 2.5 samples/s
+Step 2400/37465 | Loss: 2.5405 | LR: 4.98e-04 | 8160.4ms/step | 2.5 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-2400
+Step 2410/37465 | Loss: 2.7121 | LR: 4.97e-04 | 9147.8ms/step | 2.2 samples/s
+Step 2420/37465 | Loss: 2.8993 | LR: 4.97e-04 | 8209.1ms/step | 2.4 samples/s
+Step 2430/37465 | Loss: 2.8897 | LR: 4.97e-04 | 8201.6ms/step | 2.4 samples/s
+Step 2440/37465 | Loss: 2.8287 | LR: 4.97e-04 | 8208.0ms/step | 2.4 samples/s
+Step 2450/37465 | Loss: 2.8289 | LR: 4.97e-04 | 8194.5ms/step | 2.4 samples/s
+Step 2460/37465 | Loss: 2.8179 | LR: 4.97e-04 | 8186.6ms/step | 2.4 samples/s
+Step 2470/37465 | Loss: 2.8386 | LR: 4.97e-04 | 8189.0ms/step | 2.4 samples/s
+Step 2480/37465 | Loss: 2.6495 | LR: 4.97e-04 | 8177.5ms/step | 2.4 samples/s
+Step 2490/37465 | Loss: 2.6424 | LR: 4.97e-04 | 8143.2ms/step | 2.5 samples/s
+Step 2500/37465 | Loss: 2.7739 | LR: 4.97e-04 | 8146.2ms/step | 2.5 samples/s
+Validation at step 2500: Loss = 2.7044, Perplexity = 14.95
+Step 2510/37465 | Loss: 2.8365 | LR: 4.97e-04 | 8863.9ms/step | 2.3 samples/s
+Step 2520/37465 | Loss: 2.5305 | LR: 4.97e-04 | 8186.7ms/step | 2.4 samples/s
+Step 2530/37465 | Loss: 2.6727 | LR: 4.97e-04 | 8181.0ms/step | 2.4 samples/s
+Step 2540/37465 | Loss: 2.9635 | LR: 4.97e-04 | 8212.1ms/step | 2.4 samples/s
+Step 2550/37465 | Loss: 2.4608 | LR: 4.97e-04 | 8267.2ms/step | 2.4 samples/s
+Step 2560/37465 | Loss: 2.7541 | LR: 4.97e-04 | 8212.9ms/step | 2.4 samples/s
+Step 2570/37465 | Loss: 2.7161 | LR: 4.97e-04 | 8190.1ms/step | 2.4 samples/s
+Step 2580/37465 | Loss: 2.5075 | LR: 4.97e-04 | 8142.6ms/step | 2.5 samples/s
+Step 2590/37465 | Loss: 2.7348 | LR: 4.97e-04 | 8212.6ms/step | 2.4 samples/s
+Step 2600/37465 | Loss: 2.8617 | LR: 4.97e-04 | 8269.4ms/step | 2.4 samples/s
+Step 2610/37465 | Loss: 2.5875 | LR: 4.97e-04 | 8246.0ms/step | 2.4 samples/s
+Step 2620/37465 | Loss: 2.8683 | LR: 4.97e-04 | 8259.7ms/step | 2.4 samples/s
+Step 2630/37465 | Loss: 2.6856 | LR: 4.97e-04 | 8280.1ms/step | 2.4 samples/s
+Step 2640/37465 | Loss: 2.6261 | LR: 4.97e-04 | 8257.1ms/step | 2.4 samples/s
+Step 2650/37465 | Loss: 2.6231 | LR: 4.97e-04 | 8230.0ms/step | 2.4 samples/s
+Step 2660/37465 | Loss: 2.5173 | LR: 4.97e-04 | 8192.3ms/step | 2.4 samples/s
+Step 2670/37465 | Loss: 2.5940 | LR: 4.97e-04 | 8198.6ms/step | 2.4 samples/s
+Step 2680/37465 | Loss: 2.8824 | LR: 4.97e-04 | 8178.3ms/step | 2.4 samples/s
+Step 2690/37465 | Loss: 2.6206 | LR: 4.97e-04 | 8214.1ms/step | 2.4 samples/s
+Step 2700/37465 | Loss: 2.7812 | LR: 4.97e-04 | 8207.4ms/step | 2.4 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-2700
+Step 2710/37465 | Loss: 2.6251 | LR: 4.96e-04 | 9090.7ms/step | 2.2 samples/s
+Step 2720/37465 | Loss: 2.5969 | LR: 4.96e-04 | 8188.6ms/step | 2.4 samples/s
+Step 2730/37465 | Loss: 2.6373 | LR: 4.96e-04 | 8201.9ms/step | 2.4 samples/s
+Step 2740/37465 | Loss: 2.7168 | LR: 4.96e-04 | 8243.4ms/step | 2.4 samples/s
+Step 2750/37465 | Loss: 2.6317 | LR: 4.96e-04 | 8148.0ms/step | 2.5 samples/s
+Step 2760/37465 | Loss: 2.8555 | LR: 4.96e-04 | 8174.9ms/step | 2.4 samples/s
+Step 2770/37465 | Loss: 2.6921 | LR: 4.96e-04 | 8191.4ms/step | 2.4 samples/s
+Step 2780/37465 | Loss: 2.7705 | LR: 4.96e-04 | 8183.9ms/step | 2.4 samples/s
+Step 2790/37465 | Loss: 2.5200 | LR: 4.96e-04 | 8201.0ms/step | 2.4 samples/s
+Step 2800/37465 | Loss: 2.5776 | LR: 4.96e-04 | 8170.4ms/step | 2.4 samples/s
+Step 2810/37465 | Loss: 2.5499 | LR: 4.96e-04 | 8221.1ms/step | 2.4 samples/s
+Step 2820/37465 | Loss: 2.3869 | LR: 4.96e-04 | 8160.0ms/step | 2.5 samples/s
+Step 2830/37465 | Loss: 2.4888 | LR: 4.96e-04 | 8190.5ms/step | 2.4 samples/s
+Step 2840/37465 | Loss: 2.9341 | LR: 4.96e-04 | 8169.4ms/step | 2.4 samples/s
+Step 2850/37465 | Loss: 2.5894 | LR: 4.96e-04 | 8177.4ms/step | 2.4 samples/s
+Step 2860/37465 | Loss: 2.5536 | LR: 4.96e-04 | 8209.8ms/step | 2.4 samples/s
+Step 2870/37465 | Loss: 2.7532 | LR: 4.96e-04 | 8209.5ms/step | 2.4 samples/s
+Step 2880/37465 | Loss: 2.7046 | LR: 4.96e-04 | 8178.5ms/step | 2.4 samples/s
+Step 2890/37465 | Loss: 2.6350 | LR: 4.96e-04 | 8177.0ms/step | 2.4 samples/s
+Step 2900/37465 | Loss: 2.5862 | LR: 4.96e-04 | 8166.6ms/step | 2.4 samples/s
+Step 2910/37465 | Loss: 2.6961 | LR: 4.96e-04 | 8188.5ms/step | 2.4 samples/s
+Step 2920/37465 | Loss: 2.4846 | LR: 4.96e-04 | 8231.4ms/step | 2.4 samples/s
+Step 2930/37465 | Loss: 2.6273 | LR: 4.96e-04 | 8167.2ms/step | 2.4 samples/s
+Step 2940/37465 | Loss: 2.5379 | LR: 4.96e-04 | 8220.1ms/step | 2.4 samples/s
+Step 2950/37465 | Loss: 2.5631 | LR: 4.96e-04 | 8201.4ms/step | 2.4 samples/s
+Step 2960/37465 | Loss: 2.5970 | LR: 4.96e-04 | 8254.7ms/step | 2.4 samples/s
+Step 2970/37465 | Loss: 2.7225 | LR: 4.96e-04 | 8209.2ms/step | 2.4 samples/s
+Step 2980/37465 | Loss: 2.3602 | LR: 4.95e-04 | 8186.2ms/step | 2.4 samples/s
+Step 2990/37465 | Loss: 2.7677 | LR: 4.95e-04 | 8197.3ms/step | 2.4 samples/s
+Step 3000/37465 | Loss: 2.6670 | LR: 4.95e-04 | 8146.9ms/step | 2.5 samples/s
+Validation at step 3000: Loss = 2.6412, Perplexity = 14.03
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-3000
+Step 3010/37465 | Loss: 2.6457 | LR: 4.95e-04 | 9731.0ms/step | 2.1 samples/s
+Step 3020/37465 | Loss: 2.6357 | LR: 4.95e-04 | 8155.9ms/step | 2.5 samples/s
+Step 3030/37465 | Loss: 2.8313 | LR: 4.95e-04 | 8174.5ms/step | 2.4 samples/s
+Step 3040/37465 | Loss: 2.7540 | LR: 4.95e-04 | 8178.7ms/step | 2.4 samples/s
+Step 3050/37465 | Loss: 2.6483 | LR: 4.95e-04 | 8186.1ms/step | 2.4 samples/s
+Step 3060/37465 | Loss: 2.6350 | LR: 4.95e-04 | 8176.8ms/step | 2.4 samples/s
+Step 3070/37465 | Loss: 2.4596 | LR: 4.95e-04 | 8184.4ms/step | 2.4 samples/s
+Step 3080/37465 | Loss: 2.4220 | LR: 4.95e-04 | 8187.9ms/step | 2.4 samples/s
+Step 3090/37465 | Loss: 2.5939 | LR: 4.95e-04 | 8211.1ms/step | 2.4 samples/s
+Step 3100/37465 | Loss: 2.6991 | LR: 4.95e-04 | 8172.4ms/step | 2.4 samples/s
+Step 3110/37465 | Loss: 2.6370 | LR: 4.95e-04 | 8190.8ms/step | 2.4 samples/s
+Step 3120/37465 | Loss: 2.5206 | LR: 4.95e-04 | 8185.5ms/step | 2.4 samples/s
+Step 3130/37465 | Loss: 2.6252 | LR: 4.95e-04 | 8191.0ms/step | 2.4 samples/s
+Step 3140/37465 | Loss: 2.4345 | LR: 4.95e-04 | 8203.0ms/step | 2.4 samples/s
+Step 3150/37465 | Loss: 2.4018 | LR: 4.95e-04 | 8216.1ms/step | 2.4 samples/s
+Step 3160/37465 | Loss: 2.5793 | LR: 4.95e-04 | 8243.0ms/step | 2.4 samples/s
+Step 3170/37465 | Loss: 2.8797 | LR: 4.95e-04 | 8210.6ms/step | 2.4 samples/s
+Step 3180/37465 | Loss: 2.8203 | LR: 4.95e-04 | 8221.0ms/step | 2.4 samples/s
+Step 3190/37465 | Loss: 2.2988 | LR: 4.95e-04 | 8160.3ms/step | 2.5 samples/s
+Step 3200/37465 | Loss: 2.4824 | LR: 4.95e-04 | 8160.1ms/step | 2.5 samples/s
+Step 3210/37465 | Loss: 2.4128 | LR: 4.94e-04 | 8168.4ms/step | 2.4 samples/s
+Step 3220/37465 | Loss: 2.6652 | LR: 4.94e-04 | 8158.0ms/step | 2.5 samples/s
+Step 3230/37465 | Loss: 2.6337 | LR: 4.94e-04 | 8203.8ms/step | 2.4 samples/s
+Step 3240/37465 | Loss: 2.5037 | LR: 4.94e-04 | 8205.8ms/step | 2.4 samples/s
+Step 3250/37465 | Loss: 2.5130 | LR: 4.94e-04 | 8188.6ms/step | 2.4 samples/s
+Step 3260/37465 | Loss: 2.6979 | LR: 4.94e-04 | 8187.4ms/step | 2.4 samples/s
+Step 3270/37465 | Loss: 2.6510 | LR: 4.94e-04 | 8178.9ms/step | 2.4 samples/s
+Step 3280/37465 | Loss: 2.6239 | LR: 4.94e-04 | 8159.1ms/step | 2.5 samples/s
+Step 3290/37465 | Loss: 2.5189 | LR: 4.94e-04 | 8208.9ms/step | 2.4 samples/s
+Step 3300/37465 | Loss: 2.4943 | LR: 4.94e-04 | 8171.3ms/step | 2.4 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-3300
+Step 3310/37465 | Loss: 2.6931 | LR: 4.94e-04 | 9073.2ms/step | 2.2 samples/s
+Step 3320/37465 | Loss: 2.3834 | LR: 4.94e-04 | 8187.4ms/step | 2.4 samples/s
+Step 3330/37465 | Loss: 2.7048 | LR: 4.94e-04 | 8187.4ms/step | 2.4 samples/s
+Step 3340/37465 | Loss: 2.6655 | LR: 4.94e-04 | 8175.8ms/step | 2.4 samples/s
+Step 3350/37465 | Loss: 2.9431 | LR: 4.94e-04 | 8177.7ms/step | 2.4 samples/s
+Step 3360/37465 | Loss: 2.5442 | LR: 4.94e-04 | 8161.1ms/step | 2.5 samples/s
+Step 3370/37465 | Loss: 2.5679 | LR: 4.94e-04 | 8210.9ms/step | 2.4 samples/s
+Step 3380/37465 | Loss: 2.6058 | LR: 4.94e-04 | 8218.3ms/step | 2.4 samples/s
+Step 3390/37465 | Loss: 2.4915 | LR: 4.94e-04 | 8171.6ms/step | 2.4 samples/s
+Step 3400/37465 | Loss: 2.4060 | LR: 4.94e-04 | 8274.7ms/step | 2.4 samples/s
+Step 3410/37465 | Loss: 2.4611 | LR: 4.94e-04 | 8204.0ms/step | 2.4 samples/s
+Step 3420/37465 | Loss: 2.5989 | LR: 4.94e-04 | 8171.0ms/step | 2.4 samples/s
+Step 3430/37465 | Loss: 2.7648 | LR: 4.93e-04 | 8210.8ms/step | 2.4 samples/s
+Step 3440/37465 | Loss: 2.6462 | LR: 4.93e-04 | 8200.1ms/step | 2.4 samples/s
+Step 3450/37465 | Loss: 2.6194 | LR: 4.93e-04 | 8196.0ms/step | 2.4 samples/s
+Step 3460/37465 | Loss: 2.4992 | LR: 4.93e-04 | 8168.0ms/step | 2.4 samples/s
+Step 3470/37465 | Loss: 2.3703 | LR: 4.93e-04 | 8201.8ms/step | 2.4 samples/s
+Step 3480/37465 | Loss: 2.5212 | LR: 4.93e-04 | 8209.2ms/step | 2.4 samples/s
+Step 3490/37465 | Loss: 2.6557 | LR: 4.93e-04 | 8226.1ms/step | 2.4 samples/s
+Step 3500/37465 | Loss: 2.6127 | LR: 4.93e-04 | 8160.5ms/step | 2.5 samples/s
+Validation at step 3500: Loss = 2.5816, Perplexity = 13.22
+Step 3510/37465 | Loss: 2.8043 | LR: 4.93e-04 | 8885.6ms/step | 2.3 samples/s
+Step 3520/37465 | Loss: 2.4846 | LR: 4.93e-04 | 8180.8ms/step | 2.4 samples/s
+Step 3530/37465 | Loss: 2.5388 | LR: 4.93e-04 | 8201.8ms/step | 2.4 samples/s
+Step 3540/37465 | Loss: 2.6978 | LR: 4.93e-04 | 8211.1ms/step | 2.4 samples/s
+Step 3550/37465 | Loss: 2.5573 | LR: 4.93e-04 | 8159.3ms/step | 2.5 samples/s
+Step 3560/37465 | Loss: 2.4730 | LR: 4.93e-04 | 8189.3ms/step | 2.4 samples/s
+Step 3570/37465 | Loss: 2.4706 | LR: 4.93e-04 | 8171.7ms/step | 2.4 samples/s
+Step 3580/37465 | Loss: 2.9061 | LR: 4.93e-04 | 8210.1ms/step | 2.4 samples/s
+Step 3590/37465 | Loss: 2.5361 | LR: 4.93e-04 | 8176.7ms/step | 2.4 samples/s
+Step 3600/37465 | Loss: 2.6657 | LR: 4.93e-04 | 8179.6ms/step | 2.4 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-3600
+Step 3610/37465 | Loss: 2.5515 | LR: 4.93e-04 | 9090.3ms/step | 2.2 samples/s
+Step 3620/37465 | Loss: 2.4754 | LR: 4.92e-04 | 8207.8ms/step | 2.4 samples/s
+Step 3630/37465 | Loss: 2.4045 | LR: 4.92e-04 | 8200.0ms/step | 2.4 samples/s
+Step 3640/37465 | Loss: 2.5239 | LR: 4.92e-04 | 8183.5ms/step | 2.4 samples/s
+Step 3650/37465 | Loss: 2.6176 | LR: 4.92e-04 | 8192.5ms/step | 2.4 samples/s
+Step 3660/37465 | Loss: 2.4970 | LR: 4.92e-04 | 8157.9ms/step | 2.5 samples/s
+Step 3670/37465 | Loss: 2.3967 | LR: 4.92e-04 | 8211.1ms/step | 2.4 samples/s
+Step 3680/37465 | Loss: 2.7111 | LR: 4.92e-04 | 8202.3ms/step | 2.4 samples/s
+Step 3690/37465 | Loss: 2.8511 | LR: 4.92e-04 | 8151.4ms/step | 2.5 samples/s
+Step 3700/37465 | Loss: 2.5237 | LR: 4.92e-04 | 8154.5ms/step | 2.5 samples/s
+Step 3710/37465 | Loss: 2.8043 | LR: 4.92e-04 | 8200.6ms/step | 2.4 samples/s
+Step 3720/37465 | Loss: 2.5724 | LR: 4.92e-04 | 8169.6ms/step | 2.4 samples/s
+Step 3730/37465 | Loss: 2.7419 | LR: 4.92e-04 | 8174.8ms/step | 2.4 samples/s
+Step 3740/37465 | Loss: 2.3886 | LR: 4.92e-04 | 8216.2ms/step | 2.4 samples/s
+Step 3750/37465 | Loss: 2.5297 | LR: 4.92e-04 | 8188.2ms/step | 2.4 samples/s
+Step 3760/37465 | Loss: 2.7628 | LR: 4.92e-04 | 8224.9ms/step | 2.4 samples/s
+Step 3770/37465 | Loss: 2.7325 | LR: 4.92e-04 | 8192.5ms/step | 2.4 samples/s
+Step 3780/37465 | Loss: 2.6115 | LR: 4.92e-04 | 8203.5ms/step | 2.4 samples/s
+Step 3790/37465 | Loss: 2.4737 | LR: 4.92e-04 | 8170.1ms/step | 2.4 samples/s
+Step 3800/37465 | Loss: 2.5705 | LR: 4.92e-04 | 8233.0ms/step | 2.4 samples/s
+Step 3810/37465 | Loss: 2.6723 | LR: 4.91e-04 | 8211.9ms/step | 2.4 samples/s
+Step 3820/37465 | Loss: 2.6029 | LR: 4.91e-04 | 8161.2ms/step | 2.5 samples/s
+Step 3830/37465 | Loss: 2.7360 | LR: 4.91e-04 | 8201.3ms/step | 2.4 samples/s
+Step 3840/37465 | Loss: 2.3041 | LR: 4.91e-04 | 8176.9ms/step | 2.4 samples/s
+Step 3850/37465 | Loss: 2.6950 | LR: 4.91e-04 | 8144.0ms/step | 2.5 samples/s
+Step 3860/37465 | Loss: 2.4204 | LR: 4.91e-04 | 8207.5ms/step | 2.4 samples/s
+Step 3870/37465 | Loss: 2.7476 | LR: 4.91e-04 | 8175.0ms/step | 2.4 samples/s
+Step 3880/37465 | Loss: 2.3562 | LR: 4.91e-04 | 8189.6ms/step | 2.4 samples/s
+Step 3890/37465 | Loss: 2.3554 | LR: 4.91e-04 | 8200.8ms/step | 2.4 samples/s
+Step 3900/37465 | Loss: 2.4197 | LR: 4.91e-04 | 8250.1ms/step | 2.4 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-3900
+Step 3910/37465 | Loss: 2.6054 | LR: 4.91e-04 | 9080.9ms/step | 2.2 samples/s
+Step 3920/37465 | Loss: 2.5965 | LR: 4.91e-04 | 8191.7ms/step | 2.4 samples/s
+Step 3930/37465 | Loss: 2.6539 | LR: 4.91e-04 | 8170.0ms/step | 2.4 samples/s
+Step 3940/37465 | Loss: 2.8478 | LR: 4.91e-04 | 8180.8ms/step | 2.4 samples/s
+Step 3950/37465 | Loss: 2.3553 | LR: 4.91e-04 | 8138.7ms/step | 2.5 samples/s
+Step 3960/37465 | Loss: 2.5775 | LR: 4.91e-04 | 8147.7ms/step | 2.5 samples/s
+Step 3970/37465 | Loss: 2.3752 | LR: 4.91e-04 | 8177.8ms/step | 2.4 samples/s
+Step 3980/37465 | Loss: 2.5759 | LR: 4.91e-04 | 8183.7ms/step | 2.4 samples/s
+Step 3990/37465 | Loss: 2.5541 | LR: 4.90e-04 | 8148.6ms/step | 2.5 samples/s
+Step 4000/37465 | Loss: 2.5379 | LR: 4.90e-04 | 8175.9ms/step | 2.4 samples/s
+Validation at step 4000: Loss = 2.5357, Perplexity = 12.63
+Step 4010/37465 | Loss: 2.1452 | LR: 4.90e-04 | 8859.4ms/step | 2.3 samples/s
+Step 4020/37465 | Loss: 2.3252 | LR: 4.90e-04 | 8164.0ms/step | 2.4 samples/s
+Step 4030/37465 | Loss: 2.4662 | LR: 4.90e-04 | 8179.1ms/step | 2.4 samples/s
+Step 4040/37465 | Loss: 2.4781 | LR: 4.90e-04 | 8182.5ms/step | 2.4 samples/s
+Step 4050/37465 | Loss: 2.6928 | LR: 4.90e-04 | 8178.8ms/step | 2.4 samples/s
+Step 4060/37465 | Loss: 2.5012 | LR: 4.90e-04 | 8156.5ms/step | 2.5 samples/s
+Step 4070/37465 | Loss: 2.4975 | LR: 4.90e-04 | 8180.8ms/step | 2.4 samples/s
+Step 4080/37465 | Loss: 2.4839 | LR: 4.90e-04 | 8149.3ms/step | 2.5 samples/s
+Step 4090/37465 | Loss: 2.5882 | LR: 4.90e-04 | 8141.3ms/step | 2.5 samples/s
+Step 4100/37465 | Loss: 2.4278 | LR: 4.90e-04 | 8172.1ms/step | 2.4 samples/s
+Step 4110/37465 | Loss: 2.5256 | LR: 4.90e-04 | 8153.0ms/step | 2.5 samples/s
+Step 4120/37465 | Loss: 2.3251 | LR: 4.90e-04 | 8208.3ms/step | 2.4 samples/s
+Step 4130/37465 | Loss: 2.3016 | LR: 4.90e-04 | 8167.1ms/step | 2.4 samples/s
+Step 4140/37465 | Loss: 2.5975 | LR: 4.90e-04 | 8171.8ms/step | 2.4 samples/s
+Step 4150/37465 | Loss: 2.5831 | LR: 4.89e-04 | 8212.5ms/step | 2.4 samples/s
+Step 4160/37465 | Loss: 2.6225 | LR: 4.89e-04 | 8244.9ms/step | 2.4 samples/s
+Step 4170/37465 | Loss: 2.5197 | LR: 4.89e-04 | 8204.5ms/step | 2.4 samples/s
+Step 4180/37465 | Loss: 2.3115 | LR: 4.89e-04 | 8180.7ms/step | 2.4 samples/s
+Step 4190/37465 | Loss: 2.5864 | LR: 4.89e-04 | 8196.8ms/step | 2.4 samples/s
+Step 4200/37465 | Loss: 2.5527 | LR: 4.89e-04 | 8226.0ms/step | 2.4 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-4200
+Step 4210/37465 | Loss: 2.1629 | LR: 4.89e-04 | 9110.9ms/step | 2.2 samples/s
+Step 4220/37465 | Loss: 2.5146 | LR: 4.89e-04 | 8213.3ms/step | 2.4 samples/s
+Step 4230/37465 | Loss: 2.4114 | LR: 4.89e-04 | 8219.3ms/step | 2.4 samples/s
+Step 4240/37465 | Loss: 2.4552 | LR: 4.89e-04 | 8197.1ms/step | 2.4 samples/s
+Step 4250/37465 | Loss: 2.3725 | LR: 4.89e-04 | 8180.0ms/step | 2.4 samples/s
+Step 4260/37465 | Loss: 2.4324 | LR: 4.89e-04 | 8227.9ms/step | 2.4 samples/s
+Step 4270/37465 | Loss: 2.5080 | LR: 4.89e-04 | 8160.4ms/step | 2.5 samples/s
+Step 4280/37465 | Loss: 2.4876 | LR: 4.89e-04 | 8213.9ms/step | 2.4 samples/s
+Step 4290/37465 | Loss: 2.5656 | LR: 4.89e-04 | 8184.7ms/step | 2.4 samples/s
+Step 4300/37465 | Loss: 2.4473 | LR: 4.89e-04 | 8167.4ms/step | 2.4 samples/s
+Step 4310/37465 | Loss: 2.6317 | LR: 4.88e-04 | 8177.3ms/step | 2.4 samples/s
+Step 4320/37465 | Loss: 2.3662 | LR: 4.88e-04 | 8154.0ms/step | 2.5 samples/s
+Step 4330/37465 | Loss: 2.2315 | LR: 4.88e-04 | 8174.0ms/step | 2.4 samples/s
+Step 4340/37465 | Loss: 2.5357 | LR: 4.88e-04 | 8260.1ms/step | 2.4 samples/s
+Step 4350/37465 | Loss: 2.6435 | LR: 4.88e-04 | 8219.4ms/step | 2.4 samples/s
+Step 4360/37465 | Loss: 2.6113 | LR: 4.88e-04 | 8226.3ms/step | 2.4 samples/s
+Step 4370/37465 | Loss: 2.4357 | LR: 4.88e-04 | 8260.2ms/step | 2.4 samples/s
+Step 4380/37465 | Loss: 2.4313 | LR: 4.88e-04 | 8229.2ms/step | 2.4 samples/s
+Step 4390/37465 | Loss: 2.5864 | LR: 4.88e-04 | 8291.7ms/step | 2.4 samples/s
+Step 4400/37465 | Loss: 2.3635 | LR: 4.88e-04 | 8218.0ms/step | 2.4 samples/s
+Step 4410/37465 | Loss: 2.6025 | LR: 4.88e-04 | 8207.8ms/step | 2.4 samples/s
+Step 4420/37465 | Loss: 2.3309 | LR: 4.88e-04 | 8289.2ms/step | 2.4 samples/s
+Step 4430/37465 | Loss: 2.5159 | LR: 4.88e-04 | 8263.9ms/step | 2.4 samples/s

epfl_final/SparseAtt/wandb/run-20260708_112402-hn868f80/files/requirements.txt

@@ -0,0 +1,141 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5

epfl_final/SparseAtt/wandb/run-20260708_112402-hn868f80/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-07-08T11:24:02.703771Z",
+  "args":  [
+    "--config",
+    "configs/train_config_30k_resampled_b128.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "88298c7a418622530ca5426b22b1e5ebd10ca0cb"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-train-4-gpu-0-0",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "4939366932480"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-85844f64-d144-1c84-306d-f88b5f232a77"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-4e4c2343-4f5b-a2ff-312e-03d402a27803"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-fbf0ccd1-8bd4-5f57-6e74-6c0cf7eb70c9"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-410cba62-f098-3454-c960-ebc55bd081af"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "mh3uq51nuri6r7wm4dlcg4wax0lhetbv"
+}

epfl_final/SparseAtt/wandb/run-20260708_112402-hn868f80/logs/debug.log

@@ -0,0 +1,19 @@
+2026-07-08 11:24:02,726 INFO    MainThread:30965 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-07-08 11:24:02,726 INFO    MainThread:30965 [wandb_setup.py:_flush():81] Configure stats pid to 30965
+2026-07-08 11:24:02,726 INFO    MainThread:30965 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-08 11:24:02,726 INFO    MainThread:30965 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_112402-hn868f80/logs/debug.log
+2026-07-08 11:24:02,726 INFO    MainThread:30965 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_112402-hn868f80/logs/debug-internal.log
+2026-07-08 11:24:02,726 INFO    MainThread:30965 [wandb_init.py:init():850] calling init triggers
+2026-07-08 11:24:02,726 INFO    MainThread:30965 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/longctx30k/tokenized', 'val_path': None, 'max_seq_length': 30720, 'num_workers': 4, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209', 'resume_from_checkpoint': None, 'force_new_run': False, 'num_epochs': 1, 'max_steps': None, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 5, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 300, 'eval_steps': 500, 'eval_batches': 20, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b resampled 30k longctx', 'wandb_tags': ['qwen3', '1.2b', 'resampled', '30k', 'longctx', 'b128', 'scaleladder'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 120, 'sparse_mask_pool_size': 1000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_30k_b128_nrb120', 'sparse_mask_seed': 0, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-07-08 11:24:02,726 INFO    MainThread:30965 [wandb_init.py:init():898] starting backend
+2026-07-08 11:24:03,038 INFO    MainThread:30965 [wandb_init.py:init():913] sending inform_init request
+2026-07-08 11:24:03,626 INFO    MainThread:30965 [wandb_init.py:init():918] backend started and connected
+2026-07-08 11:24:03,629 INFO    MainThread:30965 [wandb_init.py:init():988] updated telemetry
+2026-07-08 11:24:03,655 INFO    MainThread:30965 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-07-08 11:24:04,892 INFO    MainThread:30965 [wandb_init.py:init():1056] starting run threads in backend
+2026-07-08 11:24:05,143 INFO    MainThread:30965 [wandb_run.py:_console_start():2554] atexit reg
+2026-07-08 11:24:05,143 INFO    MainThread:30965 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-07-08 11:24:05,143 INFO    MainThread:30965 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-07-08 11:24:05,143 INFO    MainThread:30965 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-07-08 11:24:05,151 INFO    MainThread:30965 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/run-20260708_112402-hn868f80/run-hn868f80.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:adf1890b5e7c9781314ec19b81eff60844f2848a4ce75d0429120dd43fa828fe
+size 11927552

epfl_final/SparseAtt/wandb/run-20260708_173250-cd66e13p/files/output.log

@@ -0,0 +1,176 @@
+Starting training...
+Validation at step 3300: Loss = 2.5783, Perplexity = 13.17
+wandb: WARNING Tried to log to step 3300 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 3301: dense (dense step interval)
+Step 3310/37465 | Loss: 2.6606 | LR: 4.94e-04 | 16887.6ms/step | 1.2 samples/s
+wandb: WARNING Tried to log to step 3310 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 3320/37465 | Loss: 2.3535 | LR: 4.94e-04 | 8333.4ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 3320 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 3330/37465 | Loss: 2.6802 | LR: 4.94e-04 | 8352.0ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 3330 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 3340/37465 | Loss: 2.6431 | LR: 4.94e-04 | 8344.2ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 3340 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 3350/37465 | Loss: 2.9170 | LR: 4.94e-04 | 8354.5ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 3350 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 3360/37465 | Loss: 2.5131 | LR: 4.94e-04 | 8360.2ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 3360 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 3370/37465 | Loss: 2.5432 | LR: 4.94e-04 | 8350.8ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 3370 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 3380/37465 | Loss: 2.5797 | LR: 4.94e-04 | 8388.2ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 3380 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 3390/37465 | Loss: 2.4650 | LR: 4.94e-04 | 8355.6ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 3390 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 3400/37465 | Loss: 2.3816 | LR: 4.94e-04 | 8338.4ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 3400 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 3410/37465 | Loss: 2.4314 | LR: 4.94e-04 | 8332.0ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 3410 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 3420/37465 | Loss: 2.5678 | LR: 4.94e-04 | 8368.0ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 3420 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 3430/37465 | Loss: 2.7321 | LR: 4.93e-04 | 8339.8ms/step | 2.4 samples/s
+wandb: WARNING Tried to log to step 3430 that is less than the current step 3431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 3440/37465 | Loss: 2.6158 | LR: 4.93e-04 | 8360.0ms/step | 2.4 samples/s
+Step 3450/37465 | Loss: 2.5854 | LR: 4.93e-04 | 8369.7ms/step | 2.4 samples/s
+Step 3460/37465 | Loss: 2.4706 | LR: 4.93e-04 | 8346.4ms/step | 2.4 samples/s
+Step 3470/37465 | Loss: 2.3381 | LR: 4.93e-04 | 8365.9ms/step | 2.4 samples/s
+Step 3480/37465 | Loss: 2.4953 | LR: 4.93e-04 | 8343.1ms/step | 2.4 samples/s
+Step 3490/37465 | Loss: 2.6281 | LR: 4.93e-04 | 8368.3ms/step | 2.4 samples/s
+Step 3500/37465 | Loss: 2.5841 | LR: 4.93e-04 | 8351.0ms/step | 2.4 samples/s
+Validation at step 3500: Loss = 2.5524, Perplexity = 12.84
+Step 3510/37465 | Loss: 2.7820 | LR: 4.93e-04 | 9361.4ms/step | 2.1 samples/s
+Step 3520/37465 | Loss: 2.4614 | LR: 4.93e-04 | 8371.8ms/step | 2.4 samples/s
+Step 3530/37465 | Loss: 2.5079 | LR: 4.93e-04 | 8353.9ms/step | 2.4 samples/s
+Step 3540/37465 | Loss: 2.6692 | LR: 4.93e-04 | 8370.6ms/step | 2.4 samples/s
+Step 3550/37465 | Loss: 2.5327 | LR: 4.93e-04 | 8378.5ms/step | 2.4 samples/s
+Step 3560/37465 | Loss: 2.4459 | LR: 4.93e-04 | 8348.7ms/step | 2.4 samples/s
+Step 3570/37465 | Loss: 2.4465 | LR: 4.93e-04 | 8366.9ms/step | 2.4 samples/s
+Step 3580/37465 | Loss: 2.8736 | LR: 4.93e-04 | 8373.4ms/step | 2.4 samples/s
+Step 3590/37465 | Loss: 2.5086 | LR: 4.93e-04 | 8355.8ms/step | 2.4 samples/s
+Step 3600/37465 | Loss: 2.6325 | LR: 4.93e-04 | 8345.8ms/step | 2.4 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-3600
+Step 3610/37465 | Loss: 2.5135 | LR: 4.93e-04 | 9375.9ms/step | 2.1 samples/s
+Step 3620/37465 | Loss: 2.4450 | LR: 4.92e-04 | 8396.5ms/step | 2.4 samples/s
+Step 3630/37465 | Loss: 2.3774 | LR: 4.92e-04 | 8368.6ms/step | 2.4 samples/s
+Step 3640/37465 | Loss: 2.4915 | LR: 4.92e-04 | 8383.0ms/step | 2.4 samples/s
+Step 3650/37465 | Loss: 2.5852 | LR: 4.92e-04 | 8368.3ms/step | 2.4 samples/s
+Step 3660/37465 | Loss: 2.4715 | LR: 4.92e-04 | 8369.6ms/step | 2.4 samples/s
+Step 3670/37465 | Loss: 2.3686 | LR: 4.92e-04 | 8378.3ms/step | 2.4 samples/s
+Step 3680/37465 | Loss: 2.6847 | LR: 4.92e-04 | 8372.5ms/step | 2.4 samples/s
+Step 3690/37465 | Loss: 2.8277 | LR: 4.92e-04 | 8361.6ms/step | 2.4 samples/s
+Step 3700/37465 | Loss: 2.4927 | LR: 4.92e-04 | 8381.6ms/step | 2.4 samples/s
+Step 3710/37465 | Loss: 2.7802 | LR: 4.92e-04 | 8385.2ms/step | 2.4 samples/s
+Step 3720/37465 | Loss: 2.5450 | LR: 4.92e-04 | 8361.8ms/step | 2.4 samples/s
+Step 3730/37465 | Loss: 2.7159 | LR: 4.92e-04 | 8368.8ms/step | 2.4 samples/s
+Step 3740/37465 | Loss: 2.3565 | LR: 4.92e-04 | 8356.7ms/step | 2.4 samples/s
+Step 3750/37465 | Loss: 2.5005 | LR: 4.92e-04 | 8359.8ms/step | 2.4 samples/s
+Step 3760/37465 | Loss: 2.7336 | LR: 4.92e-04 | 8391.1ms/step | 2.4 samples/s
+Step 3770/37465 | Loss: 2.7054 | LR: 4.92e-04 | 8368.4ms/step | 2.4 samples/s
+Step 3780/37465 | Loss: 2.5842 | LR: 4.92e-04 | 8387.8ms/step | 2.4 samples/s
+Step 3790/37465 | Loss: 2.4455 | LR: 4.92e-04 | 8385.2ms/step | 2.4 samples/s
+Step 3800/37465 | Loss: 2.5453 | LR: 4.92e-04 | 8380.5ms/step | 2.4 samples/s
+Step 3810/37465 | Loss: 2.6389 | LR: 4.91e-04 | 8374.7ms/step | 2.4 samples/s
+Step 3820/37465 | Loss: 2.5859 | LR: 4.91e-04 | 8377.2ms/step | 2.4 samples/s
+Step 3830/37465 | Loss: 2.7157 | LR: 4.91e-04 | 8396.2ms/step | 2.4 samples/s
+Step 3840/37465 | Loss: 2.2813 | LR: 4.91e-04 | 8359.1ms/step | 2.4 samples/s
+Step 3850/37465 | Loss: 2.6667 | LR: 4.91e-04 | 8374.8ms/step | 2.4 samples/s
+Step 3860/37465 | Loss: 2.3919 | LR: 4.91e-04 | 8379.6ms/step | 2.4 samples/s
+Step 3870/37465 | Loss: 2.7215 | LR: 4.91e-04 | 8365.4ms/step | 2.4 samples/s
+Step 3880/37465 | Loss: 2.3145 | LR: 4.91e-04 | 8378.0ms/step | 2.4 samples/s
+Step 3890/37465 | Loss: 2.3002 | LR: 4.91e-04 | 8357.4ms/step | 2.4 samples/s
+Step 3900/37465 | Loss: 2.3930 | LR: 4.91e-04 | 8363.6ms/step | 2.4 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-3900
+Step 3910/37465 | Loss: 2.5740 | LR: 4.91e-04 | 9211.5ms/step | 2.2 samples/s
+Step 3920/37465 | Loss: 2.5662 | LR: 4.91e-04 | 8365.1ms/step | 2.4 samples/s
+Step 3930/37465 | Loss: 2.6290 | LR: 4.91e-04 | 8380.3ms/step | 2.4 samples/s
+Step 3940/37465 | Loss: 2.8159 | LR: 4.91e-04 | 8356.3ms/step | 2.4 samples/s
+Step 3950/37465 | Loss: 2.3214 | LR: 4.91e-04 | 8372.7ms/step | 2.4 samples/s
+Step 3960/37465 | Loss: 2.5481 | LR: 4.91e-04 | 8354.2ms/step | 2.4 samples/s
+Step 3970/37465 | Loss: 2.3466 | LR: 4.91e-04 | 8354.4ms/step | 2.4 samples/s
+Step 3980/37465 | Loss: 2.5501 | LR: 4.91e-04 | 8370.4ms/step | 2.4 samples/s
+Step 3990/37465 | Loss: 2.5272 | LR: 4.90e-04 | 8368.3ms/step | 2.4 samples/s
+Step 4000/37465 | Loss: 2.5115 | LR: 4.90e-04 | 8344.0ms/step | 2.4 samples/s
+Validation at step 4000: Loss = 2.5080, Perplexity = 12.28
+Step 4010/37465 | Loss: 2.1113 | LR: 4.90e-04 | 9366.2ms/step | 2.1 samples/s
+Step 4020/37465 | Loss: 2.2947 | LR: 4.90e-04 | 8339.8ms/step | 2.4 samples/s
+Step 4030/37465 | Loss: 2.4372 | LR: 4.90e-04 | 8352.1ms/step | 2.4 samples/s
+Step 4040/37465 | Loss: 2.4509 | LR: 4.90e-04 | 8351.1ms/step | 2.4 samples/s
+Step 4050/37465 | Loss: 2.6699 | LR: 4.90e-04 | 8366.0ms/step | 2.4 samples/s
+Step 4060/37465 | Loss: 2.5102 | LR: 4.90e-04 | 8357.5ms/step | 2.4 samples/s
+Step 4070/37465 | Loss: 2.4825 | LR: 4.90e-04 | 8367.2ms/step | 2.4 samples/s
+Step 4080/37465 | Loss: 2.4585 | LR: 4.90e-04 | 8340.0ms/step | 2.4 samples/s
+Step 4090/37465 | Loss: 2.5597 | LR: 4.90e-04 | 8359.6ms/step | 2.4 samples/s
+Step 4100/37465 | Loss: 2.3972 | LR: 4.90e-04 | 8364.1ms/step | 2.4 samples/s
+Step 4110/37465 | Loss: 2.5406 | LR: 4.90e-04 | 8352.6ms/step | 2.4 samples/s
+Step 4120/37465 | Loss: 2.3038 | LR: 4.90e-04 | 8352.6ms/step | 2.4 samples/s
+Step 4130/37465 | Loss: 2.2725 | LR: 4.90e-04 | 8359.4ms/step | 2.4 samples/s
+Step 4140/37465 | Loss: 2.5664 | LR: 4.90e-04 | 8358.2ms/step | 2.4 samples/s
+Step 4150/37465 | Loss: 2.5551 | LR: 4.89e-04 | 8347.4ms/step | 2.4 samples/s
+Step 4160/37465 | Loss: 2.5973 | LR: 4.89e-04 | 8338.1ms/step | 2.4 samples/s
+Step 4170/37465 | Loss: 2.4961 | LR: 4.89e-04 | 8352.2ms/step | 2.4 samples/s
+Step 4180/37465 | Loss: 2.2842 | LR: 4.89e-04 | 8351.2ms/step | 2.4 samples/s
+Step 4190/37465 | Loss: 2.5623 | LR: 4.89e-04 | 8356.2ms/step | 2.4 samples/s
+Step 4200/37465 | Loss: 2.5244 | LR: 4.89e-04 | 8359.8ms/step | 2.4 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-4200
+Step 4210/37465 | Loss: 2.1315 | LR: 4.89e-04 | 9181.9ms/step | 2.2 samples/s
+Step 4220/37465 | Loss: 2.4843 | LR: 4.89e-04 | 8349.2ms/step | 2.4 samples/s
+Step 4230/37465 | Loss: 2.3858 | LR: 4.89e-04 | 8357.1ms/step | 2.4 samples/s
+Step 4240/37465 | Loss: 2.4300 | LR: 4.89e-04 | 8369.7ms/step | 2.4 samples/s
+Step 4250/37465 | Loss: 2.3458 | LR: 4.89e-04 | 8377.7ms/step | 2.4 samples/s
+Step 4260/37465 | Loss: 2.4023 | LR: 4.89e-04 | 8367.7ms/step | 2.4 samples/s
+Step 4270/37465 | Loss: 2.4831 | LR: 4.89e-04 | 8347.0ms/step | 2.4 samples/s
+Step 4280/37465 | Loss: 2.4608 | LR: 4.89e-04 | 8362.1ms/step | 2.4 samples/s
+Step 4290/37465 | Loss: 2.5375 | LR: 4.89e-04 | 8377.8ms/step | 2.4 samples/s
+Step 4300/37465 | Loss: 2.4230 | LR: 4.89e-04 | 8363.5ms/step | 2.4 samples/s
+Step 4310/37465 | Loss: 2.6084 | LR: 4.88e-04 | 8364.0ms/step | 2.4 samples/s
+Step 4320/37465 | Loss: 2.3435 | LR: 4.88e-04 | 8351.6ms/step | 2.4 samples/s
+Step 4330/37465 | Loss: 2.2068 | LR: 4.88e-04 | 8377.3ms/step | 2.4 samples/s
+Step 4340/37465 | Loss: 2.5103 | LR: 4.88e-04 | 8364.9ms/step | 2.4 samples/s
+Step 4350/37465 | Loss: 2.6194 | LR: 4.88e-04 | 8333.5ms/step | 2.4 samples/s
+Step 4360/37465 | Loss: 2.5892 | LR: 4.88e-04 | 8381.0ms/step | 2.4 samples/s
+Step 4370/37465 | Loss: 2.4106 | LR: 4.88e-04 | 8374.3ms/step | 2.4 samples/s
+Step 4380/37465 | Loss: 2.4141 | LR: 4.88e-04 | 8380.8ms/step | 2.4 samples/s
+Step 4390/37465 | Loss: 2.5619 | LR: 4.88e-04 | 8334.0ms/step | 2.4 samples/s
+Step 4400/37465 | Loss: 2.3260 | LR: 4.88e-04 | 8380.4ms/step | 2.4 samples/s
+Step 4410/37465 | Loss: 2.5833 | LR: 4.88e-04 | 8362.6ms/step | 2.4 samples/s
+Step 4420/37465 | Loss: 2.3065 | LR: 4.88e-04 | 8388.2ms/step | 2.4 samples/s
+Step 4430/37465 | Loss: 2.5054 | LR: 4.88e-04 | 8416.8ms/step | 2.4 samples/s
+Step 4440/37465 | Loss: 2.3477 | LR: 4.88e-04 | 8370.3ms/step | 2.4 samples/s
+Step 4450/37465 | Loss: 2.4271 | LR: 4.88e-04 | 8381.1ms/step | 2.4 samples/s
+Step 4460/37465 | Loss: 2.3098 | LR: 4.88e-04 | 8334.6ms/step | 2.4 samples/s
+Step 4470/37465 | Loss: 2.3284 | LR: 4.87e-04 | 8364.4ms/step | 2.4 samples/s
+Step 4480/37465 | Loss: 2.3389 | LR: 4.87e-04 | 8353.3ms/step | 2.4 samples/s
+Step 4490/37465 | Loss: 2.4233 | LR: 4.87e-04 | 8360.4ms/step | 2.4 samples/s
+Step 4500/37465 | Loss: 2.6390 | LR: 4.87e-04 | 8391.1ms/step | 2.4 samples/s
+Validation at step 4500: Loss = 2.4759, Perplexity = 11.89
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-4500
+Step 4510/37465 | Loss: 2.3560 | LR: 4.87e-04 | 10250.6ms/step | 2.0 samples/s
+Step 4520/37465 | Loss: 2.4278 | LR: 4.87e-04 | 8351.7ms/step | 2.4 samples/s
+Step 4530/37465 | Loss: 2.5453 | LR: 4.87e-04 | 8374.4ms/step | 2.4 samples/s
+Step 4540/37465 | Loss: 2.4651 | LR: 4.87e-04 | 8386.8ms/step | 2.4 samples/s
+Step 4550/37465 | Loss: 2.4067 | LR: 4.87e-04 | 8401.2ms/step | 2.4 samples/s
+Step 4560/37465 | Loss: 2.4239 | LR: 4.87e-04 | 8392.5ms/step | 2.4 samples/s
+Step 4570/37465 | Loss: 2.2015 | LR: 4.87e-04 | 8395.1ms/step | 2.4 samples/s
+Step 4580/37465 | Loss: 2.5254 | LR: 4.87e-04 | 8394.8ms/step | 2.4 samples/s
+Step 4590/37465 | Loss: 2.4194 | LR: 4.87e-04 | 8396.8ms/step | 2.4 samples/s
+Step 4600/37465 | Loss: 2.6416 | LR: 4.87e-04 | 8384.7ms/step | 2.4 samples/s
+Step 4610/37465 | Loss: 2.3960 | LR: 4.86e-04 | 8380.4ms/step | 2.4 samples/s
+Step 4620/37465 | Loss: 2.5731 | LR: 4.86e-04 | 8385.8ms/step | 2.4 samples/s
+Step 4630/37465 | Loss: 2.5278 | LR: 4.86e-04 | 8355.9ms/step | 2.4 samples/s
+Step 4640/37465 | Loss: 2.3128 | LR: 4.86e-04 | 8368.9ms/step | 2.4 samples/s
+Step 4650/37465 | Loss: 2.2687 | LR: 4.86e-04 | 8420.4ms/step | 2.4 samples/s
+Step 4660/37465 | Loss: 2.4507 | LR: 4.86e-04 | 8403.7ms/step | 2.4 samples/s
+Step 4670/37465 | Loss: 2.4984 | LR: 4.86e-04 | 8415.4ms/step | 2.4 samples/s
+Step 4680/37465 | Loss: 2.4701 | LR: 4.86e-04 | 8359.8ms/step | 2.4 samples/s
+Step 4690/37465 | Loss: 2.6007 | LR: 4.86e-04 | 8353.6ms/step | 2.4 samples/s
+Step 4700/37465 | Loss: 2.4826 | LR: 4.86e-04 | 8402.5ms/step | 2.4 samples/s
+Step 4710/37465 | Loss: 2.2913 | LR: 4.86e-04 | 8369.6ms/step | 2.4 samples/s
+Step 4720/37465 | Loss: 2.5531 | LR: 4.86e-04 | 8378.1ms/step | 2.4 samples/s
+Step 4730/37465 | Loss: 2.3186 | LR: 4.86e-04 | 8362.9ms/step | 2.4 samples/s
+Step 4740/37465 | Loss: 2.4826 | LR: 4.86e-04 | 8392.5ms/step | 2.4 samples/s
+Step 4750/37465 | Loss: 2.4423 | LR: 4.85e-04 | 8381.7ms/step | 2.4 samples/s
+Step 4760/37465 | Loss: 2.3839 | LR: 4.85e-04 | 8358.8ms/step | 2.4 samples/s
+Step 4770/37465 | Loss: 2.3475 | LR: 4.85e-04 | 8386.0ms/step | 2.4 samples/s
+Step 4780/37465 | Loss: 2.5026 | LR: 4.85e-04 | 8373.9ms/step | 2.4 samples/s
+Step 4790/37465 | Loss: 2.6148 | LR: 4.85e-04 | 8389.5ms/step | 2.4 samples/s
+Step 4800/37465 | Loss: 2.4976 | LR: 4.85e-04 | 8363.1ms/step | 2.4 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-4800
+Step 4810/37465 | Loss: 2.5211 | LR: 4.85e-04 | 9312.5ms/step | 2.1 samples/s

epfl_final/SparseAtt/wandb/run-20260708_173250-cd66e13p/files/requirements.txt

@@ -0,0 +1,141 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5

epfl_final/SparseAtt/wandb/run-20260708_173250-cd66e13p/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-07-08T17:32:50.854199Z",
+  "args":  [
+    "--config",
+    "configs/train_config_30k_dense_b128.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "88298c7a418622530ca5426b22b1e5ebd10ca0cb"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-train-4-gpu-2-0-0",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "9371709865984",
+      "used":  "5899450421248"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-d0294574-a180-e984-6ca3-0573e9724f2e"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-b87e8111-63b6-b364-28b9-c4ce747aec08"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-e4ba0f44-6ff8-2e43-5721-346902be5759"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bbbaaf81-9234-a1c6-0093-6d87b4e6af4c"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "8dpbzntyr16jvdat5t8w07rp1yc1vqwl"
+}

epfl_final/SparseAtt/wandb/run-20260708_173250-cd66e13p/logs/debug.log

@@ -0,0 +1,20 @@
+2026-07-08 17:32:50,904 INFO    MainThread:6183 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-07-08 17:32:50,904 INFO    MainThread:6183 [wandb_setup.py:_flush():81] Configure stats pid to 6183
+2026-07-08 17:32:50,905 INFO    MainThread:6183 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-08 17:32:50,906 INFO    MainThread:6183 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_173250-cd66e13p/logs/debug.log
+2026-07-08 17:32:50,908 INFO    MainThread:6183 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_173250-cd66e13p/logs/debug-internal.log
+2026-07-08 17:32:50,909 INFO    MainThread:6183 [wandb_init.py:init():850] calling init triggers
+2026-07-08 17:32:50,909 INFO    MainThread:6183 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/longctx30k/tokenized', 'val_path': None, 'max_seq_length': 30720, 'num_workers': 4, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-3300', 'force_new_run': False, 'num_epochs': 1, 'max_steps': None, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 5, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 300, 'eval_steps': 500, 'eval_batches': 20, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b dense 30k longctx', 'wandb_tags': ['qwen3', '1.2b', 'dense', '30k', 'longctx', 'b128', 'scaleladder'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 120, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
+2026-07-08 17:32:50,917 INFO    MainThread:6183 [wandb_init.py:init():898] starting backend
+2026-07-08 17:32:52,419 INFO    MainThread:6183 [wandb_init.py:init():913] sending inform_init request
+2026-07-08 17:32:53,102 INFO    MainThread:6183 [wandb_init.py:init():918] backend started and connected
+2026-07-08 17:32:53,110 INFO    MainThread:6183 [wandb_init.py:init():988] updated telemetry
+2026-07-08 17:32:53,154 INFO    MainThread:6183 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-07-08 17:32:53,906 INFO    MainThread:6183 [wandb_init.py:init():1051] run resumed
+2026-07-08 17:32:53,909 INFO    MainThread:6183 [wandb_init.py:init():1056] starting run threads in backend
+2026-07-08 17:32:54,959 INFO    MainThread:6183 [wandb_run.py:_console_start():2554] atexit reg
+2026-07-08 17:32:54,959 INFO    MainThread:6183 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-07-08 17:32:54,959 INFO    MainThread:6183 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-07-08 17:32:54,960 INFO    MainThread:6183 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-07-08 17:32:54,978 INFO    MainThread:6183 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/run-20260708_173250-cd66e13p/run-cd66e13p.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8650b490758606a8a1836cc8704c2d779cb946070ddb028194222f25e6fca52e
+size 4096000

epfl_final/SparseAtt/wandb/run-20260708_205109-82re6m3i/files/output.log

@@ -0,0 +1,854 @@
+Starting training...
+Switching attention mode for step 1: sparse_stochastic (configured sparse attention)
+Step 10/44000 | Loss: 12.0398 | LR: 6.67e-06 | 6736.3ms/step | 3.0 samples/s
+Step 20/44000 | Loss: 11.9261 | LR: 1.33e-05 | 677.8ms/step | 29.5 samples/s
+Step 30/44000 | Loss: 11.5485 | LR: 2.00e-05 | 680.1ms/step | 29.4 samples/s
+Step 40/44000 | Loss: 11.0225 | LR: 2.67e-05 | 674.7ms/step | 29.6 samples/s
+Step 50/44000 | Loss: 10.7137 | LR: 3.33e-05 | 671.4ms/step | 29.8 samples/s
+Step 60/44000 | Loss: 10.4833 | LR: 4.00e-05 | 675.4ms/step | 29.6 samples/s
+Step 70/44000 | Loss: 10.2636 | LR: 4.67e-05 | 672.6ms/step | 29.7 samples/s
+Step 80/44000 | Loss: 9.9451 | LR: 5.33e-05 | 673.1ms/step | 29.7 samples/s
+Step 90/44000 | Loss: 9.5554 | LR: 6.00e-05 | 677.6ms/step | 29.5 samples/s
+Step 100/44000 | Loss: 9.1247 | LR: 6.67e-05 | 675.0ms/step | 29.6 samples/s
+Step 110/44000 | Loss: 8.5965 | LR: 7.33e-05 | 674.8ms/step | 29.6 samples/s
+Step 120/44000 | Loss: 8.1657 | LR: 8.00e-05 | 672.8ms/step | 29.7 samples/s
+Step 130/44000 | Loss: 7.7923 | LR: 8.67e-05 | 673.8ms/step | 29.7 samples/s
+Step 140/44000 | Loss: 7.5092 | LR: 9.33e-05 | 675.6ms/step | 29.6 samples/s
+Step 150/44000 | Loss: 7.3410 | LR: 1.00e-04 | 674.4ms/step | 29.7 samples/s
+Step 160/44000 | Loss: 7.1668 | LR: 1.07e-04 | 676.6ms/step | 29.6 samples/s
+Step 170/44000 | Loss: 7.1116 | LR: 1.13e-04 | 677.2ms/step | 29.5 samples/s
+Step 180/44000 | Loss: 7.0514 | LR: 1.20e-04 | 678.2ms/step | 29.5 samples/s
+Step 190/44000 | Loss: 6.9679 | LR: 1.27e-04 | 678.7ms/step | 29.5 samples/s
+Step 200/44000 | Loss: 6.8446 | LR: 1.33e-04 | 680.1ms/step | 29.4 samples/s
+Step 210/44000 | Loss: 6.7549 | LR: 1.40e-04 | 677.8ms/step | 29.5 samples/s
+Step 220/44000 | Loss: 6.6656 | LR: 1.47e-04 | 680.1ms/step | 29.4 samples/s
+Step 230/44000 | Loss: 6.5696 | LR: 1.53e-04 | 679.5ms/step | 29.4 samples/s
+Step 240/44000 | Loss: 6.5532 | LR: 1.60e-04 | 680.1ms/step | 29.4 samples/s
+Step 250/44000 | Loss: 6.4335 | LR: 1.67e-04 | 680.9ms/step | 29.4 samples/s
+Validation at step 250: Loss = 6.4401, Perplexity = 626.49
+Step 260/44000 | Loss: 6.4447 | LR: 1.73e-04 | 3834.4ms/step | 5.2 samples/s
+Step 270/44000 | Loss: 6.3853 | LR: 1.80e-04 | 680.5ms/step | 29.4 samples/s
+Step 280/44000 | Loss: 6.2990 | LR: 1.87e-04 | 681.3ms/step | 29.4 samples/s
+Step 290/44000 | Loss: 6.2483 | LR: 1.93e-04 | 679.4ms/step | 29.4 samples/s
+Step 300/44000 | Loss: 6.2525 | LR: 2.00e-04 | 680.0ms/step | 29.4 samples/s
+Step 310/44000 | Loss: 6.2115 | LR: 2.07e-04 | 680.8ms/step | 29.4 samples/s
+Step 320/44000 | Loss: 6.1328 | LR: 2.13e-04 | 680.6ms/step | 29.4 samples/s
+Step 330/44000 | Loss: 6.0866 | LR: 2.20e-04 | 680.0ms/step | 29.4 samples/s
+Step 340/44000 | Loss: 6.0659 | LR: 2.27e-04 | 683.8ms/step | 29.2 samples/s
+Step 350/44000 | Loss: 5.9946 | LR: 2.33e-04 | 683.4ms/step | 29.3 samples/s
+Step 360/44000 | Loss: 5.9899 | LR: 2.40e-04 | 680.1ms/step | 29.4 samples/s
+Step 370/44000 | Loss: 5.8784 | LR: 2.47e-04 | 679.2ms/step | 29.4 samples/s
+Step 380/44000 | Loss: 5.8986 | LR: 2.53e-04 | 679.1ms/step | 29.4 samples/s
+Step 390/44000 | Loss: 5.8632 | LR: 2.60e-04 | 681.4ms/step | 29.4 samples/s
+Step 400/44000 | Loss: 5.8139 | LR: 2.67e-04 | 681.8ms/step | 29.3 samples/s
+Step 410/44000 | Loss: 5.7832 | LR: 2.73e-04 | 681.2ms/step | 29.4 samples/s
+Step 420/44000 | Loss: 5.7248 | LR: 2.80e-04 | 680.6ms/step | 29.4 samples/s
+Step 430/44000 | Loss: 5.7328 | LR: 2.87e-04 | 681.6ms/step | 29.3 samples/s
+Step 440/44000 | Loss: 5.7227 | LR: 2.93e-04 | 681.9ms/step | 29.3 samples/s
+Step 450/44000 | Loss: 5.6031 | LR: 3.00e-04 | 682.6ms/step | 29.3 samples/s
+Step 460/44000 | Loss: 5.6141 | LR: 3.07e-04 | 681.7ms/step | 29.3 samples/s
+Step 470/44000 | Loss: 5.5835 | LR: 3.13e-04 | 680.8ms/step | 29.4 samples/s
+Step 480/44000 | Loss: 5.5358 | LR: 3.20e-04 | 680.0ms/step | 29.4 samples/s
+Step 490/44000 | Loss: 5.5257 | LR: 3.27e-04 | 680.0ms/step | 29.4 samples/s
+Step 500/44000 | Loss: 5.4933 | LR: 3.33e-04 | 680.0ms/step | 29.4 samples/s
+Validation at step 500: Loss = 5.4703, Perplexity = 237.54
+Step 510/44000 | Loss: 5.3939 | LR: 3.40e-04 | 1247.2ms/step | 16.0 samples/s
+Step 520/44000 | Loss: 5.3948 | LR: 3.47e-04 | 684.7ms/step | 29.2 samples/s
+Step 530/44000 | Loss: 5.3818 | LR: 3.53e-04 | 682.7ms/step | 29.3 samples/s
+Step 540/44000 | Loss: 5.3525 | LR: 3.60e-04 | 682.7ms/step | 29.3 samples/s
+Step 550/44000 | Loss: 5.3688 | LR: 3.67e-04 | 683.7ms/step | 29.3 samples/s
+Step 560/44000 | Loss: 5.3101 | LR: 3.73e-04 | 681.4ms/step | 29.3 samples/s
+Step 570/44000 | Loss: 5.3016 | LR: 3.80e-04 | 680.8ms/step | 29.4 samples/s
+Step 580/44000 | Loss: 5.2444 | LR: 3.87e-04 | 680.8ms/step | 29.4 samples/s
+Step 590/44000 | Loss: 5.2354 | LR: 3.93e-04 | 681.7ms/step | 29.3 samples/s
+Step 600/44000 | Loss: 5.2160 | LR: 4.00e-04 | 682.2ms/step | 29.3 samples/s
+Step 610/44000 | Loss: 5.1322 | LR: 4.07e-04 | 681.8ms/step | 29.3 samples/s
+Step 620/44000 | Loss: 5.1132 | LR: 4.13e-04 | 682.4ms/step | 29.3 samples/s
+Step 630/44000 | Loss: 5.1564 | LR: 4.20e-04 | 681.6ms/step | 29.3 samples/s
+Step 640/44000 | Loss: 5.1229 | LR: 4.27e-04 | 681.7ms/step | 29.3 samples/s
+Step 650/44000 | Loss: 5.0708 | LR: 4.33e-04 | 680.7ms/step | 29.4 samples/s
+Step 660/44000 | Loss: 5.0615 | LR: 4.40e-04 | 680.9ms/step | 29.4 samples/s
+Step 670/44000 | Loss: 5.0260 | LR: 4.47e-04 | 680.7ms/step | 29.4 samples/s
+Step 680/44000 | Loss: 5.0432 | LR: 4.53e-04 | 683.5ms/step | 29.3 samples/s
+Step 690/44000 | Loss: 5.0021 | LR: 4.60e-04 | 681.8ms/step | 29.3 samples/s
+Step 700/44000 | Loss: 4.9098 | LR: 4.67e-04 | 683.3ms/step | 29.3 samples/s
+Step 710/44000 | Loss: 4.9521 | LR: 4.73e-04 | 679.8ms/step | 29.4 samples/s
+Step 720/44000 | Loss: 4.8679 | LR: 4.80e-04 | 683.1ms/step | 29.3 samples/s
+Step 730/44000 | Loss: 4.8684 | LR: 4.87e-04 | 682.5ms/step | 29.3 samples/s
+Step 740/44000 | Loss: 4.8072 | LR: 4.93e-04 | 681.6ms/step | 29.3 samples/s
+Step 750/44000 | Loss: 4.7878 | LR: 5.00e-04 | 682.0ms/step | 29.3 samples/s
+Validation at step 750: Loss = 4.8024, Perplexity = 121.80
+Step 760/44000 | Loss: 4.7690 | LR: 5.00e-04 | 1242.5ms/step | 16.1 samples/s
+Step 770/44000 | Loss: 4.7786 | LR: 5.00e-04 | 682.1ms/step | 29.3 samples/s
+Step 780/44000 | Loss: 4.6998 | LR: 5.00e-04 | 682.8ms/step | 29.3 samples/s
+Step 790/44000 | Loss: 4.7071 | LR: 5.00e-04 | 682.4ms/step | 29.3 samples/s
+Step 800/44000 | Loss: 4.7447 | LR: 5.00e-04 | 682.0ms/step | 29.3 samples/s
+Step 810/44000 | Loss: 4.6695 | LR: 5.00e-04 | 682.6ms/step | 29.3 samples/s
+Step 820/44000 | Loss: 4.6305 | LR: 5.00e-04 | 683.6ms/step | 29.3 samples/s
+Step 830/44000 | Loss: 4.6075 | LR: 5.00e-04 | 682.6ms/step | 29.3 samples/s
+Step 840/44000 | Loss: 4.5450 | LR: 5.00e-04 | 681.3ms/step | 29.4 samples/s
+Step 850/44000 | Loss: 4.5586 | LR: 5.00e-04 | 681.8ms/step | 29.3 samples/s
+Step 860/44000 | Loss: 4.5464 | LR: 5.00e-04 | 682.2ms/step | 29.3 samples/s
+Step 870/44000 | Loss: 4.5607 | LR: 5.00e-04 | 682.8ms/step | 29.3 samples/s
+Step 880/44000 | Loss: 4.4897 | LR: 5.00e-04 | 682.5ms/step | 29.3 samples/s
+Step 890/44000 | Loss: 4.4793 | LR: 5.00e-04 | 681.4ms/step | 29.4 samples/s
+Step 900/44000 | Loss: 4.4737 | LR: 5.00e-04 | 681.3ms/step | 29.4 samples/s
+Step 910/44000 | Loss: 4.4645 | LR: 5.00e-04 | 680.8ms/step | 29.4 samples/s
+Step 920/44000 | Loss: 4.4159 | LR: 5.00e-04 | 683.0ms/step | 29.3 samples/s
+Step 930/44000 | Loss: 4.4167 | LR: 5.00e-04 | 684.6ms/step | 29.2 samples/s
+Step 940/44000 | Loss: 4.3917 | LR: 5.00e-04 | 682.0ms/step | 29.3 samples/s
+Step 950/44000 | Loss: 4.4081 | LR: 5.00e-04 | 680.1ms/step | 29.4 samples/s
+Step 960/44000 | Loss: 4.3439 | LR: 5.00e-04 | 681.0ms/step | 29.4 samples/s
+Step 970/44000 | Loss: 4.3519 | LR: 5.00e-04 | 681.6ms/step | 29.3 samples/s
+Step 980/44000 | Loss: 4.3508 | LR: 5.00e-04 | 682.8ms/step | 29.3 samples/s
+Step 990/44000 | Loss: 4.3235 | LR: 5.00e-04 | 681.1ms/step | 29.4 samples/s
+Step 1000/44000 | Loss: 4.3257 | LR: 5.00e-04 | 681.5ms/step | 29.3 samples/s
+Validation at step 1000: Loss = 4.3220, Perplexity = 75.34
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-1000
+Step 1010/44000 | Loss: 4.3460 | LR: 5.00e-04 | 1526.2ms/step | 13.1 samples/s
+Step 1020/44000 | Loss: 4.2801 | LR: 5.00e-04 | 683.0ms/step | 29.3 samples/s
+Step 1030/44000 | Loss: 4.2858 | LR: 5.00e-04 | 682.9ms/step | 29.3 samples/s
+Step 1040/44000 | Loss: 4.2180 | LR: 5.00e-04 | 684.1ms/step | 29.2 samples/s
+Step 1050/44000 | Loss: 4.2685 | LR: 5.00e-04 | 680.8ms/step | 29.4 samples/s
+Step 1060/44000 | Loss: 4.2945 | LR: 5.00e-04 | 681.1ms/step | 29.4 samples/s
+Step 1070/44000 | Loss: 4.2088 | LR: 5.00e-04 | 683.5ms/step | 29.3 samples/s
+Step 1080/44000 | Loss: 4.2116 | LR: 5.00e-04 | 681.3ms/step | 29.4 samples/s
+Step 1090/44000 | Loss: 4.2323 | LR: 5.00e-04 | 682.0ms/step | 29.3 samples/s
+Step 1100/44000 | Loss: 4.2100 | LR: 5.00e-04 | 680.2ms/step | 29.4 samples/s
+Step 1110/44000 | Loss: 4.2157 | LR: 5.00e-04 | 682.8ms/step | 29.3 samples/s
+Step 1120/44000 | Loss: 4.1160 | LR: 5.00e-04 | 681.8ms/step | 29.3 samples/s
+Step 1130/44000 | Loss: 4.1421 | LR: 5.00e-04 | 683.4ms/step | 29.3 samples/s
+Step 1140/44000 | Loss: 4.1314 | LR: 5.00e-04 | 683.8ms/step | 29.2 samples/s
+Step 1150/44000 | Loss: 4.1335 | LR: 5.00e-04 | 680.6ms/step | 29.4 samples/s
+Step 1160/44000 | Loss: 4.1396 | LR: 5.00e-04 | 680.8ms/step | 29.4 samples/s
+Step 1170/44000 | Loss: 4.1272 | LR: 5.00e-04 | 683.0ms/step | 29.3 samples/s
+Step 1180/44000 | Loss: 4.1398 | LR: 5.00e-04 | 681.4ms/step | 29.4 samples/s
+Step 1190/44000 | Loss: 4.1006 | LR: 5.00e-04 | 682.9ms/step | 29.3 samples/s
+Step 1200/44000 | Loss: 4.1044 | LR: 5.00e-04 | 680.9ms/step | 29.4 samples/s
+Step 1210/44000 | Loss: 4.1278 | LR: 5.00e-04 | 682.4ms/step | 29.3 samples/s
+Step 1220/44000 | Loss: 4.1129 | LR: 5.00e-04 | 683.0ms/step | 29.3 samples/s
+Step 1230/44000 | Loss: 4.0767 | LR: 5.00e-04 | 684.0ms/step | 29.2 samples/s
+Step 1240/44000 | Loss: 4.0811 | LR: 5.00e-04 | 684.2ms/step | 29.2 samples/s
+Step 1250/44000 | Loss: 4.0561 | LR: 5.00e-04 | 681.9ms/step | 29.3 samples/s
+Validation at step 1250: Loss = 4.0741, Perplexity = 58.80
+Step 1260/44000 | Loss: 4.1101 | LR: 5.00e-04 | 1243.5ms/step | 16.1 samples/s
+Step 1270/44000 | Loss: 4.0343 | LR: 5.00e-04 | 683.4ms/step | 29.3 samples/s
+Step 1280/44000 | Loss: 4.0511 | LR: 5.00e-04 | 681.2ms/step | 29.4 samples/s
+Step 1290/44000 | Loss: 4.0439 | LR: 5.00e-04 | 682.5ms/step | 29.3 samples/s
+Step 1300/44000 | Loss: 4.0494 | LR: 5.00e-04 | 680.3ms/step | 29.4 samples/s
+Step 1310/44000 | Loss: 3.9890 | LR: 5.00e-04 | 682.4ms/step | 29.3 samples/s
+Step 1320/44000 | Loss: 3.9759 | LR: 5.00e-04 | 684.4ms/step | 29.2 samples/s
+Step 1330/44000 | Loss: 3.9987 | LR: 5.00e-04 | 682.3ms/step | 29.3 samples/s
+Step 1340/44000 | Loss: 4.0415 | LR: 5.00e-04 | 680.5ms/step | 29.4 samples/s
+Step 1350/44000 | Loss: 3.9718 | LR: 5.00e-04 | 682.6ms/step | 29.3 samples/s
+Step 1360/44000 | Loss: 3.9880 | LR: 5.00e-04 | 682.0ms/step | 29.3 samples/s
+Step 1370/44000 | Loss: 3.9674 | LR: 5.00e-04 | 680.9ms/step | 29.4 samples/s
+Step 1380/44000 | Loss: 3.9866 | LR: 5.00e-04 | 681.2ms/step | 29.4 samples/s
+Step 1390/44000 | Loss: 3.9734 | LR: 5.00e-04 | 682.4ms/step | 29.3 samples/s
+Step 1400/44000 | Loss: 3.9759 | LR: 5.00e-04 | 682.9ms/step | 29.3 samples/s
+Step 1410/44000 | Loss: 3.9504 | LR: 5.00e-04 | 682.9ms/step | 29.3 samples/s
+Step 1420/44000 | Loss: 3.9534 | LR: 5.00e-04 | 682.0ms/step | 29.3 samples/s
+Step 1430/44000 | Loss: 3.9652 | LR: 5.00e-04 | 682.6ms/step | 29.3 samples/s
+Step 1440/44000 | Loss: 3.8943 | LR: 5.00e-04 | 683.2ms/step | 29.3 samples/s
+Step 1450/44000 | Loss: 3.9158 | LR: 5.00e-04 | 682.0ms/step | 29.3 samples/s
+Step 1460/44000 | Loss: 3.9641 | LR: 5.00e-04 | 683.4ms/step | 29.3 samples/s
+Step 1470/44000 | Loss: 3.9075 | LR: 5.00e-04 | 681.9ms/step | 29.3 samples/s
+Step 1480/44000 | Loss: 3.8820 | LR: 5.00e-04 | 683.5ms/step | 29.3 samples/s
+Step 1490/44000 | Loss: 3.8792 | LR: 5.00e-04 | 681.9ms/step | 29.3 samples/s
+Step 1500/44000 | Loss: 3.9346 | LR: 5.00e-04 | 682.0ms/step | 29.3 samples/s
+Validation at step 1500: Loss = 3.9142, Perplexity = 50.11
+Step 1510/44000 | Loss: 3.9090 | LR: 5.00e-04 | 1238.7ms/step | 16.1 samples/s
+Step 1520/44000 | Loss: 3.8981 | LR: 5.00e-04 | 683.3ms/step | 29.3 samples/s
+Step 1530/44000 | Loss: 3.9228 | LR: 5.00e-04 | 682.7ms/step | 29.3 samples/s
+Step 1540/44000 | Loss: 3.8443 | LR: 5.00e-04 | 682.3ms/step | 29.3 samples/s
+Step 1550/44000 | Loss: 3.9214 | LR: 5.00e-04 | 684.9ms/step | 29.2 samples/s
+Step 1560/44000 | Loss: 3.8821 | LR: 5.00e-04 | 681.7ms/step | 29.3 samples/s
+Step 1570/44000 | Loss: 3.9016 | LR: 5.00e-04 | 683.0ms/step | 29.3 samples/s
+Step 1580/44000 | Loss: 3.8545 | LR: 5.00e-04 | 683.4ms/step | 29.3 samples/s
+Step 1590/44000 | Loss: 3.8580 | LR: 5.00e-04 | 682.3ms/step | 29.3 samples/s
+Step 1600/44000 | Loss: 3.8513 | LR: 5.00e-04 | 680.0ms/step | 29.4 samples/s
+Step 1610/44000 | Loss: 3.8633 | LR: 5.00e-04 | 681.7ms/step | 29.3 samples/s
+Step 1620/44000 | Loss: 3.8403 | LR: 5.00e-04 | 681.2ms/step | 29.4 samples/s
+Step 1630/44000 | Loss: 3.8547 | LR: 4.99e-04 | 680.4ms/step | 29.4 samples/s
+Step 1640/44000 | Loss: 3.8142 | LR: 4.99e-04 | 682.2ms/step | 29.3 samples/s
+Step 1650/44000 | Loss: 3.8556 | LR: 4.99e-04 | 681.9ms/step | 29.3 samples/s
+Step 1660/44000 | Loss: 3.8659 | LR: 4.99e-04 | 683.0ms/step | 29.3 samples/s
+Step 1670/44000 | Loss: 3.8726 | LR: 4.99e-04 | 680.3ms/step | 29.4 samples/s
+Step 1680/44000 | Loss: 3.8139 | LR: 4.99e-04 | 683.3ms/step | 29.3 samples/s
+Step 1690/44000 | Loss: 3.7974 | LR: 4.99e-04 | 680.8ms/step | 29.4 samples/s
+Step 1700/44000 | Loss: 3.8406 | LR: 4.99e-04 | 681.2ms/step | 29.4 samples/s
+Step 1710/44000 | Loss: 3.7913 | LR: 4.99e-04 | 682.3ms/step | 29.3 samples/s
+Step 1720/44000 | Loss: 3.8454 | LR: 4.99e-04 | 681.0ms/step | 29.4 samples/s
+Step 1730/44000 | Loss: 3.8109 | LR: 4.99e-04 | 681.5ms/step | 29.3 samples/s
+Step 1740/44000 | Loss: 3.7950 | LR: 4.99e-04 | 682.6ms/step | 29.3 samples/s
+Step 1750/44000 | Loss: 3.7722 | LR: 4.99e-04 | 682.2ms/step | 29.3 samples/s
+Validation at step 1750: Loss = 3.8081, Perplexity = 45.06
+Step 1760/44000 | Loss: 3.8124 | LR: 4.99e-04 | 1244.5ms/step | 16.1 samples/s
+Step 1770/44000 | Loss: 3.8140 | LR: 4.99e-04 | 683.5ms/step | 29.3 samples/s
+Step 1780/44000 | Loss: 3.8096 | LR: 4.99e-04 | 682.1ms/step | 29.3 samples/s
+Step 1790/44000 | Loss: 3.8036 | LR: 4.99e-04 | 682.3ms/step | 29.3 samples/s
+Step 1800/44000 | Loss: 3.7815 | LR: 4.99e-04 | 682.7ms/step | 29.3 samples/s
+Step 1810/44000 | Loss: 3.7622 | LR: 4.99e-04 | 683.0ms/step | 29.3 samples/s
+Step 1820/44000 | Loss: 3.7738 | LR: 4.99e-04 | 684.6ms/step | 29.2 samples/s
+Step 1830/44000 | Loss: 3.7561 | LR: 4.99e-04 | 682.1ms/step | 29.3 samples/s
+Step 1840/44000 | Loss: 3.7443 | LR: 4.99e-04 | 684.2ms/step | 29.2 samples/s
+Step 1850/44000 | Loss: 3.7809 | LR: 4.99e-04 | 683.0ms/step | 29.3 samples/s
+Step 1860/44000 | Loss: 3.8033 | LR: 4.99e-04 | 683.7ms/step | 29.3 samples/s
+Step 1870/44000 | Loss: 3.7895 | LR: 4.99e-04 | 684.3ms/step | 29.2 samples/s
+Step 1880/44000 | Loss: 3.7483 | LR: 4.99e-04 | 681.8ms/step | 29.3 samples/s
+Step 1890/44000 | Loss: 3.7258 | LR: 4.99e-04 | 684.1ms/step | 29.2 samples/s
+Step 1900/44000 | Loss: 3.7610 | LR: 4.99e-04 | 681.5ms/step | 29.3 samples/s
+Step 1910/44000 | Loss: 3.7677 | LR: 4.99e-04 | 681.1ms/step | 29.4 samples/s
+Step 1920/44000 | Loss: 3.7417 | LR: 4.99e-04 | 682.8ms/step | 29.3 samples/s
+Step 1930/44000 | Loss: 3.7539 | LR: 4.99e-04 | 681.8ms/step | 29.3 samples/s
+Step 1940/44000 | Loss: 3.7155 | LR: 4.99e-04 | 682.7ms/step | 29.3 samples/s
+Step 1950/44000 | Loss: 3.6881 | LR: 4.99e-04 | 686.7ms/step | 29.1 samples/s
+Step 1960/44000 | Loss: 3.7373 | LR: 4.99e-04 | 684.7ms/step | 29.2 samples/s
+Step 1970/44000 | Loss: 3.7051 | LR: 4.99e-04 | 682.5ms/step | 29.3 samples/s
+Step 1980/44000 | Loss: 3.6944 | LR: 4.99e-04 | 682.8ms/step | 29.3 samples/s
+Step 1990/44000 | Loss: 3.7627 | LR: 4.99e-04 | 682.4ms/step | 29.3 samples/s
+Step 2000/44000 | Loss: 3.7249 | LR: 4.99e-04 | 683.7ms/step | 29.3 samples/s
+Validation at step 2000: Loss = 3.7284, Perplexity = 41.61
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-2000
+Step 2010/44000 | Loss: 3.7331 | LR: 4.99e-04 | 1527.0ms/step | 13.1 samples/s
+Step 2020/44000 | Loss: 3.7163 | LR: 4.99e-04 | 683.5ms/step | 29.3 samples/s
+Step 2030/44000 | Loss: 3.6928 | LR: 4.99e-04 | 680.5ms/step | 29.4 samples/s
+Step 2040/44000 | Loss: 3.7274 | LR: 4.99e-04 | 680.8ms/step | 29.4 samples/s
+Step 2050/44000 | Loss: 3.7085 | LR: 4.99e-04 | 680.4ms/step | 29.4 samples/s
+Step 2060/44000 | Loss: 3.7294 | LR: 4.99e-04 | 683.1ms/step | 29.3 samples/s
+Step 2070/44000 | Loss: 3.7066 | LR: 4.99e-04 | 678.3ms/step | 29.5 samples/s
+Step 2080/44000 | Loss: 3.7279 | LR: 4.99e-04 | 679.7ms/step | 29.4 samples/s
+Step 2090/44000 | Loss: 3.7261 | LR: 4.99e-04 | 681.6ms/step | 29.3 samples/s
+Step 2100/44000 | Loss: 3.6805 | LR: 4.99e-04 | 681.8ms/step | 29.3 samples/s
+Step 2110/44000 | Loss: 3.7071 | LR: 4.99e-04 | 682.7ms/step | 29.3 samples/s
+Step 2120/44000 | Loss: 3.6959 | LR: 4.99e-04 | 681.3ms/step | 29.4 samples/s
+Step 2130/44000 | Loss: 3.6898 | LR: 4.99e-04 | 684.0ms/step | 29.2 samples/s
+Step 2140/44000 | Loss: 3.6711 | LR: 4.99e-04 | 681.0ms/step | 29.4 samples/s
+Step 2150/44000 | Loss: 3.6808 | LR: 4.99e-04 | 683.0ms/step | 29.3 samples/s
+Step 2160/44000 | Loss: 3.6943 | LR: 4.99e-04 | 684.8ms/step | 29.2 samples/s
+Step 2170/44000 | Loss: 3.6677 | LR: 4.99e-04 | 683.1ms/step | 29.3 samples/s
+Step 2180/44000 | Loss: 3.6871 | LR: 4.99e-04 | 683.6ms/step | 29.3 samples/s
+Step 2190/44000 | Loss: 3.6678 | LR: 4.99e-04 | 684.5ms/step | 29.2 samples/s
+Step 2200/44000 | Loss: 3.6547 | LR: 4.99e-04 | 682.4ms/step | 29.3 samples/s
+Step 2210/44000 | Loss: 3.6378 | LR: 4.99e-04 | 682.6ms/step | 29.3 samples/s
+Step 2220/44000 | Loss: 3.6328 | LR: 4.99e-04 | 680.6ms/step | 29.4 samples/s
+Step 2230/44000 | Loss: 3.6869 | LR: 4.99e-04 | 682.1ms/step | 29.3 samples/s
+Step 2240/44000 | Loss: 3.6823 | LR: 4.99e-04 | 682.6ms/step | 29.3 samples/s
+Step 2250/44000 | Loss: 3.6684 | LR: 4.99e-04 | 684.5ms/step | 29.2 samples/s
+Validation at step 2250: Loss = 3.6606, Perplexity = 38.88
+Step 2260/44000 | Loss: 3.6578 | LR: 4.98e-04 | 1239.2ms/step | 16.1 samples/s
+Step 2270/44000 | Loss: 3.6413 | LR: 4.98e-04 | 681.9ms/step | 29.3 samples/s
+Step 2280/44000 | Loss: 3.6586 | LR: 4.98e-04 | 682.8ms/step | 29.3 samples/s
+Step 2290/44000 | Loss: 3.6776 | LR: 4.98e-04 | 684.0ms/step | 29.2 samples/s
+Step 2300/44000 | Loss: 3.6054 | LR: 4.98e-04 | 682.7ms/step | 29.3 samples/s
+Step 2310/44000 | Loss: 3.6456 | LR: 4.98e-04 | 683.2ms/step | 29.3 samples/s
+Step 2320/44000 | Loss: 3.6547 | LR: 4.98e-04 | 683.7ms/step | 29.3 samples/s
+Step 2330/44000 | Loss: 3.6483 | LR: 4.98e-04 | 683.6ms/step | 29.3 samples/s
+Step 2340/44000 | Loss: 3.6473 | LR: 4.98e-04 | 683.1ms/step | 29.3 samples/s
+Step 2350/44000 | Loss: 3.6125 | LR: 4.98e-04 | 682.8ms/step | 29.3 samples/s
+Step 2360/44000 | Loss: 3.6355 | LR: 4.98e-04 | 682.0ms/step | 29.3 samples/s
+Step 2370/44000 | Loss: 3.6366 | LR: 4.98e-04 | 679.0ms/step | 29.5 samples/s
+Step 2380/44000 | Loss: 3.5960 | LR: 4.98e-04 | 682.2ms/step | 29.3 samples/s
+Step 2390/44000 | Loss: 3.5817 | LR: 4.98e-04 | 681.4ms/step | 29.4 samples/s
+Step 2400/44000 | Loss: 3.6369 | LR: 4.98e-04 | 681.5ms/step | 29.3 samples/s
+Step 2410/44000 | Loss: 3.6284 | LR: 4.98e-04 | 680.6ms/step | 29.4 samples/s
+Step 2420/44000 | Loss: 3.6282 | LR: 4.98e-04 | 682.3ms/step | 29.3 samples/s
+Step 2430/44000 | Loss: 3.6212 | LR: 4.98e-04 | 681.0ms/step | 29.4 samples/s
+Step 2440/44000 | Loss: 3.6275 | LR: 4.98e-04 | 682.0ms/step | 29.3 samples/s
+Step 2450/44000 | Loss: 3.6034 | LR: 4.98e-04 | 680.9ms/step | 29.4 samples/s
+Step 2460/44000 | Loss: 3.6057 | LR: 4.98e-04 | 679.2ms/step | 29.4 samples/s
+Step 2470/44000 | Loss: 3.6119 | LR: 4.98e-04 | 682.4ms/step | 29.3 samples/s
+Step 2480/44000 | Loss: 3.6256 | LR: 4.98e-04 | 679.7ms/step | 29.4 samples/s
+Step 2490/44000 | Loss: 3.5894 | LR: 4.98e-04 | 680.6ms/step | 29.4 samples/s
+Step 2500/44000 | Loss: 3.6012 | LR: 4.98e-04 | 681.7ms/step | 29.3 samples/s
+Validation at step 2500: Loss = 3.6062, Perplexity = 36.83
+Step 2510/44000 | Loss: 3.6057 | LR: 4.98e-04 | 1247.7ms/step | 16.0 samples/s
+Step 2520/44000 | Loss: 3.5934 | LR: 4.98e-04 | 682.6ms/step | 29.3 samples/s
+Step 2530/44000 | Loss: 3.6051 | LR: 4.98e-04 | 683.6ms/step | 29.3 samples/s
+Step 2540/44000 | Loss: 3.5801 | LR: 4.98e-04 | 681.8ms/step | 29.3 samples/s
+Step 2550/44000 | Loss: 3.5463 | LR: 4.98e-04 | 680.3ms/step | 29.4 samples/s
+Step 2560/44000 | Loss: 3.5690 | LR: 4.98e-04 | 681.8ms/step | 29.3 samples/s
+Step 2570/44000 | Loss: 3.5804 | LR: 4.98e-04 | 683.3ms/step | 29.3 samples/s
+Step 2580/44000 | Loss: 3.6015 | LR: 4.98e-04 | 683.3ms/step | 29.3 samples/s
+Step 2590/44000 | Loss: 3.5855 | LR: 4.98e-04 | 681.3ms/step | 29.4 samples/s
+Step 2600/44000 | Loss: 3.5511 | LR: 4.98e-04 | 682.0ms/step | 29.3 samples/s
+Step 2610/44000 | Loss: 3.5176 | LR: 4.98e-04 | 682.8ms/step | 29.3 samples/s
+Step 2620/44000 | Loss: 3.5812 | LR: 4.98e-04 | 684.3ms/step | 29.2 samples/s
+Step 2630/44000 | Loss: 3.5415 | LR: 4.98e-04 | 681.5ms/step | 29.3 samples/s
+Step 2640/44000 | Loss: 3.5606 | LR: 4.98e-04 | 680.9ms/step | 29.4 samples/s
+Step 2650/44000 | Loss: 3.5750 | LR: 4.98e-04 | 683.7ms/step | 29.3 samples/s
+Step 2660/44000 | Loss: 3.5457 | LR: 4.98e-04 | 683.1ms/step | 29.3 samples/s
+Step 2670/44000 | Loss: 3.5489 | LR: 4.98e-04 | 683.9ms/step | 29.2 samples/s
+Step 2680/44000 | Loss: 3.5814 | LR: 4.98e-04 | 680.7ms/step | 29.4 samples/s
+Step 2690/44000 | Loss: 3.5940 | LR: 4.98e-04 | 682.2ms/step | 29.3 samples/s
+Step 2700/44000 | Loss: 3.5307 | LR: 4.97e-04 | 682.5ms/step | 29.3 samples/s
+Step 2710/44000 | Loss: 3.5850 | LR: 4.97e-04 | 682.3ms/step | 29.3 samples/s
+Step 2720/44000 | Loss: 3.5988 | LR: 4.97e-04 | 684.4ms/step | 29.2 samples/s
+Step 2730/44000 | Loss: 3.5787 | LR: 4.97e-04 | 681.8ms/step | 29.3 samples/s
+Step 2740/44000 | Loss: 3.5510 | LR: 4.97e-04 | 680.6ms/step | 29.4 samples/s
+Step 2750/44000 | Loss: 3.5302 | LR: 4.97e-04 | 682.7ms/step | 29.3 samples/s
+Validation at step 2750: Loss = 3.5617, Perplexity = 35.22
+Step 2760/44000 | Loss: 3.5235 | LR: 4.97e-04 | 1241.4ms/step | 16.1 samples/s
+Step 2770/44000 | Loss: 3.6002 | LR: 4.97e-04 | 682.0ms/step | 29.3 samples/s
+Step 2780/44000 | Loss: 3.5929 | LR: 4.97e-04 | 683.1ms/step | 29.3 samples/s
+Step 2790/44000 | Loss: 3.5685 | LR: 4.97e-04 | 680.6ms/step | 29.4 samples/s
+Step 2800/44000 | Loss: 3.5706 | LR: 4.97e-04 | 680.6ms/step | 29.4 samples/s
+Step 2810/44000 | Loss: 3.5281 | LR: 4.97e-04 | 683.2ms/step | 29.3 samples/s
+Step 2820/44000 | Loss: 3.5558 | LR: 4.97e-04 | 682.6ms/step | 29.3 samples/s
+Step 2830/44000 | Loss: 3.5401 | LR: 4.97e-04 | 683.1ms/step | 29.3 samples/s
+Step 2840/44000 | Loss: 3.5146 | LR: 4.97e-04 | 681.8ms/step | 29.3 samples/s
+Step 2850/44000 | Loss: 3.5859 | LR: 4.97e-04 | 681.0ms/step | 29.4 samples/s
+Step 2860/44000 | Loss: 3.5575 | LR: 4.97e-04 | 681.0ms/step | 29.4 samples/s
+Step 2870/44000 | Loss: 3.5329 | LR: 4.97e-04 | 681.6ms/step | 29.3 samples/s
+Step 2880/44000 | Loss: 3.4896 | LR: 4.97e-04 | 681.3ms/step | 29.4 samples/s
+Step 2890/44000 | Loss: 3.5236 | LR: 4.97e-04 | 682.2ms/step | 29.3 samples/s
+Step 2900/44000 | Loss: 3.5765 | LR: 4.97e-04 | 682.9ms/step | 29.3 samples/s
+Step 2910/44000 | Loss: 3.5248 | LR: 4.97e-04 | 680.6ms/step | 29.4 samples/s
+Step 2920/44000 | Loss: 3.5438 | LR: 4.97e-04 | 679.7ms/step | 29.4 samples/s
+Step 2930/44000 | Loss: 3.5284 | LR: 4.97e-04 | 681.1ms/step | 29.4 samples/s
+Step 2940/44000 | Loss: 3.5211 | LR: 4.97e-04 | 681.1ms/step | 29.4 samples/s
+Step 2950/44000 | Loss: 3.5509 | LR: 4.97e-04 | 681.7ms/step | 29.3 samples/s
+Step 2960/44000 | Loss: 3.5302 | LR: 4.97e-04 | 682.7ms/step | 29.3 samples/s
+Step 2970/44000 | Loss: 3.5520 | LR: 4.97e-04 | 681.8ms/step | 29.3 samples/s
+Step 2980/44000 | Loss: 3.6119 | LR: 4.97e-04 | 680.9ms/step | 29.4 samples/s
+Step 2990/44000 | Loss: 3.5136 | LR: 4.97e-04 | 681.0ms/step | 29.4 samples/s
+Step 3000/44000 | Loss: 3.5309 | LR: 4.97e-04 | 683.2ms/step | 29.3 samples/s
+Validation at step 3000: Loss = 3.5276, Perplexity = 34.04
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-3000
+Step 3010/44000 | Loss: 3.5184 | LR: 4.97e-04 | 1522.3ms/step | 13.1 samples/s
+Step 3020/44000 | Loss: 3.5128 | LR: 4.97e-04 | 681.8ms/step | 29.3 samples/s
+Step 3030/44000 | Loss: 3.5592 | LR: 4.97e-04 | 680.6ms/step | 29.4 samples/s
+Step 3040/44000 | Loss: 3.5153 | LR: 4.97e-04 | 680.5ms/step | 29.4 samples/s
+Step 3050/44000 | Loss: 3.5552 | LR: 4.97e-04 | 682.6ms/step | 29.3 samples/s
+Step 3060/44000 | Loss: 3.4928 | LR: 4.96e-04 | 683.5ms/step | 29.3 samples/s
+Step 3070/44000 | Loss: 3.5235 | LR: 4.96e-04 | 681.3ms/step | 29.4 samples/s
+Step 3080/44000 | Loss: 3.5322 | LR: 4.96e-04 | 680.0ms/step | 29.4 samples/s
+Step 3090/44000 | Loss: 3.4932 | LR: 4.96e-04 | 681.9ms/step | 29.3 samples/s
+Step 3100/44000 | Loss: 3.5170 | LR: 4.96e-04 | 681.1ms/step | 29.4 samples/s
+Step 3110/44000 | Loss: 3.4852 | LR: 4.96e-04 | 681.0ms/step | 29.4 samples/s
+Step 3120/44000 | Loss: 3.4994 | LR: 4.96e-04 | 680.1ms/step | 29.4 samples/s
+Step 3130/44000 | Loss: 3.4935 | LR: 4.96e-04 | 680.4ms/step | 29.4 samples/s
+Step 3140/44000 | Loss: 3.4892 | LR: 4.96e-04 | 680.6ms/step | 29.4 samples/s
+Step 3150/44000 | Loss: 3.4834 | LR: 4.96e-04 | 680.2ms/step | 29.4 samples/s
+Step 3160/44000 | Loss: 3.5026 | LR: 4.96e-04 | 680.4ms/step | 29.4 samples/s
+Step 3170/44000 | Loss: 3.4916 | LR: 4.96e-04 | 680.2ms/step | 29.4 samples/s
+Step 3180/44000 | Loss: 3.5293 | LR: 4.96e-04 | 681.0ms/step | 29.4 samples/s
+Step 3190/44000 | Loss: 3.5257 | LR: 4.96e-04 | 681.1ms/step | 29.4 samples/s
+Step 3200/44000 | Loss: 3.4865 | LR: 4.96e-04 | 681.5ms/step | 29.3 samples/s
+Step 3210/44000 | Loss: 3.4721 | LR: 4.96e-04 | 680.4ms/step | 29.4 samples/s
+Step 3220/44000 | Loss: 3.5098 | LR: 4.96e-04 | 682.8ms/step | 29.3 samples/s
+Step 3230/44000 | Loss: 3.4769 | LR: 4.96e-04 | 681.6ms/step | 29.3 samples/s
+Step 3240/44000 | Loss: 3.4926 | LR: 4.96e-04 | 680.8ms/step | 29.4 samples/s
+Step 3250/44000 | Loss: 3.4725 | LR: 4.96e-04 | 679.6ms/step | 29.4 samples/s
+Validation at step 3250: Loss = 3.4923, Perplexity = 32.86
+Step 3260/44000 | Loss: 3.5206 | LR: 4.96e-04 | 1247.0ms/step | 16.0 samples/s
+Step 3270/44000 | Loss: 3.4743 | LR: 4.96e-04 | 682.1ms/step | 29.3 samples/s
+Step 3280/44000 | Loss: 3.4761 | LR: 4.96e-04 | 683.5ms/step | 29.3 samples/s
+Step 3290/44000 | Loss: 3.4805 | LR: 4.96e-04 | 683.7ms/step | 29.3 samples/s
+Step 3300/44000 | Loss: 3.4668 | LR: 4.96e-04 | 682.3ms/step | 29.3 samples/s
+Step 3310/44000 | Loss: 3.4832 | LR: 4.96e-04 | 681.9ms/step | 29.3 samples/s
+Step 3320/44000 | Loss: 3.4688 | LR: 4.96e-04 | 681.0ms/step | 29.4 samples/s
+Step 3330/44000 | Loss: 3.4872 | LR: 4.96e-04 | 679.4ms/step | 29.4 samples/s
+Step 3340/44000 | Loss: 3.4631 | LR: 4.96e-04 | 681.3ms/step | 29.4 samples/s
+Step 3350/44000 | Loss: 3.4930 | LR: 4.96e-04 | 682.6ms/step | 29.3 samples/s
+Step 3360/44000 | Loss: 3.4527 | LR: 4.96e-04 | 683.3ms/step | 29.3 samples/s
+Step 3370/44000 | Loss: 3.4798 | LR: 4.95e-04 | 683.8ms/step | 29.3 samples/s
+Step 3380/44000 | Loss: 3.4679 | LR: 4.95e-04 | 682.5ms/step | 29.3 samples/s
+Step 3390/44000 | Loss: 3.4798 | LR: 4.95e-04 | 684.0ms/step | 29.2 samples/s
+Step 3400/44000 | Loss: 3.4594 | LR: 4.95e-04 | 682.8ms/step | 29.3 samples/s
+Step 3410/44000 | Loss: 3.4802 | LR: 4.95e-04 | 680.9ms/step | 29.4 samples/s
+Step 3420/44000 | Loss: 3.3880 | LR: 4.95e-04 | 681.2ms/step | 29.4 samples/s
+Step 3430/44000 | Loss: 3.4410 | LR: 4.95e-04 | 679.7ms/step | 29.4 samples/s
+Step 3440/44000 | Loss: 3.4704 | LR: 4.95e-04 | 682.3ms/step | 29.3 samples/s
+Step 3450/44000 | Loss: 3.4364 | LR: 4.95e-04 | 681.3ms/step | 29.4 samples/s
+Step 3460/44000 | Loss: 3.4823 | LR: 4.95e-04 | 682.2ms/step | 29.3 samples/s
+Step 3470/44000 | Loss: 3.4830 | LR: 4.95e-04 | 681.6ms/step | 29.3 samples/s
+Step 3480/44000 | Loss: 3.4622 | LR: 4.95e-04 | 683.7ms/step | 29.3 samples/s
+Step 3490/44000 | Loss: 3.4707 | LR: 4.95e-04 | 681.4ms/step | 29.4 samples/s
+Step 3500/44000 | Loss: 3.4223 | LR: 4.95e-04 | 682.5ms/step | 29.3 samples/s
+Validation at step 3500: Loss = 3.4642, Perplexity = 31.95
+Step 3510/44000 | Loss: 3.4733 | LR: 4.95e-04 | 1252.3ms/step | 16.0 samples/s
+Step 3520/44000 | Loss: 3.4341 | LR: 4.95e-04 | 685.1ms/step | 29.2 samples/s
+Step 3530/44000 | Loss: 3.4105 | LR: 4.95e-04 | 683.0ms/step | 29.3 samples/s
+Step 3540/44000 | Loss: 3.4402 | LR: 4.95e-04 | 682.7ms/step | 29.3 samples/s
+Step 3550/44000 | Loss: 3.4576 | LR: 4.95e-04 | 682.6ms/step | 29.3 samples/s
+Step 3560/44000 | Loss: 3.4948 | LR: 4.95e-04 | 681.3ms/step | 29.4 samples/s
+Step 3570/44000 | Loss: 3.4166 | LR: 4.95e-04 | 681.1ms/step | 29.4 samples/s
+Step 3580/44000 | Loss: 3.4434 | LR: 4.95e-04 | 681.7ms/step | 29.3 samples/s
+Step 3590/44000 | Loss: 3.4437 | LR: 4.95e-04 | 682.6ms/step | 29.3 samples/s
+Step 3600/44000 | Loss: 3.4644 | LR: 4.95e-04 | 682.3ms/step | 29.3 samples/s
+Step 3610/44000 | Loss: 3.4921 | LR: 4.95e-04 | 683.2ms/step | 29.3 samples/s
+Step 3620/44000 | Loss: 3.4481 | LR: 4.95e-04 | 681.9ms/step | 29.3 samples/s
+Step 3630/44000 | Loss: 3.4289 | LR: 4.95e-04 | 681.9ms/step | 29.3 samples/s
+Step 3640/44000 | Loss: 3.4517 | LR: 4.95e-04 | 681.0ms/step | 29.4 samples/s
+Step 3650/44000 | Loss: 3.4796 | LR: 4.94e-04 | 680.6ms/step | 29.4 samples/s
+Step 3660/44000 | Loss: 3.4184 | LR: 4.94e-04 | 680.2ms/step | 29.4 samples/s
+Step 3670/44000 | Loss: 3.4607 | LR: 4.94e-04 | 680.8ms/step | 29.4 samples/s
+Step 3680/44000 | Loss: 3.4027 | LR: 4.94e-04 | 680.4ms/step | 29.4 samples/s
+Step 3690/44000 | Loss: 3.4122 | LR: 4.94e-04 | 679.8ms/step | 29.4 samples/s
+Step 3700/44000 | Loss: 3.4601 | LR: 4.94e-04 | 679.5ms/step | 29.4 samples/s
+Step 3710/44000 | Loss: 3.4568 | LR: 4.94e-04 | 681.4ms/step | 29.4 samples/s
+Step 3720/44000 | Loss: 3.4345 | LR: 4.94e-04 | 681.3ms/step | 29.4 samples/s
+Step 3730/44000 | Loss: 3.4070 | LR: 4.94e-04 | 682.4ms/step | 29.3 samples/s
+Step 3740/44000 | Loss: 3.4669 | LR: 4.94e-04 | 683.0ms/step | 29.3 samples/s
+Step 3750/44000 | Loss: 3.4222 | LR: 4.94e-04 | 681.6ms/step | 29.3 samples/s
+Validation at step 3750: Loss = 3.4390, Perplexity = 31.16
+Step 3760/44000 | Loss: 3.4512 | LR: 4.94e-04 | 1249.4ms/step | 16.0 samples/s
+Step 3770/44000 | Loss: 3.4291 | LR: 4.94e-04 | 682.9ms/step | 29.3 samples/s
+Step 3780/44000 | Loss: 3.4285 | LR: 4.94e-04 | 682.6ms/step | 29.3 samples/s
+Step 3790/44000 | Loss: 3.4337 | LR: 4.94e-04 | 683.0ms/step | 29.3 samples/s
+Step 3800/44000 | Loss: 3.4043 | LR: 4.94e-04 | 683.9ms/step | 29.2 samples/s
+Step 3810/44000 | Loss: 3.3895 | LR: 4.94e-04 | 682.0ms/step | 29.3 samples/s
+Step 3820/44000 | Loss: 3.3728 | LR: 4.94e-04 | 681.3ms/step | 29.4 samples/s
+Step 3830/44000 | Loss: 3.4081 | LR: 4.94e-04 | 683.8ms/step | 29.2 samples/s
+Step 3840/44000 | Loss: 3.4717 | LR: 4.94e-04 | 683.2ms/step | 29.3 samples/s
+Step 3850/44000 | Loss: 3.4292 | LR: 4.94e-04 | 683.5ms/step | 29.3 samples/s
+Step 3860/44000 | Loss: 3.4177 | LR: 4.94e-04 | 681.0ms/step | 29.4 samples/s
+Step 3870/44000 | Loss: 3.4681 | LR: 4.94e-04 | 682.8ms/step | 29.3 samples/s
+Step 3880/44000 | Loss: 3.4396 | LR: 4.94e-04 | 685.1ms/step | 29.2 samples/s
+Step 3890/44000 | Loss: 3.4616 | LR: 4.94e-04 | 681.3ms/step | 29.4 samples/s
+Step 3900/44000 | Loss: 3.4096 | LR: 4.93e-04 | 681.0ms/step | 29.4 samples/s
+Step 3910/44000 | Loss: 3.4503 | LR: 4.93e-04 | 679.3ms/step | 29.4 samples/s
+Step 3920/44000 | Loss: 3.4124 | LR: 4.93e-04 | 679.4ms/step | 29.4 samples/s
+Step 3930/44000 | Loss: 3.4170 | LR: 4.93e-04 | 680.3ms/step | 29.4 samples/s
+Step 3940/44000 | Loss: 3.4201 | LR: 4.93e-04 | 679.7ms/step | 29.4 samples/s
+Step 3950/44000 | Loss: 3.3592 | LR: 4.93e-04 | 679.5ms/step | 29.4 samples/s
+Step 3960/44000 | Loss: 3.4302 | LR: 4.93e-04 | 679.4ms/step | 29.4 samples/s
+Step 3970/44000 | Loss: 3.4533 | LR: 4.93e-04 | 679.9ms/step | 29.4 samples/s
+Step 3980/44000 | Loss: 3.3968 | LR: 4.93e-04 | 678.2ms/step | 29.5 samples/s
+Step 3990/44000 | Loss: 3.3934 | LR: 4.93e-04 | 677.5ms/step | 29.5 samples/s
+Step 4000/44000 | Loss: 3.4114 | LR: 4.93e-04 | 677.6ms/step | 29.5 samples/s
+Validation at step 4000: Loss = 3.4178, Perplexity = 30.50
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-4000
+Step 4010/44000 | Loss: 3.4281 | LR: 4.93e-04 | 1525.1ms/step | 13.1 samples/s
+Step 4020/44000 | Loss: 3.4596 | LR: 4.93e-04 | 678.3ms/step | 29.5 samples/s
+Step 4030/44000 | Loss: 3.3729 | LR: 4.93e-04 | 678.4ms/step | 29.5 samples/s
+Step 4040/44000 | Loss: 3.3798 | LR: 4.93e-04 | 678.3ms/step | 29.5 samples/s
+Step 4050/44000 | Loss: 3.3952 | LR: 4.93e-04 | 678.3ms/step | 29.5 samples/s
+Step 4060/44000 | Loss: 3.3878 | LR: 4.93e-04 | 679.3ms/step | 29.4 samples/s
+Step 4070/44000 | Loss: 3.4152 | LR: 4.93e-04 | 678.7ms/step | 29.5 samples/s
+Step 4080/44000 | Loss: 3.4150 | LR: 4.93e-04 | 678.4ms/step | 29.5 samples/s
+Step 4090/44000 | Loss: 3.4078 | LR: 4.93e-04 | 679.0ms/step | 29.5 samples/s
+Step 4100/44000 | Loss: 3.3983 | LR: 4.93e-04 | 680.2ms/step | 29.4 samples/s
+Step 4110/44000 | Loss: 3.4223 | LR: 4.93e-04 | 678.7ms/step | 29.5 samples/s
+Step 4120/44000 | Loss: 3.4187 | LR: 4.93e-04 | 678.8ms/step | 29.5 samples/s
+Step 4130/44000 | Loss: 3.4097 | LR: 4.93e-04 | 679.2ms/step | 29.4 samples/s
+Step 4140/44000 | Loss: 3.4083 | LR: 4.92e-04 | 678.3ms/step | 29.5 samples/s
+Step 4150/44000 | Loss: 3.3893 | LR: 4.92e-04 | 679.5ms/step | 29.4 samples/s
+Step 4160/44000 | Loss: 3.4045 | LR: 4.92e-04 | 678.8ms/step | 29.5 samples/s
+Step 4170/44000 | Loss: 3.3740 | LR: 4.92e-04 | 677.5ms/step | 29.5 samples/s
+Step 4180/44000 | Loss: 3.3596 | LR: 4.92e-04 | 679.7ms/step | 29.4 samples/s
+Step 4190/44000 | Loss: 3.4140 | LR: 4.92e-04 | 679.8ms/step | 29.4 samples/s
+Step 4200/44000 | Loss: 3.3799 | LR: 4.92e-04 | 679.7ms/step | 29.4 samples/s
+Step 4210/44000 | Loss: 3.4241 | LR: 4.92e-04 | 680.1ms/step | 29.4 samples/s
+Step 4220/44000 | Loss: 3.3931 | LR: 4.92e-04 | 679.3ms/step | 29.4 samples/s
+Step 4230/44000 | Loss: 3.4336 | LR: 4.92e-04 | 678.8ms/step | 29.5 samples/s
+Step 4240/44000 | Loss: 3.3840 | LR: 4.92e-04 | 679.9ms/step | 29.4 samples/s
+Step 4250/44000 | Loss: 3.4068 | LR: 4.92e-04 | 679.6ms/step | 29.4 samples/s
+Validation at step 4250: Loss = 3.3950, Perplexity = 29.81
+Step 4260/44000 | Loss: 3.3487 | LR: 4.92e-04 | 1233.9ms/step | 16.2 samples/s
+Step 4270/44000 | Loss: 3.3628 | LR: 4.92e-04 | 678.4ms/step | 29.5 samples/s
+Step 4280/44000 | Loss: 3.4044 | LR: 4.92e-04 | 679.8ms/step | 29.4 samples/s
+Step 4290/44000 | Loss: 3.3746 | LR: 4.92e-04 | 680.6ms/step | 29.4 samples/s
+Step 4300/44000 | Loss: 3.4053 | LR: 4.92e-04 | 680.0ms/step | 29.4 samples/s
+Step 4310/44000 | Loss: 3.3666 | LR: 4.92e-04 | 680.5ms/step | 29.4 samples/s
+Step 4320/44000 | Loss: 3.3756 | LR: 4.92e-04 | 679.0ms/step | 29.5 samples/s
+Step 4330/44000 | Loss: 3.3588 | LR: 4.92e-04 | 679.5ms/step | 29.4 samples/s
+Step 4340/44000 | Loss: 3.3874 | LR: 4.92e-04 | 679.3ms/step | 29.4 samples/s
+Step 4350/44000 | Loss: 3.3448 | LR: 4.92e-04 | 679.8ms/step | 29.4 samples/s
+Step 4360/44000 | Loss: 3.3811 | LR: 4.91e-04 | 678.5ms/step | 29.5 samples/s
+Step 4370/44000 | Loss: 3.3967 | LR: 4.91e-04 | 679.2ms/step | 29.4 samples/s
+Step 4380/44000 | Loss: 3.3458 | LR: 4.91e-04 | 678.8ms/step | 29.5 samples/s
+Step 4390/44000 | Loss: 3.3588 | LR: 4.91e-04 | 678.0ms/step | 29.5 samples/s
+Step 4400/44000 | Loss: 3.4147 | LR: 4.91e-04 | 679.2ms/step | 29.4 samples/s
+Step 4410/44000 | Loss: 3.3854 | LR: 4.91e-04 | 680.7ms/step | 29.4 samples/s
+Step 4420/44000 | Loss: 3.3861 | LR: 4.91e-04 | 678.9ms/step | 29.5 samples/s
+Step 4430/44000 | Loss: 3.3763 | LR: 4.91e-04 | 678.9ms/step | 29.5 samples/s
+Step 4440/44000 | Loss: 3.3891 | LR: 4.91e-04 | 679.0ms/step | 29.5 samples/s
+Step 4450/44000 | Loss: 3.3627 | LR: 4.91e-04 | 679.6ms/step | 29.4 samples/s
+Step 4460/44000 | Loss: 3.3368 | LR: 4.91e-04 | 679.9ms/step | 29.4 samples/s
+Step 4470/44000 | Loss: 3.4033 | LR: 4.91e-04 | 679.2ms/step | 29.4 samples/s
+Step 4480/44000 | Loss: 3.3707 | LR: 4.91e-04 | 679.3ms/step | 29.4 samples/s
+Step 4490/44000 | Loss: 3.3335 | LR: 4.91e-04 | 679.8ms/step | 29.4 samples/s
+Step 4500/44000 | Loss: 3.3609 | LR: 4.91e-04 | 678.6ms/step | 29.5 samples/s
+Validation at step 4500: Loss = 3.3769, Perplexity = 29.28
+Step 4510/44000 | Loss: 3.3556 | LR: 4.91e-04 | 1234.8ms/step | 16.2 samples/s
+Step 4520/44000 | Loss: 3.3521 | LR: 4.91e-04 | 679.6ms/step | 29.4 samples/s
+Step 4530/44000 | Loss: 3.3342 | LR: 4.91e-04 | 679.8ms/step | 29.4 samples/s
+Step 4540/44000 | Loss: 3.3765 | LR: 4.91e-04 | 678.8ms/step | 29.5 samples/s
+Step 4550/44000 | Loss: 3.3709 | LR: 4.91e-04 | 678.2ms/step | 29.5 samples/s
+Step 4560/44000 | Loss: 3.3867 | LR: 4.90e-04 | 678.6ms/step | 29.5 samples/s
+Step 4570/44000 | Loss: 3.3832 | LR: 4.90e-04 | 677.5ms/step | 29.5 samples/s
+Step 4580/44000 | Loss: 3.3673 | LR: 4.90e-04 | 678.4ms/step | 29.5 samples/s
+Step 4590/44000 | Loss: 3.3171 | LR: 4.90e-04 | 678.3ms/step | 29.5 samples/s
+Step 4600/44000 | Loss: 3.3516 | LR: 4.90e-04 | 679.5ms/step | 29.4 samples/s
+Step 4610/44000 | Loss: 3.3609 | LR: 4.90e-04 | 679.1ms/step | 29.5 samples/s
+Step 4620/44000 | Loss: 3.3258 | LR: 4.90e-04 | 677.5ms/step | 29.5 samples/s
+Step 4630/44000 | Loss: 3.3486 | LR: 4.90e-04 | 678.4ms/step | 29.5 samples/s
+Step 4640/44000 | Loss: 3.3631 | LR: 4.90e-04 | 677.2ms/step | 29.5 samples/s
+Step 4650/44000 | Loss: 3.3265 | LR: 4.90e-04 | 678.7ms/step | 29.5 samples/s
+Step 4660/44000 | Loss: 3.3211 | LR: 4.90e-04 | 679.8ms/step | 29.4 samples/s
+Step 4670/44000 | Loss: 3.3957 | LR: 4.90e-04 | 680.1ms/step | 29.4 samples/s
+Step 4680/44000 | Loss: 3.3563 | LR: 4.90e-04 | 678.8ms/step | 29.5 samples/s
+Step 4690/44000 | Loss: 3.4132 | LR: 4.90e-04 | 679.6ms/step | 29.4 samples/s
+Step 4700/44000 | Loss: 3.3966 | LR: 4.90e-04 | 679.9ms/step | 29.4 samples/s
+Step 4710/44000 | Loss: 3.3839 | LR: 4.90e-04 | 679.2ms/step | 29.4 samples/s
+Step 4720/44000 | Loss: 3.3321 | LR: 4.90e-04 | 678.9ms/step | 29.5 samples/s
+Step 4730/44000 | Loss: 3.3661 | LR: 4.90e-04 | 679.5ms/step | 29.4 samples/s
+Step 4740/44000 | Loss: 3.3577 | LR: 4.90e-04 | 680.3ms/step | 29.4 samples/s
+Step 4750/44000 | Loss: 3.3139 | LR: 4.90e-04 | 679.6ms/step | 29.4 samples/s
+Validation at step 4750: Loss = 3.3614, Perplexity = 28.83
+Step 4760/44000 | Loss: 3.3747 | LR: 4.89e-04 | 1233.8ms/step | 16.2 samples/s
+Step 4770/44000 | Loss: 3.3652 | LR: 4.89e-04 | 679.9ms/step | 29.4 samples/s
+Step 4780/44000 | Loss: 3.3694 | LR: 4.89e-04 | 679.8ms/step | 29.4 samples/s
+Step 4790/44000 | Loss: 3.3512 | LR: 4.89e-04 | 680.1ms/step | 29.4 samples/s
+Step 4800/44000 | Loss: 3.3583 | LR: 4.89e-04 | 679.8ms/step | 29.4 samples/s
+Step 4810/44000 | Loss: 3.3428 | LR: 4.89e-04 | 680.2ms/step | 29.4 samples/s
+Step 4820/44000 | Loss: 3.3505 | LR: 4.89e-04 | 679.0ms/step | 29.5 samples/s
+Step 4830/44000 | Loss: 3.3398 | LR: 4.89e-04 | 679.3ms/step | 29.4 samples/s
+Step 4840/44000 | Loss: 3.3197 | LR: 4.89e-04 | 680.0ms/step | 29.4 samples/s
+Step 4850/44000 | Loss: 3.3333 | LR: 4.89e-04 | 681.2ms/step | 29.4 samples/s
+Step 4860/44000 | Loss: 3.3329 | LR: 4.89e-04 | 678.1ms/step | 29.5 samples/s
+Step 4870/44000 | Loss: 3.3272 | LR: 4.89e-04 | 678.5ms/step | 29.5 samples/s
+Step 4880/44000 | Loss: 3.3495 | LR: 4.89e-04 | 679.3ms/step | 29.4 samples/s
+Step 4890/44000 | Loss: 3.3169 | LR: 4.89e-04 | 678.7ms/step | 29.5 samples/s
+Step 4900/44000 | Loss: 3.3860 | LR: 4.89e-04 | 678.3ms/step | 29.5 samples/s
+Step 4910/44000 | Loss: 3.3662 | LR: 4.89e-04 | 678.1ms/step | 29.5 samples/s
+Step 4920/44000 | Loss: 3.3651 | LR: 4.89e-04 | 677.5ms/step | 29.5 samples/s
+Step 4930/44000 | Loss: 3.3539 | LR: 4.89e-04 | 678.1ms/step | 29.5 samples/s
+Step 4940/44000 | Loss: 3.3554 | LR: 4.89e-04 | 678.7ms/step | 29.5 samples/s
+Step 4950/44000 | Loss: 3.3109 | LR: 4.88e-04 | 680.9ms/step | 29.4 samples/s
+Step 4960/44000 | Loss: 3.3611 | LR: 4.88e-04 | 679.2ms/step | 29.4 samples/s
+Step 4970/44000 | Loss: 3.3116 | LR: 4.88e-04 | 679.0ms/step | 29.5 samples/s
+Step 4980/44000 | Loss: 3.3370 | LR: 4.88e-04 | 678.5ms/step | 29.5 samples/s
+Step 4990/44000 | Loss: 3.3416 | LR: 4.88e-04 | 679.5ms/step | 29.4 samples/s
+Step 5000/44000 | Loss: 3.3401 | LR: 4.88e-04 | 679.2ms/step | 29.4 samples/s
+Validation at step 5000: Loss = 3.3442, Perplexity = 28.34
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-5000
+Step 5010/44000 | Loss: 3.3201 | LR: 4.88e-04 | 1521.3ms/step | 13.1 samples/s
+Step 5020/44000 | Loss: 3.3306 | LR: 4.88e-04 | 678.3ms/step | 29.5 samples/s
+Step 5030/44000 | Loss: 3.3327 | LR: 4.88e-04 | 677.6ms/step | 29.5 samples/s
+Step 5040/44000 | Loss: 3.3609 | LR: 4.88e-04 | 678.3ms/step | 29.5 samples/s
+Step 5050/44000 | Loss: 3.3322 | LR: 4.88e-04 | 677.3ms/step | 29.5 samples/s
+Step 5060/44000 | Loss: 3.3437 | LR: 4.88e-04 | 679.3ms/step | 29.4 samples/s
+Step 5070/44000 | Loss: 3.3212 | LR: 4.88e-04 | 679.2ms/step | 29.4 samples/s
+Step 5080/44000 | Loss: 3.3374 | LR: 4.88e-04 | 679.5ms/step | 29.4 samples/s
+Step 5090/44000 | Loss: 3.3319 | LR: 4.88e-04 | 679.7ms/step | 29.4 samples/s
+Step 5100/44000 | Loss: 3.3516 | LR: 4.88e-04 | 679.6ms/step | 29.4 samples/s
+Step 5110/44000 | Loss: 3.3098 | LR: 4.88e-04 | 679.5ms/step | 29.4 samples/s
+Step 5120/44000 | Loss: 3.3821 | LR: 4.88e-04 | 678.1ms/step | 29.5 samples/s
+Step 5130/44000 | Loss: 3.3418 | LR: 4.87e-04 | 679.4ms/step | 29.4 samples/s
+Step 5140/44000 | Loss: 3.3210 | LR: 4.87e-04 | 679.5ms/step | 29.4 samples/s
+Step 5150/44000 | Loss: 3.3319 | LR: 4.87e-04 | 679.4ms/step | 29.4 samples/s
+Step 5160/44000 | Loss: 3.3373 | LR: 4.87e-04 | 679.4ms/step | 29.4 samples/s
+Step 5170/44000 | Loss: 3.2874 | LR: 4.87e-04 | 678.7ms/step | 29.5 samples/s
+Step 5180/44000 | Loss: 3.3305 | LR: 4.87e-04 | 678.4ms/step | 29.5 samples/s
+Step 5190/44000 | Loss: 3.3494 | LR: 4.87e-04 | 678.4ms/step | 29.5 samples/s
+Step 5200/44000 | Loss: 3.3278 | LR: 4.87e-04 | 678.9ms/step | 29.5 samples/s
+Step 5210/44000 | Loss: 3.3478 | LR: 4.87e-04 | 677.8ms/step | 29.5 samples/s
+Step 5220/44000 | Loss: 3.3379 | LR: 4.87e-04 | 678.1ms/step | 29.5 samples/s
+Step 5230/44000 | Loss: 3.3458 | LR: 4.87e-04 | 678.0ms/step | 29.5 samples/s
+Step 5240/44000 | Loss: 3.3979 | LR: 4.87e-04 | 677.1ms/step | 29.5 samples/s
+Step 5250/44000 | Loss: 3.3364 | LR: 4.87e-04 | 678.5ms/step | 29.5 samples/s
+Validation at step 5250: Loss = 3.3295, Perplexity = 27.93
+Step 5260/44000 | Loss: 3.3595 | LR: 4.87e-04 | 1234.7ms/step | 16.2 samples/s
+Step 5270/44000 | Loss: 3.3485 | LR: 4.87e-04 | 677.6ms/step | 29.5 samples/s
+Step 5280/44000 | Loss: 3.3431 | LR: 4.87e-04 | 678.4ms/step | 29.5 samples/s
+Step 5290/44000 | Loss: 3.3174 | LR: 4.87e-04 | 679.0ms/step | 29.5 samples/s
+Step 5300/44000 | Loss: 3.3188 | LR: 4.86e-04 | 681.3ms/step | 29.4 samples/s
+Step 5310/44000 | Loss: 3.3468 | LR: 4.86e-04 | 679.8ms/step | 29.4 samples/s
+Step 5320/44000 | Loss: 3.2971 | LR: 4.86e-04 | 678.9ms/step | 29.5 samples/s
+Step 5330/44000 | Loss: 3.3454 | LR: 4.86e-04 | 679.3ms/step | 29.4 samples/s
+Step 5340/44000 | Loss: 3.3099 | LR: 4.86e-04 | 679.7ms/step | 29.4 samples/s
+Step 5350/44000 | Loss: 3.3012 | LR: 4.86e-04 | 680.3ms/step | 29.4 samples/s
+Step 5360/44000 | Loss: 3.3069 | LR: 4.86e-04 | 679.6ms/step | 29.4 samples/s
+Step 5370/44000 | Loss: 3.3328 | LR: 4.86e-04 | 680.9ms/step | 29.4 samples/s
+Step 5380/44000 | Loss: 3.3176 | LR: 4.86e-04 | 679.3ms/step | 29.4 samples/s
+Step 5390/44000 | Loss: 3.2883 | LR: 4.86e-04 | 680.2ms/step | 29.4 samples/s
+Step 5400/44000 | Loss: 3.3253 | LR: 4.86e-04 | 679.8ms/step | 29.4 samples/s
+Step 5410/44000 | Loss: 3.2998 | LR: 4.86e-04 | 679.2ms/step | 29.4 samples/s
+Step 5420/44000 | Loss: 3.3220 | LR: 4.86e-04 | 679.0ms/step | 29.5 samples/s
+Step 5430/44000 | Loss: 3.3162 | LR: 4.86e-04 | 679.4ms/step | 29.4 samples/s
+Step 5440/44000 | Loss: 3.3488 | LR: 4.86e-04 | 678.1ms/step | 29.5 samples/s
+Step 5450/44000 | Loss: 3.2814 | LR: 4.86e-04 | 679.2ms/step | 29.4 samples/s
+Step 5460/44000 | Loss: 3.2891 | LR: 4.86e-04 | 678.3ms/step | 29.5 samples/s
+Step 5470/44000 | Loss: 3.2895 | LR: 4.85e-04 | 678.2ms/step | 29.5 samples/s
+Step 5480/44000 | Loss: 3.3356 | LR: 4.85e-04 | 678.8ms/step | 29.5 samples/s
+Step 5490/44000 | Loss: 3.3519 | LR: 4.85e-04 | 678.6ms/step | 29.5 samples/s
+Step 5500/44000 | Loss: 3.3232 | LR: 4.85e-04 | 679.5ms/step | 29.4 samples/s
+Validation at step 5500: Loss = 3.3156, Perplexity = 27.54
+Step 5510/44000 | Loss: 3.2666 | LR: 4.85e-04 | 1233.2ms/step | 16.2 samples/s
+Step 5520/44000 | Loss: 3.2916 | LR: 4.85e-04 | 678.6ms/step | 29.5 samples/s
+Step 5530/44000 | Loss: 3.2834 | LR: 4.85e-04 | 678.9ms/step | 29.5 samples/s
+Step 5540/44000 | Loss: 3.2878 | LR: 4.85e-04 | 679.0ms/step | 29.5 samples/s
+Step 5550/44000 | Loss: 3.3214 | LR: 4.85e-04 | 679.9ms/step | 29.4 samples/s
+Step 5560/44000 | Loss: 3.3214 | LR: 4.85e-04 | 678.2ms/step | 29.5 samples/s
+Step 5570/44000 | Loss: 3.3011 | LR: 4.85e-04 | 678.8ms/step | 29.5 samples/s
+Step 5580/44000 | Loss: 3.3161 | LR: 4.85e-04 | 679.8ms/step | 29.4 samples/s
+Step 5590/44000 | Loss: 3.3378 | LR: 4.85e-04 | 679.8ms/step | 29.4 samples/s
+Step 5600/44000 | Loss: 3.3209 | LR: 4.85e-04 | 678.2ms/step | 29.5 samples/s
+Step 5610/44000 | Loss: 3.2975 | LR: 4.85e-04 | 679.3ms/step | 29.4 samples/s
+Step 5620/44000 | Loss: 3.3005 | LR: 4.85e-04 | 678.3ms/step | 29.5 samples/s
+Step 5630/44000 | Loss: 3.2914 | LR: 4.84e-04 | 678.4ms/step | 29.5 samples/s
+Step 5640/44000 | Loss: 3.3362 | LR: 4.84e-04 | 678.0ms/step | 29.5 samples/s
+Step 5650/44000 | Loss: 3.2457 | LR: 4.84e-04 | 678.3ms/step | 29.5 samples/s
+Step 5660/44000 | Loss: 3.2908 | LR: 4.84e-04 | 678.8ms/step | 29.5 samples/s
+Step 5670/44000 | Loss: 3.3123 | LR: 4.84e-04 | 678.7ms/step | 29.5 samples/s
+Step 5680/44000 | Loss: 3.2936 | LR: 4.84e-04 | 680.3ms/step | 29.4 samples/s
+Step 5690/44000 | Loss: 3.2909 | LR: 4.84e-04 | 679.8ms/step | 29.4 samples/s
+Step 5700/44000 | Loss: 3.2340 | LR: 4.84e-04 | 679.9ms/step | 29.4 samples/s
+Step 5710/44000 | Loss: 3.3004 | LR: 4.84e-04 | 680.5ms/step | 29.4 samples/s
+Step 5720/44000 | Loss: 3.3141 | LR: 4.84e-04 | 679.5ms/step | 29.4 samples/s
+Step 5730/44000 | Loss: 3.2876 | LR: 4.84e-04 | 679.8ms/step | 29.4 samples/s
+Step 5740/44000 | Loss: 3.2518 | LR: 4.84e-04 | 679.4ms/step | 29.4 samples/s
+Step 5750/44000 | Loss: 3.2692 | LR: 4.84e-04 | 679.0ms/step | 29.5 samples/s
+Validation at step 5750: Loss = 3.3021, Perplexity = 27.17
+Step 5760/44000 | Loss: 3.3389 | LR: 4.84e-04 | 1232.6ms/step | 16.2 samples/s
+Step 5770/44000 | Loss: 3.3094 | LR: 4.84e-04 | 679.0ms/step | 29.5 samples/s
+Step 5780/44000 | Loss: 3.2895 | LR: 4.83e-04 | 679.7ms/step | 29.4 samples/s
+Step 5790/44000 | Loss: 3.2590 | LR: 4.83e-04 | 681.2ms/step | 29.4 samples/s
+Step 5800/44000 | Loss: 3.3042 | LR: 4.83e-04 | 678.2ms/step | 29.5 samples/s
+Step 5810/44000 | Loss: 3.3517 | LR: 4.83e-04 | 680.2ms/step | 29.4 samples/s
+Step 5820/44000 | Loss: 3.3069 | LR: 4.83e-04 | 678.7ms/step | 29.5 samples/s
+Step 5830/44000 | Loss: 3.3161 | LR: 4.83e-04 | 679.4ms/step | 29.4 samples/s
+Step 5840/44000 | Loss: 3.2882 | LR: 4.83e-04 | 679.3ms/step | 29.4 samples/s
+Step 5850/44000 | Loss: 3.2929 | LR: 4.83e-04 | 677.5ms/step | 29.5 samples/s
+Step 5860/44000 | Loss: 3.2733 | LR: 4.83e-04 | 679.7ms/step | 29.4 samples/s
+Step 5870/44000 | Loss: 3.2655 | LR: 4.83e-04 | 679.0ms/step | 29.5 samples/s
+Step 5880/44000 | Loss: 3.3012 | LR: 4.83e-04 | 678.2ms/step | 29.5 samples/s
+Step 5890/44000 | Loss: 3.3211 | LR: 4.83e-04 | 677.8ms/step | 29.5 samples/s
+Step 5900/44000 | Loss: 3.3090 | LR: 4.83e-04 | 679.8ms/step | 29.4 samples/s
+Step 5910/44000 | Loss: 3.3210 | LR: 4.83e-04 | 679.5ms/step | 29.4 samples/s
+Step 5920/44000 | Loss: 3.3013 | LR: 4.83e-04 | 679.9ms/step | 29.4 samples/s
+Step 5930/44000 | Loss: 3.2569 | LR: 4.83e-04 | 679.5ms/step | 29.4 samples/s
+Step 5940/44000 | Loss: 3.2824 | LR: 4.82e-04 | 679.2ms/step | 29.4 samples/s
+Step 5950/44000 | Loss: 3.2879 | LR: 4.82e-04 | 679.6ms/step | 29.4 samples/s
+Step 5960/44000 | Loss: 3.2794 | LR: 4.82e-04 | 678.7ms/step | 29.5 samples/s
+Step 5970/44000 | Loss: 3.3082 | LR: 4.82e-04 | 679.7ms/step | 29.4 samples/s
+Step 5980/44000 | Loss: 3.2952 | LR: 4.82e-04 | 679.8ms/step | 29.4 samples/s
+Step 5990/44000 | Loss: 3.2891 | LR: 4.82e-04 | 678.9ms/step | 29.5 samples/s
+Step 6000/44000 | Loss: 3.2904 | LR: 4.82e-04 | 679.5ms/step | 29.4 samples/s
+Validation at step 6000: Loss = 3.2899, Perplexity = 26.84
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-6000
+Step 6010/44000 | Loss: 3.3171 | LR: 4.82e-04 | 1517.4ms/step | 13.2 samples/s
+Step 6020/44000 | Loss: 3.3210 | LR: 4.82e-04 | 678.6ms/step | 29.5 samples/s
+Step 6030/44000 | Loss: 3.2763 | LR: 4.82e-04 | 678.5ms/step | 29.5 samples/s
+Step 6040/44000 | Loss: 3.2888 | LR: 4.82e-04 | 680.5ms/step | 29.4 samples/s
+Step 6050/44000 | Loss: 3.2496 | LR: 4.82e-04 | 678.9ms/step | 29.5 samples/s
+Step 6060/44000 | Loss: 3.2594 | LR: 4.82e-04 | 679.1ms/step | 29.4 samples/s
+Step 6070/44000 | Loss: 3.2980 | LR: 4.82e-04 | 680.3ms/step | 29.4 samples/s
+Step 6080/44000 | Loss: 3.2949 | LR: 4.81e-04 | 678.5ms/step | 29.5 samples/s
+Step 6090/44000 | Loss: 3.3161 | LR: 4.81e-04 | 678.7ms/step | 29.5 samples/s
+Step 6100/44000 | Loss: 3.2980 | LR: 4.81e-04 | 679.2ms/step | 29.4 samples/s
+Step 6110/44000 | Loss: 3.2402 | LR: 4.81e-04 | 679.3ms/step | 29.4 samples/s
+Step 6120/44000 | Loss: 3.2767 | LR: 4.81e-04 | 678.6ms/step | 29.5 samples/s
+Step 6130/44000 | Loss: 3.2639 | LR: 4.81e-04 | 679.2ms/step | 29.4 samples/s
+Step 6140/44000 | Loss: 3.2620 | LR: 4.81e-04 | 678.9ms/step | 29.5 samples/s
+Step 6150/44000 | Loss: 3.2545 | LR: 4.81e-04 | 678.6ms/step | 29.5 samples/s
+Step 6160/44000 | Loss: 3.2771 | LR: 4.81e-04 | 677.4ms/step | 29.5 samples/s
+Step 6170/44000 | Loss: 3.2956 | LR: 4.81e-04 | 676.4ms/step | 29.6 samples/s
+Step 6180/44000 | Loss: 3.3112 | LR: 4.81e-04 | 678.4ms/step | 29.5 samples/s
+Step 6190/44000 | Loss: 3.2732 | LR: 4.81e-04 | 678.4ms/step | 29.5 samples/s
+Step 6200/44000 | Loss: 3.2504 | LR: 4.81e-04 | 678.6ms/step | 29.5 samples/s
+Step 6210/44000 | Loss: 3.2548 | LR: 4.81e-04 | 676.9ms/step | 29.5 samples/s
+Step 6220/44000 | Loss: 3.2342 | LR: 4.81e-04 | 679.1ms/step | 29.4 samples/s
+Step 6230/44000 | Loss: 3.2800 | LR: 4.80e-04 | 680.0ms/step | 29.4 samples/s
+Step 6240/44000 | Loss: 3.2352 | LR: 4.80e-04 | 679.0ms/step | 29.5 samples/s
+Step 6250/44000 | Loss: 3.2904 | LR: 4.80e-04 | 679.4ms/step | 29.4 samples/s
+Validation at step 6250: Loss = 3.2793, Perplexity = 26.56
+Step 6260/44000 | Loss: 3.2713 | LR: 4.80e-04 | 1233.3ms/step | 16.2 samples/s
+Step 6270/44000 | Loss: 3.2917 | LR: 4.80e-04 | 679.7ms/step | 29.4 samples/s
+Step 6280/44000 | Loss: 3.2853 | LR: 4.80e-04 | 680.2ms/step | 29.4 samples/s
+Step 6290/44000 | Loss: 3.2928 | LR: 4.80e-04 | 678.7ms/step | 29.5 samples/s
+Step 6300/44000 | Loss: 3.2695 | LR: 4.80e-04 | 680.0ms/step | 29.4 samples/s
+Step 6310/44000 | Loss: 3.3031 | LR: 4.80e-04 | 679.5ms/step | 29.4 samples/s
+Step 6320/44000 | Loss: 3.2441 | LR: 4.80e-04 | 678.8ms/step | 29.5 samples/s
+Step 6330/44000 | Loss: 3.3167 | LR: 4.80e-04 | 679.1ms/step | 29.5 samples/s
+Step 6340/44000 | Loss: 3.2719 | LR: 4.80e-04 | 680.6ms/step | 29.4 samples/s
+Step 6350/44000 | Loss: 3.3179 | LR: 4.80e-04 | 679.4ms/step | 29.4 samples/s
+Step 6360/44000 | Loss: 3.2857 | LR: 4.80e-04 | 679.0ms/step | 29.5 samples/s
+Step 6370/44000 | Loss: 3.2397 | LR: 4.79e-04 | 679.6ms/step | 29.4 samples/s
+Step 6380/44000 | Loss: 3.2720 | LR: 4.79e-04 | 680.0ms/step | 29.4 samples/s
+Step 6390/44000 | Loss: 3.2851 | LR: 4.79e-04 | 679.6ms/step | 29.4 samples/s
+Step 6400/44000 | Loss: 3.2461 | LR: 4.79e-04 | 680.4ms/step | 29.4 samples/s
+Step 6410/44000 | Loss: 3.2709 | LR: 4.79e-04 | 679.2ms/step | 29.4 samples/s
+Step 6420/44000 | Loss: 3.2656 | LR: 4.79e-04 | 679.7ms/step | 29.4 samples/s
+Step 6430/44000 | Loss: 3.2963 | LR: 4.79e-04 | 679.1ms/step | 29.4 samples/s
+Step 6440/44000 | Loss: 3.2586 | LR: 4.79e-04 | 678.3ms/step | 29.5 samples/s
+Step 6450/44000 | Loss: 3.2651 | LR: 4.79e-04 | 679.1ms/step | 29.5 samples/s
+Step 6460/44000 | Loss: 3.2751 | LR: 4.79e-04 | 680.5ms/step | 29.4 samples/s
+Step 6470/44000 | Loss: 3.2956 | LR: 4.79e-04 | 679.2ms/step | 29.4 samples/s
+Step 6480/44000 | Loss: 3.3148 | LR: 4.79e-04 | 678.9ms/step | 29.5 samples/s
+Step 6490/44000 | Loss: 3.2306 | LR: 4.79e-04 | 679.1ms/step | 29.4 samples/s
+Step 6500/44000 | Loss: 3.3101 | LR: 4.79e-04 | 678.8ms/step | 29.5 samples/s
+Validation at step 6500: Loss = 3.2681, Perplexity = 26.26
+Step 6510/44000 | Loss: 3.2772 | LR: 4.78e-04 | 1233.9ms/step | 16.2 samples/s
+Step 6520/44000 | Loss: 3.2638 | LR: 4.78e-04 | 677.9ms/step | 29.5 samples/s
+Step 6530/44000 | Loss: 3.2889 | LR: 4.78e-04 | 678.3ms/step | 29.5 samples/s
+Step 6540/44000 | Loss: 3.2582 | LR: 4.78e-04 | 678.4ms/step | 29.5 samples/s
+Step 6550/44000 | Loss: 3.2878 | LR: 4.78e-04 | 679.0ms/step | 29.5 samples/s
+Step 6560/44000 | Loss: 3.2331 | LR: 4.78e-04 | 678.5ms/step | 29.5 samples/s
+Step 6570/44000 | Loss: 3.2258 | LR: 4.78e-04 | 679.0ms/step | 29.5 samples/s
+Step 6580/44000 | Loss: 3.2472 | LR: 4.78e-04 | 678.6ms/step | 29.5 samples/s
+Step 6590/44000 | Loss: 3.2573 | LR: 4.78e-04 | 679.0ms/step | 29.5 samples/s
+Step 6600/44000 | Loss: 3.2273 | LR: 4.78e-04 | 677.9ms/step | 29.5 samples/s
+Step 6610/44000 | Loss: 3.2580 | LR: 4.78e-04 | 679.4ms/step | 29.4 samples/s
+Step 6620/44000 | Loss: 3.2500 | LR: 4.78e-04 | 679.0ms/step | 29.5 samples/s
+Step 6630/44000 | Loss: 3.2876 | LR: 4.78e-04 | 678.3ms/step | 29.5 samples/s
+Step 6640/44000 | Loss: 3.2677 | LR: 4.77e-04 | 677.6ms/step | 29.5 samples/s
+Step 6650/44000 | Loss: 3.2883 | LR: 4.77e-04 | 678.5ms/step | 29.5 samples/s
+Step 6660/44000 | Loss: 3.2172 | LR: 4.77e-04 | 678.0ms/step | 29.5 samples/s
+Step 6670/44000 | Loss: 3.2684 | LR: 4.77e-04 | 678.6ms/step | 29.5 samples/s
+Step 6680/44000 | Loss: 3.2422 | LR: 4.77e-04 | 678.4ms/step | 29.5 samples/s
+Step 6690/44000 | Loss: 3.2733 | LR: 4.77e-04 | 678.5ms/step | 29.5 samples/s
+Step 6700/44000 | Loss: 3.2605 | LR: 4.77e-04 | 678.6ms/step | 29.5 samples/s
+Step 6710/44000 | Loss: 3.2407 | LR: 4.77e-04 | 678.1ms/step | 29.5 samples/s
+Step 6720/44000 | Loss: 3.2555 | LR: 4.77e-04 | 677.8ms/step | 29.5 samples/s
+Step 6730/44000 | Loss: 3.3198 | LR: 4.77e-04 | 678.6ms/step | 29.5 samples/s
+Step 6740/44000 | Loss: 3.2470 | LR: 4.77e-04 | 678.2ms/step | 29.5 samples/s
+Step 6750/44000 | Loss: 3.2217 | LR: 4.77e-04 | 679.2ms/step | 29.4 samples/s
+Validation at step 6750: Loss = 3.2617, Perplexity = 26.10
+Step 6760/44000 | Loss: 3.2398 | LR: 4.77e-04 | 1234.5ms/step | 16.2 samples/s
+Step 6770/44000 | Loss: 3.2717 | LR: 4.76e-04 | 679.8ms/step | 29.4 samples/s
+Step 6780/44000 | Loss: 3.2467 | LR: 4.76e-04 | 680.6ms/step | 29.4 samples/s
+Step 6790/44000 | Loss: 3.2670 | LR: 4.76e-04 | 679.7ms/step | 29.4 samples/s
+Step 6800/44000 | Loss: 3.2245 | LR: 4.76e-04 | 680.1ms/step | 29.4 samples/s
+Step 6810/44000 | Loss: 3.2664 | LR: 4.76e-04 | 680.6ms/step | 29.4 samples/s
+Step 6820/44000 | Loss: 3.2656 | LR: 4.76e-04 | 679.5ms/step | 29.4 samples/s
+Step 6830/44000 | Loss: 3.2434 | LR: 4.76e-04 | 679.8ms/step | 29.4 samples/s
+Step 6840/44000 | Loss: 3.2242 | LR: 4.76e-04 | 680.1ms/step | 29.4 samples/s
+Step 6850/44000 | Loss: 3.2333 | LR: 4.76e-04 | 678.3ms/step | 29.5 samples/s
+Step 6860/44000 | Loss: 3.2091 | LR: 4.76e-04 | 678.8ms/step | 29.5 samples/s
+Step 6870/44000 | Loss: 3.2832 | LR: 4.76e-04 | 679.0ms/step | 29.5 samples/s
+Step 6880/44000 | Loss: 3.2373 | LR: 4.76e-04 | 679.5ms/step | 29.4 samples/s
+Step 6890/44000 | Loss: 3.2338 | LR: 4.76e-04 | 678.7ms/step | 29.5 samples/s
+Step 6900/44000 | Loss: 3.2429 | LR: 4.75e-04 | 681.7ms/step | 29.3 samples/s
+Step 6910/44000 | Loss: 3.2843 | LR: 4.75e-04 | 679.7ms/step | 29.4 samples/s
+Step 6920/44000 | Loss: 3.2883 | LR: 4.75e-04 | 678.5ms/step | 29.5 samples/s
+Step 6930/44000 | Loss: 3.2477 | LR: 4.75e-04 | 680.7ms/step | 29.4 samples/s
+Step 6940/44000 | Loss: 3.2472 | LR: 4.75e-04 | 680.6ms/step | 29.4 samples/s
+Step 6950/44000 | Loss: 3.2425 | LR: 4.75e-04 | 680.5ms/step | 29.4 samples/s
+Step 6960/44000 | Loss: 3.2633 | LR: 4.75e-04 | 678.1ms/step | 29.5 samples/s
+Step 6970/44000 | Loss: 3.2067 | LR: 4.75e-04 | 679.0ms/step | 29.5 samples/s
+Step 6980/44000 | Loss: 3.2616 | LR: 4.75e-04 | 678.8ms/step | 29.5 samples/s
+Step 6990/44000 | Loss: 3.2119 | LR: 4.75e-04 | 679.1ms/step | 29.5 samples/s
+Step 7000/44000 | Loss: 3.2409 | LR: 4.75e-04 | 678.9ms/step | 29.5 samples/s
+Validation at step 7000: Loss = 3.2491, Perplexity = 25.77
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-7000
+Step 7010/44000 | Loss: 3.2437 | LR: 4.75e-04 | 1513.2ms/step | 13.2 samples/s
+Step 7020/44000 | Loss: 3.2494 | LR: 4.75e-04 | 679.5ms/step | 29.4 samples/s
+Step 7030/44000 | Loss: 3.2273 | LR: 4.74e-04 | 678.0ms/step | 29.5 samples/s
+Step 7040/44000 | Loss: 3.2369 | LR: 4.74e-04 | 677.2ms/step | 29.5 samples/s
+Step 7050/44000 | Loss: 3.2484 | LR: 4.74e-04 | 678.2ms/step | 29.5 samples/s
+Step 7060/44000 | Loss: 3.2558 | LR: 4.74e-04 | 677.9ms/step | 29.5 samples/s
+Step 7070/44000 | Loss: 3.1837 | LR: 4.74e-04 | 679.2ms/step | 29.4 samples/s
+Step 7080/44000 | Loss: 3.2312 | LR: 4.74e-04 | 677.3ms/step | 29.5 samples/s
+Step 7090/44000 | Loss: 3.2743 | LR: 4.74e-04 | 676.6ms/step | 29.6 samples/s
+Step 7100/44000 | Loss: 3.2342 | LR: 4.74e-04 | 676.8ms/step | 29.6 samples/s
+Step 7110/44000 | Loss: 3.2184 | LR: 4.74e-04 | 677.9ms/step | 29.5 samples/s
+Step 7120/44000 | Loss: 3.2133 | LR: 4.74e-04 | 677.0ms/step | 29.5 samples/s
+Step 7130/44000 | Loss: 3.1702 | LR: 4.74e-04 | 678.6ms/step | 29.5 samples/s
+Step 7140/44000 | Loss: 3.2252 | LR: 4.74e-04 | 678.4ms/step | 29.5 samples/s
+Step 7150/44000 | Loss: 3.2479 | LR: 4.73e-04 | 678.2ms/step | 29.5 samples/s
+Step 7160/44000 | Loss: 3.2362 | LR: 4.73e-04 | 678.0ms/step | 29.5 samples/s
+Step 7170/44000 | Loss: 3.2109 | LR: 4.73e-04 | 678.4ms/step | 29.5 samples/s
+Step 7180/44000 | Loss: 3.2289 | LR: 4.73e-04 | 678.7ms/step | 29.5 samples/s
+Step 7190/44000 | Loss: 3.2519 | LR: 4.73e-04 | 678.4ms/step | 29.5 samples/s
+Step 7200/44000 | Loss: 3.2563 | LR: 4.73e-04 | 678.0ms/step | 29.5 samples/s
+Step 7210/44000 | Loss: 3.2352 | LR: 4.73e-04 | 677.6ms/step | 29.5 samples/s
+Step 7220/44000 | Loss: 3.2157 | LR: 4.73e-04 | 677.8ms/step | 29.5 samples/s
+Step 7230/44000 | Loss: 3.2302 | LR: 4.73e-04 | 677.1ms/step | 29.5 samples/s
+Step 7240/44000 | Loss: 3.2174 | LR: 4.73e-04 | 678.0ms/step | 29.5 samples/s
+Step 7250/44000 | Loss: 3.2539 | LR: 4.73e-04 | 677.8ms/step | 29.5 samples/s
+Validation at step 7250: Loss = 3.2385, Perplexity = 25.50
+Step 7260/44000 | Loss: 3.2460 | LR: 4.73e-04 | 1233.0ms/step | 16.2 samples/s
+Step 7270/44000 | Loss: 3.2156 | LR: 4.72e-04 | 678.9ms/step | 29.5 samples/s
+Step 7280/44000 | Loss: 3.2339 | LR: 4.72e-04 | 679.9ms/step | 29.4 samples/s
+Step 7290/44000 | Loss: 3.2530 | LR: 4.72e-04 | 680.5ms/step | 29.4 samples/s
+Step 7300/44000 | Loss: 3.2496 | LR: 4.72e-04 | 680.4ms/step | 29.4 samples/s
+Step 7310/44000 | Loss: 3.2499 | LR: 4.72e-04 | 679.6ms/step | 29.4 samples/s
+Step 7320/44000 | Loss: 3.2096 | LR: 4.72e-04 | 680.8ms/step | 29.4 samples/s
+Step 7330/44000 | Loss: 3.2435 | LR: 4.72e-04 | 679.0ms/step | 29.5 samples/s
+Step 7340/44000 | Loss: 3.2338 | LR: 4.72e-04 | 680.0ms/step | 29.4 samples/s
+Step 7350/44000 | Loss: 3.2175 | LR: 4.72e-04 | 680.6ms/step | 29.4 samples/s
+Step 7360/44000 | Loss: 3.2012 | LR: 4.72e-04 | 679.2ms/step | 29.4 samples/s
+Step 7370/44000 | Loss: 3.2560 | LR: 4.72e-04 | 678.3ms/step | 29.5 samples/s
+Step 7380/44000 | Loss: 3.2539 | LR: 4.72e-04 | 678.0ms/step | 29.5 samples/s
+Step 7390/44000 | Loss: 3.2795 | LR: 4.71e-04 | 677.7ms/step | 29.5 samples/s
+Step 7400/44000 | Loss: 3.2290 | LR: 4.71e-04 | 678.1ms/step | 29.5 samples/s
+Step 7410/44000 | Loss: 3.2432 | LR: 4.71e-04 | 678.1ms/step | 29.5 samples/s
+Step 7420/44000 | Loss: 3.2311 | LR: 4.71e-04 | 680.0ms/step | 29.4 samples/s
+Step 7430/44000 | Loss: 3.2289 | LR: 4.71e-04 | 678.3ms/step | 29.5 samples/s
+Step 7440/44000 | Loss: 3.2376 | LR: 4.71e-04 | 680.8ms/step | 29.4 samples/s
+Step 7450/44000 | Loss: 3.2647 | LR: 4.71e-04 | 679.5ms/step | 29.4 samples/s
+Step 7460/44000 | Loss: 3.2444 | LR: 4.71e-04 | 680.9ms/step | 29.4 samples/s
+Step 7470/44000 | Loss: 3.2463 | LR: 4.71e-04 | 680.5ms/step | 29.4 samples/s
+Step 7480/44000 | Loss: 3.2525 | LR: 4.71e-04 | 679.5ms/step | 29.4 samples/s
+Step 7490/44000 | Loss: 3.2028 | LR: 4.71e-04 | 681.6ms/step | 29.3 samples/s
+Step 7500/44000 | Loss: 3.1953 | LR: 4.71e-04 | 679.6ms/step | 29.4 samples/s
+Validation at step 7500: Loss = 3.2309, Perplexity = 25.30
+Step 7510/44000 | Loss: 3.2101 | LR: 4.70e-04 | 1237.4ms/step | 16.2 samples/s
+Step 7520/44000 | Loss: 3.2320 | LR: 4.70e-04 | 679.5ms/step | 29.4 samples/s
+Step 7530/44000 | Loss: 3.2523 | LR: 4.70e-04 | 679.8ms/step | 29.4 samples/s
+Step 7540/44000 | Loss: 3.2104 | LR: 4.70e-04 | 680.7ms/step | 29.4 samples/s
+Step 7550/44000 | Loss: 3.2043 | LR: 4.70e-04 | 680.1ms/step | 29.4 samples/s
+Step 7560/44000 | Loss: 3.2283 | LR: 4.70e-04 | 679.4ms/step | 29.4 samples/s
+Step 7570/44000 | Loss: 3.1746 | LR: 4.70e-04 | 680.1ms/step | 29.4 samples/s
+Step 7580/44000 | Loss: 3.1486 | LR: 4.70e-04 | 679.5ms/step | 29.4 samples/s
+Step 7590/44000 | Loss: 3.2140 | LR: 4.70e-04 | 678.8ms/step | 29.5 samples/s
+Step 7600/44000 | Loss: 3.2231 | LR: 4.70e-04 | 678.4ms/step | 29.5 samples/s
+Step 7610/44000 | Loss: 3.2206 | LR: 4.70e-04 | 678.7ms/step | 29.5 samples/s
+Step 7620/44000 | Loss: 3.2409 | LR: 4.70e-04 | 678.6ms/step | 29.5 samples/s
+Step 7630/44000 | Loss: 3.2120 | LR: 4.69e-04 | 679.0ms/step | 29.5 samples/s
+Step 7640/44000 | Loss: 3.1782 | LR: 4.69e-04 | 678.9ms/step | 29.5 samples/s
+Step 7650/44000 | Loss: 3.1998 | LR: 4.69e-04 | 680.3ms/step | 29.4 samples/s
+Step 7660/44000 | Loss: 3.2252 | LR: 4.69e-04 | 679.8ms/step | 29.4 samples/s
+Step 7670/44000 | Loss: 3.1973 | LR: 4.69e-04 | 678.9ms/step | 29.5 samples/s
+Step 7680/44000 | Loss: 3.2567 | LR: 4.69e-04 | 679.4ms/step | 29.4 samples/s
+Step 7690/44000 | Loss: 3.2256 | LR: 4.69e-04 | 680.2ms/step | 29.4 samples/s
+Step 7700/44000 | Loss: 3.1972 | LR: 4.69e-04 | 680.0ms/step | 29.4 samples/s
+Step 7710/44000 | Loss: 3.1891 | LR: 4.69e-04 | 680.3ms/step | 29.4 samples/s
+Step 7720/44000 | Loss: 3.2200 | LR: 4.69e-04 | 680.3ms/step | 29.4 samples/s
+Step 7730/44000 | Loss: 3.2071 | LR: 4.69e-04 | 678.8ms/step | 29.5 samples/s
+Step 7740/44000 | Loss: 3.1694 | LR: 4.68e-04 | 680.0ms/step | 29.4 samples/s
+Step 7750/44000 | Loss: 3.2097 | LR: 4.68e-04 | 681.5ms/step | 29.3 samples/s
+Validation at step 7750: Loss = 3.2223, Perplexity = 25.09
+Step 7760/44000 | Loss: 3.2055 | LR: 4.68e-04 | 1233.8ms/step | 16.2 samples/s
+Step 7770/44000 | Loss: 3.2165 | LR: 4.68e-04 | 678.0ms/step | 29.5 samples/s
+Step 7780/44000 | Loss: 3.2132 | LR: 4.68e-04 | 679.0ms/step | 29.5 samples/s
+Step 7790/44000 | Loss: 3.2165 | LR: 4.68e-04 | 679.5ms/step | 29.4 samples/s
+Step 7800/44000 | Loss: 3.2271 | LR: 4.68e-04 | 679.9ms/step | 29.4 samples/s
+Step 7810/44000 | Loss: 3.1735 | LR: 4.68e-04 | 678.1ms/step | 29.5 samples/s
+Step 7820/44000 | Loss: 3.2187 | LR: 4.68e-04 | 677.3ms/step | 29.5 samples/s
+Step 7830/44000 | Loss: 3.2056 | LR: 4.68e-04 | 678.4ms/step | 29.5 samples/s
+Step 7840/44000 | Loss: 3.1888 | LR: 4.68e-04 | 678.7ms/step | 29.5 samples/s
+Step 7850/44000 | Loss: 3.1858 | LR: 4.67e-04 | 679.5ms/step | 29.4 samples/s
+Step 7860/44000 | Loss: 3.2051 | LR: 4.67e-04 | 680.9ms/step | 29.4 samples/s
+Step 7870/44000 | Loss: 3.2081 | LR: 4.67e-04 | 679.9ms/step | 29.4 samples/s
+Step 7880/44000 | Loss: 3.2378 | LR: 4.67e-04 | 679.6ms/step | 29.4 samples/s
+Step 7890/44000 | Loss: 3.2040 | LR: 4.67e-04 | 680.0ms/step | 29.4 samples/s
+Step 7900/44000 | Loss: 3.2436 | LR: 4.67e-04 | 680.0ms/step | 29.4 samples/s
+Step 7910/44000 | Loss: 3.2239 | LR: 4.67e-04 | 678.6ms/step | 29.5 samples/s
+Step 7920/44000 | Loss: 3.2372 | LR: 4.67e-04 | 678.4ms/step | 29.5 samples/s
+Step 7930/44000 | Loss: 3.2606 | LR: 4.67e-04 | 678.1ms/step | 29.5 samples/s
+Step 7940/44000 | Loss: 3.2614 | LR: 4.67e-04 | 678.3ms/step | 29.5 samples/s
+Step 7950/44000 | Loss: 3.1729 | LR: 4.67e-04 | 679.1ms/step | 29.4 samples/s
+Step 7960/44000 | Loss: 3.2248 | LR: 4.66e-04 | 679.1ms/step | 29.5 samples/s
+Step 7970/44000 | Loss: 3.1600 | LR: 4.66e-04 | 680.0ms/step | 29.4 samples/s
+Step 7980/44000 | Loss: 3.1919 | LR: 4.66e-04 | 678.3ms/step | 29.5 samples/s
+Step 7990/44000 | Loss: 3.2139 | LR: 4.66e-04 | 680.5ms/step | 29.4 samples/s
+Step 8000/44000 | Loss: 3.2483 | LR: 4.66e-04 | 678.2ms/step | 29.5 samples/s
+Validation at step 8000: Loss = 3.2159, Perplexity = 24.93
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822/checkpoint-8000
+Step 8010/44000 | Loss: 3.2012 | LR: 4.66e-04 | 1521.3ms/step | 13.1 samples/s
+Step 8020/44000 | Loss: 3.2198 | LR: 4.66e-04 | 678.7ms/step | 29.5 samples/s
+Step 8030/44000 | Loss: 3.2153 | LR: 4.66e-04 | 679.2ms/step | 29.4 samples/s
+Step 8040/44000 | Loss: 3.2348 | LR: 4.66e-04 | 677.0ms/step | 29.5 samples/s
+Step 8050/44000 | Loss: 3.2063 | LR: 4.66e-04 | 676.9ms/step | 29.5 samples/s
+Step 8060/44000 | Loss: 3.2254 | LR: 4.66e-04 | 677.9ms/step | 29.5 samples/s
+Step 8070/44000 | Loss: 3.1721 | LR: 4.65e-04 | 678.9ms/step | 29.5 samples/s
+Step 8080/44000 | Loss: 3.1908 | LR: 4.65e-04 | 678.1ms/step | 29.5 samples/s
+Step 8090/44000 | Loss: 3.2304 | LR: 4.65e-04 | 679.3ms/step | 29.4 samples/s
+Step 8100/44000 | Loss: 3.2392 | LR: 4.65e-04 | 679.7ms/step | 29.4 samples/s
+Step 8110/44000 | Loss: 3.2200 | LR: 4.65e-04 | 679.5ms/step | 29.4 samples/s
+Step 8120/44000 | Loss: 3.1675 | LR: 4.65e-04 | 678.8ms/step | 29.5 samples/s

epfl_final/SparseAtt/wandb/run-20260708_205109-82re6m3i/files/requirements.txt

@@ -0,0 +1,141 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5

epfl_final/SparseAtt/wandb/run-20260708_205109-82re6m3i/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-07-08T20:51:09.024310Z",
+  "args":  [
+    "--config",
+    "configs/ladder8k/pool100.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "88298c7a418622530ca5426b22b1e5ebd10ca0cb"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-workspace-4-gpu-0-5",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "6202832625664"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-b817d241-acce-e5dd-a749-3447239c5207"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-165543e5-e049-5219-e717-e1216ac960de"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-47444fc7-ea8c-cef6-aac8-ef408dc8047b"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-37dc97ad-46ef-50d9-3619-3a6eaea1b6aa"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "3run8gukvolnx9v22g5316khqmraopq1"
+}

epfl_final/SparseAtt/wandb/run-20260708_205109-82re6m3i/logs/debug.log

@@ -0,0 +1,19 @@
+2026-07-08 20:51:09,052 INFO    MainThread:388 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-07-08 20:51:09,053 INFO    MainThread:388 [wandb_setup.py:_flush():81] Configure stats pid to 388
+2026-07-08 20:51:09,053 INFO    MainThread:388 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-08 20:51:09,053 INFO    MainThread:388 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_205109-82re6m3i/logs/debug.log
+2026-07-08 20:51:09,053 INFO    MainThread:388 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_205109-82re6m3i/logs/debug-internal.log
+2026-07-08 20:51:09,053 INFO    MainThread:388 [wandb_init.py:init():850] calling init triggers
+2026-07-08 20:51:09,053 INFO    MainThread:388 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool100/run_20260708_204822', 'resume_from_checkpoint': None, 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 5, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m pool100 8k nrb31 freqabl', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'resampled', 'freqablation', 'pool100'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 100, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_32', 'sparse_mask_seed': 0, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-07-08 20:51:09,053 INFO    MainThread:388 [wandb_init.py:init():898] starting backend
+2026-07-08 20:51:09,802 INFO    MainThread:388 [wandb_init.py:init():913] sending inform_init request
+2026-07-08 20:51:10,402 INFO    MainThread:388 [wandb_init.py:init():918] backend started and connected
+2026-07-08 20:51:10,406 INFO    MainThread:388 [wandb_init.py:init():988] updated telemetry
+2026-07-08 20:51:10,424 INFO    MainThread:388 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-07-08 20:51:12,169 INFO    MainThread:388 [wandb_init.py:init():1056] starting run threads in backend
+2026-07-08 20:51:12,432 INFO    MainThread:388 [wandb_run.py:_console_start():2554] atexit reg
+2026-07-08 20:51:12,432 INFO    MainThread:388 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-07-08 20:51:12,432 INFO    MainThread:388 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-07-08 20:51:12,432 INFO    MainThread:388 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-07-08 20:51:12,436 INFO    MainThread:388 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/run-20260708_205109-82re6m3i/run-82re6m3i.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6dffedc49f0fb5fdf06336ec159ee1639219c2ac7b66a086fe5305ecb2cca34f
+size 2588672

epfl_final/SparseAtt/wandb/run-20260708_205109-ti3a1x4c/files/output.log

@@ -0,0 +1,854 @@
+Starting training...
+Switching attention mode for step 1: sparse_stochastic (configured sparse attention)
+Step 10/44000 | Loss: 12.0382 | LR: 6.67e-06 | 6783.2ms/step | 2.9 samples/s
+Step 20/44000 | Loss: 11.9252 | LR: 1.33e-05 | 671.2ms/step | 29.8 samples/s
+Step 30/44000 | Loss: 11.5508 | LR: 2.00e-05 | 670.7ms/step | 29.8 samples/s
+Step 40/44000 | Loss: 11.0227 | LR: 2.67e-05 | 668.7ms/step | 29.9 samples/s
+Step 50/44000 | Loss: 10.7142 | LR: 3.33e-05 | 667.4ms/step | 30.0 samples/s
+Step 60/44000 | Loss: 10.4834 | LR: 4.00e-05 | 666.6ms/step | 30.0 samples/s
+Step 70/44000 | Loss: 10.2641 | LR: 4.67e-05 | 667.3ms/step | 30.0 samples/s
+Step 80/44000 | Loss: 9.9467 | LR: 5.33e-05 | 665.5ms/step | 30.1 samples/s
+Step 90/44000 | Loss: 9.5576 | LR: 6.00e-05 | 670.1ms/step | 29.8 samples/s
+Step 100/44000 | Loss: 9.1273 | LR: 6.67e-05 | 669.9ms/step | 29.9 samples/s
+Step 110/44000 | Loss: 8.6009 | LR: 7.33e-05 | 669.8ms/step | 29.9 samples/s
+Step 120/44000 | Loss: 8.1706 | LR: 8.00e-05 | 670.7ms/step | 29.8 samples/s
+Step 130/44000 | Loss: 7.8000 | LR: 8.67e-05 | 669.7ms/step | 29.9 samples/s
+Step 140/44000 | Loss: 7.5213 | LR: 9.33e-05 | 671.1ms/step | 29.8 samples/s
+Step 150/44000 | Loss: 7.3526 | LR: 1.00e-04 | 671.4ms/step | 29.8 samples/s
+Step 160/44000 | Loss: 7.1745 | LR: 1.07e-04 | 672.9ms/step | 29.7 samples/s
+Step 170/44000 | Loss: 7.1121 | LR: 1.13e-04 | 672.2ms/step | 29.8 samples/s
+Step 180/44000 | Loss: 7.0473 | LR: 1.20e-04 | 673.6ms/step | 29.7 samples/s
+Step 190/44000 | Loss: 6.9656 | LR: 1.27e-04 | 674.7ms/step | 29.6 samples/s
+Step 200/44000 | Loss: 6.8609 | LR: 1.33e-04 | 674.4ms/step | 29.7 samples/s
+Step 210/44000 | Loss: 6.7756 | LR: 1.40e-04 | 675.2ms/step | 29.6 samples/s
+Step 220/44000 | Loss: 6.6990 | LR: 1.47e-04 | 675.0ms/step | 29.6 samples/s
+Step 230/44000 | Loss: 6.5972 | LR: 1.53e-04 | 676.2ms/step | 29.6 samples/s
+Step 240/44000 | Loss: 6.5657 | LR: 1.60e-04 | 676.3ms/step | 29.6 samples/s
+Step 250/44000 | Loss: 6.4418 | LR: 1.67e-04 | 675.6ms/step | 29.6 samples/s
+Validation at step 250: Loss = 6.4439, Perplexity = 628.86
+Step 260/44000 | Loss: 6.4414 | LR: 1.73e-04 | 3935.5ms/step | 5.1 samples/s
+Step 270/44000 | Loss: 6.3787 | LR: 1.80e-04 | 676.7ms/step | 29.6 samples/s
+Step 280/44000 | Loss: 6.2945 | LR: 1.87e-04 | 676.6ms/step | 29.6 samples/s
+Step 290/44000 | Loss: 6.2513 | LR: 1.93e-04 | 678.2ms/step | 29.5 samples/s
+Step 300/44000 | Loss: 6.2507 | LR: 2.00e-04 | 676.8ms/step | 29.6 samples/s
+Step 310/44000 | Loss: 6.2086 | LR: 2.07e-04 | 679.3ms/step | 29.4 samples/s
+Step 320/44000 | Loss: 6.1304 | LR: 2.13e-04 | 677.8ms/step | 29.5 samples/s
+Step 330/44000 | Loss: 6.0840 | LR: 2.20e-04 | 678.7ms/step | 29.5 samples/s
+Step 340/44000 | Loss: 6.0627 | LR: 2.27e-04 | 678.2ms/step | 29.5 samples/s
+Step 350/44000 | Loss: 5.9852 | LR: 2.33e-04 | 679.5ms/step | 29.4 samples/s
+Step 360/44000 | Loss: 5.9827 | LR: 2.40e-04 | 679.9ms/step | 29.4 samples/s
+Step 370/44000 | Loss: 5.8705 | LR: 2.47e-04 | 678.0ms/step | 29.5 samples/s
+Step 380/44000 | Loss: 5.8938 | LR: 2.53e-04 | 677.8ms/step | 29.5 samples/s
+Step 390/44000 | Loss: 5.8599 | LR: 2.60e-04 | 678.1ms/step | 29.5 samples/s
+Step 400/44000 | Loss: 5.8131 | LR: 2.67e-04 | 678.1ms/step | 29.5 samples/s
+Step 410/44000 | Loss: 5.7854 | LR: 2.73e-04 | 678.6ms/step | 29.5 samples/s
+Step 420/44000 | Loss: 5.7321 | LR: 2.80e-04 | 678.1ms/step | 29.5 samples/s
+Step 430/44000 | Loss: 5.7388 | LR: 2.87e-04 | 678.6ms/step | 29.5 samples/s
+Step 440/44000 | Loss: 5.7220 | LR: 2.93e-04 | 678.3ms/step | 29.5 samples/s
+Step 450/44000 | Loss: 5.6070 | LR: 3.00e-04 | 680.3ms/step | 29.4 samples/s
+Step 460/44000 | Loss: 5.6095 | LR: 3.07e-04 | 677.9ms/step | 29.5 samples/s
+Step 470/44000 | Loss: 5.5903 | LR: 3.13e-04 | 678.9ms/step | 29.5 samples/s
+Step 480/44000 | Loss: 5.5445 | LR: 3.20e-04 | 679.7ms/step | 29.4 samples/s
+Step 490/44000 | Loss: 5.5292 | LR: 3.27e-04 | 679.7ms/step | 29.4 samples/s
+Step 500/44000 | Loss: 5.4936 | LR: 3.33e-04 | 678.5ms/step | 29.5 samples/s
+Validation at step 500: Loss = 5.4865, Perplexity = 241.40
+Step 510/44000 | Loss: 5.3963 | LR: 3.40e-04 | 1233.9ms/step | 16.2 samples/s
+Step 520/44000 | Loss: 5.3932 | LR: 3.47e-04 | 680.1ms/step | 29.4 samples/s
+Step 530/44000 | Loss: 5.3959 | LR: 3.53e-04 | 679.0ms/step | 29.5 samples/s
+Step 540/44000 | Loss: 5.3567 | LR: 3.60e-04 | 678.1ms/step | 29.5 samples/s
+Step 550/44000 | Loss: 5.3682 | LR: 3.67e-04 | 679.1ms/step | 29.5 samples/s
+Step 560/44000 | Loss: 5.3081 | LR: 3.73e-04 | 679.1ms/step | 29.4 samples/s
+Step 570/44000 | Loss: 5.3110 | LR: 3.80e-04 | 678.3ms/step | 29.5 samples/s
+Step 580/44000 | Loss: 5.2360 | LR: 3.87e-04 | 678.2ms/step | 29.5 samples/s
+Step 590/44000 | Loss: 5.2242 | LR: 3.93e-04 | 680.0ms/step | 29.4 samples/s
+Step 600/44000 | Loss: 5.2097 | LR: 4.00e-04 | 678.8ms/step | 29.5 samples/s
+Step 610/44000 | Loss: 5.1316 | LR: 4.07e-04 | 679.2ms/step | 29.4 samples/s
+Step 620/44000 | Loss: 5.1077 | LR: 4.13e-04 | 679.4ms/step | 29.4 samples/s
+Step 630/44000 | Loss: 5.1647 | LR: 4.20e-04 | 678.6ms/step | 29.5 samples/s
+Step 640/44000 | Loss: 5.1062 | LR: 4.27e-04 | 678.7ms/step | 29.5 samples/s
+Step 650/44000 | Loss: 5.0631 | LR: 4.33e-04 | 677.8ms/step | 29.5 samples/s
+Step 660/44000 | Loss: 5.0732 | LR: 4.40e-04 | 678.6ms/step | 29.5 samples/s
+Step 670/44000 | Loss: 5.0441 | LR: 4.47e-04 | 679.0ms/step | 29.5 samples/s
+Step 680/44000 | Loss: 5.0482 | LR: 4.53e-04 | 680.4ms/step | 29.4 samples/s
+Step 690/44000 | Loss: 4.9983 | LR: 4.60e-04 | 681.3ms/step | 29.4 samples/s
+Step 700/44000 | Loss: 4.9409 | LR: 4.67e-04 | 679.7ms/step | 29.4 samples/s
+Step 710/44000 | Loss: 4.9727 | LR: 4.73e-04 | 680.2ms/step | 29.4 samples/s
+Step 720/44000 | Loss: 4.8840 | LR: 4.80e-04 | 680.4ms/step | 29.4 samples/s
+Step 730/44000 | Loss: 4.9085 | LR: 4.87e-04 | 679.2ms/step | 29.4 samples/s
+Step 740/44000 | Loss: 4.8286 | LR: 4.93e-04 | 678.7ms/step | 29.5 samples/s
+Step 750/44000 | Loss: 4.7973 | LR: 5.00e-04 | 679.9ms/step | 29.4 samples/s
+Validation at step 750: Loss = 4.8014, Perplexity = 121.68
+Step 760/44000 | Loss: 4.7792 | LR: 5.00e-04 | 1233.5ms/step | 16.2 samples/s
+Step 770/44000 | Loss: 4.8051 | LR: 5.00e-04 | 680.5ms/step | 29.4 samples/s
+Step 780/44000 | Loss: 4.7135 | LR: 5.00e-04 | 680.4ms/step | 29.4 samples/s
+Step 790/44000 | Loss: 4.7176 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s
+Step 800/44000 | Loss: 4.7575 | LR: 5.00e-04 | 681.4ms/step | 29.4 samples/s
+Step 810/44000 | Loss: 4.6824 | LR: 5.00e-04 | 678.3ms/step | 29.5 samples/s
+Step 820/44000 | Loss: 4.6420 | LR: 5.00e-04 | 678.4ms/step | 29.5 samples/s
+Step 830/44000 | Loss: 4.6077 | LR: 5.00e-04 | 680.1ms/step | 29.4 samples/s
+Step 840/44000 | Loss: 4.5562 | LR: 5.00e-04 | 680.1ms/step | 29.4 samples/s
+Step 850/44000 | Loss: 4.5649 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s
+Step 860/44000 | Loss: 4.5487 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s
+Step 870/44000 | Loss: 4.5591 | LR: 5.00e-04 | 680.0ms/step | 29.4 samples/s
+Step 880/44000 | Loss: 4.4944 | LR: 5.00e-04 | 679.8ms/step | 29.4 samples/s
+Step 890/44000 | Loss: 4.4827 | LR: 5.00e-04 | 680.1ms/step | 29.4 samples/s
+Step 900/44000 | Loss: 4.4674 | LR: 5.00e-04 | 679.9ms/step | 29.4 samples/s
+Step 910/44000 | Loss: 4.4617 | LR: 5.00e-04 | 678.5ms/step | 29.5 samples/s
+Step 920/44000 | Loss: 4.4134 | LR: 5.00e-04 | 679.3ms/step | 29.4 samples/s
+Step 930/44000 | Loss: 4.4133 | LR: 5.00e-04 | 679.9ms/step | 29.4 samples/s
+Step 940/44000 | Loss: 4.3863 | LR: 5.00e-04 | 678.5ms/step | 29.5 samples/s
+Step 950/44000 | Loss: 4.4044 | LR: 5.00e-04 | 677.9ms/step | 29.5 samples/s
+Step 960/44000 | Loss: 4.3391 | LR: 5.00e-04 | 678.5ms/step | 29.5 samples/s
+Step 970/44000 | Loss: 4.3446 | LR: 5.00e-04 | 678.9ms/step | 29.5 samples/s
+Step 980/44000 | Loss: 4.3478 | LR: 5.00e-04 | 677.2ms/step | 29.5 samples/s
+Step 990/44000 | Loss: 4.3144 | LR: 5.00e-04 | 678.0ms/step | 29.5 samples/s
+Step 1000/44000 | Loss: 4.3165 | LR: 5.00e-04 | 678.5ms/step | 29.5 samples/s
+Validation at step 1000: Loss = 4.3113, Perplexity = 74.54
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-1000
+Step 1010/44000 | Loss: 4.3391 | LR: 5.00e-04 | 1524.7ms/step | 13.1 samples/s
+Step 1020/44000 | Loss: 4.2807 | LR: 5.00e-04 | 678.7ms/step | 29.5 samples/s
+Step 1030/44000 | Loss: 4.2830 | LR: 5.00e-04 | 678.1ms/step | 29.5 samples/s
+Step 1040/44000 | Loss: 4.2130 | LR: 5.00e-04 | 680.0ms/step | 29.4 samples/s
+Step 1050/44000 | Loss: 4.2590 | LR: 5.00e-04 | 679.8ms/step | 29.4 samples/s
+Step 1060/44000 | Loss: 4.2902 | LR: 5.00e-04 | 679.4ms/step | 29.4 samples/s
+Step 1070/44000 | Loss: 4.2019 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s
+Step 1080/44000 | Loss: 4.2037 | LR: 5.00e-04 | 680.9ms/step | 29.4 samples/s
+Step 1090/44000 | Loss: 4.2202 | LR: 5.00e-04 | 680.4ms/step | 29.4 samples/s
+Step 1100/44000 | Loss: 4.1972 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s
+Step 1110/44000 | Loss: 4.2082 | LR: 5.00e-04 | 678.7ms/step | 29.5 samples/s
+Step 1120/44000 | Loss: 4.1097 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s
+Step 1130/44000 | Loss: 4.1362 | LR: 5.00e-04 | 680.1ms/step | 29.4 samples/s
+Step 1140/44000 | Loss: 4.1240 | LR: 5.00e-04 | 679.8ms/step | 29.4 samples/s
+Step 1150/44000 | Loss: 4.1284 | LR: 5.00e-04 | 678.9ms/step | 29.5 samples/s
+Step 1160/44000 | Loss: 4.1355 | LR: 5.00e-04 | 679.7ms/step | 29.4 samples/s
+Step 1170/44000 | Loss: 4.1230 | LR: 5.00e-04 | 679.1ms/step | 29.5 samples/s
+Step 1180/44000 | Loss: 4.1392 | LR: 5.00e-04 | 681.1ms/step | 29.4 samples/s
+Step 1190/44000 | Loss: 4.0935 | LR: 5.00e-04 | 679.1ms/step | 29.4 samples/s
+Step 1200/44000 | Loss: 4.0989 | LR: 5.00e-04 | 678.7ms/step | 29.5 samples/s
+Step 1210/44000 | Loss: 4.1222 | LR: 5.00e-04 | 679.0ms/step | 29.5 samples/s
+Step 1220/44000 | Loss: 4.1045 | LR: 5.00e-04 | 680.0ms/step | 29.4 samples/s
+Step 1230/44000 | Loss: 4.0698 | LR: 5.00e-04 | 679.8ms/step | 29.4 samples/s
+Step 1240/44000 | Loss: 4.0740 | LR: 5.00e-04 | 680.5ms/step | 29.4 samples/s
+Step 1250/44000 | Loss: 4.0511 | LR: 5.00e-04 | 680.4ms/step | 29.4 samples/s
+Validation at step 1250: Loss = 4.0719, Perplexity = 58.67
+Step 1260/44000 | Loss: 4.1055 | LR: 5.00e-04 | 1235.6ms/step | 16.2 samples/s
+Step 1270/44000 | Loss: 4.0299 | LR: 5.00e-04 | 679.9ms/step | 29.4 samples/s
+Step 1280/44000 | Loss: 4.0458 | LR: 5.00e-04 | 678.7ms/step | 29.5 samples/s
+Step 1290/44000 | Loss: 4.0374 | LR: 5.00e-04 | 678.7ms/step | 29.5 samples/s
+Step 1300/44000 | Loss: 4.0425 | LR: 5.00e-04 | 679.4ms/step | 29.4 samples/s
+Step 1310/44000 | Loss: 3.9859 | LR: 5.00e-04 | 678.7ms/step | 29.5 samples/s
+Step 1320/44000 | Loss: 3.9726 | LR: 5.00e-04 | 679.5ms/step | 29.4 samples/s
+Step 1330/44000 | Loss: 3.9942 | LR: 5.00e-04 | 678.5ms/step | 29.5 samples/s
+Step 1340/44000 | Loss: 4.0381 | LR: 5.00e-04 | 680.3ms/step | 29.4 samples/s
+Step 1350/44000 | Loss: 3.9667 | LR: 5.00e-04 | 680.4ms/step | 29.4 samples/s
+Step 1360/44000 | Loss: 3.9881 | LR: 5.00e-04 | 680.4ms/step | 29.4 samples/s
+Step 1370/44000 | Loss: 3.9616 | LR: 5.00e-04 | 679.1ms/step | 29.4 samples/s
+Step 1380/44000 | Loss: 3.9808 | LR: 5.00e-04 | 678.5ms/step | 29.5 samples/s
+Step 1390/44000 | Loss: 3.9663 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s
+Step 1400/44000 | Loss: 3.9699 | LR: 5.00e-04 | 680.1ms/step | 29.4 samples/s
+Step 1410/44000 | Loss: 3.9459 | LR: 5.00e-04 | 679.9ms/step | 29.4 samples/s
+Step 1420/44000 | Loss: 3.9498 | LR: 5.00e-04 | 679.4ms/step | 29.4 samples/s
+Step 1430/44000 | Loss: 3.9607 | LR: 5.00e-04 | 677.7ms/step | 29.5 samples/s
+Step 1440/44000 | Loss: 3.8915 | LR: 5.00e-04 | 678.4ms/step | 29.5 samples/s
+Step 1450/44000 | Loss: 3.9141 | LR: 5.00e-04 | 678.3ms/step | 29.5 samples/s
+Step 1460/44000 | Loss: 3.9595 | LR: 5.00e-04 | 678.8ms/step | 29.5 samples/s
+Step 1470/44000 | Loss: 3.9038 | LR: 5.00e-04 | 679.2ms/step | 29.4 samples/s
+Step 1480/44000 | Loss: 3.8795 | LR: 5.00e-04 | 682.4ms/step | 29.3 samples/s
+Step 1490/44000 | Loss: 3.8771 | LR: 5.00e-04 | 679.9ms/step | 29.4 samples/s
+Step 1500/44000 | Loss: 3.9326 | LR: 5.00e-04 | 680.5ms/step | 29.4 samples/s
+Validation at step 1500: Loss = 3.9134, Perplexity = 50.07
+Step 1510/44000 | Loss: 3.9028 | LR: 5.00e-04 | 1233.6ms/step | 16.2 samples/s
+Step 1520/44000 | Loss: 3.8939 | LR: 5.00e-04 | 679.8ms/step | 29.4 samples/s
+Step 1530/44000 | Loss: 3.9195 | LR: 5.00e-04 | 679.9ms/step | 29.4 samples/s
+Step 1540/44000 | Loss: 3.8397 | LR: 5.00e-04 | 680.2ms/step | 29.4 samples/s
+Step 1550/44000 | Loss: 3.9178 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s
+Step 1560/44000 | Loss: 3.8780 | LR: 5.00e-04 | 678.8ms/step | 29.5 samples/s
+Step 1570/44000 | Loss: 3.8966 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s
+Step 1580/44000 | Loss: 3.8516 | LR: 5.00e-04 | 679.6ms/step | 29.4 samples/s
+Step 1590/44000 | Loss: 3.8534 | LR: 5.00e-04 | 678.3ms/step | 29.5 samples/s
+Step 1600/44000 | Loss: 3.8467 | LR: 5.00e-04 | 679.4ms/step | 29.4 samples/s
+Step 1610/44000 | Loss: 3.8594 | LR: 5.00e-04 | 679.7ms/step | 29.4 samples/s
+Step 1620/44000 | Loss: 3.8355 | LR: 5.00e-04 | 678.7ms/step | 29.5 samples/s
+Step 1630/44000 | Loss: 3.8507 | LR: 4.99e-04 | 678.7ms/step | 29.5 samples/s
+Step 1640/44000 | Loss: 3.8121 | LR: 4.99e-04 | 679.5ms/step | 29.4 samples/s
+Step 1650/44000 | Loss: 3.8513 | LR: 4.99e-04 | 677.8ms/step | 29.5 samples/s
+Step 1660/44000 | Loss: 3.8623 | LR: 4.99e-04 | 677.8ms/step | 29.5 samples/s
+Step 1670/44000 | Loss: 3.8700 | LR: 4.99e-04 | 678.7ms/step | 29.5 samples/s
+Step 1680/44000 | Loss: 3.8120 | LR: 4.99e-04 | 678.2ms/step | 29.5 samples/s
+Step 1690/44000 | Loss: 3.7932 | LR: 4.99e-04 | 678.1ms/step | 29.5 samples/s
+Step 1700/44000 | Loss: 3.8350 | LR: 4.99e-04 | 679.8ms/step | 29.4 samples/s
+Step 1710/44000 | Loss: 3.7861 | LR: 4.99e-04 | 679.7ms/step | 29.4 samples/s
+Step 1720/44000 | Loss: 3.8389 | LR: 4.99e-04 | 678.5ms/step | 29.5 samples/s
+Step 1730/44000 | Loss: 3.8071 | LR: 4.99e-04 | 678.6ms/step | 29.5 samples/s
+Step 1740/44000 | Loss: 3.7915 | LR: 4.99e-04 | 678.5ms/step | 29.5 samples/s
+Step 1750/44000 | Loss: 3.7694 | LR: 4.99e-04 | 678.9ms/step | 29.5 samples/s
+Validation at step 1750: Loss = 3.8036, Perplexity = 44.86
+Step 1760/44000 | Loss: 3.8084 | LR: 4.99e-04 | 1234.0ms/step | 16.2 samples/s
+Step 1770/44000 | Loss: 3.8108 | LR: 4.99e-04 | 679.8ms/step | 29.4 samples/s
+Step 1780/44000 | Loss: 3.8069 | LR: 4.99e-04 | 679.3ms/step | 29.4 samples/s
+Step 1790/44000 | Loss: 3.7994 | LR: 4.99e-04 | 679.4ms/step | 29.4 samples/s
+Step 1800/44000 | Loss: 3.7784 | LR: 4.99e-04 | 680.0ms/step | 29.4 samples/s
+Step 1810/44000 | Loss: 3.7567 | LR: 4.99e-04 | 678.3ms/step | 29.5 samples/s
+Step 1820/44000 | Loss: 3.7668 | LR: 4.99e-04 | 678.6ms/step | 29.5 samples/s
+Step 1830/44000 | Loss: 3.7511 | LR: 4.99e-04 | 678.6ms/step | 29.5 samples/s
+Step 1840/44000 | Loss: 3.7417 | LR: 4.99e-04 | 678.8ms/step | 29.5 samples/s
+Step 1850/44000 | Loss: 3.7737 | LR: 4.99e-04 | 678.3ms/step | 29.5 samples/s
+Step 1860/44000 | Loss: 3.8001 | LR: 4.99e-04 | 678.0ms/step | 29.5 samples/s
+Step 1870/44000 | Loss: 3.7855 | LR: 4.99e-04 | 678.1ms/step | 29.5 samples/s
+Step 1880/44000 | Loss: 3.7448 | LR: 4.99e-04 | 680.2ms/step | 29.4 samples/s
+Step 1890/44000 | Loss: 3.7230 | LR: 4.99e-04 | 678.4ms/step | 29.5 samples/s
+Step 1900/44000 | Loss: 3.7565 | LR: 4.99e-04 | 679.6ms/step | 29.4 samples/s
+Step 1910/44000 | Loss: 3.7632 | LR: 4.99e-04 | 680.1ms/step | 29.4 samples/s
+Step 1920/44000 | Loss: 3.7395 | LR: 4.99e-04 | 679.9ms/step | 29.4 samples/s
+Step 1930/44000 | Loss: 3.7510 | LR: 4.99e-04 | 679.1ms/step | 29.5 samples/s
+Step 1940/44000 | Loss: 3.7140 | LR: 4.99e-04 | 679.0ms/step | 29.5 samples/s
+Step 1950/44000 | Loss: 3.6843 | LR: 4.99e-04 | 679.7ms/step | 29.4 samples/s
+Step 1960/44000 | Loss: 3.7313 | LR: 4.99e-04 | 680.3ms/step | 29.4 samples/s
+Step 1970/44000 | Loss: 3.6992 | LR: 4.99e-04 | 679.1ms/step | 29.5 samples/s
+Step 1980/44000 | Loss: 3.6899 | LR: 4.99e-04 | 679.1ms/step | 29.5 samples/s
+Step 1990/44000 | Loss: 3.7579 | LR: 4.99e-04 | 679.2ms/step | 29.4 samples/s
+Step 2000/44000 | Loss: 3.7184 | LR: 4.99e-04 | 678.3ms/step | 29.5 samples/s
+Validation at step 2000: Loss = 3.7220, Perplexity = 41.35
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-2000
+Step 2010/44000 | Loss: 3.7271 | LR: 4.99e-04 | 1526.4ms/step | 13.1 samples/s
+Step 2020/44000 | Loss: 3.7128 | LR: 4.99e-04 | 679.4ms/step | 29.4 samples/s
+Step 2030/44000 | Loss: 3.6913 | LR: 4.99e-04 | 677.8ms/step | 29.5 samples/s
+Step 2040/44000 | Loss: 3.7242 | LR: 4.99e-04 | 676.4ms/step | 29.6 samples/s
+Step 2050/44000 | Loss: 3.7057 | LR: 4.99e-04 | 677.0ms/step | 29.5 samples/s
+Step 2060/44000 | Loss: 3.7248 | LR: 4.99e-04 | 676.5ms/step | 29.6 samples/s
+Step 2070/44000 | Loss: 3.7039 | LR: 4.99e-04 | 677.8ms/step | 29.5 samples/s
+Step 2080/44000 | Loss: 3.7238 | LR: 4.99e-04 | 678.7ms/step | 29.5 samples/s
+Step 2090/44000 | Loss: 3.7226 | LR: 4.99e-04 | 677.6ms/step | 29.5 samples/s
+Step 2100/44000 | Loss: 3.6758 | LR: 4.99e-04 | 678.8ms/step | 29.5 samples/s
+Step 2110/44000 | Loss: 3.7050 | LR: 4.99e-04 | 678.3ms/step | 29.5 samples/s
+Step 2120/44000 | Loss: 3.6907 | LR: 4.99e-04 | 678.7ms/step | 29.5 samples/s
+Step 2130/44000 | Loss: 3.6842 | LR: 4.99e-04 | 679.1ms/step | 29.4 samples/s
+Step 2140/44000 | Loss: 3.6678 | LR: 4.99e-04 | 680.2ms/step | 29.4 samples/s
+Step 2150/44000 | Loss: 3.6774 | LR: 4.99e-04 | 679.5ms/step | 29.4 samples/s
+Step 2160/44000 | Loss: 3.6910 | LR: 4.99e-04 | 678.2ms/step | 29.5 samples/s
+Step 2170/44000 | Loss: 3.6656 | LR: 4.99e-04 | 677.4ms/step | 29.5 samples/s
+Step 2180/44000 | Loss: 3.6832 | LR: 4.99e-04 | 678.5ms/step | 29.5 samples/s
+Step 2190/44000 | Loss: 3.6646 | LR: 4.99e-04 | 678.6ms/step | 29.5 samples/s
+Step 2200/44000 | Loss: 3.6518 | LR: 4.99e-04 | 678.5ms/step | 29.5 samples/s
+Step 2210/44000 | Loss: 3.6337 | LR: 4.99e-04 | 681.1ms/step | 29.4 samples/s
+Step 2220/44000 | Loss: 3.6290 | LR: 4.99e-04 | 679.5ms/step | 29.4 samples/s
+Step 2230/44000 | Loss: 3.6832 | LR: 4.99e-04 | 679.4ms/step | 29.4 samples/s
+Step 2240/44000 | Loss: 3.6783 | LR: 4.99e-04 | 678.9ms/step | 29.5 samples/s
+Step 2250/44000 | Loss: 3.6675 | LR: 4.99e-04 | 679.2ms/step | 29.4 samples/s
+Validation at step 2250: Loss = 3.6566, Perplexity = 38.73
+Step 2260/44000 | Loss: 3.6536 | LR: 4.98e-04 | 1235.3ms/step | 16.2 samples/s
+Step 2270/44000 | Loss: 3.6374 | LR: 4.98e-04 | 678.3ms/step | 29.5 samples/s
+Step 2280/44000 | Loss: 3.6526 | LR: 4.98e-04 | 684.8ms/step | 29.2 samples/s
+Step 2290/44000 | Loss: 3.6724 | LR: 4.98e-04 | 679.6ms/step | 29.4 samples/s
+Step 2300/44000 | Loss: 3.6020 | LR: 4.98e-04 | 678.9ms/step | 29.5 samples/s
+Step 2310/44000 | Loss: 3.6434 | LR: 4.98e-04 | 679.8ms/step | 29.4 samples/s
+Step 2320/44000 | Loss: 3.6527 | LR: 4.98e-04 | 680.1ms/step | 29.4 samples/s
+Step 2330/44000 | Loss: 3.6444 | LR: 4.98e-04 | 679.8ms/step | 29.4 samples/s
+Step 2340/44000 | Loss: 3.6432 | LR: 4.98e-04 | 679.7ms/step | 29.4 samples/s
+Step 2350/44000 | Loss: 3.6070 | LR: 4.98e-04 | 679.9ms/step | 29.4 samples/s
+Step 2360/44000 | Loss: 3.6312 | LR: 4.98e-04 | 678.6ms/step | 29.5 samples/s
+Step 2370/44000 | Loss: 3.6329 | LR: 4.98e-04 | 680.4ms/step | 29.4 samples/s
+Step 2380/44000 | Loss: 3.5914 | LR: 4.98e-04 | 679.4ms/step | 29.4 samples/s
+Step 2390/44000 | Loss: 3.5777 | LR: 4.98e-04 | 680.4ms/step | 29.4 samples/s
+Step 2400/44000 | Loss: 3.6346 | LR: 4.98e-04 | 679.8ms/step | 29.4 samples/s
+Step 2410/44000 | Loss: 3.6268 | LR: 4.98e-04 | 680.8ms/step | 29.4 samples/s
+Step 2420/44000 | Loss: 3.6248 | LR: 4.98e-04 | 678.7ms/step | 29.5 samples/s
+Step 2430/44000 | Loss: 3.6167 | LR: 4.98e-04 | 680.8ms/step | 29.4 samples/s
+Step 2440/44000 | Loss: 3.6231 | LR: 4.98e-04 | 679.7ms/step | 29.4 samples/s
+Step 2450/44000 | Loss: 3.5986 | LR: 4.98e-04 | 680.0ms/step | 29.4 samples/s
+Step 2460/44000 | Loss: 3.6018 | LR: 4.98e-04 | 679.5ms/step | 29.4 samples/s
+Step 2470/44000 | Loss: 3.6084 | LR: 4.98e-04 | 680.4ms/step | 29.4 samples/s
+Step 2480/44000 | Loss: 3.6210 | LR: 4.98e-04 | 680.5ms/step | 29.4 samples/s
+Step 2490/44000 | Loss: 3.5856 | LR: 4.98e-04 | 679.8ms/step | 29.4 samples/s
+Step 2500/44000 | Loss: 3.5987 | LR: 4.98e-04 | 679.8ms/step | 29.4 samples/s
+Validation at step 2500: Loss = 3.6030, Perplexity = 36.71
+Step 2510/44000 | Loss: 3.6005 | LR: 4.98e-04 | 1235.9ms/step | 16.2 samples/s
+Step 2520/44000 | Loss: 3.5890 | LR: 4.98e-04 | 680.2ms/step | 29.4 samples/s
+Step 2530/44000 | Loss: 3.6016 | LR: 4.98e-04 | 680.3ms/step | 29.4 samples/s
+Step 2540/44000 | Loss: 3.5777 | LR: 4.98e-04 | 680.4ms/step | 29.4 samples/s
+Step 2550/44000 | Loss: 3.5435 | LR: 4.98e-04 | 677.7ms/step | 29.5 samples/s
+Step 2560/44000 | Loss: 3.5676 | LR: 4.98e-04 | 680.0ms/step | 29.4 samples/s
+Step 2570/44000 | Loss: 3.5774 | LR: 4.98e-04 | 680.2ms/step | 29.4 samples/s
+Step 2580/44000 | Loss: 3.5951 | LR: 4.98e-04 | 680.0ms/step | 29.4 samples/s
+Step 2590/44000 | Loss: 3.5799 | LR: 4.98e-04 | 679.3ms/step | 29.4 samples/s
+Step 2600/44000 | Loss: 3.5484 | LR: 4.98e-04 | 678.9ms/step | 29.5 samples/s
+Step 2610/44000 | Loss: 3.5159 | LR: 4.98e-04 | 679.9ms/step | 29.4 samples/s
+Step 2620/44000 | Loss: 3.5791 | LR: 4.98e-04 | 679.9ms/step | 29.4 samples/s
+Step 2630/44000 | Loss: 3.5372 | LR: 4.98e-04 | 679.8ms/step | 29.4 samples/s
+Step 2640/44000 | Loss: 3.5582 | LR: 4.98e-04 | 679.5ms/step | 29.4 samples/s
+Step 2650/44000 | Loss: 3.5712 | LR: 4.98e-04 | 679.3ms/step | 29.4 samples/s
+Step 2660/44000 | Loss: 3.5420 | LR: 4.98e-04 | 680.6ms/step | 29.4 samples/s
+Step 2670/44000 | Loss: 3.5468 | LR: 4.98e-04 | 680.2ms/step | 29.4 samples/s
+Step 2680/44000 | Loss: 3.5787 | LR: 4.98e-04 | 680.2ms/step | 29.4 samples/s
+Step 2690/44000 | Loss: 3.5915 | LR: 4.98e-04 | 679.5ms/step | 29.4 samples/s
+Step 2700/44000 | Loss: 3.5287 | LR: 4.97e-04 | 680.3ms/step | 29.4 samples/s
+Step 2710/44000 | Loss: 3.5807 | LR: 4.97e-04 | 679.9ms/step | 29.4 samples/s
+Step 2720/44000 | Loss: 3.5960 | LR: 4.97e-04 | 680.0ms/step | 29.4 samples/s
+Step 2730/44000 | Loss: 3.5774 | LR: 4.97e-04 | 680.2ms/step | 29.4 samples/s
+Step 2740/44000 | Loss: 3.5472 | LR: 4.97e-04 | 680.0ms/step | 29.4 samples/s
+Step 2750/44000 | Loss: 3.5272 | LR: 4.97e-04 | 678.7ms/step | 29.5 samples/s
+Validation at step 2750: Loss = 3.5600, Perplexity = 35.16
+Step 2760/44000 | Loss: 3.5219 | LR: 4.97e-04 | 1235.3ms/step | 16.2 samples/s
+Step 2770/44000 | Loss: 3.5977 | LR: 4.97e-04 | 679.5ms/step | 29.4 samples/s
+Step 2780/44000 | Loss: 3.5897 | LR: 4.97e-04 | 679.2ms/step | 29.4 samples/s
+Step 2790/44000 | Loss: 3.5675 | LR: 4.97e-04 | 679.6ms/step | 29.4 samples/s
+Step 2800/44000 | Loss: 3.5683 | LR: 4.97e-04 | 679.1ms/step | 29.4 samples/s
+Step 2810/44000 | Loss: 3.5227 | LR: 4.97e-04 | 681.0ms/step | 29.4 samples/s
+Step 2820/44000 | Loss: 3.5505 | LR: 4.97e-04 | 678.3ms/step | 29.5 samples/s
+Step 2830/44000 | Loss: 3.5359 | LR: 4.97e-04 | 678.1ms/step | 29.5 samples/s
+Step 2840/44000 | Loss: 3.5107 | LR: 4.97e-04 | 680.1ms/step | 29.4 samples/s
+Step 2850/44000 | Loss: 3.5808 | LR: 4.97e-04 | 679.7ms/step | 29.4 samples/s
+Step 2860/44000 | Loss: 3.5556 | LR: 4.97e-04 | 679.6ms/step | 29.4 samples/s
+Step 2870/44000 | Loss: 3.5299 | LR: 4.97e-04 | 681.9ms/step | 29.3 samples/s
+Step 2880/44000 | Loss: 3.4852 | LR: 4.97e-04 | 678.8ms/step | 29.5 samples/s
+Step 2890/44000 | Loss: 3.5212 | LR: 4.97e-04 | 679.0ms/step | 29.5 samples/s
+Step 2900/44000 | Loss: 3.5748 | LR: 4.97e-04 | 679.8ms/step | 29.4 samples/s
+Step 2910/44000 | Loss: 3.5205 | LR: 4.97e-04 | 679.5ms/step | 29.4 samples/s
+Step 2920/44000 | Loss: 3.5393 | LR: 4.97e-04 | 681.4ms/step | 29.4 samples/s
+Step 2930/44000 | Loss: 3.5234 | LR: 4.97e-04 | 681.2ms/step | 29.4 samples/s
+Step 2940/44000 | Loss: 3.5169 | LR: 4.97e-04 | 679.8ms/step | 29.4 samples/s
+Step 2950/44000 | Loss: 3.5483 | LR: 4.97e-04 | 679.1ms/step | 29.5 samples/s
+Step 2960/44000 | Loss: 3.5279 | LR: 4.97e-04 | 679.4ms/step | 29.4 samples/s
+Step 2970/44000 | Loss: 3.5482 | LR: 4.97e-04 | 679.9ms/step | 29.4 samples/s
+Step 2980/44000 | Loss: 3.6133 | LR: 4.97e-04 | 679.4ms/step | 29.4 samples/s
+Step 2990/44000 | Loss: 3.5098 | LR: 4.97e-04 | 679.9ms/step | 29.4 samples/s
+Step 3000/44000 | Loss: 3.5279 | LR: 4.97e-04 | 678.3ms/step | 29.5 samples/s
+Validation at step 3000: Loss = 3.5247, Perplexity = 33.94
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-3000
+Step 3010/44000 | Loss: 3.5169 | LR: 4.97e-04 | 1523.9ms/step | 13.1 samples/s
+Step 3020/44000 | Loss: 3.5089 | LR: 4.97e-04 | 679.6ms/step | 29.4 samples/s
+Step 3030/44000 | Loss: 3.5554 | LR: 4.97e-04 | 679.5ms/step | 29.4 samples/s
+Step 3040/44000 | Loss: 3.5123 | LR: 4.97e-04 | 679.2ms/step | 29.4 samples/s
+Step 3050/44000 | Loss: 3.5534 | LR: 4.97e-04 | 678.6ms/step | 29.5 samples/s
+Step 3060/44000 | Loss: 3.4931 | LR: 4.96e-04 | 678.3ms/step | 29.5 samples/s
+Step 3070/44000 | Loss: 3.5219 | LR: 4.96e-04 | 679.7ms/step | 29.4 samples/s
+Step 3080/44000 | Loss: 3.5287 | LR: 4.96e-04 | 680.1ms/step | 29.4 samples/s
+Step 3090/44000 | Loss: 3.4914 | LR: 4.96e-04 | 680.2ms/step | 29.4 samples/s
+Step 3100/44000 | Loss: 3.5144 | LR: 4.96e-04 | 680.0ms/step | 29.4 samples/s
+Step 3110/44000 | Loss: 3.4814 | LR: 4.96e-04 | 679.9ms/step | 29.4 samples/s
+Step 3120/44000 | Loss: 3.4979 | LR: 4.96e-04 | 679.5ms/step | 29.4 samples/s
+Step 3130/44000 | Loss: 3.4916 | LR: 4.96e-04 | 679.8ms/step | 29.4 samples/s
+Step 3140/44000 | Loss: 3.4857 | LR: 4.96e-04 | 678.6ms/step | 29.5 samples/s
+Step 3150/44000 | Loss: 3.4805 | LR: 4.96e-04 | 680.9ms/step | 29.4 samples/s
+Step 3160/44000 | Loss: 3.5001 | LR: 4.96e-04 | 679.5ms/step | 29.4 samples/s
+Step 3170/44000 | Loss: 3.4908 | LR: 4.96e-04 | 679.1ms/step | 29.4 samples/s
+Step 3180/44000 | Loss: 3.5278 | LR: 4.96e-04 | 678.6ms/step | 29.5 samples/s
+Step 3190/44000 | Loss: 3.5230 | LR: 4.96e-04 | 679.3ms/step | 29.4 samples/s
+Step 3200/44000 | Loss: 3.4853 | LR: 4.96e-04 | 679.8ms/step | 29.4 samples/s
+Step 3210/44000 | Loss: 3.4717 | LR: 4.96e-04 | 679.9ms/step | 29.4 samples/s
+Step 3220/44000 | Loss: 3.5071 | LR: 4.96e-04 | 679.2ms/step | 29.4 samples/s
+Step 3230/44000 | Loss: 3.4747 | LR: 4.96e-04 | 679.4ms/step | 29.4 samples/s
+Step 3240/44000 | Loss: 3.4899 | LR: 4.96e-04 | 678.4ms/step | 29.5 samples/s
+Step 3250/44000 | Loss: 3.4701 | LR: 4.96e-04 | 678.3ms/step | 29.5 samples/s
+Validation at step 3250: Loss = 3.4909, Perplexity = 32.82
+Step 3260/44000 | Loss: 3.5190 | LR: 4.96e-04 | 1243.6ms/step | 16.1 samples/s
+Step 3270/44000 | Loss: 3.4723 | LR: 4.96e-04 | 681.4ms/step | 29.4 samples/s
+Step 3280/44000 | Loss: 3.4731 | LR: 4.96e-04 | 680.0ms/step | 29.4 samples/s
+Step 3290/44000 | Loss: 3.4796 | LR: 4.96e-04 | 679.9ms/step | 29.4 samples/s
+Step 3300/44000 | Loss: 3.4657 | LR: 4.96e-04 | 679.5ms/step | 29.4 samples/s
+Step 3310/44000 | Loss: 3.4818 | LR: 4.96e-04 | 678.8ms/step | 29.5 samples/s
+Step 3320/44000 | Loss: 3.4657 | LR: 4.96e-04 | 681.1ms/step | 29.4 samples/s
+Step 3330/44000 | Loss: 3.4865 | LR: 4.96e-04 | 679.2ms/step | 29.4 samples/s
+Step 3340/44000 | Loss: 3.4610 | LR: 4.96e-04 | 679.4ms/step | 29.4 samples/s
+Step 3350/44000 | Loss: 3.4906 | LR: 4.96e-04 | 679.6ms/step | 29.4 samples/s
+Step 3360/44000 | Loss: 3.4482 | LR: 4.96e-04 | 679.8ms/step | 29.4 samples/s
+Step 3370/44000 | Loss: 3.4783 | LR: 4.95e-04 | 680.1ms/step | 29.4 samples/s
+Step 3380/44000 | Loss: 3.4658 | LR: 4.95e-04 | 678.9ms/step | 29.5 samples/s
+Step 3390/44000 | Loss: 3.4788 | LR: 4.95e-04 | 679.5ms/step | 29.4 samples/s
+Step 3400/44000 | Loss: 3.4584 | LR: 4.95e-04 | 680.0ms/step | 29.4 samples/s
+Step 3410/44000 | Loss: 3.4768 | LR: 4.95e-04 | 679.1ms/step | 29.4 samples/s
+Step 3420/44000 | Loss: 3.3836 | LR: 4.95e-04 | 680.6ms/step | 29.4 samples/s
+Step 3430/44000 | Loss: 3.4377 | LR: 4.95e-04 | 680.8ms/step | 29.4 samples/s
+Step 3440/44000 | Loss: 3.4689 | LR: 4.95e-04 | 678.5ms/step | 29.5 samples/s
+Step 3450/44000 | Loss: 3.4345 | LR: 4.95e-04 | 678.8ms/step | 29.5 samples/s
+Step 3460/44000 | Loss: 3.4805 | LR: 4.95e-04 | 678.5ms/step | 29.5 samples/s
+Step 3470/44000 | Loss: 3.4827 | LR: 4.95e-04 | 680.1ms/step | 29.4 samples/s
+Step 3480/44000 | Loss: 3.4605 | LR: 4.95e-04 | 680.3ms/step | 29.4 samples/s
+Step 3490/44000 | Loss: 3.4687 | LR: 4.95e-04 | 679.3ms/step | 29.4 samples/s
+Step 3500/44000 | Loss: 3.4217 | LR: 4.95e-04 | 680.3ms/step | 29.4 samples/s
+Validation at step 3500: Loss = 3.4627, Perplexity = 31.90
+Step 3510/44000 | Loss: 3.4708 | LR: 4.95e-04 | 1234.0ms/step | 16.2 samples/s
+Step 3520/44000 | Loss: 3.4309 | LR: 4.95e-04 | 679.2ms/step | 29.4 samples/s
+Step 3530/44000 | Loss: 3.4098 | LR: 4.95e-04 | 679.4ms/step | 29.4 samples/s
+Step 3540/44000 | Loss: 3.4374 | LR: 4.95e-04 | 679.3ms/step | 29.4 samples/s
+Step 3550/44000 | Loss: 3.4533 | LR: 4.95e-04 | 680.2ms/step | 29.4 samples/s
+Step 3560/44000 | Loss: 3.4913 | LR: 4.95e-04 | 680.6ms/step | 29.4 samples/s
+Step 3570/44000 | Loss: 3.4153 | LR: 4.95e-04 | 680.2ms/step | 29.4 samples/s
+Step 3580/44000 | Loss: 3.4416 | LR: 4.95e-04 | 678.9ms/step | 29.5 samples/s
+Step 3590/44000 | Loss: 3.4475 | LR: 4.95e-04 | 678.5ms/step | 29.5 samples/s
+Step 3600/44000 | Loss: 3.4678 | LR: 4.95e-04 | 678.7ms/step | 29.5 samples/s
+Step 3610/44000 | Loss: 3.4923 | LR: 4.95e-04 | 681.2ms/step | 29.4 samples/s
+Step 3620/44000 | Loss: 3.4472 | LR: 4.95e-04 | 678.6ms/step | 29.5 samples/s
+Step 3630/44000 | Loss: 3.4275 | LR: 4.95e-04 | 678.9ms/step | 29.5 samples/s
+Step 3640/44000 | Loss: 3.4488 | LR: 4.95e-04 | 679.7ms/step | 29.4 samples/s
+Step 3650/44000 | Loss: 3.4783 | LR: 4.94e-04 | 679.3ms/step | 29.4 samples/s
+Step 3660/44000 | Loss: 3.4170 | LR: 4.94e-04 | 678.3ms/step | 29.5 samples/s
+Step 3670/44000 | Loss: 3.4606 | LR: 4.94e-04 | 679.0ms/step | 29.5 samples/s
+Step 3680/44000 | Loss: 3.4019 | LR: 4.94e-04 | 678.2ms/step | 29.5 samples/s
+Step 3690/44000 | Loss: 3.4091 | LR: 4.94e-04 | 679.1ms/step | 29.4 samples/s
+Step 3700/44000 | Loss: 3.4554 | LR: 4.94e-04 | 678.5ms/step | 29.5 samples/s
+Step 3710/44000 | Loss: 3.4502 | LR: 4.94e-04 | 679.8ms/step | 29.4 samples/s
+Step 3720/44000 | Loss: 3.4324 | LR: 4.94e-04 | 679.4ms/step | 29.4 samples/s
+Step 3730/44000 | Loss: 3.4059 | LR: 4.94e-04 | 680.7ms/step | 29.4 samples/s
+Step 3740/44000 | Loss: 3.4658 | LR: 4.94e-04 | 679.9ms/step | 29.4 samples/s
+Step 3750/44000 | Loss: 3.4212 | LR: 4.94e-04 | 682.5ms/step | 29.3 samples/s
+Validation at step 3750: Loss = 3.4375, Perplexity = 31.11
+Step 3760/44000 | Loss: 3.4491 | LR: 4.94e-04 | 1234.2ms/step | 16.2 samples/s
+Step 3770/44000 | Loss: 3.4293 | LR: 4.94e-04 | 679.2ms/step | 29.4 samples/s
+Step 3780/44000 | Loss: 3.4265 | LR: 4.94e-04 | 679.0ms/step | 29.5 samples/s
+Step 3790/44000 | Loss: 3.4338 | LR: 4.94e-04 | 680.9ms/step | 29.4 samples/s
+Step 3800/44000 | Loss: 3.4025 | LR: 4.94e-04 | 680.3ms/step | 29.4 samples/s
+Step 3810/44000 | Loss: 3.3858 | LR: 4.94e-04 | 680.1ms/step | 29.4 samples/s
+Step 3820/44000 | Loss: 3.3710 | LR: 4.94e-04 | 679.0ms/step | 29.5 samples/s
+Step 3830/44000 | Loss: 3.4053 | LR: 4.94e-04 | 680.0ms/step | 29.4 samples/s
+Step 3840/44000 | Loss: 3.4694 | LR: 4.94e-04 | 679.6ms/step | 29.4 samples/s
+Step 3850/44000 | Loss: 3.4273 | LR: 4.94e-04 | 679.0ms/step | 29.5 samples/s
+Step 3860/44000 | Loss: 3.4149 | LR: 4.94e-04 | 679.7ms/step | 29.4 samples/s
+Step 3870/44000 | Loss: 3.4650 | LR: 4.94e-04 | 678.9ms/step | 29.5 samples/s
+Step 3880/44000 | Loss: 3.4368 | LR: 4.94e-04 | 678.9ms/step | 29.5 samples/s
+Step 3890/44000 | Loss: 3.4628 | LR: 4.94e-04 | 681.1ms/step | 29.4 samples/s
+Step 3900/44000 | Loss: 3.4075 | LR: 4.93e-04 | 679.9ms/step | 29.4 samples/s
+Step 3910/44000 | Loss: 3.4479 | LR: 4.93e-04 | 677.4ms/step | 29.5 samples/s
+Step 3920/44000 | Loss: 3.4100 | LR: 4.93e-04 | 678.3ms/step | 29.5 samples/s
+Step 3930/44000 | Loss: 3.4134 | LR: 4.93e-04 | 679.7ms/step | 29.4 samples/s
+Step 3940/44000 | Loss: 3.4183 | LR: 4.93e-04 | 680.4ms/step | 29.4 samples/s
+Step 3950/44000 | Loss: 3.3578 | LR: 4.93e-04 | 681.2ms/step | 29.4 samples/s
+Step 3960/44000 | Loss: 3.4280 | LR: 4.93e-04 | 680.6ms/step | 29.4 samples/s
+Step 3970/44000 | Loss: 3.4507 | LR: 4.93e-04 | 679.3ms/step | 29.4 samples/s
+Step 3980/44000 | Loss: 3.3956 | LR: 4.93e-04 | 679.0ms/step | 29.5 samples/s
+Step 3990/44000 | Loss: 3.3910 | LR: 4.93e-04 | 680.8ms/step | 29.4 samples/s
+Step 4000/44000 | Loss: 3.4093 | LR: 4.93e-04 | 679.1ms/step | 29.4 samples/s
+Validation at step 4000: Loss = 3.4173, Perplexity = 30.49
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-4000
+Step 4010/44000 | Loss: 3.4274 | LR: 4.93e-04 | 1541.5ms/step | 13.0 samples/s
+Step 4020/44000 | Loss: 3.4592 | LR: 4.93e-04 | 680.5ms/step | 29.4 samples/s
+Step 4030/44000 | Loss: 3.3700 | LR: 4.93e-04 | 679.1ms/step | 29.5 samples/s
+Step 4040/44000 | Loss: 3.3770 | LR: 4.93e-04 | 679.2ms/step | 29.4 samples/s
+Step 4050/44000 | Loss: 3.3930 | LR: 4.93e-04 | 677.3ms/step | 29.5 samples/s
+Step 4060/44000 | Loss: 3.3864 | LR: 4.93e-04 | 678.8ms/step | 29.5 samples/s
+Step 4070/44000 | Loss: 3.4125 | LR: 4.93e-04 | 679.3ms/step | 29.4 samples/s
+Step 4080/44000 | Loss: 3.4098 | LR: 4.93e-04 | 681.7ms/step | 29.3 samples/s
+Step 4090/44000 | Loss: 3.4028 | LR: 4.93e-04 | 678.7ms/step | 29.5 samples/s
+Step 4100/44000 | Loss: 3.3925 | LR: 4.93e-04 | 679.3ms/step | 29.4 samples/s
+Step 4110/44000 | Loss: 3.4158 | LR: 4.93e-04 | 679.5ms/step | 29.4 samples/s
+Step 4120/44000 | Loss: 3.4153 | LR: 4.93e-04 | 680.1ms/step | 29.4 samples/s
+Step 4130/44000 | Loss: 3.4069 | LR: 4.93e-04 | 680.1ms/step | 29.4 samples/s
+Step 4140/44000 | Loss: 3.4074 | LR: 4.92e-04 | 680.0ms/step | 29.4 samples/s
+Step 4150/44000 | Loss: 3.3860 | LR: 4.92e-04 | 678.3ms/step | 29.5 samples/s
+Step 4160/44000 | Loss: 3.4024 | LR: 4.92e-04 | 677.9ms/step | 29.5 samples/s
+Step 4170/44000 | Loss: 3.3716 | LR: 4.92e-04 | 678.8ms/step | 29.5 samples/s
+Step 4180/44000 | Loss: 3.3577 | LR: 4.92e-04 | 679.4ms/step | 29.4 samples/s
+Step 4190/44000 | Loss: 3.4122 | LR: 4.92e-04 | 679.2ms/step | 29.4 samples/s
+Step 4200/44000 | Loss: 3.3779 | LR: 4.92e-04 | 679.2ms/step | 29.4 samples/s
+Step 4210/44000 | Loss: 3.4211 | LR: 4.92e-04 | 679.9ms/step | 29.4 samples/s
+Step 4220/44000 | Loss: 3.3901 | LR: 4.92e-04 | 680.5ms/step | 29.4 samples/s
+Step 4230/44000 | Loss: 3.4316 | LR: 4.92e-04 | 678.8ms/step | 29.5 samples/s
+Step 4240/44000 | Loss: 3.3808 | LR: 4.92e-04 | 680.2ms/step | 29.4 samples/s
+Step 4250/44000 | Loss: 3.4069 | LR: 4.92e-04 | 678.8ms/step | 29.5 samples/s
+Validation at step 4250: Loss = 3.3935, Perplexity = 29.77
+Step 4260/44000 | Loss: 3.3471 | LR: 4.92e-04 | 1234.2ms/step | 16.2 samples/s
+Step 4270/44000 | Loss: 3.3606 | LR: 4.92e-04 | 679.8ms/step | 29.4 samples/s
+Step 4280/44000 | Loss: 3.4021 | LR: 4.92e-04 | 679.5ms/step | 29.4 samples/s
+Step 4290/44000 | Loss: 3.3672 | LR: 4.92e-04 | 678.8ms/step | 29.5 samples/s
+Step 4300/44000 | Loss: 3.4024 | LR: 4.92e-04 | 678.4ms/step | 29.5 samples/s
+Step 4310/44000 | Loss: 3.3616 | LR: 4.92e-04 | 679.2ms/step | 29.4 samples/s
+Step 4320/44000 | Loss: 3.3728 | LR: 4.92e-04 | 680.0ms/step | 29.4 samples/s
+Step 4330/44000 | Loss: 3.3557 | LR: 4.92e-04 | 677.8ms/step | 29.5 samples/s
+Step 4340/44000 | Loss: 3.3848 | LR: 4.92e-04 | 679.7ms/step | 29.4 samples/s
+Step 4350/44000 | Loss: 3.3405 | LR: 4.92e-04 | 679.4ms/step | 29.4 samples/s
+Step 4360/44000 | Loss: 3.3781 | LR: 4.91e-04 | 678.0ms/step | 29.5 samples/s
+Step 4370/44000 | Loss: 3.3946 | LR: 4.91e-04 | 680.6ms/step | 29.4 samples/s
+Step 4380/44000 | Loss: 3.3441 | LR: 4.91e-04 | 679.8ms/step | 29.4 samples/s
+Step 4390/44000 | Loss: 3.3535 | LR: 4.91e-04 | 679.8ms/step | 29.4 samples/s
+Step 4400/44000 | Loss: 3.4129 | LR: 4.91e-04 | 678.8ms/step | 29.5 samples/s
+Step 4410/44000 | Loss: 3.3832 | LR: 4.91e-04 | 677.8ms/step | 29.5 samples/s
+Step 4420/44000 | Loss: 3.3845 | LR: 4.91e-04 | 679.2ms/step | 29.4 samples/s
+Step 4430/44000 | Loss: 3.3751 | LR: 4.91e-04 | 679.1ms/step | 29.5 samples/s
+Step 4440/44000 | Loss: 3.3846 | LR: 4.91e-04 | 679.8ms/step | 29.4 samples/s
+Step 4450/44000 | Loss: 3.3631 | LR: 4.91e-04 | 678.6ms/step | 29.5 samples/s
+Step 4460/44000 | Loss: 3.3365 | LR: 4.91e-04 | 678.0ms/step | 29.5 samples/s
+Step 4470/44000 | Loss: 3.3994 | LR: 4.91e-04 | 679.4ms/step | 29.4 samples/s
+Step 4480/44000 | Loss: 3.3678 | LR: 4.91e-04 | 680.4ms/step | 29.4 samples/s
+Step 4490/44000 | Loss: 3.3312 | LR: 4.91e-04 | 679.9ms/step | 29.4 samples/s
+Step 4500/44000 | Loss: 3.3581 | LR: 4.91e-04 | 679.9ms/step | 29.4 samples/s
+Validation at step 4500: Loss = 3.3756, Perplexity = 29.24
+Step 4510/44000 | Loss: 3.3550 | LR: 4.91e-04 | 1235.7ms/step | 16.2 samples/s
+Step 4520/44000 | Loss: 3.3489 | LR: 4.91e-04 | 680.2ms/step | 29.4 samples/s
+Step 4530/44000 | Loss: 3.3322 | LR: 4.91e-04 | 680.4ms/step | 29.4 samples/s
+Step 4540/44000 | Loss: 3.3758 | LR: 4.91e-04 | 679.2ms/step | 29.4 samples/s
+Step 4550/44000 | Loss: 3.3705 | LR: 4.91e-04 | 678.8ms/step | 29.5 samples/s
+Step 4560/44000 | Loss: 3.3866 | LR: 4.90e-04 | 678.9ms/step | 29.5 samples/s
+Step 4570/44000 | Loss: 3.3803 | LR: 4.90e-04 | 679.3ms/step | 29.4 samples/s
+Step 4580/44000 | Loss: 3.3635 | LR: 4.90e-04 | 679.8ms/step | 29.4 samples/s
+Step 4590/44000 | Loss: 3.3132 | LR: 4.90e-04 | 679.9ms/step | 29.4 samples/s
+Step 4600/44000 | Loss: 3.3504 | LR: 4.90e-04 | 679.7ms/step | 29.4 samples/s
+Step 4610/44000 | Loss: 3.3584 | LR: 4.90e-04 | 678.7ms/step | 29.5 samples/s
+Step 4620/44000 | Loss: 3.3225 | LR: 4.90e-04 | 679.3ms/step | 29.4 samples/s
+Step 4630/44000 | Loss: 3.3467 | LR: 4.90e-04 | 678.1ms/step | 29.5 samples/s
+Step 4640/44000 | Loss: 3.3604 | LR: 4.90e-04 | 679.9ms/step | 29.4 samples/s
+Step 4650/44000 | Loss: 3.3247 | LR: 4.90e-04 | 680.5ms/step | 29.4 samples/s
+Step 4660/44000 | Loss: 3.3196 | LR: 4.90e-04 | 679.1ms/step | 29.5 samples/s
+Step 4670/44000 | Loss: 3.3940 | LR: 4.90e-04 | 678.5ms/step | 29.5 samples/s
+Step 4680/44000 | Loss: 3.3543 | LR: 4.90e-04 | 680.0ms/step | 29.4 samples/s
+Step 4690/44000 | Loss: 3.4117 | LR: 4.90e-04 | 678.3ms/step | 29.5 samples/s
+Step 4700/44000 | Loss: 3.3923 | LR: 4.90e-04 | 679.6ms/step | 29.4 samples/s
+Step 4710/44000 | Loss: 3.3810 | LR: 4.90e-04 | 678.0ms/step | 29.5 samples/s
+Step 4720/44000 | Loss: 3.3317 | LR: 4.90e-04 | 678.5ms/step | 29.5 samples/s
+Step 4730/44000 | Loss: 3.3653 | LR: 4.90e-04 | 678.9ms/step | 29.5 samples/s
+Step 4740/44000 | Loss: 3.3559 | LR: 4.90e-04 | 678.2ms/step | 29.5 samples/s
+Step 4750/44000 | Loss: 3.3137 | LR: 4.90e-04 | 678.6ms/step | 29.5 samples/s
+Validation at step 4750: Loss = 3.3607, Perplexity = 28.81
+Step 4760/44000 | Loss: 3.3728 | LR: 4.89e-04 | 1234.3ms/step | 16.2 samples/s
+Step 4770/44000 | Loss: 3.3637 | LR: 4.89e-04 | 678.5ms/step | 29.5 samples/s
+Step 4780/44000 | Loss: 3.3674 | LR: 4.89e-04 | 679.8ms/step | 29.4 samples/s
+Step 4790/44000 | Loss: 3.3470 | LR: 4.89e-04 | 678.9ms/step | 29.5 samples/s
+Step 4800/44000 | Loss: 3.3552 | LR: 4.89e-04 | 679.3ms/step | 29.4 samples/s
+Step 4810/44000 | Loss: 3.3407 | LR: 4.89e-04 | 679.7ms/step | 29.4 samples/s
+Step 4820/44000 | Loss: 3.3467 | LR: 4.89e-04 | 680.5ms/step | 29.4 samples/s
+Step 4830/44000 | Loss: 3.3377 | LR: 4.89e-04 | 678.5ms/step | 29.5 samples/s
+Step 4840/44000 | Loss: 3.3188 | LR: 4.89e-04 | 679.7ms/step | 29.4 samples/s
+Step 4850/44000 | Loss: 3.3334 | LR: 4.89e-04 | 683.2ms/step | 29.3 samples/s
+Step 4860/44000 | Loss: 3.3322 | LR: 4.89e-04 | 681.0ms/step | 29.4 samples/s
+Step 4870/44000 | Loss: 3.3272 | LR: 4.89e-04 | 680.9ms/step | 29.4 samples/s
+Step 4880/44000 | Loss: 3.3476 | LR: 4.89e-04 | 681.2ms/step | 29.4 samples/s
+Step 4890/44000 | Loss: 3.3145 | LR: 4.89e-04 | 680.5ms/step | 29.4 samples/s
+Step 4900/44000 | Loss: 3.3838 | LR: 4.89e-04 | 685.0ms/step | 29.2 samples/s
+Step 4910/44000 | Loss: 3.3637 | LR: 4.89e-04 | 688.5ms/step | 29.1 samples/s
+Step 4920/44000 | Loss: 3.3630 | LR: 4.89e-04 | 689.4ms/step | 29.0 samples/s
+Step 4930/44000 | Loss: 3.3524 | LR: 4.89e-04 | 687.1ms/step | 29.1 samples/s
+Step 4940/44000 | Loss: 3.3524 | LR: 4.89e-04 | 683.6ms/step | 29.3 samples/s
+Step 4950/44000 | Loss: 3.3078 | LR: 4.88e-04 | 683.8ms/step | 29.2 samples/s
+Step 4960/44000 | Loss: 3.3596 | LR: 4.88e-04 | 684.7ms/step | 29.2 samples/s
+Step 4970/44000 | Loss: 3.3109 | LR: 4.88e-04 | 684.1ms/step | 29.2 samples/s
+Step 4980/44000 | Loss: 3.3340 | LR: 4.88e-04 | 683.6ms/step | 29.3 samples/s
+Step 4990/44000 | Loss: 3.3376 | LR: 4.88e-04 | 682.6ms/step | 29.3 samples/s
+Step 5000/44000 | Loss: 3.3362 | LR: 4.88e-04 | 684.5ms/step | 29.2 samples/s
+Validation at step 5000: Loss = 3.3428, Perplexity = 28.30
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-5000
+Step 5010/44000 | Loss: 3.3187 | LR: 4.88e-04 | 1530.9ms/step | 13.1 samples/s
+Step 5020/44000 | Loss: 3.3284 | LR: 4.88e-04 | 683.4ms/step | 29.3 samples/s
+Step 5030/44000 | Loss: 3.3320 | LR: 4.88e-04 | 682.6ms/step | 29.3 samples/s
+Step 5040/44000 | Loss: 3.3599 | LR: 4.88e-04 | 682.4ms/step | 29.3 samples/s
+Step 5050/44000 | Loss: 3.3305 | LR: 4.88e-04 | 681.7ms/step | 29.3 samples/s
+Step 5060/44000 | Loss: 3.3416 | LR: 4.88e-04 | 683.2ms/step | 29.3 samples/s
+Step 5070/44000 | Loss: 3.3174 | LR: 4.88e-04 | 685.3ms/step | 29.2 samples/s
+Step 5080/44000 | Loss: 3.3351 | LR: 4.88e-04 | 684.8ms/step | 29.2 samples/s
+Step 5090/44000 | Loss: 3.3322 | LR: 4.88e-04 | 683.2ms/step | 29.3 samples/s
+Step 5100/44000 | Loss: 3.3526 | LR: 4.88e-04 | 682.2ms/step | 29.3 samples/s
+Step 5110/44000 | Loss: 3.3092 | LR: 4.88e-04 | 684.6ms/step | 29.2 samples/s
+Step 5120/44000 | Loss: 3.3815 | LR: 4.88e-04 | 683.5ms/step | 29.3 samples/s
+Step 5130/44000 | Loss: 3.3388 | LR: 4.87e-04 | 683.6ms/step | 29.3 samples/s
+Step 5140/44000 | Loss: 3.3184 | LR: 4.87e-04 | 683.6ms/step | 29.3 samples/s
+Step 5150/44000 | Loss: 3.3296 | LR: 4.87e-04 | 683.3ms/step | 29.3 samples/s
+Step 5160/44000 | Loss: 3.3362 | LR: 4.87e-04 | 683.3ms/step | 29.3 samples/s
+Step 5170/44000 | Loss: 3.2838 | LR: 4.87e-04 | 684.1ms/step | 29.2 samples/s
+Step 5180/44000 | Loss: 3.3286 | LR: 4.87e-04 | 682.8ms/step | 29.3 samples/s
+Step 5190/44000 | Loss: 3.3495 | LR: 4.87e-04 | 684.0ms/step | 29.2 samples/s
+Step 5200/44000 | Loss: 3.3265 | LR: 4.87e-04 | 682.3ms/step | 29.3 samples/s
+Step 5210/44000 | Loss: 3.3450 | LR: 4.87e-04 | 682.5ms/step | 29.3 samples/s
+Step 5220/44000 | Loss: 3.3372 | LR: 4.87e-04 | 684.2ms/step | 29.2 samples/s
+Step 5230/44000 | Loss: 3.3439 | LR: 4.87e-04 | 683.1ms/step | 29.3 samples/s
+Step 5240/44000 | Loss: 3.3951 | LR: 4.87e-04 | 684.0ms/step | 29.2 samples/s
+Step 5250/44000 | Loss: 3.3348 | LR: 4.87e-04 | 684.0ms/step | 29.2 samples/s
+Validation at step 5250: Loss = 3.3283, Perplexity = 27.89
+Step 5260/44000 | Loss: 3.3578 | LR: 4.87e-04 | 1248.7ms/step | 16.0 samples/s
+Step 5270/44000 | Loss: 3.3456 | LR: 4.87e-04 | 687.0ms/step | 29.1 samples/s
+Step 5280/44000 | Loss: 3.3424 | LR: 4.87e-04 | 685.5ms/step | 29.2 samples/s
+Step 5290/44000 | Loss: 3.3156 | LR: 4.87e-04 | 683.2ms/step | 29.3 samples/s
+Step 5300/44000 | Loss: 3.3169 | LR: 4.86e-04 | 684.7ms/step | 29.2 samples/s
+Step 5310/44000 | Loss: 3.3450 | LR: 4.86e-04 | 683.3ms/step | 29.3 samples/s
+Step 5320/44000 | Loss: 3.2940 | LR: 4.86e-04 | 683.7ms/step | 29.3 samples/s
+Step 5330/44000 | Loss: 3.3424 | LR: 4.86e-04 | 684.4ms/step | 29.2 samples/s
+Step 5340/44000 | Loss: 3.3076 | LR: 4.86e-04 | 683.0ms/step | 29.3 samples/s
+Step 5350/44000 | Loss: 3.2986 | LR: 4.86e-04 | 684.6ms/step | 29.2 samples/s
+Step 5360/44000 | Loss: 3.3042 | LR: 4.86e-04 | 682.7ms/step | 29.3 samples/s
+Step 5370/44000 | Loss: 3.3298 | LR: 4.86e-04 | 684.5ms/step | 29.2 samples/s
+Step 5380/44000 | Loss: 3.3142 | LR: 4.86e-04 | 684.7ms/step | 29.2 samples/s
+Step 5390/44000 | Loss: 3.2867 | LR: 4.86e-04 | 682.6ms/step | 29.3 samples/s
+Step 5400/44000 | Loss: 3.3236 | LR: 4.86e-04 | 682.8ms/step | 29.3 samples/s
+Step 5410/44000 | Loss: 3.2990 | LR: 4.86e-04 | 683.0ms/step | 29.3 samples/s
+Step 5420/44000 | Loss: 3.3215 | LR: 4.86e-04 | 680.0ms/step | 29.4 samples/s
+Step 5430/44000 | Loss: 3.3168 | LR: 4.86e-04 | 680.1ms/step | 29.4 samples/s
+Step 5440/44000 | Loss: 3.3473 | LR: 4.86e-04 | 679.8ms/step | 29.4 samples/s
+Step 5450/44000 | Loss: 3.2748 | LR: 4.86e-04 | 680.3ms/step | 29.4 samples/s
+Step 5460/44000 | Loss: 3.2848 | LR: 4.86e-04 | 679.7ms/step | 29.4 samples/s
+Step 5470/44000 | Loss: 3.2872 | LR: 4.85e-04 | 679.0ms/step | 29.5 samples/s
+Step 5480/44000 | Loss: 3.3316 | LR: 4.85e-04 | 680.5ms/step | 29.4 samples/s
+Step 5490/44000 | Loss: 3.3487 | LR: 4.85e-04 | 679.2ms/step | 29.4 samples/s
+Step 5500/44000 | Loss: 3.3209 | LR: 4.85e-04 | 679.5ms/step | 29.4 samples/s
+Validation at step 5500: Loss = 3.3158, Perplexity = 27.55
+Step 5510/44000 | Loss: 3.2648 | LR: 4.85e-04 | 1242.4ms/step | 16.1 samples/s
+Step 5520/44000 | Loss: 3.2905 | LR: 4.85e-04 | 683.8ms/step | 29.2 samples/s
+Step 5530/44000 | Loss: 3.2811 | LR: 4.85e-04 | 682.1ms/step | 29.3 samples/s
+Step 5540/44000 | Loss: 3.2864 | LR: 4.85e-04 | 682.0ms/step | 29.3 samples/s
+Step 5550/44000 | Loss: 3.3184 | LR: 4.85e-04 | 682.8ms/step | 29.3 samples/s
+Step 5560/44000 | Loss: 3.3184 | LR: 4.85e-04 | 682.0ms/step | 29.3 samples/s
+Step 5570/44000 | Loss: 3.2975 | LR: 4.85e-04 | 682.0ms/step | 29.3 samples/s
+Step 5580/44000 | Loss: 3.3139 | LR: 4.85e-04 | 682.6ms/step | 29.3 samples/s
+Step 5590/44000 | Loss: 3.3363 | LR: 4.85e-04 | 681.7ms/step | 29.3 samples/s
+Step 5600/44000 | Loss: 3.3200 | LR: 4.85e-04 | 682.2ms/step | 29.3 samples/s
+Step 5610/44000 | Loss: 3.2936 | LR: 4.85e-04 | 680.5ms/step | 29.4 samples/s
+Step 5620/44000 | Loss: 3.2959 | LR: 4.85e-04 | 680.5ms/step | 29.4 samples/s
+Step 5630/44000 | Loss: 3.2888 | LR: 4.84e-04 | 679.6ms/step | 29.4 samples/s
+Step 5640/44000 | Loss: 3.3318 | LR: 4.84e-04 | 682.2ms/step | 29.3 samples/s
+Step 5650/44000 | Loss: 3.2412 | LR: 4.84e-04 | 682.8ms/step | 29.3 samples/s
+Step 5660/44000 | Loss: 3.2890 | LR: 4.84e-04 | 682.7ms/step | 29.3 samples/s
+Step 5670/44000 | Loss: 3.3116 | LR: 4.84e-04 | 682.6ms/step | 29.3 samples/s
+Step 5680/44000 | Loss: 3.2914 | LR: 4.84e-04 | 683.7ms/step | 29.3 samples/s
+Step 5690/44000 | Loss: 3.2913 | LR: 4.84e-04 | 682.6ms/step | 29.3 samples/s
+Step 5700/44000 | Loss: 3.2328 | LR: 4.84e-04 | 681.8ms/step | 29.3 samples/s
+Step 5710/44000 | Loss: 3.2992 | LR: 4.84e-04 | 683.0ms/step | 29.3 samples/s
+Step 5720/44000 | Loss: 3.3119 | LR: 4.84e-04 | 681.8ms/step | 29.3 samples/s
+Step 5730/44000 | Loss: 3.2860 | LR: 4.84e-04 | 682.5ms/step | 29.3 samples/s
+Step 5740/44000 | Loss: 3.2499 | LR: 4.84e-04 | 682.7ms/step | 29.3 samples/s
+Step 5750/44000 | Loss: 3.2661 | LR: 4.84e-04 | 681.6ms/step | 29.3 samples/s
+Validation at step 5750: Loss = 3.2998, Perplexity = 27.11
+Step 5760/44000 | Loss: 3.3369 | LR: 4.84e-04 | 1237.9ms/step | 16.2 samples/s
+Step 5770/44000 | Loss: 3.3070 | LR: 4.84e-04 | 680.1ms/step | 29.4 samples/s
+Step 5780/44000 | Loss: 3.2880 | LR: 4.83e-04 | 679.3ms/step | 29.4 samples/s
+Step 5790/44000 | Loss: 3.2553 | LR: 4.83e-04 | 680.5ms/step | 29.4 samples/s
+Step 5800/44000 | Loss: 3.3024 | LR: 4.83e-04 | 680.0ms/step | 29.4 samples/s
+Step 5810/44000 | Loss: 3.3459 | LR: 4.83e-04 | 680.1ms/step | 29.4 samples/s
+Step 5820/44000 | Loss: 3.3006 | LR: 4.83e-04 | 680.9ms/step | 29.4 samples/s
+Step 5830/44000 | Loss: 3.3122 | LR: 4.83e-04 | 680.9ms/step | 29.4 samples/s
+Step 5840/44000 | Loss: 3.2838 | LR: 4.83e-04 | 680.8ms/step | 29.4 samples/s
+Step 5850/44000 | Loss: 3.2904 | LR: 4.83e-04 | 681.9ms/step | 29.3 samples/s
+Step 5860/44000 | Loss: 3.2707 | LR: 4.83e-04 | 679.1ms/step | 29.5 samples/s
+Step 5870/44000 | Loss: 3.2643 | LR: 4.83e-04 | 680.5ms/step | 29.4 samples/s
+Step 5880/44000 | Loss: 3.2992 | LR: 4.83e-04 | 681.2ms/step | 29.4 samples/s
+Step 5890/44000 | Loss: 3.3205 | LR: 4.83e-04 | 681.4ms/step | 29.4 samples/s
+Step 5900/44000 | Loss: 3.3075 | LR: 4.83e-04 | 681.2ms/step | 29.4 samples/s
+Step 5910/44000 | Loss: 3.3168 | LR: 4.83e-04 | 680.2ms/step | 29.4 samples/s
+Step 5920/44000 | Loss: 3.2993 | LR: 4.83e-04 | 679.7ms/step | 29.4 samples/s
+Step 5930/44000 | Loss: 3.2549 | LR: 4.83e-04 | 680.4ms/step | 29.4 samples/s
+Step 5940/44000 | Loss: 3.2812 | LR: 4.82e-04 | 679.9ms/step | 29.4 samples/s
+Step 5950/44000 | Loss: 3.2856 | LR: 4.82e-04 | 680.8ms/step | 29.4 samples/s
+Step 5960/44000 | Loss: 3.2767 | LR: 4.82e-04 | 678.6ms/step | 29.5 samples/s
+Step 5970/44000 | Loss: 3.3060 | LR: 4.82e-04 | 680.9ms/step | 29.4 samples/s
+Step 5980/44000 | Loss: 3.2944 | LR: 4.82e-04 | 681.5ms/step | 29.3 samples/s
+Step 5990/44000 | Loss: 3.2877 | LR: 4.82e-04 | 680.4ms/step | 29.4 samples/s
+Step 6000/44000 | Loss: 3.2886 | LR: 4.82e-04 | 681.8ms/step | 29.3 samples/s
+Validation at step 6000: Loss = 3.2881, Perplexity = 26.79
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-6000
+Step 6010/44000 | Loss: 3.3164 | LR: 4.82e-04 | 1529.9ms/step | 13.1 samples/s
+Step 6020/44000 | Loss: 3.3169 | LR: 4.82e-04 | 680.4ms/step | 29.4 samples/s
+Step 6030/44000 | Loss: 3.2748 | LR: 4.82e-04 | 681.6ms/step | 29.3 samples/s
+Step 6040/44000 | Loss: 3.2842 | LR: 4.82e-04 | 680.4ms/step | 29.4 samples/s
+Step 6050/44000 | Loss: 3.2479 | LR: 4.82e-04 | 681.6ms/step | 29.3 samples/s
+Step 6060/44000 | Loss: 3.2579 | LR: 4.82e-04 | 682.3ms/step | 29.3 samples/s
+Step 6070/44000 | Loss: 3.2966 | LR: 4.82e-04 | 682.2ms/step | 29.3 samples/s
+Step 6080/44000 | Loss: 3.2917 | LR: 4.81e-04 | 681.9ms/step | 29.3 samples/s
+Step 6090/44000 | Loss: 3.3131 | LR: 4.81e-04 | 682.0ms/step | 29.3 samples/s
+Step 6100/44000 | Loss: 3.2950 | LR: 4.81e-04 | 683.6ms/step | 29.3 samples/s
+Step 6110/44000 | Loss: 3.2380 | LR: 4.81e-04 | 681.4ms/step | 29.4 samples/s
+Step 6120/44000 | Loss: 3.2746 | LR: 4.81e-04 | 682.8ms/step | 29.3 samples/s
+Step 6130/44000 | Loss: 3.2626 | LR: 4.81e-04 | 682.2ms/step | 29.3 samples/s
+Step 6140/44000 | Loss: 3.2603 | LR: 4.81e-04 | 682.0ms/step | 29.3 samples/s
+Step 6150/44000 | Loss: 3.2538 | LR: 4.81e-04 | 683.3ms/step | 29.3 samples/s
+Step 6160/44000 | Loss: 3.2764 | LR: 4.81e-04 | 683.2ms/step | 29.3 samples/s
+Step 6170/44000 | Loss: 3.2947 | LR: 4.81e-04 | 681.6ms/step | 29.3 samples/s
+Step 6180/44000 | Loss: 3.3079 | LR: 4.81e-04 | 680.8ms/step | 29.4 samples/s
+Step 6190/44000 | Loss: 3.2699 | LR: 4.81e-04 | 682.7ms/step | 29.3 samples/s
+Step 6200/44000 | Loss: 3.2474 | LR: 4.81e-04 | 681.2ms/step | 29.4 samples/s
+Step 6210/44000 | Loss: 3.2506 | LR: 4.81e-04 | 682.0ms/step | 29.3 samples/s
+Step 6220/44000 | Loss: 3.2324 | LR: 4.81e-04 | 681.5ms/step | 29.3 samples/s
+Step 6230/44000 | Loss: 3.2779 | LR: 4.80e-04 | 681.7ms/step | 29.3 samples/s
+Step 6240/44000 | Loss: 3.2363 | LR: 4.80e-04 | 680.4ms/step | 29.4 samples/s
+Step 6250/44000 | Loss: 3.2897 | LR: 4.80e-04 | 682.7ms/step | 29.3 samples/s
+Validation at step 6250: Loss = 3.2784, Perplexity = 26.53
+Step 6260/44000 | Loss: 3.2699 | LR: 4.80e-04 | 1239.7ms/step | 16.1 samples/s
+Step 6270/44000 | Loss: 3.2876 | LR: 4.80e-04 | 681.1ms/step | 29.4 samples/s
+Step 6280/44000 | Loss: 3.2835 | LR: 4.80e-04 | 681.9ms/step | 29.3 samples/s
+Step 6290/44000 | Loss: 3.2915 | LR: 4.80e-04 | 682.5ms/step | 29.3 samples/s
+Step 6300/44000 | Loss: 3.2651 | LR: 4.80e-04 | 681.2ms/step | 29.4 samples/s
+Step 6310/44000 | Loss: 3.3021 | LR: 4.80e-04 | 682.1ms/step | 29.3 samples/s
+Step 6320/44000 | Loss: 3.2442 | LR: 4.80e-04 | 682.3ms/step | 29.3 samples/s
+Step 6330/44000 | Loss: 3.3132 | LR: 4.80e-04 | 682.4ms/step | 29.3 samples/s
+Step 6340/44000 | Loss: 3.2685 | LR: 4.80e-04 | 681.7ms/step | 29.3 samples/s
+Step 6350/44000 | Loss: 3.3161 | LR: 4.80e-04 | 682.1ms/step | 29.3 samples/s
+Step 6360/44000 | Loss: 3.2853 | LR: 4.80e-04 | 681.2ms/step | 29.4 samples/s
+Step 6370/44000 | Loss: 3.2403 | LR: 4.79e-04 | 681.5ms/step | 29.3 samples/s
+Step 6380/44000 | Loss: 3.2693 | LR: 4.79e-04 | 683.7ms/step | 29.3 samples/s
+Step 6390/44000 | Loss: 3.2859 | LR: 4.79e-04 | 682.0ms/step | 29.3 samples/s
+Step 6400/44000 | Loss: 3.2454 | LR: 4.79e-04 | 681.8ms/step | 29.3 samples/s
+Step 6410/44000 | Loss: 3.2697 | LR: 4.79e-04 | 683.0ms/step | 29.3 samples/s
+Step 6420/44000 | Loss: 3.2635 | LR: 4.79e-04 | 681.2ms/step | 29.4 samples/s
+Step 6430/44000 | Loss: 3.2927 | LR: 4.79e-04 | 682.3ms/step | 29.3 samples/s
+Step 6440/44000 | Loss: 3.2558 | LR: 4.79e-04 | 681.4ms/step | 29.4 samples/s
+Step 6450/44000 | Loss: 3.2614 | LR: 4.79e-04 | 682.0ms/step | 29.3 samples/s
+Step 6460/44000 | Loss: 3.2724 | LR: 4.79e-04 | 682.3ms/step | 29.3 samples/s
+Step 6470/44000 | Loss: 3.2957 | LR: 4.79e-04 | 682.6ms/step | 29.3 samples/s
+Step 6480/44000 | Loss: 3.3140 | LR: 4.79e-04 | 682.1ms/step | 29.3 samples/s
+Step 6490/44000 | Loss: 3.2279 | LR: 4.79e-04 | 680.6ms/step | 29.4 samples/s
+Step 6500/44000 | Loss: 3.3076 | LR: 4.79e-04 | 682.7ms/step | 29.3 samples/s
+Validation at step 6500: Loss = 3.2658, Perplexity = 26.20
+Step 6510/44000 | Loss: 3.2742 | LR: 4.78e-04 | 1242.0ms/step | 16.1 samples/s
+Step 6520/44000 | Loss: 3.2622 | LR: 4.78e-04 | 681.7ms/step | 29.3 samples/s
+Step 6530/44000 | Loss: 3.2865 | LR: 4.78e-04 | 681.1ms/step | 29.4 samples/s
+Step 6540/44000 | Loss: 3.2561 | LR: 4.78e-04 | 681.4ms/step | 29.4 samples/s
+Step 6550/44000 | Loss: 3.2848 | LR: 4.78e-04 | 682.8ms/step | 29.3 samples/s
+Step 6560/44000 | Loss: 3.2322 | LR: 4.78e-04 | 682.1ms/step | 29.3 samples/s
+Step 6570/44000 | Loss: 3.2234 | LR: 4.78e-04 | 683.3ms/step | 29.3 samples/s
+Step 6580/44000 | Loss: 3.2458 | LR: 4.78e-04 | 682.2ms/step | 29.3 samples/s
+Step 6590/44000 | Loss: 3.2562 | LR: 4.78e-04 | 682.4ms/step | 29.3 samples/s
+Step 6600/44000 | Loss: 3.2259 | LR: 4.78e-04 | 682.5ms/step | 29.3 samples/s
+Step 6610/44000 | Loss: 3.2552 | LR: 4.78e-04 | 682.0ms/step | 29.3 samples/s
+Step 6620/44000 | Loss: 3.2465 | LR: 4.78e-04 | 682.3ms/step | 29.3 samples/s
+Step 6630/44000 | Loss: 3.2883 | LR: 4.78e-04 | 682.3ms/step | 29.3 samples/s
+Step 6640/44000 | Loss: 3.2659 | LR: 4.77e-04 | 683.8ms/step | 29.2 samples/s
+Step 6650/44000 | Loss: 3.2864 | LR: 4.77e-04 | 681.4ms/step | 29.4 samples/s
+Step 6660/44000 | Loss: 3.2136 | LR: 4.77e-04 | 681.9ms/step | 29.3 samples/s
+Step 6670/44000 | Loss: 3.2661 | LR: 4.77e-04 | 682.2ms/step | 29.3 samples/s
+Step 6680/44000 | Loss: 3.2368 | LR: 4.77e-04 | 681.5ms/step | 29.3 samples/s
+Step 6690/44000 | Loss: 3.2720 | LR: 4.77e-04 | 682.4ms/step | 29.3 samples/s
+Step 6700/44000 | Loss: 3.2586 | LR: 4.77e-04 | 680.6ms/step | 29.4 samples/s
+Step 6710/44000 | Loss: 3.2374 | LR: 4.77e-04 | 680.0ms/step | 29.4 samples/s
+Step 6720/44000 | Loss: 3.2537 | LR: 4.77e-04 | 682.3ms/step | 29.3 samples/s
+Step 6730/44000 | Loss: 3.3197 | LR: 4.77e-04 | 681.2ms/step | 29.4 samples/s
+Step 6740/44000 | Loss: 3.2437 | LR: 4.77e-04 | 681.2ms/step | 29.4 samples/s
+Step 6750/44000 | Loss: 3.2199 | LR: 4.77e-04 | 682.5ms/step | 29.3 samples/s
+Validation at step 6750: Loss = 3.2597, Perplexity = 26.04
+Step 6760/44000 | Loss: 3.2379 | LR: 4.77e-04 | 1240.3ms/step | 16.1 samples/s
+Step 6770/44000 | Loss: 3.2695 | LR: 4.76e-04 | 682.8ms/step | 29.3 samples/s
+Step 6780/44000 | Loss: 3.2444 | LR: 4.76e-04 | 681.2ms/step | 29.4 samples/s
+Step 6790/44000 | Loss: 3.2643 | LR: 4.76e-04 | 682.6ms/step | 29.3 samples/s
+Step 6800/44000 | Loss: 3.2226 | LR: 4.76e-04 | 682.1ms/step | 29.3 samples/s
+Step 6810/44000 | Loss: 3.2656 | LR: 4.76e-04 | 681.0ms/step | 29.4 samples/s
+Step 6820/44000 | Loss: 3.2647 | LR: 4.76e-04 | 681.9ms/step | 29.3 samples/s
+Step 6830/44000 | Loss: 3.2415 | LR: 4.76e-04 | 680.8ms/step | 29.4 samples/s
+Step 6840/44000 | Loss: 3.2216 | LR: 4.76e-04 | 681.0ms/step | 29.4 samples/s
+Step 6850/44000 | Loss: 3.2313 | LR: 4.76e-04 | 681.1ms/step | 29.4 samples/s
+Step 6860/44000 | Loss: 3.2074 | LR: 4.76e-04 | 681.7ms/step | 29.3 samples/s
+Step 6870/44000 | Loss: 3.2803 | LR: 4.76e-04 | 680.0ms/step | 29.4 samples/s
+Step 6880/44000 | Loss: 3.2353 | LR: 4.76e-04 | 680.1ms/step | 29.4 samples/s
+Step 6890/44000 | Loss: 3.2318 | LR: 4.76e-04 | 681.3ms/step | 29.4 samples/s
+Step 6900/44000 | Loss: 3.2416 | LR: 4.75e-04 | 681.1ms/step | 29.4 samples/s
+Step 6910/44000 | Loss: 3.2817 | LR: 4.75e-04 | 680.9ms/step | 29.4 samples/s
+Step 6920/44000 | Loss: 3.2866 | LR: 4.75e-04 | 681.9ms/step | 29.3 samples/s
+Step 6930/44000 | Loss: 3.2466 | LR: 4.75e-04 | 682.2ms/step | 29.3 samples/s
+Step 6940/44000 | Loss: 3.2453 | LR: 4.75e-04 | 682.6ms/step | 29.3 samples/s
+Step 6950/44000 | Loss: 3.2409 | LR: 4.75e-04 | 682.5ms/step | 29.3 samples/s
+Step 6960/44000 | Loss: 3.2616 | LR: 4.75e-04 | 682.2ms/step | 29.3 samples/s
+Step 6970/44000 | Loss: 3.2052 | LR: 4.75e-04 | 682.7ms/step | 29.3 samples/s
+Step 6980/44000 | Loss: 3.2595 | LR: 4.75e-04 | 681.7ms/step | 29.3 samples/s
+Step 6990/44000 | Loss: 3.2086 | LR: 4.75e-04 | 682.2ms/step | 29.3 samples/s
+Step 7000/44000 | Loss: 3.2390 | LR: 4.75e-04 | 681.5ms/step | 29.3 samples/s
+Validation at step 7000: Loss = 3.2479, Perplexity = 25.74
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-7000
+Step 7010/44000 | Loss: 3.2416 | LR: 4.75e-04 | 1513.6ms/step | 13.2 samples/s
+Step 7020/44000 | Loss: 3.2459 | LR: 4.75e-04 | 681.5ms/step | 29.3 samples/s
+Step 7030/44000 | Loss: 3.2243 | LR: 4.74e-04 | 679.4ms/step | 29.4 samples/s
+Step 7040/44000 | Loss: 3.2355 | LR: 4.74e-04 | 680.1ms/step | 29.4 samples/s
+Step 7050/44000 | Loss: 3.2453 | LR: 4.74e-04 | 681.1ms/step | 29.4 samples/s
+Step 7060/44000 | Loss: 3.2557 | LR: 4.74e-04 | 680.9ms/step | 29.4 samples/s
+Step 7070/44000 | Loss: 3.1812 | LR: 4.74e-04 | 679.9ms/step | 29.4 samples/s
+Step 7080/44000 | Loss: 3.2296 | LR: 4.74e-04 | 678.6ms/step | 29.5 samples/s
+Step 7090/44000 | Loss: 3.2749 | LR: 4.74e-04 | 678.8ms/step | 29.5 samples/s
+Step 7100/44000 | Loss: 3.2316 | LR: 4.74e-04 | 678.9ms/step | 29.5 samples/s
+Step 7110/44000 | Loss: 3.2163 | LR: 4.74e-04 | 677.4ms/step | 29.5 samples/s
+Step 7120/44000 | Loss: 3.2116 | LR: 4.74e-04 | 676.9ms/step | 29.5 samples/s
+Step 7130/44000 | Loss: 3.1702 | LR: 4.74e-04 | 677.5ms/step | 29.5 samples/s
+Step 7140/44000 | Loss: 3.2219 | LR: 4.74e-04 | 677.5ms/step | 29.5 samples/s
+Step 7150/44000 | Loss: 3.2445 | LR: 4.73e-04 | 678.5ms/step | 29.5 samples/s
+Step 7160/44000 | Loss: 3.2340 | LR: 4.73e-04 | 677.7ms/step | 29.5 samples/s
+Step 7170/44000 | Loss: 3.2090 | LR: 4.73e-04 | 677.9ms/step | 29.5 samples/s
+Step 7180/44000 | Loss: 3.2265 | LR: 4.73e-04 | 680.2ms/step | 29.4 samples/s
+Step 7190/44000 | Loss: 3.2487 | LR: 4.73e-04 | 679.9ms/step | 29.4 samples/s
+Step 7200/44000 | Loss: 3.2554 | LR: 4.73e-04 | 679.0ms/step | 29.5 samples/s
+Step 7210/44000 | Loss: 3.2330 | LR: 4.73e-04 | 679.6ms/step | 29.4 samples/s
+Step 7220/44000 | Loss: 3.2147 | LR: 4.73e-04 | 678.6ms/step | 29.5 samples/s
+Step 7230/44000 | Loss: 3.2292 | LR: 4.73e-04 | 679.8ms/step | 29.4 samples/s
+Step 7240/44000 | Loss: 3.2154 | LR: 4.73e-04 | 679.5ms/step | 29.4 samples/s
+Step 7250/44000 | Loss: 3.2517 | LR: 4.73e-04 | 678.5ms/step | 29.5 samples/s
+Validation at step 7250: Loss = 3.2378, Perplexity = 25.48
+Step 7260/44000 | Loss: 3.2445 | LR: 4.73e-04 | 1235.9ms/step | 16.2 samples/s
+Step 7270/44000 | Loss: 3.2137 | LR: 4.72e-04 | 679.9ms/step | 29.4 samples/s
+Step 7280/44000 | Loss: 3.2321 | LR: 4.72e-04 | 679.2ms/step | 29.4 samples/s
+Step 7290/44000 | Loss: 3.2509 | LR: 4.72e-04 | 678.9ms/step | 29.5 samples/s
+Step 7300/44000 | Loss: 3.2473 | LR: 4.72e-04 | 677.3ms/step | 29.5 samples/s
+Step 7310/44000 | Loss: 3.2488 | LR: 4.72e-04 | 678.8ms/step | 29.5 samples/s
+Step 7320/44000 | Loss: 3.2084 | LR: 4.72e-04 | 678.0ms/step | 29.5 samples/s
+Step 7330/44000 | Loss: 3.2435 | LR: 4.72e-04 | 678.0ms/step | 29.5 samples/s
+Step 7340/44000 | Loss: 3.2316 | LR: 4.72e-04 | 678.2ms/step | 29.5 samples/s
+Step 7350/44000 | Loss: 3.2156 | LR: 4.72e-04 | 678.1ms/step | 29.5 samples/s
+Step 7360/44000 | Loss: 3.2001 | LR: 4.72e-04 | 678.6ms/step | 29.5 samples/s
+Step 7370/44000 | Loss: 3.2559 | LR: 4.72e-04 | 678.2ms/step | 29.5 samples/s
+Step 7380/44000 | Loss: 3.2513 | LR: 4.72e-04 | 678.6ms/step | 29.5 samples/s
+Step 7390/44000 | Loss: 3.2778 | LR: 4.71e-04 | 678.8ms/step | 29.5 samples/s
+Step 7400/44000 | Loss: 3.2280 | LR: 4.71e-04 | 677.9ms/step | 29.5 samples/s
+Step 7410/44000 | Loss: 3.2428 | LR: 4.71e-04 | 678.4ms/step | 29.5 samples/s
+Step 7420/44000 | Loss: 3.2287 | LR: 4.71e-04 | 678.7ms/step | 29.5 samples/s
+Step 7430/44000 | Loss: 3.2288 | LR: 4.71e-04 | 679.1ms/step | 29.5 samples/s
+Step 7440/44000 | Loss: 3.2346 | LR: 4.71e-04 | 677.6ms/step | 29.5 samples/s
+Step 7450/44000 | Loss: 3.2619 | LR: 4.71e-04 | 678.1ms/step | 29.5 samples/s
+Step 7460/44000 | Loss: 3.2427 | LR: 4.71e-04 | 679.7ms/step | 29.4 samples/s
+Step 7470/44000 | Loss: 3.2438 | LR: 4.71e-04 | 678.4ms/step | 29.5 samples/s
+Step 7480/44000 | Loss: 3.2501 | LR: 4.71e-04 | 677.5ms/step | 29.5 samples/s
+Step 7490/44000 | Loss: 3.2011 | LR: 4.71e-04 | 678.5ms/step | 29.5 samples/s
+Step 7500/44000 | Loss: 3.1936 | LR: 4.71e-04 | 678.5ms/step | 29.5 samples/s
+Validation at step 7500: Loss = 3.2298, Perplexity = 25.27
+Step 7510/44000 | Loss: 3.2095 | LR: 4.70e-04 | 1239.6ms/step | 16.1 samples/s
+Step 7520/44000 | Loss: 3.2317 | LR: 4.70e-04 | 682.3ms/step | 29.3 samples/s
+Step 7530/44000 | Loss: 3.2518 | LR: 4.70e-04 | 682.0ms/step | 29.3 samples/s
+Step 7540/44000 | Loss: 3.2086 | LR: 4.70e-04 | 683.8ms/step | 29.3 samples/s
+Step 7550/44000 | Loss: 3.2021 | LR: 4.70e-04 | 682.6ms/step | 29.3 samples/s
+Step 7560/44000 | Loss: 3.2269 | LR: 4.70e-04 | 680.4ms/step | 29.4 samples/s
+Step 7570/44000 | Loss: 3.1721 | LR: 4.70e-04 | 679.8ms/step | 29.4 samples/s
+Step 7580/44000 | Loss: 3.1467 | LR: 4.70e-04 | 682.4ms/step | 29.3 samples/s
+Step 7590/44000 | Loss: 3.2108 | LR: 4.70e-04 | 681.0ms/step | 29.4 samples/s
+Step 7600/44000 | Loss: 3.2213 | LR: 4.70e-04 | 681.0ms/step | 29.4 samples/s
+Step 7610/44000 | Loss: 3.2189 | LR: 4.70e-04 | 682.2ms/step | 29.3 samples/s
+Step 7620/44000 | Loss: 3.2377 | LR: 4.70e-04 | 682.4ms/step | 29.3 samples/s
+Step 7630/44000 | Loss: 3.2080 | LR: 4.69e-04 | 680.9ms/step | 29.4 samples/s
+Step 7640/44000 | Loss: 3.1766 | LR: 4.69e-04 | 679.9ms/step | 29.4 samples/s
+Step 7650/44000 | Loss: 3.1990 | LR: 4.69e-04 | 681.9ms/step | 29.3 samples/s
+Step 7660/44000 | Loss: 3.2231 | LR: 4.69e-04 | 682.3ms/step | 29.3 samples/s
+Step 7670/44000 | Loss: 3.1943 | LR: 4.69e-04 | 683.1ms/step | 29.3 samples/s
+Step 7680/44000 | Loss: 3.2572 | LR: 4.69e-04 | 682.7ms/step | 29.3 samples/s
+Step 7690/44000 | Loss: 3.2244 | LR: 4.69e-04 | 682.1ms/step | 29.3 samples/s
+Step 7700/44000 | Loss: 3.1975 | LR: 4.69e-04 | 681.8ms/step | 29.3 samples/s
+Step 7710/44000 | Loss: 3.1890 | LR: 4.69e-04 | 681.1ms/step | 29.4 samples/s
+Step 7720/44000 | Loss: 3.2167 | LR: 4.69e-04 | 683.3ms/step | 29.3 samples/s
+Step 7730/44000 | Loss: 3.2047 | LR: 4.69e-04 | 681.9ms/step | 29.3 samples/s
+Step 7740/44000 | Loss: 3.1646 | LR: 4.68e-04 | 682.6ms/step | 29.3 samples/s
+Step 7750/44000 | Loss: 3.2065 | LR: 4.68e-04 | 684.8ms/step | 29.2 samples/s
+Validation at step 7750: Loss = 3.2211, Perplexity = 25.06
+Step 7760/44000 | Loss: 3.2048 | LR: 4.68e-04 | 1241.5ms/step | 16.1 samples/s
+Step 7770/44000 | Loss: 3.2135 | LR: 4.68e-04 | 682.3ms/step | 29.3 samples/s
+Step 7780/44000 | Loss: 3.2123 | LR: 4.68e-04 | 682.3ms/step | 29.3 samples/s
+Step 7790/44000 | Loss: 3.2149 | LR: 4.68e-04 | 681.1ms/step | 29.4 samples/s
+Step 7800/44000 | Loss: 3.2257 | LR: 4.68e-04 | 681.3ms/step | 29.4 samples/s
+Step 7810/44000 | Loss: 3.1717 | LR: 4.68e-04 | 680.5ms/step | 29.4 samples/s
+Step 7820/44000 | Loss: 3.2175 | LR: 4.68e-04 | 681.5ms/step | 29.3 samples/s
+Step 7830/44000 | Loss: 3.2034 | LR: 4.68e-04 | 681.7ms/step | 29.3 samples/s
+Step 7840/44000 | Loss: 3.1849 | LR: 4.68e-04 | 682.0ms/step | 29.3 samples/s
+Step 7850/44000 | Loss: 3.1836 | LR: 4.67e-04 | 681.3ms/step | 29.4 samples/s
+Step 7860/44000 | Loss: 3.2015 | LR: 4.67e-04 | 682.0ms/step | 29.3 samples/s
+Step 7870/44000 | Loss: 3.2070 | LR: 4.67e-04 | 683.2ms/step | 29.3 samples/s
+Step 7880/44000 | Loss: 3.2347 | LR: 4.67e-04 | 681.5ms/step | 29.3 samples/s
+Step 7890/44000 | Loss: 3.2008 | LR: 4.67e-04 | 682.0ms/step | 29.3 samples/s
+Step 7900/44000 | Loss: 3.2439 | LR: 4.67e-04 | 681.8ms/step | 29.3 samples/s
+Step 7910/44000 | Loss: 3.2194 | LR: 4.67e-04 | 682.7ms/step | 29.3 samples/s
+Step 7920/44000 | Loss: 3.2346 | LR: 4.67e-04 | 683.1ms/step | 29.3 samples/s
+Step 7930/44000 | Loss: 3.2598 | LR: 4.67e-04 | 682.1ms/step | 29.3 samples/s
+Step 7940/44000 | Loss: 3.2585 | LR: 4.67e-04 | 682.5ms/step | 29.3 samples/s
+Step 7950/44000 | Loss: 3.1714 | LR: 4.67e-04 | 682.1ms/step | 29.3 samples/s
+Step 7960/44000 | Loss: 3.2250 | LR: 4.66e-04 | 682.0ms/step | 29.3 samples/s
+Step 7970/44000 | Loss: 3.1593 | LR: 4.66e-04 | 682.2ms/step | 29.3 samples/s
+Step 7980/44000 | Loss: 3.1908 | LR: 4.66e-04 | 682.0ms/step | 29.3 samples/s
+Step 7990/44000 | Loss: 3.2133 | LR: 4.66e-04 | 682.5ms/step | 29.3 samples/s
+Step 8000/44000 | Loss: 3.2482 | LR: 4.66e-04 | 682.3ms/step | 29.3 samples/s
+Validation at step 8000: Loss = 3.2132, Perplexity = 24.86
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822/checkpoint-8000
+Step 8010/44000 | Loss: 3.2003 | LR: 4.66e-04 | 1522.8ms/step | 13.1 samples/s
+Step 8020/44000 | Loss: 3.2183 | LR: 4.66e-04 | 682.3ms/step | 29.3 samples/s
+Step 8030/44000 | Loss: 3.2136 | LR: 4.66e-04 | 682.5ms/step | 29.3 samples/s
+Step 8040/44000 | Loss: 3.2349 | LR: 4.66e-04 | 680.9ms/step | 29.4 samples/s
+Step 8050/44000 | Loss: 3.2064 | LR: 4.66e-04 | 680.0ms/step | 29.4 samples/s
+Step 8060/44000 | Loss: 3.2226 | LR: 4.66e-04 | 680.0ms/step | 29.4 samples/s
+Step 8070/44000 | Loss: 3.1707 | LR: 4.65e-04 | 681.7ms/step | 29.3 samples/s
+Step 8080/44000 | Loss: 3.1858 | LR: 4.65e-04 | 682.5ms/step | 29.3 samples/s
+Step 8090/44000 | Loss: 3.2295 | LR: 4.65e-04 | 681.9ms/step | 29.3 samples/s
+Step 8100/44000 | Loss: 3.2361 | LR: 4.65e-04 | 680.9ms/step | 29.4 samples/s
+Step 8110/44000 | Loss: 3.2172 | LR: 4.65e-04 | 683.2ms/step | 29.3 samples/s
+Step 8120/44000 | Loss: 3.1681 | LR: 4.65e-04 | 681.7ms/step | 29.3 samples/s

epfl_final/SparseAtt/wandb/run-20260708_205109-ti3a1x4c/files/requirements.txt

@@ -0,0 +1,141 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5

epfl_final/SparseAtt/wandb/run-20260708_205109-ti3a1x4c/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-07-08T20:51:09.024134Z",
+  "args":  [
+    "--config",
+    "configs/ladder8k/pool10.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "88298c7a418622530ca5426b22b1e5ebd10ca0cb"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-workspace-4-gpu-0-5",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "6202832625664"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-b817d241-acce-e5dd-a749-3447239c5207"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-165543e5-e049-5219-e717-e1216ac960de"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-47444fc7-ea8c-cef6-aac8-ef408dc8047b"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-37dc97ad-46ef-50d9-3619-3a6eaea1b6aa"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "csm3va7p33ujcg8bji88epodpi8fvr5c"
+}

epfl_final/SparseAtt/wandb/run-20260708_205109-ti3a1x4c/logs/debug.log

@@ -0,0 +1,19 @@
+2026-07-08 20:51:09,051 INFO    MainThread:386 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-07-08 20:51:09,051 INFO    MainThread:386 [wandb_setup.py:_flush():81] Configure stats pid to 386
+2026-07-08 20:51:09,051 INFO    MainThread:386 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-08 20:51:09,051 INFO    MainThread:386 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_205109-ti3a1x4c/logs/debug.log
+2026-07-08 20:51:09,051 INFO    MainThread:386 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_205109-ti3a1x4c/logs/debug-internal.log
+2026-07-08 20:51:09,051 INFO    MainThread:386 [wandb_init.py:init():850] calling init triggers
+2026-07-08 20:51:09,051 INFO    MainThread:386 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_pool10/run_20260708_204822', 'resume_from_checkpoint': None, 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 5, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m pool10 8k nrb31 freqabl', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'resampled', 'freqablation', 'pool10'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 10, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_32', 'sparse_mask_seed': 0, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-07-08 20:51:09,051 INFO    MainThread:386 [wandb_init.py:init():898] starting backend
+2026-07-08 20:51:09,802 INFO    MainThread:386 [wandb_init.py:init():913] sending inform_init request
+2026-07-08 20:51:10,386 INFO    MainThread:386 [wandb_init.py:init():918] backend started and connected
+2026-07-08 20:51:10,389 INFO    MainThread:386 [wandb_init.py:init():988] updated telemetry
+2026-07-08 20:51:10,424 INFO    MainThread:386 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-07-08 20:51:11,689 INFO    MainThread:386 [wandb_init.py:init():1056] starting run threads in backend
+2026-07-08 20:51:11,908 INFO    MainThread:386 [wandb_run.py:_console_start():2554] atexit reg
+2026-07-08 20:51:11,908 INFO    MainThread:386 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-07-08 20:51:11,908 INFO    MainThread:386 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-07-08 20:51:11,908 INFO    MainThread:386 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-07-08 20:51:11,915 INFO    MainThread:386 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/run-20260708_205109-ti3a1x4c/run-ti3a1x4c.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ebaf37de873574efbc2d29107496cde30ed164cecd850594639f4d36296b3af
+size 2588672

epfl_final/SparseAtt/wandb/run-20260708_234025-hn868f80/files/config.yaml

@@ -0,0 +1,189 @@
+_wandb:
+    value:
+        cli_version: 0.26.1
+        e:
+            zel9q53vtad0ytaeeuqautdqrgn188st:
+                args:
+                    - --config
+                    - configs/train_config_30k_resampled_b128.json
+                codePath: train.py
+                codePathLocal: train.py
+                cpu_count: 96
+                cpu_count_logical: 192
+                cudaVersion: "13.0"
+                disk:
+                    /:
+                        total: "9374931091456"
+                        used: "6203676491776"
+                email: ruben.navasardyan@mail.utoronto.ca
+                executable: /mloscratch/homes/navasard/conda/bin/python3.13
+                git:
+                    commit: 88298c7a418622530ca5426b22b1e5ebd10ca0cb
+                    remote: https://github.com/rubentium/TokenPrune.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 4
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-b817d241-acce-e5dd-a749-3447239c5207
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-165543e5-e049-5219-e717-e1216ac960de
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-47444fc7-ea8c-cef6-aac8-ef408dc8047b
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-37dc97ad-46ef-50d9-3619-3a6eaea1b6aa
+                host: h100-workspace-4-gpu-0-5
+                memory:
+                    total: "1623005732864"
+                os: Linux-6.8.0-124-generic-x86_64-with-glibc2.35
+                program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
+                python: CPython 3.13.13
+                root: /mloscratch/homes/navasard/sparse/SparseAtt
+                startedAt: "2026-07-08T23:40:25.467535Z"
+                writerId: zel9q53vtad0ytaeeuqautdqrgn188st
+        m: []
+        python_version: 3.13.13
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 5
+                - 13
+                - 14
+                - 15
+                - 16
+                - 41
+                - 62
+            "4": 3.13.13
+            "5": 0.26.1
+            "6": 5.7.0
+            "12": 0.26.1
+            "13": linux-x86_64
+attention_type:
+    value: sparse_stochastic
+checkpoint_activations:
+    value: true
+dense_mode:
+    value: false
+dense_step_intervals:
+    value: null
+eval_batches:
+    value: 20
+eval_steps:
+    value: 500
+force_new_run:
+    value: false
+gradient_accumulation_steps:
+    value: 5
+head_dim:
+    value: null
+hidden_size:
+    value: 1536
+init_from_scratch:
+    value: true
+intermediate_size:
+    value: 6144
+learning_rate:
+    value: 0.0005
+logging_steps:
+    value: 10
+lr_scheduler:
+    value: cosine
+max_grad_norm:
+    value: 1
+max_seq_length:
+    value: 30720
+max_steps:
+    value: null
+mixed_precision:
+    value: bf16
+model_dtype:
+    value: bfloat16
+model_path:
+    value: /mloscratch/homes/navasard/qwen3_06b_base
+num_attention_heads:
+    value: 12
+num_epochs:
+    value: 1
+num_hidden_layers:
+    value: null
+num_key_value_heads:
+    value: 4
+num_workers:
+    value: 4
+output_dir:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209
+per_device_batch_size:
+    value: 1
+resume_from_checkpoint:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-4200
+save_steps:
+    value: 300
+seed:
+    value: 42
+sparse_block_size:
+    value: 128
+sparse_layer_indices:
+    value: null
+sparse_mask_cache_dir:
+    value: /mloscratch/homes/navasard/sparse/mask_cache_30k_b128_nrb120
+sparse_mask_pool_size:
+    value: 1000
+sparse_mask_seed:
+    value: 0
+sparse_num_random_blocks:
+    value: 120
+sparse_val_dense_compare:
+    value: false
+strategy:
+    value: ddp
+train_path:
+    value: /mloscratch/homes/navasard/data/longctx30k/tokenized
+val_path:
+    value: null
+verbose:
+    value: true
+wandb_enabled:
+    value: true
+wandb_name:
+    value: 1.2b resampled 30k longctx
+wandb_project:
+    value: stochastic_sparsity
+wandb_tags:
+    value:
+        - qwen3
+        - 1.2b
+        - resampled
+        - 30k
+        - longctx
+        - b128
+        - scaleladder
+warmup_steps:
+    value: 750
+weight_decay:
+    value: 0.01

epfl_final/SparseAtt/wandb/run-20260708_234025-hn868f80/files/output.log

@@ -0,0 +1,473 @@
+Starting training...
+Traceback (most recent call last):
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1463, in <module>
+    main()
+    ~~~~^^
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1460, in main
+    train(config)
+    ~~~~~^^^^^^^^
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1274, in train
+    run_validation_step(
+    ~~~~~~~~~~~~~~~~~~~^
+        model=model,
+        ^^^^^^^^^^^^
+    ...<7 lines>...
+        dense_step_intervals=dense_step_intervals,
+        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+    )
+    ^
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 629, in run_validation_step
+    eval_loss, eval_perplexity = evaluate(
+                                 ~~~~~~~~^
+        model,
+        ^^^^^^
+    ...<6 lines>...
+        step_for_mode=global_step,
+        ^^^^^^^^^^^^^^^^^^^^^^^^^^
+    )
+    ^
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 588, in evaluate
+    outputs = model(
+        input_ids=input_ids,
+    ...<2 lines>...
+        sparse_indices=sparse_indices,
+    )
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1779, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1790, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/parallel/distributed.py", line 1699, in forward
+    else self._run_ddp_forward(*inputs, **kwargs)
+         ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/parallel/distributed.py", line 1524, in _run_ddp_forward
+    return self.module(*inputs, **kwargs)  # type: ignore[index]
+           ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1779, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1790, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/sparseatt/model.py", line 311, in forward
+    hidden_states, next_cache, all_hidden_states, all_attentions = self.model(
+                                                                   ~~~~~~~~~~^
+        input_ids=input_ids,
+        ^^^^^^^^^^^^^^^^^^^^
+    ...<6 lines>...
+        sparse_indices=sparse_indices,
+        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+    )
+    ^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1779, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1790, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/sparseatt/model.py", line 189, in forward
+    hidden_states, attn_weights, present_key_value = checkpoint(
+                                                     ~~~~~~~~~~^
+        decoder_layer,
+        ^^^^^^^^^^^^^^
+    ...<7 lines>...
+        use_reentrant=False
+        ^^^^^^^^^^^^^^^^^^^
+    )
+    ^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_compile.py", line 54, in inner
+    return disable_fn(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1263, in _fn
+    return fn(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/utils/checkpoint.py", line 512, in checkpoint
+    ret = function(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1779, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1790, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/sparseatt/model.py", line 78, in forward
+    hidden_states, attn_weights, present_key_value = self.self_attn(**attn_kwargs)
+                                                     ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1779, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1790, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/sparseatt/sparse_attention.py", line 313, in forward
+    attn_output = self._flex_attention(
+        query_states,
+    ...<4 lines>...
+        kernel_options={"BACKEND": "TRITON"},
+    )
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py", line 1731, in flex_attention
+    out, lse, max_scores = flex_fn(
+                           ~~~~~~~^
+        query,
+        ^^^^^^
+    ...<5 lines>...
+        kernel_options,
+        ^^^^^^^^^^^^^^^
+    )
+    ^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py", line 1721, in _flex_attention_hop_wrapper
+    return flex_attention_hop(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_higher_order_ops/flex_attention.py", line 112, in __call__
+    return super().__call__(
+           ~~~~~~~~~~~~~~~~^
+        query,
+        ^^^^^^
+    ...<7 lines>...
+        mask_mod_other_buffers,
+        ^^^^^^^^^^^^^^^^^^^^^^^
+    )
+    ^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_ops.py", line 533, in __call__
+    return self.dispatch(dispatch_key_set.highestPriorityTypeId(), *args, **kwargs)
+           ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_ops.py", line 522, in dispatch
+    return kernel(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_higher_order_ops/flex_attention.py", line 341, in flex_attention_autocast_cuda
+    return _flex_attention_autocast_impl(
+        query,
+    ...<7 lines>...
+        mask_mod_other_buffers,
+    )
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_higher_order_ops/flex_attention.py", line 316, in _flex_attention_autocast_impl
+    return flex_attention(
+        query,
+    ...<7 lines>...
+        mask_mod_other_buffers,
+    )
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_higher_order_ops/flex_attention.py", line 112, in __call__
+    return super().__call__(
+           ~~~~~~~~~~~~~~~~^
+        query,
+        ^^^^^^
+    ...<7 lines>...
+        mask_mod_other_buffers,
+        ^^^^^^^^^^^^^^^^^^^^^^^
+    )
+    ^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_ops.py", line 533, in __call__
+    return self.dispatch(dispatch_key_set.highestPriorityTypeId(), *args, **kwargs)
+           ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_ops.py", line 380, in dispatch
+    return kernel(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_higher_order_ops/flex_attention.py", line 928, in flex_attention_autograd
+    out, logsumexp, max_scores = FlexAttentionAutogradOp.apply(
+                                 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
+        query,
+        ^^^^^^
+    ...<8 lines>...
+        *score_mod_other_buffers,
+        ^^^^^^^^^^^^^^^^^^^^^^^^^
+    )
+    ^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/autograd/function.py", line 596, in apply
+    return super().apply(*args, **kwargs)  # type: ignore[misc]
+           ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_higher_order_ops/flex_attention.py", line 780, in forward
+    out, logsumexp, max_scores = flex_attention(
+                                 ~~~~~~~~~~~~~~^
+        query,
+        ^^^^^^
+    ...<7 lines>...
+        mask_mod_other_buffers,
+        ^^^^^^^^^^^^^^^^^^^^^^^
+    )
+    ^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_higher_order_ops/flex_attention.py", line 112, in __call__
+    return super().__call__(
+           ~~~~~~~~~~~~~~~~^
+        query,
+        ^^^^^^
+    ...<7 lines>...
+        mask_mod_other_buffers,
+        ^^^^^^^^^^^^^^^^^^^^^^^
+    )
+    ^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_ops.py", line 528, in __call__
+    return torch.overrides.handle_torch_function(
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
+        self, flat_args, *args, **kwargs
+        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+    )
+    ^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/overrides.py", line 1741, in handle_torch_function
+    result = mode.__torch_function__(public_api, types, args, kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/_trace_wrapped_higher_order_op.py", line 158, in __torch_function__
+    return func(*args, **(kwargs or {}))
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_higher_order_ops/flex_attention.py", line 112, in __call__
+    return super().__call__(
+           ~~~~~~~~~~~~~~~~^
+        query,
+        ^^^^^^
+    ...<7 lines>...
+        mask_mod_other_buffers,
+        ^^^^^^^^^^^^^^^^^^^^^^^
+    )
+    ^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_ops.py", line 533, in __call__
+    return self.dispatch(dispatch_key_set.highestPriorityTypeId(), *args, **kwargs)
+           ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_ops.py", line 380, in dispatch
+    return kernel(*args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_higher_order_ops/flex_attention.py", line 391, in sdpa_dense
+    out, lse, max_scores = math_attention(
+                           ~~~~~~~~~~~~~~^
+        query,
+        ^^^^^^
+    ...<7 lines>...
+        mask_mod_other_buffers,
+        ^^^^^^^^^^^^^^^^^^^^^^^
+    )
+    ^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_higher_order_ops/flex_attention.py", line 258, in math_attention
+    _, post_mod_scores = _math_attention_inner(
+                         ~~~~~~~~~~~~~~~~~~~~~^
+        query,
+        ^^^^^^
+    ...<7 lines>...
+        mask_mod_other_buffers,
+        ^^^^^^^^^^^^^^^^^^^^^^^
+    )
+    ^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_higher_order_ops/flex_attention.py", line 190, in _math_attention_inner
+    scores = query.to(working_precision) @ key.to(working_precision).transpose(-2, -1)
+             ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 42.19 GiB. GPU 0 has a total capacity of 79.18 GiB of which 30.87 GiB is free. Including non-PyTorch memory, this process has 48.30 GiB memory in use. Of the allocated memory 11.48 GiB is allocated by PyTorch, and 35.29 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://docs.pytorch.org/docs/stable/notes/cuda.html#optimizing-memory-usage-with-pytorch-cuda-alloc-conf)
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1463, in <module>
+[rank0]:     main()
+[rank0]:     ~~~~^^
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1460, in main
+[rank0]:     train(config)
+[rank0]:     ~~~~~^^^^^^^^
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1274, in train
+[rank0]:     run_validation_step(
+[rank0]:     ~~~~~~~~~~~~~~~~~~~^
+[rank0]:         model=model,
+[rank0]:         ^^^^^^^^^^^^
+[rank0]:     ...<7 lines>...
+[rank0]:         dense_step_intervals=dense_step_intervals,
+[rank0]:         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:     )
+[rank0]:     ^
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 629, in run_validation_step
+[rank0]:     eval_loss, eval_perplexity = evaluate(
+[rank0]:                                  ~~~~~~~~^
+[rank0]:         model,
+[rank0]:         ^^^^^^
+[rank0]:     ...<6 lines>...
+[rank0]:         step_for_mode=global_step,
+[rank0]:         ^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:     )
+[rank0]:     ^
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 588, in evaluate
+[rank0]:     outputs = model(
+[rank0]:         input_ids=input_ids,
+[rank0]:     ...<2 lines>...
+[rank0]:         sparse_indices=sparse_indices,
+[rank0]:     )
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1779, in _wrapped_call_impl
+[rank0]:     return self._call_impl(*args, **kwargs)
+[rank0]:            ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1790, in _call_impl
+[rank0]:     return forward_call(*args, **kwargs)
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/parallel/distributed.py", line 1699, in forward
+[rank0]:     else self._run_ddp_forward(*inputs, **kwargs)
+[rank0]:          ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/parallel/distributed.py", line 1524, in _run_ddp_forward
+[rank0]:     return self.module(*inputs, **kwargs)  # type: ignore[index]
+[rank0]:            ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1779, in _wrapped_call_impl
+[rank0]:     return self._call_impl(*args, **kwargs)
+[rank0]:            ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1790, in _call_impl
+[rank0]:     return forward_call(*args, **kwargs)
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/sparseatt/model.py", line 311, in forward
+[rank0]:     hidden_states, next_cache, all_hidden_states, all_attentions = self.model(
+[rank0]:                                                                    ~~~~~~~~~~^
+[rank0]:         input_ids=input_ids,
+[rank0]:         ^^^^^^^^^^^^^^^^^^^^
+[rank0]:     ...<6 lines>...
+[rank0]:         sparse_indices=sparse_indices,
+[rank0]:         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:     )
+[rank0]:     ^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1779, in _wrapped_call_impl
+[rank0]:     return self._call_impl(*args, **kwargs)
+[rank0]:            ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1790, in _call_impl
+[rank0]:     return forward_call(*args, **kwargs)
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/sparseatt/model.py", line 189, in forward
+[rank0]:     hidden_states, attn_weights, present_key_value = checkpoint(
+[rank0]:                                                      ~~~~~~~~~~^
+[rank0]:         decoder_layer,
+[rank0]:         ^^^^^^^^^^^^^^
+[rank0]:     ...<7 lines>...
+[rank0]:         use_reentrant=False
+[rank0]:         ^^^^^^^^^^^^^^^^^^^
+[rank0]:     )
+[rank0]:     ^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_compile.py", line 54, in inner
+[rank0]:     return disable_fn(*args, **kwargs)
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1263, in _fn
+[rank0]:     return fn(*args, **kwargs)
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/utils/checkpoint.py", line 512, in checkpoint
+[rank0]:     ret = function(*args, **kwargs)
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1779, in _wrapped_call_impl
+[rank0]:     return self._call_impl(*args, **kwargs)
+[rank0]:            ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1790, in _call_impl
+[rank0]:     return forward_call(*args, **kwargs)
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/sparseatt/model.py", line 78, in forward
+[rank0]:     hidden_states, attn_weights, present_key_value = self.self_attn(**attn_kwargs)
+[rank0]:                                                      ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1779, in _wrapped_call_impl
+[rank0]:     return self._call_impl(*args, **kwargs)
+[rank0]:            ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1790, in _call_impl
+[rank0]:     return forward_call(*args, **kwargs)
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/sparseatt/sparse_attention.py", line 313, in forward
+[rank0]:     attn_output = self._flex_attention(
+[rank0]:         query_states,
+[rank0]:     ...<4 lines>...
+[rank0]:         kernel_options={"BACKEND": "TRITON"},
+[rank0]:     )
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py", line 1731, in flex_attention
+[rank0]:     out, lse, max_scores = flex_fn(
+[rank0]:                            ~~~~~~~^
+[rank0]:         query,
+[rank0]:         ^^^^^^
+[rank0]:     ...<5 lines>...
+[rank0]:         kernel_options,
+[rank0]:         ^^^^^^^^^^^^^^^
+[rank0]:     )
+[rank0]:     ^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py", line 1721, in _flex_attention_hop_wrapper
+[rank0]:     return flex_attention_hop(*args, **kwargs)
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_higher_order_ops/flex_attention.py", line 112, in __call__
+[rank0]:     return super().__call__(
+[rank0]:            ~~~~~~~~~~~~~~~~^
+[rank0]:         query,
+[rank0]:         ^^^^^^
+[rank0]:     ...<7 lines>...
+[rank0]:         mask_mod_other_buffers,
+[rank0]:         ^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:     )
+[rank0]:     ^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_ops.py", line 533, in __call__
+[rank0]:     return self.dispatch(dispatch_key_set.highestPriorityTypeId(), *args, **kwargs)
+[rank0]:            ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_ops.py", line 522, in dispatch
+[rank0]:     return kernel(*args, **kwargs)
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_higher_order_ops/flex_attention.py", line 341, in flex_attention_autocast_cuda
+[rank0]:     return _flex_attention_autocast_impl(
+[rank0]:         query,
+[rank0]:     ...<7 lines>...
+[rank0]:         mask_mod_other_buffers,
+[rank0]:     )
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_higher_order_ops/flex_attention.py", line 316, in _flex_attention_autocast_impl
+[rank0]:     return flex_attention(
+[rank0]:         query,
+[rank0]:     ...<7 lines>...
+[rank0]:         mask_mod_other_buffers,
+[rank0]:     )
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_higher_order_ops/flex_attention.py", line 112, in __call__
+[rank0]:     return super().__call__(
+[rank0]:            ~~~~~~~~~~~~~~~~^
+[rank0]:         query,
+[rank0]:         ^^^^^^
+[rank0]:     ...<7 lines>...
+[rank0]:         mask_mod_other_buffers,
+[rank0]:         ^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:     )
+[rank0]:     ^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_ops.py", line 533, in __call__
+[rank0]:     return self.dispatch(dispatch_key_set.highestPriorityTypeId(), *args, **kwargs)
+[rank0]:            ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_ops.py", line 380, in dispatch
+[rank0]:     return kernel(*args, **kwargs)
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_higher_order_ops/flex_attention.py", line 928, in flex_attention_autograd
+[rank0]:     out, logsumexp, max_scores = FlexAttentionAutogradOp.apply(
+[rank0]:                                  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
+[rank0]:         query,
+[rank0]:         ^^^^^^
+[rank0]:     ...<8 lines>...
+[rank0]:         *score_mod_other_buffers,
+[rank0]:         ^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:     )
+[rank0]:     ^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/autograd/function.py", line 596, in apply
+[rank0]:     return super().apply(*args, **kwargs)  # type: ignore[misc]
+[rank0]:            ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_higher_order_ops/flex_attention.py", line 780, in forward
+[rank0]:     out, logsumexp, max_scores = flex_attention(
+[rank0]:                                  ~~~~~~~~~~~~~~^
+[rank0]:         query,
+[rank0]:         ^^^^^^
+[rank0]:     ...<7 lines>...
+[rank0]:         mask_mod_other_buffers,
+[rank0]:         ^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:     )
+[rank0]:     ^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_higher_order_ops/flex_attention.py", line 112, in __call__
+[rank0]:     return super().__call__(
+[rank0]:            ~~~~~~~~~~~~~~~~^
+[rank0]:         query,
+[rank0]:         ^^^^^^
+[rank0]:     ...<7 lines>...
+[rank0]:         mask_mod_other_buffers,
+[rank0]:         ^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:     )
+[rank0]:     ^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_ops.py", line 528, in __call__
+[rank0]:     return torch.overrides.handle_torch_function(
+[rank0]:            ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
+[rank0]:         self, flat_args, *args, **kwargs
+[rank0]:         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:     )
+[rank0]:     ^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/overrides.py", line 1741, in handle_torch_function
+[rank0]:     result = mode.__torch_function__(public_api, types, args, kwargs)
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/_trace_wrapped_higher_order_op.py", line 158, in __torch_function__
+[rank0]:     return func(*args, **(kwargs or {}))
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_higher_order_ops/flex_attention.py", line 112, in __call__
+[rank0]:     return super().__call__(
+[rank0]:            ~~~~~~~~~~~~~~~~^
+[rank0]:         query,
+[rank0]:         ^^^^^^
+[rank0]:     ...<7 lines>...
+[rank0]:         mask_mod_other_buffers,
+[rank0]:         ^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:     )
+[rank0]:     ^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_ops.py", line 533, in __call__
+[rank0]:     return self.dispatch(dispatch_key_set.highestPriorityTypeId(), *args, **kwargs)
+[rank0]:            ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_ops.py", line 380, in dispatch
+[rank0]:     return kernel(*args, **kwargs)
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_higher_order_ops/flex_attention.py", line 391, in sdpa_dense
+[rank0]:     out, lse, max_scores = math_attention(
+[rank0]:                            ~~~~~~~~~~~~~~^
+[rank0]:         query,
+[rank0]:         ^^^^^^
+[rank0]:     ...<7 lines>...
+[rank0]:         mask_mod_other_buffers,
+[rank0]:         ^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:     )
+[rank0]:     ^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_higher_order_ops/flex_attention.py", line 258, in math_attention
+[rank0]:     _, post_mod_scores = _math_attention_inner(
+[rank0]:                          ~~~~~~~~~~~~~~~~~~~~~^
+[rank0]:         query,
+[rank0]:         ^^^^^^
+[rank0]:     ...<7 lines>...
+[rank0]:         mask_mod_other_buffers,
+[rank0]:         ^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:     )
+[rank0]:     ^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_higher_order_ops/flex_attention.py", line 190, in _math_attention_inner
+[rank0]:     scores = query.to(working_precision) @ key.to(working_precision).transpose(-2, -1)
+[rank0]:              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 42.19 GiB. GPU 0 has a total capacity of 79.18 GiB of which 30.87 GiB is free. Including non-PyTorch memory, this process has 48.30 GiB memory in use. Of the allocated memory 11.48 GiB is allocated by PyTorch, and 35.29 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://docs.pytorch.org/docs/stable/notes/cuda.html#optimizing-memory-usage-with-pytorch-cuda-alloc-conf)

epfl_final/SparseAtt/wandb/run-20260708_234025-hn868f80/files/requirements.txt

@@ -0,0 +1,141 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5

epfl_final/SparseAtt/wandb/run-20260708_234025-hn868f80/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-07-08T23:40:25.467535Z",
+  "args":  [
+    "--config",
+    "configs/train_config_30k_resampled_b128.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "88298c7a418622530ca5426b22b1e5ebd10ca0cb"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-workspace-4-gpu-0-5",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "6203676491776"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-b817d241-acce-e5dd-a749-3447239c5207"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-165543e5-e049-5219-e717-e1216ac960de"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-47444fc7-ea8c-cef6-aac8-ef408dc8047b"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-37dc97ad-46ef-50d9-3619-3a6eaea1b6aa"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "zel9q53vtad0ytaeeuqautdqrgn188st"
+}

epfl_final/SparseAtt/wandb/run-20260708_234025-hn868f80/files/wandb-summary.json

@@ -0,0 +1 @@
+{"speed/samples_per_sec":2.412778261212465,"train/global_step":4420,"speed/steps_per_sec":0.12063891306062324,"eval/uses_sparse_attention":1,"eval/loss":2.5357428550720216,"eval/dense_interval_end":-1,"speed/ms_per_step":8289.199352264404,"_timestamp":1.783546406725075e+09,"train/epoch":0.117976778326438,"_runtime":36665,"_wandb":{"runtime":36665},"eval/uses_dense_attention":0,"_step":4420,"train/loss":2.3308617448806763,"eval/dense_interval_active":0,"train/learning_rate":0.0004877740427375342,"eval/dense_interval_start":-1,"eval/perplexity":12.625806509630907}

epfl_final/SparseAtt/wandb/run-20260708_234025-hn868f80/logs/debug-internal.log

@@ -0,0 +1,20 @@
+{"time":"2026-07-08T23:40:25.774961776Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-07-08T23:40:25.775471679Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-07-08T23:40:26.09549968Z","level":"INFO","msg":"stream: created new stream","id":"hn868f80"}
+{"time":"2026-07-08T23:40:26.095589325Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-07-08T23:40:26.099215286Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-07-08T23:40:26.099250168Z","level":"INFO","msg":"writer: started","stream_id":"hn868f80"}
+{"time":"2026-07-08T23:40:26.099284339Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-07-08T23:40:26.714914305Z","level":"INFO","msg":"filestream: sending request","total_files":1}
+{"time":"2026-07-08T23:40:27.072142827Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T23:40:28.365321062Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":94}
+{"time":"2026-07-08T23:40:28.372006516Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":15}
+{"time":"2026-07-08T23:40:41.716949421Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":473,"uploaded_len":2}
+{"time":"2026-07-08T23:40:42.116152053Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T23:40:49.246194624Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-07-08T23:40:49.246468655Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"uploaded_len":3,"complete":true,"exit_code":1}
+{"time":"2026-07-08T23:40:49.653643458Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-08T23:40:49.6538052Z","level":"INFO","msg":"stream: finishing up"}
+{"time":"2026-07-08T23:40:49.653821224Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-07-08T23:40:49.654922203Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-07-08T23:40:49.654941762Z","level":"INFO","msg":"stream: all finished"}

epfl_final/SparseAtt/wandb/run-20260708_234025-hn868f80/logs/debug.log

@@ -0,0 +1,22 @@
+2026-07-08 23:40:25,488 INFO    MainThread:4356 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-07-08 23:40:25,488 INFO    MainThread:4356 [wandb_setup.py:_flush():81] Configure stats pid to 4356
+2026-07-08 23:40:25,488 INFO    MainThread:4356 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-08 23:40:25,488 INFO    MainThread:4356 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_234025-hn868f80/logs/debug.log
+2026-07-08 23:40:25,488 INFO    MainThread:4356 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260708_234025-hn868f80/logs/debug-internal.log
+2026-07-08 23:40:25,488 INFO    MainThread:4356 [wandb_init.py:init():850] calling init triggers
+2026-07-08 23:40:25,488 INFO    MainThread:4356 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/longctx30k/tokenized', 'val_path': None, 'max_seq_length': 30720, 'num_workers': 4, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-4200', 'force_new_run': False, 'num_epochs': 1, 'max_steps': None, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 5, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 300, 'eval_steps': 500, 'eval_batches': 20, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b resampled 30k longctx', 'wandb_tags': ['qwen3', '1.2b', 'resampled', '30k', 'longctx', 'b128', 'scaleladder'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 120, 'sparse_mask_pool_size': 1000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_30k_b128_nrb120', 'sparse_mask_seed': 0, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-07-08 23:40:25,488 INFO    MainThread:4356 [wandb_init.py:init():898] starting backend
+2026-07-08 23:40:25,764 INFO    MainThread:4356 [wandb_init.py:init():913] sending inform_init request
+2026-07-08 23:40:26,100 INFO    MainThread:4356 [wandb_init.py:init():918] backend started and connected
+2026-07-08 23:40:26,101 INFO    MainThread:4356 [wandb_init.py:init():988] updated telemetry
+2026-07-08 23:40:26,131 INFO    MainThread:4356 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-07-08 23:40:26,712 INFO    MainThread:4356 [wandb_init.py:init():1051] run resumed
+2026-07-08 23:40:26,713 INFO    MainThread:4356 [wandb_init.py:init():1056] starting run threads in backend
+2026-07-08 23:40:26,890 INFO    MainThread:4356 [wandb_run.py:_console_start():2554] atexit reg
+2026-07-08 23:40:26,890 INFO    MainThread:4356 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-07-08 23:40:26,890 INFO    MainThread:4356 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-07-08 23:40:26,890 INFO    MainThread:4356 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-07-08 23:40:26,895 INFO    MainThread:4356 [wandb_init.py:init():1094] run started, returning control to user process
+2026-07-08 23:40:48,385 INFO    wandb-AsyncioManager-main:4356 [service_client.py:_forward_responses():134] Reached EOF.
+2026-07-08 23:40:48,386 INFO    wandb-AsyncioManager-main:4356 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.

epfl_final/SparseAtt/wandb/run-20260708_235322-cd66e13p/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-07-08T23:53:22.472009Z",
+  "args":  [
+    "--config",
+    "configs/train_config_30k_dense_b128_nockpt.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "88298c7a418622530ca5426b22b1e5ebd10ca0cb"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h200-train-4-gpu-0-0",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H200",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "7679361548288",
+      "used":  "5076391157760"
+    }
+  },
+  "memory":  {
+    "total":  "1623010586624"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H200",
+      "memoryTotal":  "150754820096",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-58acfad7-62f4-2de1-416b-00b062e1ee12"
+    },
+    {
+      "name":  "NVIDIA H200",
+      "memoryTotal":  "150754820096",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-6542e3f1-939f-77c0-a351-e17d642e8d2c"
+    },
+    {
+      "name":  "NVIDIA H200",
+      "memoryTotal":  "150754820096",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-034f28b5-f7a2-1a02-3ee0-6b6f0fde8f65"
+    },
+    {
+      "name":  "NVIDIA H200",
+      "memoryTotal":  "150754820096",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-cdf4f047-f9b9-78ab-8bd3-9d70fdf02ba6"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "s5n6p8j1u1mldg5kydsros9du3hkrtdy"
+}