EPFL final checkpoints and W&B history 1151-1200 of 1463

.gitattributes

@@ -149,3 +149,9 @@ epfl_final/SparseAtt/wandb/run-20260825_211848-rgmkfpnl/run-rgmkfpnl.wandb filte
 epfl_final/SparseAtt/wandb/run-20260826_034845-rgmkfpnl/run-rgmkfpnl.wandb filter=lfs diff=lfs merge=lfs -text
 epfl_final/SparseAtt/wandb/run-20260826_041533-qctui00g/run-qctui00g.wandb filter=lfs diff=lfs merge=lfs -text
 epfl_final/SparseAtt/wandb/run-20260826_125030-qctui00g/run-qctui00g.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260826_140249-rgmkfpnl/run-rgmkfpnl.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260826_152940-i8ewyy95/run-i8ewyy95.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260826_173107-rgmkfpnl/run-rgmkfpnl.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260826_175546-8ynn1we6/run-8ynn1we6.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260827_022420-rgmkfpnl/run-rgmkfpnl.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260827_041913-8ynn1we6/run-8ynn1we6.wandb filter=lfs diff=lfs merge=lfs -text

epfl_final/SparseAtt/wandb/run-20260826_135917-rgmkfpnl/files/output.log

@@ -0,0 +1,74 @@
+Starting training...
+Validation at step 32000: Loss = 2.5320, Perplexity = 12.58
+Switching attention mode for step 32001: dense (dense step interval)
+Traceback (most recent call last):
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1506, in <module>
+    main()
+    ~~~~^^
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1503, in main
+    train(config)
+    ~~~~~^^^^^^^^
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1385, in train
+    loss.backward()
+    ~~~~~~~~~~~~~^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_tensor.py", line 631, in backward
+    torch.autograd.backward(
+    ~~~~~~~~~~~~~~~~~~~~~~~^
+        self, gradient, retain_graph, create_graph, inputs=inputs
+        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+    )
+    ^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/autograd/__init__.py", line 381, in backward
+    _engine_run_backward(
+    ~~~~~~~~~~~~~~~~~~~~^
+        tensors,
+        ^^^^^^^^
+    ...<5 lines>...
+        accumulate_grad=True,
+        ^^^^^^^^^^^^^^^^^^^^^
+    )
+    ^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/autograd/graph.py", line 869, in _engine_run_backward
+    return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+        t_outputs, *args, **kwargs
+        ^^^^^^^^^^^^^^^^^^^^^^^^^^
+    )  # Calls into the C++ engine to run the backward pass
+    ^
+torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 9.27 GiB. GPU 0 has a total capacity of 79.18 GiB of which 7.43 GiB is free. Including non-PyTorch memory, this process has 71.74 GiB memory in use. Of the allocated memory 70.02 GiB is allocated by PyTorch, and 193.03 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://docs.pytorch.org/docs/stable/notes/cuda.html#optimizing-memory-usage-with-pytorch-cuda-alloc-conf)
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1506, in <module>
+[rank0]:     main()
+[rank0]:     ~~~~^^
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1503, in main
+[rank0]:     train(config)
+[rank0]:     ~~~~~^^^^^^^^
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1385, in train
+[rank0]:     loss.backward()
+[rank0]:     ~~~~~~~~~~~~~^^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_tensor.py", line 631, in backward
+[rank0]:     torch.autograd.backward(
+[rank0]:     ~~~~~~~~~~~~~~~~~~~~~~~^
+[rank0]:         self, gradient, retain_graph, create_graph, inputs=inputs
+[rank0]:         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:     )
+[rank0]:     ^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/autograd/__init__.py", line 381, in backward
+[rank0]:     _engine_run_backward(
+[rank0]:     ~~~~~~~~~~~~~~~~~~~~^
+[rank0]:         tensors,
+[rank0]:         ^^^^^^^^
+[rank0]:     ...<5 lines>...
+[rank0]:         accumulate_grad=True,
+[rank0]:         ^^^^^^^^^^^^^^^^^^^^^
+[rank0]:     )
+[rank0]:     ^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/autograd/graph.py", line 869, in _engine_run_backward
+[rank0]:     return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+[rank0]:            ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:         t_outputs, *args, **kwargs
+[rank0]:         ^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:     )  # Calls into the C++ engine to run the backward pass
+[rank0]:     ^
+[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 9.27 GiB. GPU 0 has a total capacity of 79.18 GiB of which 7.43 GiB is free. Including non-PyTorch memory, this process has 71.74 GiB memory in use. Of the allocated memory 70.02 GiB is allocated by PyTorch, and 193.03 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://docs.pytorch.org/docs/stable/notes/cuda.html#optimizing-memory-usage-with-pytorch-cuda-alloc-conf)
+wandb: WARNING Tried to log to step 32000 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.

epfl_final/SparseAtt/wandb/run-20260826_135917-rgmkfpnl/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260826_135917-rgmkfpnl/files/wandb-metadata.json

@@ -0,0 +1,51 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-08-26T13:59:17.614076Z",
+  "args":  [
+    "--config",
+    "configs/p2_1p2b_dense_seed70_2gpu.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/SparseAtt.git",
+    "commit":  "8f862f749eb4d5c17f688eb1562235e3fe5b2550"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-workspace-2-gpu-0-20",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  2,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "5456340393984"
+    }
+  },
+  "memory":  {
+    "total":  "1623005626368"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-572b71c8-9d82-4b1c-e4f6-ed5616fd870a"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-3ea6c840-350f-c68c-7562-6e0affa4e153"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "vynmv5ghprw0cfh5acbvrcf9wtx6pvib"
+}

epfl_final/SparseAtt/wandb/run-20260826_135917-rgmkfpnl/logs/debug-internal.log

@@ -0,0 +1,28 @@
+{"time":"2026-08-26T13:59:17.901782598Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-08-26T13:59:17.902086763Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-08-26T13:59:18.196193683Z","level":"INFO","msg":"stream: created new stream","id":"rgmkfpnl"}
+{"time":"2026-08-26T13:59:18.196268575Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-08-26T13:59:18.198554223Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-08-26T13:59:18.198562205Z","level":"INFO","msg":"writer: started","stream_id":"rgmkfpnl"}
+{"time":"2026-08-26T13:59:18.198590227Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-08-26T13:59:18.746548474Z","level":"INFO","msg":"filestream: sending request","total_files":1}
+{"time":"2026-08-26T13:59:19.043151226Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T13:59:33.747425159Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1,"uploaded_len":2}
+{"time":"2026-08-26T13:59:33.931462795Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T13:59:48.747074499Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":0,"events_lines":2}
+{"time":"2026-08-26T13:59:48.948048861Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T14:00:03.747203617Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
+{"time":"2026-08-26T14:00:03.942693117Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T14:00:18.746979631Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":4,"events_lines":2}
+{"time":"2026-08-26T14:00:18.946821629Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T14:00:19.729073979Z","level":"WARN","msg":"handler: ignoring partial history record","step":32000,"current":32121}
+{"time":"2026-08-26T14:00:33.74712122Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":6,"events_lines":2,"console_offset":1,"console_lines":73}
+{"time":"2026-08-26T14:00:34.001878962Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T14:00:48.747064123Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":8,"events_lines":2}
+{"time":"2026-08-26T14:00:48.977436702Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T14:01:03.747491089Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":10,"events_lines":2}
+{"time":"2026-08-26T14:01:03.972333229Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T14:01:18.747039722Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":12,"events_lines":2}
+{"time":"2026-08-26T14:01:19.027518945Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T14:01:33.746707423Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":14,"events_lines":2}
+{"time":"2026-08-26T14:01:33.94253588Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

epfl_final/SparseAtt/wandb/run-20260826_140249-rgmkfpnl/files/output.log

@@ -0,0 +1,176 @@
+Starting training...
+Validation at step 32000: Loss = 2.5320, Perplexity = 12.58
+Switching attention mode for step 32001: dense (dense step interval)
+wandb: WARNING Tried to log to step 32000 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 32010/50462 | Loss: 2.5168 | LR: 1.52e-04 | 14380.0ms/step | 4.2 samples/s
+wandb: WARNING Tried to log to step 32010 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 32020/50462 | Loss: 2.5393 | LR: 1.51e-04 | 7968.8ms/step | 7.5 samples/s
+wandb: WARNING Tried to log to step 32020 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 32030/50462 | Loss: 2.5201 | LR: 1.51e-04 | 7955.3ms/step | 7.5 samples/s
+wandb: WARNING Tried to log to step 32030 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 32040/50462 | Loss: 2.5227 | LR: 1.51e-04 | 7953.5ms/step | 7.5 samples/s
+wandb: WARNING Tried to log to step 32040 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 32050/50462 | Loss: 2.5264 | LR: 1.51e-04 | 7952.9ms/step | 7.5 samples/s
+wandb: WARNING Tried to log to step 32050 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 32060/50462 | Loss: 2.5234 | LR: 1.51e-04 | 7953.5ms/step | 7.5 samples/s
+wandb: WARNING Tried to log to step 32060 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 32070/50462 | Loss: 2.5138 | LR: 1.51e-04 | 7954.4ms/step | 7.5 samples/s
+wandb: WARNING Tried to log to step 32070 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 32080/50462 | Loss: 2.5400 | LR: 1.51e-04 | 7955.2ms/step | 7.5 samples/s
+wandb: WARNING Tried to log to step 32080 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 32090/50462 | Loss: 2.5294 | LR: 1.50e-04 | 7956.8ms/step | 7.5 samples/s
+wandb: WARNING Tried to log to step 32090 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 32100/50462 | Loss: 2.5349 | LR: 1.50e-04 | 7954.5ms/step | 7.5 samples/s
+wandb: WARNING Tried to log to step 32100 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 32110/50462 | Loss: 2.5089 | LR: 1.50e-04 | 7952.9ms/step | 7.5 samples/s
+wandb: WARNING Tried to log to step 32110 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 32120/50462 | Loss: 2.5187 | LR: 1.50e-04 | 7955.4ms/step | 7.5 samples/s
+wandb: WARNING Tried to log to step 32120 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 32130/50462 | Loss: 2.5054 | LR: 1.50e-04 | 7955.8ms/step | 7.5 samples/s
+Step 32140/50462 | Loss: 2.5293 | LR: 1.50e-04 | 7955.8ms/step | 7.5 samples/s
+Step 32150/50462 | Loss: 2.5470 | LR: 1.50e-04 | 7955.2ms/step | 7.5 samples/s
+Step 32160/50462 | Loss: 2.5272 | LR: 1.49e-04 | 7953.8ms/step | 7.5 samples/s
+Step 32170/50462 | Loss: 2.5062 | LR: 1.49e-04 | 7954.2ms/step | 7.5 samples/s
+Step 32180/50462 | Loss: 2.5364 | LR: 1.49e-04 | 7952.2ms/step | 7.5 samples/s
+Step 32190/50462 | Loss: 2.5128 | LR: 1.49e-04 | 7952.9ms/step | 7.5 samples/s
+Step 32200/50462 | Loss: 2.4884 | LR: 1.49e-04 | 7953.6ms/step | 7.5 samples/s
+Step 32210/50462 | Loss: 2.5220 | LR: 1.49e-04 | 7952.5ms/step | 7.5 samples/s
+Step 32220/50462 | Loss: 2.5136 | LR: 1.49e-04 | 7951.2ms/step | 7.5 samples/s
+Step 32230/50462 | Loss: 2.5225 | LR: 1.48e-04 | 7954.1ms/step | 7.5 samples/s
+Step 32240/50462 | Loss: 2.5340 | LR: 1.48e-04 | 7952.1ms/step | 7.5 samples/s
+Step 32250/50462 | Loss: 2.5188 | LR: 1.48e-04 | 7951.2ms/step | 7.5 samples/s
+Validation at step 32250: Loss = 2.5308, Perplexity = 12.56
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-32250
+Step 32260/50462 | Loss: 2.5418 | LR: 1.48e-04 | 15161.9ms/step | 4.0 samples/s
+Step 32270/50462 | Loss: 2.5208 | LR: 1.48e-04 | 7951.6ms/step | 7.5 samples/s
+Step 32280/50462 | Loss: 2.5082 | LR: 1.48e-04 | 7953.3ms/step | 7.5 samples/s
+Step 32290/50462 | Loss: 2.5104 | LR: 1.48e-04 | 7952.8ms/step | 7.5 samples/s
+Step 32300/50462 | Loss: 2.5220 | LR: 1.47e-04 | 7952.8ms/step | 7.5 samples/s
+Step 32310/50462 | Loss: 2.4841 | LR: 1.47e-04 | 7951.5ms/step | 7.5 samples/s
+Step 32320/50462 | Loss: 2.5203 | LR: 1.47e-04 | 7951.6ms/step | 7.5 samples/s
+Step 32330/50462 | Loss: 2.5213 | LR: 1.47e-04 | 7951.5ms/step | 7.5 samples/s
+Step 32340/50462 | Loss: 2.5247 | LR: 1.47e-04 | 7951.4ms/step | 7.5 samples/s
+Step 32350/50462 | Loss: 2.5044 | LR: 1.47e-04 | 7950.4ms/step | 7.5 samples/s
+Step 32360/50462 | Loss: 2.5440 | LR: 1.47e-04 | 7953.0ms/step | 7.5 samples/s
+Step 32370/50462 | Loss: 2.5284 | LR: 1.46e-04 | 7953.7ms/step | 7.5 samples/s
+Step 32380/50462 | Loss: 2.4881 | LR: 1.46e-04 | 7951.9ms/step | 7.5 samples/s
+Step 32390/50462 | Loss: 2.5022 | LR: 1.46e-04 | 7952.0ms/step | 7.5 samples/s
+Step 32400/50462 | Loss: 2.5156 | LR: 1.46e-04 | 7950.8ms/step | 7.5 samples/s
+Step 32410/50462 | Loss: 2.5264 | LR: 1.46e-04 | 7951.6ms/step | 7.5 samples/s
+Step 32420/50462 | Loss: 2.5098 | LR: 1.46e-04 | 7951.2ms/step | 7.5 samples/s
+Step 32430/50462 | Loss: 2.5245 | LR: 1.45e-04 | 7953.5ms/step | 7.5 samples/s
+Step 32440/50462 | Loss: 2.5116 | LR: 1.45e-04 | 7951.5ms/step | 7.5 samples/s
+Step 32450/50462 | Loss: 2.5223 | LR: 1.45e-04 | 7952.6ms/step | 7.5 samples/s
+Step 32460/50462 | Loss: 2.5081 | LR: 1.45e-04 | 7954.2ms/step | 7.5 samples/s
+Step 32470/50462 | Loss: 2.4938 | LR: 1.45e-04 | 7954.3ms/step | 7.5 samples/s
+Step 32480/50462 | Loss: 2.5192 | LR: 1.45e-04 | 7953.9ms/step | 7.5 samples/s
+Step 32490/50462 | Loss: 2.4950 | LR: 1.45e-04 | 7956.5ms/step | 7.5 samples/s
+Step 32500/50462 | Loss: 2.5168 | LR: 1.44e-04 | 7958.4ms/step | 7.5 samples/s
+Validation at step 32500: Loss = 2.5298, Perplexity = 12.55
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-32500
+Step 32510/50462 | Loss: 2.5303 | LR: 1.44e-04 | 14978.8ms/step | 4.0 samples/s
+Step 32520/50462 | Loss: 2.5262 | LR: 1.44e-04 | 7951.0ms/step | 7.5 samples/s
+Step 32530/50462 | Loss: 2.5260 | LR: 1.44e-04 | 7957.8ms/step | 7.5 samples/s
+Step 32540/50462 | Loss: 2.5123 | LR: 1.44e-04 | 7962.7ms/step | 7.5 samples/s
+Step 32550/50462 | Loss: 2.5382 | LR: 1.44e-04 | 7963.3ms/step | 7.5 samples/s
+Step 32560/50462 | Loss: 2.5191 | LR: 1.44e-04 | 7963.1ms/step | 7.5 samples/s
+Step 32570/50462 | Loss: 2.5264 | LR: 1.43e-04 | 7965.9ms/step | 7.5 samples/s
+Step 32580/50462 | Loss: 2.5139 | LR: 1.43e-04 | 7974.8ms/step | 7.5 samples/s
+Step 32590/50462 | Loss: 2.5012 | LR: 1.43e-04 | 7958.3ms/step | 7.5 samples/s
+Step 32600/50462 | Loss: 2.5204 | LR: 1.43e-04 | 7957.7ms/step | 7.5 samples/s
+Step 32610/50462 | Loss: 2.4918 | LR: 1.43e-04 | 7957.4ms/step | 7.5 samples/s
+Step 32620/50462 | Loss: 2.5209 | LR: 1.43e-04 | 7957.7ms/step | 7.5 samples/s
+Step 32630/50462 | Loss: 2.5136 | LR: 1.43e-04 | 7957.4ms/step | 7.5 samples/s
+Step 32640/50462 | Loss: 2.4992 | LR: 1.42e-04 | 7958.1ms/step | 7.5 samples/s
+Step 32650/50462 | Loss: 2.5108 | LR: 1.42e-04 | 7959.3ms/step | 7.5 samples/s
+Step 32660/50462 | Loss: 2.4910 | LR: 1.42e-04 | 7954.9ms/step | 7.5 samples/s
+Step 32670/50462 | Loss: 2.5155 | LR: 1.42e-04 | 7957.1ms/step | 7.5 samples/s
+Step 32680/50462 | Loss: 2.5118 | LR: 1.42e-04 | 7956.2ms/step | 7.5 samples/s
+Step 32690/50462 | Loss: 2.5083 | LR: 1.42e-04 | 7956.6ms/step | 7.5 samples/s
+Step 32700/50462 | Loss: 2.5080 | LR: 1.42e-04 | 7955.3ms/step | 7.5 samples/s
+Step 32710/50462 | Loss: 2.5027 | LR: 1.41e-04 | 7953.8ms/step | 7.5 samples/s
+Step 32720/50462 | Loss: 2.5381 | LR: 1.41e-04 | 7953.4ms/step | 7.5 samples/s
+Step 32730/50462 | Loss: 2.5032 | LR: 1.41e-04 | 7952.4ms/step | 7.5 samples/s
+Step 32740/50462 | Loss: 2.5122 | LR: 1.41e-04 | 7952.8ms/step | 7.5 samples/s
+Step 32750/50462 | Loss: 2.5349 | LR: 1.41e-04 | 7969.1ms/step | 7.5 samples/s
+Validation at step 32750: Loss = 2.5289, Perplexity = 12.54
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-32750
+Step 32760/50462 | Loss: 2.4937 | LR: 1.41e-04 | 15038.0ms/step | 4.0 samples/s
+Step 32770/50462 | Loss: 2.5269 | LR: 1.41e-04 | 7973.3ms/step | 7.5 samples/s
+Step 32780/50462 | Loss: 2.5209 | LR: 1.41e-04 | 7961.5ms/step | 7.5 samples/s
+Step 32790/50462 | Loss: 2.5174 | LR: 1.40e-04 | 7968.3ms/step | 7.5 samples/s
+Step 32800/50462 | Loss: 2.5169 | LR: 1.40e-04 | 7966.2ms/step | 7.5 samples/s
+Step 32810/50462 | Loss: 2.4942 | LR: 1.40e-04 | 7958.4ms/step | 7.5 samples/s
+Step 32820/50462 | Loss: 2.5293 | LR: 1.40e-04 | 7973.1ms/step | 7.5 samples/s
+Step 32830/50462 | Loss: 2.5132 | LR: 1.40e-04 | 7966.9ms/step | 7.5 samples/s
+Step 32840/50462 | Loss: 2.5248 | LR: 1.40e-04 | 7962.6ms/step | 7.5 samples/s
+Step 32850/50462 | Loss: 2.5003 | LR: 1.40e-04 | 7970.2ms/step | 7.5 samples/s
+Step 32860/50462 | Loss: 2.5061 | LR: 1.39e-04 | 7973.4ms/step | 7.5 samples/s
+Step 32870/50462 | Loss: 2.4807 | LR: 1.39e-04 | 7965.5ms/step | 7.5 samples/s
+Step 32880/50462 | Loss: 2.4954 | LR: 1.39e-04 | 7967.8ms/step | 7.5 samples/s
+Step 32890/50462 | Loss: 2.5137 | LR: 1.39e-04 | 7968.6ms/step | 7.5 samples/s
+Step 32900/50462 | Loss: 2.5083 | LR: 1.39e-04 | 7962.0ms/step | 7.5 samples/s
+Step 32910/50462 | Loss: 2.5169 | LR: 1.39e-04 | 7967.8ms/step | 7.5 samples/s
+Step 32920/50462 | Loss: 2.5215 | LR: 1.39e-04 | 7966.9ms/step | 7.5 samples/s
+Step 32930/50462 | Loss: 2.5084 | LR: 1.38e-04 | 7965.0ms/step | 7.5 samples/s
+Step 32940/50462 | Loss: 2.5293 | LR: 1.38e-04 | 7968.1ms/step | 7.5 samples/s
+Step 32950/50462 | Loss: 2.5226 | LR: 1.38e-04 | 7966.7ms/step | 7.5 samples/s
+Step 32960/50462 | Loss: 2.5018 | LR: 1.38e-04 | 7972.1ms/step | 7.5 samples/s
+Step 32970/50462 | Loss: 2.5206 | LR: 1.38e-04 | 7979.9ms/step | 7.5 samples/s
+Step 32980/50462 | Loss: 2.5245 | LR: 1.38e-04 | 7985.1ms/step | 7.5 samples/s
+Step 32990/50462 | Loss: 2.5348 | LR: 1.38e-04 | 7982.5ms/step | 7.5 samples/s
+Step 33000/50462 | Loss: 2.5241 | LR: 1.37e-04 | 7975.0ms/step | 7.5 samples/s
+Validation at step 33000: Loss = 2.5279, Perplexity = 12.53
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-33000
+Step 33010/50462 | Loss: 2.5174 | LR: 1.37e-04 | 15020.9ms/step | 4.0 samples/s
+Step 33020/50462 | Loss: 2.5204 | LR: 1.37e-04 | 7964.6ms/step | 7.5 samples/s
+Step 33030/50462 | Loss: 2.5059 | LR: 1.37e-04 | 7972.3ms/step | 7.5 samples/s
+Step 33040/50462 | Loss: 2.5184 | LR: 1.37e-04 | 7970.3ms/step | 7.5 samples/s
+Step 33050/50462 | Loss: 2.5247 | LR: 1.37e-04 | 7968.8ms/step | 7.5 samples/s
+Step 33060/50462 | Loss: 2.5153 | LR: 1.37e-04 | 7969.8ms/step | 7.5 samples/s
+Step 33070/50462 | Loss: 2.5172 | LR: 1.36e-04 | 7964.1ms/step | 7.5 samples/s
+Step 33080/50462 | Loss: 2.5123 | LR: 1.36e-04 | 7981.1ms/step | 7.5 samples/s
+Step 33090/50462 | Loss: 2.5081 | LR: 1.36e-04 | 7982.1ms/step | 7.5 samples/s
+Step 33100/50462 | Loss: 2.4931 | LR: 1.36e-04 | 7968.4ms/step | 7.5 samples/s
+Step 33110/50462 | Loss: 2.5296 | LR: 1.36e-04 | 7961.2ms/step | 7.5 samples/s
+Step 33120/50462 | Loss: 2.5104 | LR: 1.36e-04 | 7967.5ms/step | 7.5 samples/s
+Step 33130/50462 | Loss: 2.5040 | LR: 1.36e-04 | 7964.4ms/step | 7.5 samples/s
+Step 33140/50462 | Loss: 2.5142 | LR: 1.35e-04 | 7964.6ms/step | 7.5 samples/s
+Step 33150/50462 | Loss: 2.5310 | LR: 1.35e-04 | 7968.3ms/step | 7.5 samples/s
+Step 33160/50462 | Loss: 2.5055 | LR: 1.35e-04 | 7964.9ms/step | 7.5 samples/s
+Step 33170/50462 | Loss: 2.5262 | LR: 1.35e-04 | 7967.4ms/step | 7.5 samples/s
+Step 33180/50462 | Loss: 2.5182 | LR: 1.35e-04 | 7964.2ms/step | 7.5 samples/s
+Step 33190/50462 | Loss: 2.5355 | LR: 1.35e-04 | 7963.1ms/step | 7.5 samples/s
+Step 33200/50462 | Loss: 2.5024 | LR: 1.35e-04 | 7962.0ms/step | 7.5 samples/s
+Step 33210/50462 | Loss: 2.5094 | LR: 1.34e-04 | 7961.3ms/step | 7.5 samples/s
+Step 33220/50462 | Loss: 2.5521 | LR: 1.34e-04 | 7964.9ms/step | 7.5 samples/s
+Step 33230/50462 | Loss: 2.5323 | LR: 1.34e-04 | 7974.3ms/step | 7.5 samples/s
+Step 33240/50462 | Loss: 2.5066 | LR: 1.34e-04 | 7957.4ms/step | 7.5 samples/s
+Step 33250/50462 | Loss: 2.5200 | LR: 1.34e-04 | 7964.7ms/step | 7.5 samples/s
+Validation at step 33250: Loss = 2.5268, Perplexity = 12.51
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-33250
+Step 33260/50462 | Loss: 2.5115 | LR: 1.34e-04 | 15129.2ms/step | 4.0 samples/s
+Step 33270/50462 | Loss: 2.4846 | LR: 1.34e-04 | 7963.5ms/step | 7.5 samples/s
+Step 33280/50462 | Loss: 2.5425 | LR: 1.33e-04 | 7960.2ms/step | 7.5 samples/s
+Step 33290/50462 | Loss: 2.5074 | LR: 1.33e-04 | 7957.3ms/step | 7.5 samples/s
+Step 33300/50462 | Loss: 2.5234 | LR: 1.33e-04 | 7960.6ms/step | 7.5 samples/s
+Step 33310/50462 | Loss: 2.5243 | LR: 1.33e-04 | 7964.6ms/step | 7.5 samples/s
+Step 33320/50462 | Loss: 2.5178 | LR: 1.33e-04 | 7959.8ms/step | 7.5 samples/s
+Step 33330/50462 | Loss: 2.4968 | LR: 1.33e-04 | 7966.2ms/step | 7.5 samples/s
+Step 33340/50462 | Loss: 2.4934 | LR: 1.33e-04 | 7958.7ms/step | 7.5 samples/s
+Step 33350/50462 | Loss: 2.5080 | LR: 1.32e-04 | 7964.9ms/step | 7.5 samples/s
+Step 33360/50462 | Loss: 2.5233 | LR: 1.32e-04 | 7956.9ms/step | 7.5 samples/s
+Step 33370/50462 | Loss: 2.5398 | LR: 1.32e-04 | 7958.6ms/step | 7.5 samples/s
+Step 33380/50462 | Loss: 2.5112 | LR: 1.32e-04 | 7959.7ms/step | 7.5 samples/s
+Step 33390/50462 | Loss: 2.5045 | LR: 1.32e-04 | 7959.5ms/step | 7.5 samples/s
+Step 33400/50462 | Loss: 2.5173 | LR: 1.32e-04 | 7960.4ms/step | 7.5 samples/s
+Step 33410/50462 | Loss: 2.5159 | LR: 1.32e-04 | 7959.9ms/step | 7.5 samples/s
+Step 33420/50462 | Loss: 2.5118 | LR: 1.32e-04 | 7953.2ms/step | 7.5 samples/s
+Step 33430/50462 | Loss: 2.5228 | LR: 1.31e-04 | 7953.5ms/step | 7.5 samples/s
+Step 33440/50462 | Loss: 2.5355 | LR: 1.31e-04 | 7960.5ms/step | 7.5 samples/s
+Step 33450/50462 | Loss: 2.4778 | LR: 1.31e-04 | 7965.3ms/step | 7.5 samples/s
+Step 33460/50462 | Loss: 2.5028 | LR: 1.31e-04 | 7954.6ms/step | 7.5 samples/s
+Step 33470/50462 | Loss: 2.5052 | LR: 1.31e-04 | 7961.7ms/step | 7.5 samples/s
+Step 33480/50462 | Loss: 2.5034 | LR: 1.31e-04 | 7959.8ms/step | 7.5 samples/s
+Step 33490/50462 | Loss: 2.5164 | LR: 1.31e-04 | 7957.8ms/step | 7.5 samples/s
+Step 33500/50462 | Loss: 2.5020 | LR: 1.30e-04 | 7963.1ms/step | 7.5 samples/s

epfl_final/SparseAtt/wandb/run-20260826_140249-rgmkfpnl/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260826_140249-rgmkfpnl/files/wandb-metadata.json

@@ -0,0 +1,51 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-08-26T14:02:49.592846Z",
+  "args":  [
+    "--config",
+    "configs/p2_1p2b_dense_seed70_2gpu.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/SparseAtt.git",
+    "commit":  "8f862f749eb4d5c17f688eb1562235e3fe5b2550"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-workspace-2-gpu-0-20",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  2,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "5456340410368"
+    }
+  },
+  "memory":  {
+    "total":  "1623005626368"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-572b71c8-9d82-4b1c-e4f6-ed5616fd870a"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-3ea6c840-350f-c68c-7562-6e0affa4e153"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "557nl5acy7fpu1hpwhyns1yy1m413u1w"
+}

epfl_final/SparseAtt/wandb/run-20260826_140249-rgmkfpnl/logs/debug.log

@@ -0,0 +1,20 @@
+2026-08-26 14:02:49,632 INFO    MainThread:4543 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-08-26 14:02:49,633 INFO    MainThread:4543 [wandb_setup.py:_flush():81] Configure stats pid to 4543
+2026-08-26 14:02:49,633 INFO    MainThread:4543 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-26 14:02:49,633 INFO    MainThread:4543 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260826_140249-rgmkfpnl/logs/debug.log
+2026-08-26 14:02:49,633 INFO    MainThread:4543 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260826_140249-rgmkfpnl/logs/debug-internal.log
+2026-08-26 14:02:49,633 INFO    MainThread:4543 [wandb_init.py:init():850] calling init triggers
+2026-08-26 14:02:49,633 INFO    MainThread:4543 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-32000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 30, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 70, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b dense 8k seed70 2gpu resume', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'dense', 'seed70'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
+2026-08-26 14:02:49,633 INFO    MainThread:4543 [wandb_init.py:init():898] starting backend
+2026-08-26 14:02:49,889 INFO    MainThread:4543 [wandb_init.py:init():913] sending inform_init request
+2026-08-26 14:02:50,205 INFO    MainThread:4543 [wandb_init.py:init():918] backend started and connected
+2026-08-26 14:02:50,208 INFO    MainThread:4543 [wandb_init.py:init():988] updated telemetry
+2026-08-26 14:02:50,222 INFO    MainThread:4543 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-08-26 14:02:50,735 INFO    MainThread:4543 [wandb_init.py:init():1051] run resumed
+2026-08-26 14:02:50,737 INFO    MainThread:4543 [wandb_init.py:init():1056] starting run threads in backend
+2026-08-26 14:02:50,888 INFO    MainThread:4543 [wandb_run.py:_console_start():2554] atexit reg
+2026-08-26 14:02:50,888 INFO    MainThread:4543 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-08-26 14:02:50,888 INFO    MainThread:4543 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-08-26 14:02:50,888 INFO    MainThread:4543 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-08-26 14:02:50,912 INFO    MainThread:4543 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/run-20260826_140249-rgmkfpnl/run-rgmkfpnl.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7125cddc0794462db67a8050419779c3a7fa24a2150006d6bb3df02fda621906
+size 2523136

epfl_final/SparseAtt/wandb/run-20260826_152940-i8ewyy95/files/config.yaml

@@ -0,0 +1,210 @@
+_wandb:
+    value:
+        cli_version: 0.26.1
+        e:
+            f75a5zbdrc79jw3dypdbluk1x3eamao9:
+                args:
+                    - --config
+                    - configs/p2_1p2b_mixed_seed70_6gpu_resume48750.json
+                codePath: train.py
+                codePathLocal: train.py
+                cpu_count: 96
+                cpu_count_logical: 192
+                cudaVersion: "13.0"
+                disk:
+                    /:
+                        total: "9374931091456"
+                        used: "4877579661312"
+                email: ruben.navasardyan@mail.utoronto.ca
+                executable: /mloscratch/homes/navasard/conda/bin/python3.13
+                git:
+                    commit: 8f862f749eb4d5c17f688eb1562235e3fe5b2550
+                    remote: https://github.com/rubentium/SparseAtt.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 6
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-99f99f92-e501-0b15-5782-6beb2c34fa1a
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-3f5bace4-0066-71eb-863d-73a3a6d79e3b
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-1aef7113-cfe5-3f49-53bb-a84f3c5abfa0
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-5e17ddff-2e95-8c68-b074-b0189417e834
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-61e64705-1798-f2b5-568f-877549238703
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-9cd8a6e5-05e9-cfde-0a28-d8058797b3bf
+                host: h100-train-6-gpu-0-39
+                memory:
+                    total: "1623005732864"
+                os: Linux-6.8.0-124-generic-x86_64-with-glibc2.35
+                program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
+                python: CPython 3.13.13
+                root: /mloscratch/homes/navasard/sparse/SparseAtt
+                startedAt: "2026-08-26T15:29:40.703291Z"
+                writerId: f75a5zbdrc79jw3dypdbluk1x3eamao9
+        m: []
+        python_version: 3.13.13
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 5
+                - 13
+                - 14
+                - 15
+                - 16
+                - 61
+                - 62
+            "4": 3.13.13
+            "5": 0.26.1
+            "6": 5.7.0
+            "12": 0.26.1
+            "13": linux-x86_64
+attention_dropout:
+    value: null
+attention_type:
+    value: sparse_stochastic
+checkpoint_activations:
+    value: true
+dense_mode:
+    value: false
+dense_step_intervals:
+    value: null
+eval_batches:
+    value: 50
+eval_steps:
+    value: 250
+force_new_run:
+    value: false
+gradient_accumulation_steps:
+    value: 10
+head_dim:
+    value: null
+hidden_size:
+    value: 1536
+init_from_scratch:
+    value: true
+intermediate_size:
+    value: 6144
+learning_rate:
+    value: 0.0005
+logging_steps:
+    value: 10
+lr_scheduler:
+    value: cosine
+max_grad_norm:
+    value: 1
+max_seq_length:
+    value: 8192
+max_steps:
+    value: 50462
+mixed_precision:
+    value: bf16
+model_dtype:
+    value: bfloat16
+model_path:
+    value: /mloscratch/homes/navasard/qwen3_06b_base
+num_attention_heads:
+    value: 12
+num_epochs:
+    value: 1
+num_hidden_layers:
+    value: null
+num_key_value_heads:
+    value: 4
+num_workers:
+    value: 8
+output_dir:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107
+per_device_batch_size:
+    value: 1
+resume_from_checkpoint:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-48750
+rope_train_position_scale_set:
+    value: null
+save_steps:
+    value: 250
+seed:
+    value: 70
+sparse_block_size:
+    value: 128
+sparse_layer_indices:
+    value: null
+sparse_mask_cache_dir:
+    value: /mloscratch/homes/navasard/sparse/mask_cache_8k_mixed
+sparse_mask_pool_size:
+    value: 10000
+sparse_mask_seed:
+    value: 70
+sparse_mixed_nrb_set:
+    value:
+        - 63
+        - 47
+        - 31
+        - 21
+        - 11
+sparse_num_random_blocks:
+    value: 35
+sparse_val_dense_compare:
+    value: false
+strategy:
+    value: ddp
+train_path:
+    value: /mloscratch/homes/navasard/data/fineweb_large/tokenized
+val_path:
+    value: null
+verbose:
+    value: true
+wandb_enabled:
+    value: true
+wandb_name:
+    value: 1.2b mixed 8k seed70 6gpu resume48750
+wandb_project:
+    value: stochastic_sparsity
+wandb_tags:
+    value:
+        - qwen3
+        - 1.2b
+        - 8k
+        - p2
+        - mixed
+        - seed70
+warmup_steps:
+    value: 750
+weight_decay:
+    value: 0.01

epfl_final/SparseAtt/wandb/run-20260826_152940-i8ewyy95/files/output.log

@@ -0,0 +1,47 @@
+Starting training...
+Validation at step 48750: Loss = 2.5281, Perplexity = 12.53
+wandb: WARNING Tried to log to step 48750 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 48751: sparse_stochastic (configured sparse attention)
+Step 48760/50462 | Loss: 2.5039 | LR: 1.44e-06 | 74877.1ms/step | 0.8 samples/s
+wandb: WARNING Tried to log to step 48760 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 48770/50462 | Loss: 2.5169 | LR: 1.43e-06 | 3307.9ms/step | 18.1 samples/s
+wandb: WARNING Tried to log to step 48770 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 48780/50462 | Loss: 2.5173 | LR: 1.41e-06 | 2735.2ms/step | 21.9 samples/s
+wandb: WARNING Tried to log to step 48780 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 48790/50462 | Loss: 2.5569 | LR: 1.39e-06 | 2736.4ms/step | 21.9 samples/s
+wandb: WARNING Tried to log to step 48790 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 48800/50462 | Loss: 2.5348 | LR: 1.38e-06 | 2734.6ms/step | 21.9 samples/s
+wandb: WARNING Tried to log to step 48800 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 48810/50462 | Loss: 2.5282 | LR: 1.36e-06 | 2742.2ms/step | 21.9 samples/s
+wandb: WARNING Tried to log to step 48810 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 48820/50462 | Loss: 2.5347 | LR: 1.34e-06 | 2748.7ms/step | 21.8 samples/s
+wandb: WARNING Tried to log to step 48820 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 48830/50462 | Loss: 2.5142 | LR: 1.33e-06 | 2740.7ms/step | 21.9 samples/s
+wandb: WARNING Tried to log to step 48830 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 48840/50462 | Loss: 2.5337 | LR: 1.31e-06 | 2746.6ms/step | 21.8 samples/s
+wandb: WARNING Tried to log to step 48840 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 48850/50462 | Loss: 2.5308 | LR: 1.30e-06 | 2740.9ms/step | 21.9 samples/s
+wandb: WARNING Tried to log to step 48850 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 48860/50462 | Loss: 2.4891 | LR: 1.28e-06 | 2744.3ms/step | 21.9 samples/s
+wandb: WARNING Tried to log to step 48860 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 48870/50462 | Loss: 2.5272 | LR: 1.26e-06 | 2735.8ms/step | 21.9 samples/s
+wandb: WARNING Tried to log to step 48870 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 48880/50462 | Loss: 2.5142 | LR: 1.25e-06 | 2738.2ms/step | 21.9 samples/s
+wandb: WARNING Tried to log to step 48880 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 48890/50462 | Loss: 2.5033 | LR: 1.23e-06 | 2735.5ms/step | 21.9 samples/s
+wandb: WARNING Tried to log to step 48890 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 48900/50462 | Loss: 2.5143 | LR: 1.22e-06 | 3301.6ms/step | 18.2 samples/s
+wandb: WARNING Tried to log to step 48900 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 48910/50462 | Loss: 2.5073 | LR: 1.20e-06 | 2739.0ms/step | 21.9 samples/s
+wandb: WARNING Tried to log to step 48910 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 48920/50462 | Loss: 2.4855 | LR: 1.19e-06 | 2737.2ms/step | 21.9 samples/s
+wandb: WARNING Tried to log to step 48920 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 48930/50462 | Loss: 2.5544 | LR: 1.17e-06 | 2736.7ms/step | 21.9 samples/s
+wandb: WARNING Tried to log to step 48930 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 48940/50462 | Loss: 2.4489 | LR: 1.16e-06 | 2736.7ms/step | 21.9 samples/s
+wandb: WARNING Tried to log to step 48940 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 48950/50462 | Loss: 2.5202 | LR: 1.14e-06 | 2733.4ms/step | 22.0 samples/s
+wandb: WARNING Tried to log to step 48950 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 48960/50462 | Loss: 2.5298 | LR: 1.13e-06 | 2742.7ms/step | 21.9 samples/s
+wandb: WARNING Tried to log to step 48960 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-48962

epfl_final/SparseAtt/wandb/run-20260826_152940-i8ewyy95/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260826_152940-i8ewyy95/files/wandb-metadata.json

@@ -0,0 +1,79 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-08-26T15:29:40.703291Z",
+  "args":  [
+    "--config",
+    "configs/p2_1p2b_mixed_seed70_6gpu_resume48750.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/SparseAtt.git",
+    "commit":  "8f862f749eb4d5c17f688eb1562235e3fe5b2550"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-train-6-gpu-0-39",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  6,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "4877579661312"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-99f99f92-e501-0b15-5782-6beb2c34fa1a"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-3f5bace4-0066-71eb-863d-73a3a6d79e3b"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-1aef7113-cfe5-3f49-53bb-a84f3c5abfa0"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-5e17ddff-2e95-8c68-b074-b0189417e834"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-61e64705-1798-f2b5-568f-877549238703"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-9cd8a6e5-05e9-cfde-0a28-d8058797b3bf"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "f75a5zbdrc79jw3dypdbluk1x3eamao9"
+}

epfl_final/SparseAtt/wandb/run-20260826_152940-i8ewyy95/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/loss":2.5299893152713775,"_timestamp":1.7876265843495035e+09,"eval/perplexity":12.529849258791517,"train/learning_rate":1.1253848138212951e-06,"_wandb":{"runtime":157688},"eval/dense_interval_start":-1,"_runtime":157688,"eval/loss":2.5281137384119488,"speed/samples_per_sec":21.985407094904552,"eval/uses_dense_attention":0,"train/epoch":0.9702350283381554,"_wandb.runtime":25091,"speed/steps_per_sec":0.36642345158174255,"_step":48960,"eval/uses_sparse_attention":1,"speed/ms_per_step":2729.08296585083,"eval/dense_interval_end":-1,"eval/dense_interval_active":0,"train/global_step":48960}

epfl_final/SparseAtt/wandb/run-20260826_152940-i8ewyy95/logs/debug-internal.log

@@ -0,0 +1,215 @@
+{"time":"2026-08-26T15:29:41.042986104Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-08-26T15:29:41.043598469Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-08-26T15:29:41.363957274Z","level":"INFO","msg":"stream: created new stream","id":"i8ewyy95"}
+{"time":"2026-08-26T15:29:41.364083382Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-08-26T15:29:41.366757208Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-08-26T15:29:41.366833782Z","level":"INFO","msg":"writer: started","stream_id":"i8ewyy95"}
+{"time":"2026-08-26T15:29:41.366882415Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-08-26T15:29:41.904709397Z","level":"INFO","msg":"filestream: sending request","total_files":1}
+{"time":"2026-08-26T15:29:42.19523727Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:29:56.905389426Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1,"uploaded_len":2}
+{"time":"2026-08-26T15:29:57.100838583Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:30:11.905059986Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":0,"events_lines":2}
+{"time":"2026-08-26T15:30:12.093163798Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:30:16.117293128Z","level":"WARN","msg":"handler: ignoring partial history record","step":48750,"current":48961}
+{"time":"2026-08-26T15:30:26.905376251Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":2,"events_lines":2,"console_offset":1,"console_lines":2}
+{"time":"2026-08-26T15:30:27.101141039Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:30:41.905020455Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":4,"events_lines":2}
+{"time":"2026-08-26T15:30:42.149055258Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:30:56.904715151Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":6,"events_lines":2}
+{"time":"2026-08-26T15:30:57.105857695Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:31:11.905506396Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":8,"events_lines":2}
+{"time":"2026-08-26T15:31:12.101935667Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:31:26.904940062Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":10,"events_lines":2}
+{"time":"2026-08-26T15:31:27.131814504Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:31:41.905676865Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":12,"events_lines":2}
+{"time":"2026-08-26T15:31:42.12709228Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:31:56.905388877Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":14,"events_lines":2}
+{"time":"2026-08-26T15:31:57.106680413Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:32:11.90481343Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":16,"events_lines":2}
+{"time":"2026-08-26T15:32:12.108696208Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:32:26.905636812Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":18,"events_lines":2}
+{"time":"2026-08-26T15:32:27.117520866Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:32:41.905701421Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":20,"events_lines":2}
+{"time":"2026-08-26T15:32:42.094333051Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:32:56.905583657Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":22,"events_lines":2}
+{"time":"2026-08-26T15:32:57.093295453Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:33:11.905133517Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":24,"events_lines":2}
+{"time":"2026-08-26T15:33:12.103095438Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:33:26.90484609Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":26,"events_lines":2}
+{"time":"2026-08-26T15:33:27.094091267Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:33:41.905057399Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":28,"events_lines":2}
+{"time":"2026-08-26T15:33:42.100565364Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:33:56.905334045Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":30,"events_lines":2}
+{"time":"2026-08-26T15:33:57.108633392Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:34:11.904636285Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":32,"events_lines":2}
+{"time":"2026-08-26T15:34:12.103024093Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:34:26.905163465Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":34,"events_lines":2}
+{"time":"2026-08-26T15:34:27.101591634Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:34:41.904638162Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":36,"events_lines":2}
+{"time":"2026-08-26T15:34:42.107506945Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:34:56.904832345Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":38,"events_lines":2}
+{"time":"2026-08-26T15:34:57.092272174Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:35:11.905101259Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":40,"events_lines":2}
+{"time":"2026-08-26T15:35:12.133516914Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:35:26.905077375Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":42,"events_lines":2}
+{"time":"2026-08-26T15:35:27.095970485Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:35:41.905121233Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":44,"events_lines":2}
+{"time":"2026-08-26T15:35:42.153462391Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:35:56.905444168Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":46,"events_lines":2}
+{"time":"2026-08-26T15:35:57.166020998Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:36:11.905284161Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":48,"events_lines":2}
+{"time":"2026-08-26T15:36:12.129049719Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:36:26.90513257Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":50,"events_lines":2}
+{"time":"2026-08-26T15:36:27.139101112Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:36:41.904962007Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":52,"events_lines":2}
+{"time":"2026-08-26T15:36:42.132201775Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:36:56.905827042Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":54,"events_lines":2}
+{"time":"2026-08-26T15:36:57.130979321Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:37:11.90461272Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":56,"events_lines":2}
+{"time":"2026-08-26T15:37:12.1380647Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:37:26.905060612Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":58,"events_lines":2}
+{"time":"2026-08-26T15:37:27.122189823Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:37:41.905508383Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":60,"events_lines":2}
+{"time":"2026-08-26T15:37:42.110523233Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:37:56.90478869Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":62,"events_lines":2}
+{"time":"2026-08-26T15:37:57.126877882Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:38:11.905406265Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":64,"events_lines":2}
+{"time":"2026-08-26T15:38:12.117222688Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:38:26.905349493Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":66,"events_lines":2}
+{"time":"2026-08-26T15:38:27.184207608Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:38:41.904841216Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":68,"events_lines":2}
+{"time":"2026-08-26T15:38:42.097584519Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:38:56.90474217Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":70,"events_lines":2}
+{"time":"2026-08-26T15:38:57.110454552Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:39:11.905466084Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":72,"events_lines":2}
+{"time":"2026-08-26T15:39:12.124453932Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:39:26.905138107Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":74,"events_lines":2}
+{"time":"2026-08-26T15:39:27.123381272Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:39:41.905132741Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":76,"events_lines":2}
+{"time":"2026-08-26T15:39:42.145011514Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:39:56.905307204Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":78,"events_lines":2}
+{"time":"2026-08-26T15:39:57.101340194Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:40:11.904967098Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":80,"events_lines":2}
+{"time":"2026-08-26T15:40:12.135793025Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:40:26.905146569Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":82,"events_lines":2}
+{"time":"2026-08-26T15:40:27.187855248Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:40:41.90534668Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":84,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-08-26T15:40:42.160573256Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:40:56.905077871Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":86,"events_lines":2}
+{"time":"2026-08-26T15:40:57.21819154Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:41:11.905096311Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":88,"events_lines":2}
+{"time":"2026-08-26T15:41:12.157772575Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:41:26.905257313Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":90,"events_lines":2}
+{"time":"2026-08-26T15:41:27.14387364Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:41:41.905479528Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":92,"events_lines":2}
+{"time":"2026-08-26T15:41:42.126224552Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:41:56.905257238Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":94,"events_lines":2}
+{"time":"2026-08-26T15:41:57.134850473Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:42:10.977014316Z","level":"WARN","msg":"handler: ignoring partial history record","step":48760,"current":48961}
+{"time":"2026-08-26T15:42:11.905698792Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":96,"events_lines":2,"console_offset":4,"console_lines":1}
+{"time":"2026-08-26T15:42:12.097290064Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:42:26.905008753Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":98,"events_lines":2,"console_offset":5,"console_lines":1}
+{"time":"2026-08-26T15:42:27.13559414Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:42:41.904741186Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":100,"events_lines":2}
+{"time":"2026-08-26T15:42:42.131908425Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:42:44.053618524Z","level":"WARN","msg":"handler: ignoring partial history record","step":48770,"current":48961}
+{"time":"2026-08-26T15:42:56.90557921Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":102,"events_lines":2,"console_offset":6,"console_lines":2}
+{"time":"2026-08-26T15:42:57.140729327Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:43:11.405850145Z","level":"WARN","msg":"handler: ignoring partial history record","step":48780,"current":48961}
+{"time":"2026-08-26T15:43:11.905011214Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":104,"events_lines":2,"console_offset":8,"console_lines":1}
+{"time":"2026-08-26T15:43:12.123672474Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:43:26.905572324Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":106,"events_lines":2,"console_offset":9,"console_lines":1}
+{"time":"2026-08-26T15:43:27.095436126Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:43:38.769022263Z","level":"WARN","msg":"handler: ignoring partial history record","step":48790,"current":48961}
+{"time":"2026-08-26T15:43:41.905264226Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":108,"events_lines":2,"console_offset":10,"console_lines":1}
+{"time":"2026-08-26T15:43:42.133774813Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:43:56.904924841Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":110,"events_lines":2,"console_offset":11,"console_lines":1}
+{"time":"2026-08-26T15:43:57.114560215Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:44:06.115049924Z","level":"WARN","msg":"handler: ignoring partial history record","step":48800,"current":48961}
+{"time":"2026-08-26T15:44:11.905286364Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":112,"events_lines":2,"console_offset":12,"console_lines":1}
+{"time":"2026-08-26T15:44:12.106258784Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:44:26.905429701Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":114,"events_lines":2,"console_offset":13,"console_lines":1}
+{"time":"2026-08-26T15:44:27.10891821Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:44:33.537518513Z","level":"WARN","msg":"handler: ignoring partial history record","step":48810,"current":48961}
+{"time":"2026-08-26T15:44:41.905160641Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":116,"events_lines":2,"console_offset":14,"console_lines":1}
+{"time":"2026-08-26T15:44:42.09426646Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:44:56.905244589Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":118,"events_lines":2,"console_offset":15,"console_lines":1}
+{"time":"2026-08-26T15:44:57.12583335Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:45:01.024071946Z","level":"WARN","msg":"handler: ignoring partial history record","step":48820,"current":48961}
+{"time":"2026-08-26T15:45:11.904726011Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":120,"events_lines":2,"console_offset":16,"console_lines":2}
+{"time":"2026-08-26T15:45:12.131809621Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:45:26.905640179Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":122,"events_lines":2}
+{"time":"2026-08-26T15:45:27.128532157Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:45:28.430638782Z","level":"WARN","msg":"handler: ignoring partial history record","step":48830,"current":48961}
+{"time":"2026-08-26T15:45:41.90574653Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":124,"events_lines":2,"console_offset":18,"console_lines":2}
+{"time":"2026-08-26T15:45:42.113101984Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:45:55.896524046Z","level":"WARN","msg":"handler: ignoring partial history record","step":48840,"current":48961}
+{"time":"2026-08-26T15:45:56.908754788Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":126,"events_lines":2,"console_offset":20,"console_lines":1}
+{"time":"2026-08-26T15:45:57.147020917Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:46:11.905345196Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":128,"events_lines":2,"console_offset":21,"console_lines":1}
+{"time":"2026-08-26T15:46:12.108112324Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:46:23.305018818Z","level":"WARN","msg":"handler: ignoring partial history record","step":48850,"current":48961}
+{"time":"2026-08-26T15:46:26.904858541Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":130,"events_lines":2,"console_offset":22,"console_lines":1}
+{"time":"2026-08-26T15:46:27.10328962Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:46:41.904763482Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":132,"events_lines":2,"console_offset":23,"console_lines":1}
+{"time":"2026-08-26T15:46:42.103128222Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:46:50.747682614Z","level":"WARN","msg":"handler: ignoring partial history record","step":48860,"current":48961}
+{"time":"2026-08-26T15:46:56.905177382Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":134,"events_lines":2,"console_offset":24,"console_lines":2}
+{"time":"2026-08-26T15:46:57.108494522Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:47:11.90542807Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":136,"events_lines":2}
+{"time":"2026-08-26T15:47:12.140353698Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:47:18.10528204Z","level":"WARN","msg":"handler: ignoring partial history record","step":48870,"current":48961}
+{"time":"2026-08-26T15:47:26.905434612Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":138,"events_lines":2,"console_offset":26,"console_lines":2}
+{"time":"2026-08-26T15:47:27.104149766Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:47:41.905494825Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":140,"events_lines":2}
+{"time":"2026-08-26T15:47:42.102460685Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:47:45.487050749Z","level":"WARN","msg":"handler: ignoring partial history record","step":48880,"current":48961}
+{"time":"2026-08-26T15:47:56.905444953Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":142,"events_lines":2,"console_offset":28,"console_lines":2}
+{"time":"2026-08-26T15:47:57.126017525Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:48:11.905944512Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":144,"events_lines":2}
+{"time":"2026-08-26T15:48:12.114982409Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:48:12.841771794Z","level":"WARN","msg":"handler: ignoring partial history record","step":48890,"current":48961}
+{"time":"2026-08-26T15:48:26.905519159Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":146,"events_lines":2,"console_offset":30,"console_lines":2}
+{"time":"2026-08-26T15:48:27.117916686Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:48:41.905168027Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":148,"events_lines":2}
+{"time":"2026-08-26T15:48:42.115124757Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:48:45.857900631Z","level":"WARN","msg":"handler: ignoring partial history record","step":48900,"current":48961}
+{"time":"2026-08-26T15:48:56.90536244Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":150,"events_lines":2,"console_offset":32,"console_lines":2}
+{"time":"2026-08-26T15:48:57.120380983Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:49:11.905437595Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":152,"events_lines":2}
+{"time":"2026-08-26T15:49:12.118680736Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:49:13.248589498Z","level":"WARN","msg":"handler: ignoring partial history record","step":48910,"current":48961}
+{"time":"2026-08-26T15:49:26.905694952Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":154,"events_lines":2,"console_offset":34,"console_lines":2}
+{"time":"2026-08-26T15:49:27.117506604Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:49:40.620281341Z","level":"WARN","msg":"handler: ignoring partial history record","step":48920,"current":48961}
+{"time":"2026-08-26T15:49:41.904690413Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":156,"events_lines":2,"console_offset":36,"console_lines":1}
+{"time":"2026-08-26T15:49:42.148244904Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:49:56.905219747Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":158,"events_lines":2,"console_offset":37,"console_lines":1}
+{"time":"2026-08-26T15:49:57.105811835Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:50:07.98697799Z","level":"WARN","msg":"handler: ignoring partial history record","step":48930,"current":48961}
+{"time":"2026-08-26T15:50:11.905627999Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":160,"events_lines":2,"console_offset":38,"console_lines":1}
+{"time":"2026-08-26T15:50:12.127247366Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:50:26.905509935Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":162,"events_lines":2,"console_offset":39,"console_lines":1}
+{"time":"2026-08-26T15:50:27.169261254Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:50:35.355389541Z","level":"WARN","msg":"handler: ignoring partial history record","step":48940,"current":48961}
+{"time":"2026-08-26T15:50:41.905205012Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":164,"events_lines":2,"console_offset":40,"console_lines":1}
+{"time":"2026-08-26T15:50:42.146938202Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:50:56.904923955Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":166,"events_lines":2,"console_offset":41,"console_lines":1}
+{"time":"2026-08-26T15:50:57.100951422Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:51:02.688631633Z","level":"WARN","msg":"handler: ignoring partial history record","step":48950,"current":48961}
+{"time":"2026-08-26T15:51:11.905160851Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":168,"events_lines":2,"console_offset":42,"console_lines":1}
+{"time":"2026-08-26T15:51:12.098486642Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:51:26.905478338Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":170,"events_lines":2,"console_offset":43,"console_lines":1}
+{"time":"2026-08-26T15:51:27.131681544Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:51:30.115137757Z","level":"WARN","msg":"handler: ignoring partial history record","step":48960,"current":48961}
+{"time":"2026-08-26T15:51:41.905603698Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":172,"events_lines":2,"console_offset":44,"console_lines":2}
+{"time":"2026-08-26T15:51:42.106710405Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:51:49.21688732Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-08-26T15:51:49.217132947Z","level":"INFO","msg":"filestream: sending request","total_files":3,"events_offset":174,"events_lines":2,"console_offset":46,"console_lines":1,"uploaded_len":3,"complete":true,"exit_code":0}
+{"time":"2026-08-26T15:51:49.467624403Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-26T15:51:49.469691662Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-08-26T15:51:49.478242308Z","level":"INFO","msg":"stream: finishing up"}
+{"time":"2026-08-26T15:51:49.478270441Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-08-26T15:51:49.480379572Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-08-26T15:51:49.480389917Z","level":"INFO","msg":"stream: all finished"}

epfl_final/SparseAtt/wandb/run-20260826_152940-i8ewyy95/logs/debug.log

@@ -0,0 +1,25 @@
+2026-08-26 15:29:40,738 INFO    MainThread:4329 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-08-26 15:29:40,738 INFO    MainThread:4329 [wandb_setup.py:_flush():81] Configure stats pid to 4329
+2026-08-26 15:29:40,738 INFO    MainThread:4329 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-26 15:29:40,738 INFO    MainThread:4329 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260826_152940-i8ewyy95/logs/debug.log
+2026-08-26 15:29:40,738 INFO    MainThread:4329 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260826_152940-i8ewyy95/logs/debug-internal.log
+2026-08-26 15:29:40,738 INFO    MainThread:4329 [wandb_init.py:init():850] calling init triggers
+2026-08-26 15:29:40,738 INFO    MainThread:4329 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-48750', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 70, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b mixed 8k seed70 6gpu resume48750', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'mixed', 'seed70'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed', 'sparse_mask_seed': 70, 'sparse_mixed_nrb_set': [63, 47, 31, 21, 11], 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-08-26 15:29:40,738 INFO    MainThread:4329 [wandb_init.py:init():898] starting backend
+2026-08-26 15:29:41,022 INFO    MainThread:4329 [wandb_init.py:init():913] sending inform_init request
+2026-08-26 15:29:41,367 INFO    MainThread:4329 [wandb_init.py:init():918] backend started and connected
+2026-08-26 15:29:41,369 INFO    MainThread:4329 [wandb_init.py:init():988] updated telemetry
+2026-08-26 15:29:41,385 INFO    MainThread:4329 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-08-26 15:29:41,900 INFO    MainThread:4329 [wandb_init.py:init():1051] run resumed
+2026-08-26 15:29:41,903 INFO    MainThread:4329 [wandb_init.py:init():1056] starting run threads in backend
+2026-08-26 15:29:42,196 INFO    MainThread:4329 [wandb_run.py:_console_start():2554] atexit reg
+2026-08-26 15:29:42,196 INFO    MainThread:4329 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-08-26 15:29:42,196 INFO    MainThread:4329 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-08-26 15:29:42,196 INFO    MainThread:4329 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-08-26 15:29:42,200 INFO    MainThread:4329 [wandb_init.py:init():1094] run started, returning control to user process
+2026-08-26 15:51:47,885 INFO    MainThread:4329 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/i8ewyy95
+2026-08-26 15:51:47,886 INFO    MainThread:4329 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
+2026-08-26 15:51:47,887 INFO    MainThread:4329 [wandb_run.py:_restore():2502] restore
+2026-08-26 15:51:47,887 INFO    MainThread:4329 [wandb_run.py:_restore():2508] restore done
+2026-08-26 15:51:49,472 INFO    MainThread:4329 [wandb_run.py:_footer_sync_info():3889] logging synced files

epfl_final/SparseAtt/wandb/run-20260826_152940-i8ewyy95/run-i8ewyy95.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d70efe926c35b2fb185f685b4030e96cf54faebe65b5df332974a16588db1f3a
+size 538343

epfl_final/SparseAtt/wandb/run-20260826_173107-rgmkfpnl/files/output.log

@@ -0,0 +1,958 @@
+Starting training...
+Validation at step 33250: Loss = 2.5147, Perplexity = 12.36
+wandb: WARNING Tried to log to step 33250 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 33251: dense (dense step interval)
+Step 33260/50462 | Loss: 2.5133 | LR: 1.34e-04 | 10270.9ms/step | 5.8 samples/s
+wandb: WARNING Tried to log to step 33260 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 33270/50462 | Loss: 2.4854 | LR: 1.34e-04 | 1885.7ms/step | 31.8 samples/s
+wandb: WARNING Tried to log to step 33270 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 33280/50462 | Loss: 2.4878 | LR: 1.33e-04 | 1892.0ms/step | 31.7 samples/s
+wandb: WARNING Tried to log to step 33280 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 33290/50462 | Loss: 2.5207 | LR: 1.33e-04 | 1898.2ms/step | 31.6 samples/s
+wandb: WARNING Tried to log to step 33290 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 33300/50462 | Loss: 2.5077 | LR: 1.33e-04 | 1901.1ms/step | 31.6 samples/s
+wandb: WARNING Tried to log to step 33300 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 33310/50462 | Loss: 2.5032 | LR: 1.33e-04 | 1906.6ms/step | 31.5 samples/s
+wandb: WARNING Tried to log to step 33310 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 33320/50462 | Loss: 2.4662 | LR: 1.33e-04 | 1908.1ms/step | 31.4 samples/s
+wandb: WARNING Tried to log to step 33320 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 33330/50462 | Loss: 2.5206 | LR: 1.33e-04 | 1906.4ms/step | 31.5 samples/s
+wandb: WARNING Tried to log to step 33330 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 33340/50462 | Loss: 2.4841 | LR: 1.33e-04 | 1909.1ms/step | 31.4 samples/s
+wandb: WARNING Tried to log to step 33340 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 33350/50462 | Loss: 2.5607 | LR: 1.32e-04 | 1911.8ms/step | 31.4 samples/s
+wandb: WARNING Tried to log to step 33350 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 33360/50462 | Loss: 2.4827 | LR: 1.32e-04 | 1911.0ms/step | 31.4 samples/s
+wandb: WARNING Tried to log to step 33360 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 33370/50462 | Loss: 2.4959 | LR: 1.32e-04 | 1911.6ms/step | 31.4 samples/s
+wandb: WARNING Tried to log to step 33370 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 33380/50462 | Loss: 2.5044 | LR: 1.32e-04 | 1911.6ms/step | 31.4 samples/s
+wandb: WARNING Tried to log to step 33380 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 33390/50462 | Loss: 2.5119 | LR: 1.32e-04 | 1907.2ms/step | 31.5 samples/s
+wandb: WARNING Tried to log to step 33390 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 33400/50462 | Loss: 2.4696 | LR: 1.32e-04 | 1909.8ms/step | 31.4 samples/s
+wandb: WARNING Tried to log to step 33400 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 33410/50462 | Loss: 2.5027 | LR: 1.32e-04 | 1912.1ms/step | 31.4 samples/s
+wandb: WARNING Tried to log to step 33410 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 33420/50462 | Loss: 2.4818 | LR: 1.32e-04 | 1913.3ms/step | 31.4 samples/s
+wandb: WARNING Tried to log to step 33420 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 33430/50462 | Loss: 2.4731 | LR: 1.31e-04 | 1912.7ms/step | 31.4 samples/s
+wandb: WARNING Tried to log to step 33430 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 33440/50462 | Loss: 2.4966 | LR: 1.31e-04 | 1916.5ms/step | 31.3 samples/s
+wandb: WARNING Tried to log to step 33440 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 33450/50462 | Loss: 2.5432 | LR: 1.31e-04 | 1912.2ms/step | 31.4 samples/s
+wandb: WARNING Tried to log to step 33450 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 33460/50462 | Loss: 2.4893 | LR: 1.31e-04 | 1908.7ms/step | 31.4 samples/s
+wandb: WARNING Tried to log to step 33460 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 33470/50462 | Loss: 2.5461 | LR: 1.31e-04 | 1907.2ms/step | 31.5 samples/s
+wandb: WARNING Tried to log to step 33470 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 33480/50462 | Loss: 2.4717 | LR: 1.31e-04 | 1904.0ms/step | 31.5 samples/s
+wandb: WARNING Tried to log to step 33480 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 33490/50462 | Loss: 2.5124 | LR: 1.31e-04 | 1904.8ms/step | 31.5 samples/s
+wandb: WARNING Tried to log to step 33490 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 33500/50462 | Loss: 2.5140 | LR: 1.30e-04 | 1908.0ms/step | 31.4 samples/s
+Validation at step 33500: Loss = 2.5140, Perplexity = 12.35
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-33500
+Step 33510/50462 | Loss: 2.5217 | LR: 1.30e-04 | 3288.5ms/step | 18.2 samples/s
+Step 33520/50462 | Loss: 2.5154 | LR: 1.30e-04 | 1905.7ms/step | 31.5 samples/s
+Step 33530/50462 | Loss: 2.4817 | LR: 1.30e-04 | 1910.2ms/step | 31.4 samples/s
+Step 33540/50462 | Loss: 2.5001 | LR: 1.30e-04 | 1912.9ms/step | 31.4 samples/s
+Step 33550/50462 | Loss: 2.5292 | LR: 1.30e-04 | 1914.8ms/step | 31.3 samples/s
+Step 33560/50462 | Loss: 2.5229 | LR: 1.30e-04 | 1912.4ms/step | 31.4 samples/s
+Step 33570/50462 | Loss: 2.4785 | LR: 1.29e-04 | 1912.5ms/step | 31.4 samples/s
+Step 33580/50462 | Loss: 2.5025 | LR: 1.29e-04 | 1911.5ms/step | 31.4 samples/s
+Step 33590/50462 | Loss: 2.5228 | LR: 1.29e-04 | 1913.8ms/step | 31.4 samples/s
+Step 33600/50462 | Loss: 2.5023 | LR: 1.29e-04 | 1917.7ms/step | 31.3 samples/s
+Step 33610/50462 | Loss: 2.5061 | LR: 1.29e-04 | 1911.0ms/step | 31.4 samples/s
+Step 33620/50462 | Loss: 2.4939 | LR: 1.29e-04 | 1911.7ms/step | 31.4 samples/s
+Step 33630/50462 | Loss: 2.4811 | LR: 1.29e-04 | 1912.7ms/step | 31.4 samples/s
+Step 33640/50462 | Loss: 2.4906 | LR: 1.28e-04 | 1913.6ms/step | 31.4 samples/s
+Step 33650/50462 | Loss: 2.4895 | LR: 1.28e-04 | 1916.0ms/step | 31.3 samples/s
+Step 33660/50462 | Loss: 2.5235 | LR: 1.28e-04 | 1917.7ms/step | 31.3 samples/s
+Step 33670/50462 | Loss: 2.4564 | LR: 1.28e-04 | 1917.6ms/step | 31.3 samples/s
+Step 33680/50462 | Loss: 2.4563 | LR: 1.28e-04 | 1919.6ms/step | 31.3 samples/s
+Step 33690/50462 | Loss: 2.5245 | LR: 1.28e-04 | 1922.3ms/step | 31.2 samples/s
+Step 33700/50462 | Loss: 2.5313 | LR: 1.28e-04 | 1918.9ms/step | 31.3 samples/s
+Step 33710/50462 | Loss: 2.5310 | LR: 1.27e-04 | 1913.2ms/step | 31.4 samples/s
+Step 33720/50462 | Loss: 2.4870 | LR: 1.27e-04 | 1911.7ms/step | 31.4 samples/s
+Step 33730/50462 | Loss: 2.4817 | LR: 1.27e-04 | 1912.3ms/step | 31.4 samples/s
+Step 33740/50462 | Loss: 2.5252 | LR: 1.27e-04 | 1909.0ms/step | 31.4 samples/s
+Step 33750/50462 | Loss: 2.4947 | LR: 1.27e-04 | 1908.8ms/step | 31.4 samples/s
+Validation at step 33750: Loss = 2.5121, Perplexity = 12.33
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-33750
+Step 33760/50462 | Loss: 2.4622 | LR: 1.27e-04 | 3246.6ms/step | 18.5 samples/s
+Step 33770/50462 | Loss: 2.5154 | LR: 1.27e-04 | 1904.4ms/step | 31.5 samples/s
+Step 33780/50462 | Loss: 2.4942 | LR: 1.27e-04 | 1908.5ms/step | 31.4 samples/s
+Step 33790/50462 | Loss: 2.5323 | LR: 1.26e-04 | 1907.9ms/step | 31.4 samples/s
+Step 33800/50462 | Loss: 2.5361 | LR: 1.26e-04 | 1909.8ms/step | 31.4 samples/s
+Step 33810/50462 | Loss: 2.4906 | LR: 1.26e-04 | 1915.1ms/step | 31.3 samples/s
+Step 33820/50462 | Loss: 2.5137 | LR: 1.26e-04 | 1914.6ms/step | 31.3 samples/s
+Step 33830/50462 | Loss: 2.5447 | LR: 1.26e-04 | 1919.9ms/step | 31.3 samples/s
+Step 33840/50462 | Loss: 2.5328 | LR: 1.26e-04 | 1909.1ms/step | 31.4 samples/s
+Step 33850/50462 | Loss: 2.4883 | LR: 1.26e-04 | 1914.6ms/step | 31.3 samples/s
+Step 33860/50462 | Loss: 2.4988 | LR: 1.25e-04 | 1916.7ms/step | 31.3 samples/s
+Step 33870/50462 | Loss: 2.4962 | LR: 1.25e-04 | 1914.2ms/step | 31.3 samples/s
+Step 33880/50462 | Loss: 2.4935 | LR: 1.25e-04 | 1919.0ms/step | 31.3 samples/s
+Step 33890/50462 | Loss: 2.4720 | LR: 1.25e-04 | 1909.2ms/step | 31.4 samples/s
+Step 33900/50462 | Loss: 2.4888 | LR: 1.25e-04 | 1904.4ms/step | 31.5 samples/s
+Step 33910/50462 | Loss: 2.5052 | LR: 1.25e-04 | 1908.6ms/step | 31.4 samples/s
+Step 33920/50462 | Loss: 2.4829 | LR: 1.25e-04 | 1914.9ms/step | 31.3 samples/s
+Step 33930/50462 | Loss: 2.4855 | LR: 1.24e-04 | 1917.0ms/step | 31.3 samples/s
+Step 33940/50462 | Loss: 2.4702 | LR: 1.24e-04 | 1916.3ms/step | 31.3 samples/s
+Step 33950/50462 | Loss: 2.5520 | LR: 1.24e-04 | 1917.9ms/step | 31.3 samples/s
+Step 33960/50462 | Loss: 2.5291 | LR: 1.24e-04 | 1915.6ms/step | 31.3 samples/s
+Step 33970/50462 | Loss: 2.5355 | LR: 1.24e-04 | 1910.8ms/step | 31.4 samples/s
+Step 33980/50462 | Loss: 2.4861 | LR: 1.24e-04 | 1907.2ms/step | 31.5 samples/s
+Step 33990/50462 | Loss: 2.5019 | LR: 1.24e-04 | 1915.1ms/step | 31.3 samples/s
+Step 34000/50462 | Loss: 2.5265 | LR: 1.24e-04 | 1915.9ms/step | 31.3 samples/s
+Validation at step 34000: Loss = 2.5119, Perplexity = 12.33
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-34000
+Step 34010/50462 | Loss: 2.5237 | LR: 1.23e-04 | 3237.1ms/step | 18.5 samples/s
+Step 34020/50462 | Loss: 2.4873 | LR: 1.23e-04 | 1906.4ms/step | 31.5 samples/s
+Step 34030/50462 | Loss: 2.5294 | LR: 1.23e-04 | 1910.5ms/step | 31.4 samples/s
+Step 34040/50462 | Loss: 2.4993 | LR: 1.23e-04 | 1913.1ms/step | 31.4 samples/s
+Step 34050/50462 | Loss: 2.4872 | LR: 1.23e-04 | 1904.1ms/step | 31.5 samples/s
+Step 34060/50462 | Loss: 2.5320 | LR: 1.23e-04 | 1909.3ms/step | 31.4 samples/s
+Step 34070/50462 | Loss: 2.5202 | LR: 1.23e-04 | 1908.6ms/step | 31.4 samples/s
+Step 34080/50462 | Loss: 2.5193 | LR: 1.22e-04 | 1908.1ms/step | 31.4 samples/s
+Step 34090/50462 | Loss: 2.5235 | LR: 1.22e-04 | 1910.8ms/step | 31.4 samples/s
+Step 34100/50462 | Loss: 2.5209 | LR: 1.22e-04 | 1909.2ms/step | 31.4 samples/s
+Step 34110/50462 | Loss: 2.4700 | LR: 1.22e-04 | 1909.7ms/step | 31.4 samples/s
+Step 34120/50462 | Loss: 2.4972 | LR: 1.22e-04 | 1911.2ms/step | 31.4 samples/s
+Step 34130/50462 | Loss: 2.5175 | LR: 1.22e-04 | 1914.1ms/step | 31.3 samples/s
+Step 34140/50462 | Loss: 2.4870 | LR: 1.22e-04 | 1911.0ms/step | 31.4 samples/s
+Step 34150/50462 | Loss: 2.4972 | LR: 1.21e-04 | 1914.4ms/step | 31.3 samples/s
+Step 34160/50462 | Loss: 2.5286 | LR: 1.21e-04 | 1912.4ms/step | 31.4 samples/s
+Step 34170/50462 | Loss: 2.5389 | LR: 1.21e-04 | 1910.1ms/step | 31.4 samples/s
+Step 34180/50462 | Loss: 2.5542 | LR: 1.21e-04 | 1911.4ms/step | 31.4 samples/s
+Step 34190/50462 | Loss: 2.4777 | LR: 1.21e-04 | 1915.9ms/step | 31.3 samples/s
+Step 34200/50462 | Loss: 2.4670 | LR: 1.21e-04 | 1917.0ms/step | 31.3 samples/s
+Step 34210/50462 | Loss: 2.5238 | LR: 1.21e-04 | 1913.2ms/step | 31.4 samples/s
+Step 34220/50462 | Loss: 2.4937 | LR: 1.21e-04 | 1918.5ms/step | 31.3 samples/s
+Step 34230/50462 | Loss: 2.5384 | LR: 1.20e-04 | 1919.5ms/step | 31.3 samples/s
+Step 34240/50462 | Loss: 2.5210 | LR: 1.20e-04 | 1918.2ms/step | 31.3 samples/s
+Step 34250/50462 | Loss: 2.5202 | LR: 1.20e-04 | 1917.8ms/step | 31.3 samples/s
+Validation at step 34250: Loss = 2.5109, Perplexity = 12.32
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-34250
+Step 34260/50462 | Loss: 2.5426 | LR: 1.20e-04 | 3266.0ms/step | 18.4 samples/s
+Step 34270/50462 | Loss: 2.5089 | LR: 1.20e-04 | 1900.6ms/step | 31.6 samples/s
+Step 34280/50462 | Loss: 2.5460 | LR: 1.20e-04 | 1909.8ms/step | 31.4 samples/s
+Step 34290/50462 | Loss: 2.5079 | LR: 1.20e-04 | 1906.5ms/step | 31.5 samples/s
+Step 34300/50462 | Loss: 2.4931 | LR: 1.19e-04 | 1909.6ms/step | 31.4 samples/s
+Step 34310/50462 | Loss: 2.4819 | LR: 1.19e-04 | 1913.5ms/step | 31.4 samples/s
+Step 34320/50462 | Loss: 2.5306 | LR: 1.19e-04 | 1913.1ms/step | 31.4 samples/s
+Step 34330/50462 | Loss: 2.5054 | LR: 1.19e-04 | 1912.7ms/step | 31.4 samples/s
+Step 34340/50462 | Loss: 2.5203 | LR: 1.19e-04 | 1904.2ms/step | 31.5 samples/s
+Step 34350/50462 | Loss: 2.5127 | LR: 1.19e-04 | 1906.6ms/step | 31.5 samples/s
+Step 34360/50462 | Loss: 2.5046 | LR: 1.19e-04 | 1915.5ms/step | 31.3 samples/s
+Step 34370/50462 | Loss: 2.4960 | LR: 1.19e-04 | 1911.9ms/step | 31.4 samples/s
+Step 34380/50462 | Loss: 2.5016 | LR: 1.18e-04 | 1915.1ms/step | 31.3 samples/s
+Step 34390/50462 | Loss: 2.5344 | LR: 1.18e-04 | 1913.9ms/step | 31.3 samples/s
+Step 34400/50462 | Loss: 2.5017 | LR: 1.18e-04 | 1916.5ms/step | 31.3 samples/s
+Step 34410/50462 | Loss: 2.4961 | LR: 1.18e-04 | 1913.2ms/step | 31.4 samples/s
+Step 34420/50462 | Loss: 2.5027 | LR: 1.18e-04 | 1912.9ms/step | 31.4 samples/s
+Step 34430/50462 | Loss: 2.5035 | LR: 1.18e-04 | 1914.8ms/step | 31.3 samples/s
+Step 34440/50462 | Loss: 2.4683 | LR: 1.18e-04 | 1916.8ms/step | 31.3 samples/s
+Step 34450/50462 | Loss: 2.5380 | LR: 1.17e-04 | 1916.9ms/step | 31.3 samples/s
+Step 34460/50462 | Loss: 2.4745 | LR: 1.17e-04 | 1910.0ms/step | 31.4 samples/s
+Step 34470/50462 | Loss: 2.5739 | LR: 1.17e-04 | 1906.9ms/step | 31.5 samples/s
+Step 34480/50462 | Loss: 2.5255 | LR: 1.17e-04 | 1909.8ms/step | 31.4 samples/s
+Step 34490/50462 | Loss: 2.5326 | LR: 1.17e-04 | 1912.1ms/step | 31.4 samples/s
+Step 34500/50462 | Loss: 2.5316 | LR: 1.17e-04 | 1909.1ms/step | 31.4 samples/s
+Validation at step 34500: Loss = 2.5103, Perplexity = 12.31
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-34500
+Step 34510/50462 | Loss: 2.4788 | LR: 1.17e-04 | 3224.7ms/step | 18.6 samples/s
+Step 34520/50462 | Loss: 2.5266 | LR: 1.16e-04 | 1903.0ms/step | 31.5 samples/s
+Step 34530/50462 | Loss: 2.5230 | LR: 1.16e-04 | 1908.8ms/step | 31.4 samples/s
+Step 34540/50462 | Loss: 2.5101 | LR: 1.16e-04 | 1914.6ms/step | 31.3 samples/s
+Step 34550/50462 | Loss: 2.4883 | LR: 1.16e-04 | 1912.1ms/step | 31.4 samples/s
+Step 34560/50462 | Loss: 2.4728 | LR: 1.16e-04 | 1912.6ms/step | 31.4 samples/s
+Step 34570/50462 | Loss: 2.5033 | LR: 1.16e-04 | 1916.8ms/step | 31.3 samples/s
+Step 34580/50462 | Loss: 2.5262 | LR: 1.16e-04 | 1917.8ms/step | 31.3 samples/s
+Step 34590/50462 | Loss: 2.5026 | LR: 1.16e-04 | 1919.8ms/step | 31.3 samples/s
+Step 34600/50462 | Loss: 2.5039 | LR: 1.15e-04 | 1916.8ms/step | 31.3 samples/s
+Step 34610/50462 | Loss: 2.5155 | LR: 1.15e-04 | 1917.4ms/step | 31.3 samples/s
+Step 34620/50462 | Loss: 2.4741 | LR: 1.15e-04 | 1916.6ms/step | 31.3 samples/s
+Step 34630/50462 | Loss: 2.5154 | LR: 1.15e-04 | 1921.0ms/step | 31.2 samples/s
+Step 34640/50462 | Loss: 2.4573 | LR: 1.15e-04 | 1919.7ms/step | 31.3 samples/s
+Step 34650/50462 | Loss: 2.5462 | LR: 1.15e-04 | 1919.2ms/step | 31.3 samples/s
+Step 34660/50462 | Loss: 2.5216 | LR: 1.15e-04 | 1919.8ms/step | 31.3 samples/s
+Step 34670/50462 | Loss: 2.4915 | LR: 1.15e-04 | 1919.4ms/step | 31.3 samples/s
+Step 34680/50462 | Loss: 2.5600 | LR: 1.14e-04 | 1915.1ms/step | 31.3 samples/s
+Step 34690/50462 | Loss: 2.4741 | LR: 1.14e-04 | 1914.8ms/step | 31.3 samples/s
+Step 34700/50462 | Loss: 2.4768 | LR: 1.14e-04 | 1917.9ms/step | 31.3 samples/s
+Step 34710/50462 | Loss: 2.5448 | LR: 1.14e-04 | 1919.1ms/step | 31.3 samples/s
+Step 34720/50462 | Loss: 2.5152 | LR: 1.14e-04 | 1919.4ms/step | 31.3 samples/s
+Step 34730/50462 | Loss: 2.4990 | LR: 1.14e-04 | 1917.3ms/step | 31.3 samples/s
+Step 34740/50462 | Loss: 2.5202 | LR: 1.14e-04 | 1925.5ms/step | 31.2 samples/s
+Step 34750/50462 | Loss: 2.4598 | LR: 1.13e-04 | 1920.6ms/step | 31.2 samples/s
+Validation at step 34750: Loss = 2.5094, Perplexity = 12.30
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-34750
+Step 34760/50462 | Loss: 2.5515 | LR: 1.13e-04 | 3237.6ms/step | 18.5 samples/s
+Step 34770/50462 | Loss: 2.4988 | LR: 1.13e-04 | 1905.9ms/step | 31.5 samples/s
+Step 34780/50462 | Loss: 2.5322 | LR: 1.13e-04 | 1906.0ms/step | 31.5 samples/s
+Step 34790/50462 | Loss: 2.5357 | LR: 1.13e-04 | 1913.4ms/step | 31.4 samples/s
+Step 34800/50462 | Loss: 2.5396 | LR: 1.13e-04 | 1911.3ms/step | 31.4 samples/s
+Step 34810/50462 | Loss: 2.5178 | LR: 1.13e-04 | 1910.5ms/step | 31.4 samples/s
+Step 34820/50462 | Loss: 2.5023 | LR: 1.13e-04 | 1914.4ms/step | 31.3 samples/s
+Step 34830/50462 | Loss: 2.5432 | LR: 1.12e-04 | 1918.4ms/step | 31.3 samples/s
+Step 34840/50462 | Loss: 2.4898 | LR: 1.12e-04 | 1918.1ms/step | 31.3 samples/s
+Step 34850/50462 | Loss: 2.5530 | LR: 1.12e-04 | 1919.3ms/step | 31.3 samples/s
+Step 34860/50462 | Loss: 2.5086 | LR: 1.12e-04 | 1915.8ms/step | 31.3 samples/s
+Step 34870/50462 | Loss: 2.4784 | LR: 1.12e-04 | 1919.2ms/step | 31.3 samples/s
+Step 34880/50462 | Loss: 2.4975 | LR: 1.12e-04 | 1919.6ms/step | 31.3 samples/s
+Step 34890/50462 | Loss: 2.5464 | LR: 1.12e-04 | 1920.1ms/step | 31.2 samples/s
+Step 34900/50462 | Loss: 2.4681 | LR: 1.11e-04 | 1920.0ms/step | 31.3 samples/s
+Step 34910/50462 | Loss: 2.5037 | LR: 1.11e-04 | 1920.0ms/step | 31.2 samples/s
+Step 34920/50462 | Loss: 2.5063 | LR: 1.11e-04 | 1919.6ms/step | 31.3 samples/s
+Step 34930/50462 | Loss: 2.4780 | LR: 1.11e-04 | 1920.1ms/step | 31.2 samples/s
+Step 34940/50462 | Loss: 2.5065 | LR: 1.11e-04 | 1920.3ms/step | 31.2 samples/s
+Step 34950/50462 | Loss: 2.5188 | LR: 1.11e-04 | 1921.9ms/step | 31.2 samples/s
+Step 34960/50462 | Loss: 2.5054 | LR: 1.11e-04 | 1920.4ms/step | 31.2 samples/s
+Step 34970/50462 | Loss: 2.4901 | LR: 1.11e-04 | 1919.8ms/step | 31.3 samples/s
+Step 34980/50462 | Loss: 2.5384 | LR: 1.10e-04 | 1915.2ms/step | 31.3 samples/s
+Step 34990/50462 | Loss: 2.4836 | LR: 1.10e-04 | 1908.9ms/step | 31.4 samples/s
+Step 35000/50462 | Loss: 2.5253 | LR: 1.10e-04 | 1910.8ms/step | 31.4 samples/s
+Validation at step 35000: Loss = 2.5090, Perplexity = 12.29
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-35000
+Step 35010/50462 | Loss: 2.5122 | LR: 1.10e-04 | 3235.6ms/step | 18.5 samples/s
+Step 35020/50462 | Loss: 2.4542 | LR: 1.10e-04 | 1902.9ms/step | 31.5 samples/s
+Step 35030/50462 | Loss: 2.5578 | LR: 1.10e-04 | 1908.9ms/step | 31.4 samples/s
+Step 35040/50462 | Loss: 2.4686 | LR: 1.10e-04 | 1908.4ms/step | 31.4 samples/s
+Step 35050/50462 | Loss: 2.4992 | LR: 1.09e-04 | 1909.2ms/step | 31.4 samples/s
+Step 35060/50462 | Loss: 2.5092 | LR: 1.09e-04 | 1913.8ms/step | 31.4 samples/s
+Step 35070/50462 | Loss: 2.4695 | LR: 1.09e-04 | 1919.2ms/step | 31.3 samples/s
+Step 35080/50462 | Loss: 2.5148 | LR: 1.09e-04 | 1917.4ms/step | 31.3 samples/s
+Step 35090/50462 | Loss: 2.5370 | LR: 1.09e-04 | 1916.5ms/step | 31.3 samples/s
+Step 35100/50462 | Loss: 2.5033 | LR: 1.09e-04 | 1913.9ms/step | 31.3 samples/s
+Step 35110/50462 | Loss: 2.5233 | LR: 1.09e-04 | 1907.3ms/step | 31.5 samples/s
+Step 35120/50462 | Loss: 2.5232 | LR: 1.09e-04 | 1912.2ms/step | 31.4 samples/s
+Step 35130/50462 | Loss: 2.4695 | LR: 1.08e-04 | 1916.5ms/step | 31.3 samples/s
+Step 35140/50462 | Loss: 2.5140 | LR: 1.08e-04 | 1911.0ms/step | 31.4 samples/s
+Step 35150/50462 | Loss: 2.4806 | LR: 1.08e-04 | 1908.5ms/step | 31.4 samples/s
+Step 35160/50462 | Loss: 2.4722 | LR: 1.08e-04 | 1912.2ms/step | 31.4 samples/s
+Step 35170/50462 | Loss: 2.5323 | LR: 1.08e-04 | 1916.5ms/step | 31.3 samples/s
+Step 35180/50462 | Loss: 2.5086 | LR: 1.08e-04 | 1915.1ms/step | 31.3 samples/s
+Step 35190/50462 | Loss: 2.4892 | LR: 1.08e-04 | 1913.7ms/step | 31.4 samples/s
+Step 35200/50462 | Loss: 2.4761 | LR: 1.08e-04 | 1917.6ms/step | 31.3 samples/s
+Step 35210/50462 | Loss: 2.4793 | LR: 1.07e-04 | 1921.1ms/step | 31.2 samples/s
+Step 35220/50462 | Loss: 2.4702 | LR: 1.07e-04 | 1920.3ms/step | 31.2 samples/s
+Step 35230/50462 | Loss: 2.5000 | LR: 1.07e-04 | 1922.9ms/step | 31.2 samples/s
+Step 35240/50462 | Loss: 2.5110 | LR: 1.07e-04 | 1921.1ms/step | 31.2 samples/s
+Step 35250/50462 | Loss: 2.5049 | LR: 1.07e-04 | 1917.1ms/step | 31.3 samples/s
+Validation at step 35250: Loss = 2.5081, Perplexity = 12.28
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-35250
+Step 35260/50462 | Loss: 2.5157 | LR: 1.07e-04 | 3240.8ms/step | 18.5 samples/s
+Step 35270/50462 | Loss: 2.5146 | LR: 1.07e-04 | 1900.4ms/step | 31.6 samples/s
+Step 35280/50462 | Loss: 2.5432 | LR: 1.07e-04 | 1910.4ms/step | 31.4 samples/s
+Step 35290/50462 | Loss: 2.5248 | LR: 1.06e-04 | 1911.1ms/step | 31.4 samples/s
+Step 35300/50462 | Loss: 2.5575 | LR: 1.06e-04 | 1914.6ms/step | 31.3 samples/s
+Step 35310/50462 | Loss: 2.4685 | LR: 1.06e-04 | 1915.4ms/step | 31.3 samples/s
+Step 35320/50462 | Loss: 2.4775 | LR: 1.06e-04 | 1912.5ms/step | 31.4 samples/s
+Step 35330/50462 | Loss: 2.5392 | LR: 1.06e-04 | 1917.5ms/step | 31.3 samples/s
+Step 35340/50462 | Loss: 2.5046 | LR: 1.06e-04 | 1916.1ms/step | 31.3 samples/s
+Step 35350/50462 | Loss: 2.5131 | LR: 1.06e-04 | 1916.4ms/step | 31.3 samples/s
+Step 35360/50462 | Loss: 2.4869 | LR: 1.05e-04 | 1911.7ms/step | 31.4 samples/s
+Step 35370/50462 | Loss: 2.5163 | LR: 1.05e-04 | 1913.1ms/step | 31.4 samples/s
+Step 35380/50462 | Loss: 2.4655 | LR: 1.05e-04 | 1909.3ms/step | 31.4 samples/s
+Step 35390/50462 | Loss: 2.4749 | LR: 1.05e-04 | 1910.0ms/step | 31.4 samples/s
+Step 35400/50462 | Loss: 2.5254 | LR: 1.05e-04 | 1904.6ms/step | 31.5 samples/s
+Step 35410/50462 | Loss: 2.5312 | LR: 1.05e-04 | 1907.1ms/step | 31.5 samples/s
+Step 35420/50462 | Loss: 2.5044 | LR: 1.05e-04 | 1910.2ms/step | 31.4 samples/s
+Step 35430/50462 | Loss: 2.5105 | LR: 1.05e-04 | 1909.6ms/step | 31.4 samples/s
+Step 35440/50462 | Loss: 2.4909 | LR: 1.04e-04 | 1915.3ms/step | 31.3 samples/s
+Step 35450/50462 | Loss: 2.5037 | LR: 1.04e-04 | 1915.3ms/step | 31.3 samples/s
+Step 35460/50462 | Loss: 2.4626 | LR: 1.04e-04 | 1915.4ms/step | 31.3 samples/s
+Step 35470/50462 | Loss: 2.4766 | LR: 1.04e-04 | 1914.7ms/step | 31.3 samples/s
+Step 35480/50462 | Loss: 2.4645 | LR: 1.04e-04 | 1917.5ms/step | 31.3 samples/s
+Step 35490/50462 | Loss: 2.4706 | LR: 1.04e-04 | 1916.9ms/step | 31.3 samples/s
+Step 35500/50462 | Loss: 2.4909 | LR: 1.04e-04 | 1916.1ms/step | 31.3 samples/s
+Validation at step 35500: Loss = 2.5078, Perplexity = 12.28
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-35500
+Step 35510/50462 | Loss: 2.5306 | LR: 1.04e-04 | 3252.4ms/step | 18.4 samples/s
+Step 35520/50462 | Loss: 2.4990 | LR: 1.03e-04 | 1897.6ms/step | 31.6 samples/s
+Step 35530/50462 | Loss: 2.5164 | LR: 1.03e-04 | 1904.7ms/step | 31.5 samples/s
+Step 35540/50462 | Loss: 2.5160 | LR: 1.03e-04 | 1907.5ms/step | 31.5 samples/s
+Step 35550/50462 | Loss: 2.5097 | LR: 1.03e-04 | 1910.2ms/step | 31.4 samples/s
+Step 35560/50462 | Loss: 2.4813 | LR: 1.03e-04 | 1911.0ms/step | 31.4 samples/s
+Step 35570/50462 | Loss: 2.5209 | LR: 1.03e-04 | 1909.8ms/step | 31.4 samples/s
+Step 35580/50462 | Loss: 2.4975 | LR: 1.03e-04 | 1908.4ms/step | 31.4 samples/s
+Step 35590/50462 | Loss: 2.5224 | LR: 1.03e-04 | 1911.9ms/step | 31.4 samples/s
+Step 35600/50462 | Loss: 2.4925 | LR: 1.02e-04 | 1911.8ms/step | 31.4 samples/s
+Step 35610/50462 | Loss: 2.4890 | LR: 1.02e-04 | 1911.8ms/step | 31.4 samples/s
+Step 35620/50462 | Loss: 2.4705 | LR: 1.02e-04 | 1906.9ms/step | 31.5 samples/s
+Step 35630/50462 | Loss: 2.4662 | LR: 1.02e-04 | 1911.4ms/step | 31.4 samples/s
+Step 35640/50462 | Loss: 2.4917 | LR: 1.02e-04 | 1914.5ms/step | 31.3 samples/s
+Step 35650/50462 | Loss: 2.4901 | LR: 1.02e-04 | 1915.5ms/step | 31.3 samples/s
+Step 35660/50462 | Loss: 2.5070 | LR: 1.02e-04 | 1918.7ms/step | 31.3 samples/s
+Step 35670/50462 | Loss: 2.5124 | LR: 1.02e-04 | 1919.1ms/step | 31.3 samples/s
+Step 35680/50462 | Loss: 2.5352 | LR: 1.01e-04 | 1916.9ms/step | 31.3 samples/s
+Step 35690/50462 | Loss: 2.5264 | LR: 1.01e-04 | 1910.6ms/step | 31.4 samples/s
+Step 35700/50462 | Loss: 2.5406 | LR: 1.01e-04 | 1912.7ms/step | 31.4 samples/s
+Step 35710/50462 | Loss: 2.4734 | LR: 1.01e-04 | 1919.6ms/step | 31.3 samples/s
+Step 35720/50462 | Loss: 2.4802 | LR: 1.01e-04 | 1920.3ms/step | 31.2 samples/s
+Step 35730/50462 | Loss: 2.4793 | LR: 1.01e-04 | 1919.2ms/step | 31.3 samples/s
+Step 35740/50462 | Loss: 2.5191 | LR: 1.01e-04 | 1921.4ms/step | 31.2 samples/s
+Step 35750/50462 | Loss: 2.5311 | LR: 1.00e-04 | 1914.7ms/step | 31.3 samples/s
+Validation at step 35750: Loss = 2.5075, Perplexity = 12.27
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-35750
+Step 35760/50462 | Loss: 2.5122 | LR: 1.00e-04 | 3254.8ms/step | 18.4 samples/s
+Step 35770/50462 | Loss: 2.5166 | LR: 1.00e-04 | 1905.2ms/step | 31.5 samples/s
+Step 35780/50462 | Loss: 2.5171 | LR: 1.00e-04 | 1905.4ms/step | 31.5 samples/s
+Step 35790/50462 | Loss: 2.5029 | LR: 1.00e-04 | 1910.1ms/step | 31.4 samples/s
+Step 35800/50462 | Loss: 2.4661 | LR: 9.99e-05 | 1915.4ms/step | 31.3 samples/s
+Step 35810/50462 | Loss: 2.4831 | LR: 9.97e-05 | 1914.4ms/step | 31.3 samples/s
+Step 35820/50462 | Loss: 2.5079 | LR: 9.96e-05 | 1909.1ms/step | 31.4 samples/s
+Step 35830/50462 | Loss: 2.4778 | LR: 9.95e-05 | 1909.3ms/step | 31.4 samples/s
+Step 35840/50462 | Loss: 2.4756 | LR: 9.94e-05 | 1904.4ms/step | 31.5 samples/s
+Step 35850/50462 | Loss: 2.5409 | LR: 9.92e-05 | 1905.7ms/step | 31.5 samples/s
+Step 35860/50462 | Loss: 2.5298 | LR: 9.91e-05 | 1909.5ms/step | 31.4 samples/s
+Step 35870/50462 | Loss: 2.4948 | LR: 9.90e-05 | 1913.0ms/step | 31.4 samples/s
+Step 35880/50462 | Loss: 2.4963 | LR: 9.88e-05 | 1913.4ms/step | 31.4 samples/s
+Step 35890/50462 | Loss: 2.4910 | LR: 9.87e-05 | 1910.9ms/step | 31.4 samples/s
+Step 35900/50462 | Loss: 2.4747 | LR: 9.86e-05 | 1910.0ms/step | 31.4 samples/s
+Step 35910/50462 | Loss: 2.4642 | LR: 9.85e-05 | 1915.5ms/step | 31.3 samples/s
+Step 35920/50462 | Loss: 2.5029 | LR: 9.83e-05 | 1915.8ms/step | 31.3 samples/s
+Step 35930/50462 | Loss: 2.4728 | LR: 9.82e-05 | 1913.6ms/step | 31.4 samples/s
+Step 35940/50462 | Loss: 2.4901 | LR: 9.81e-05 | 1914.5ms/step | 31.3 samples/s
+Step 35950/50462 | Loss: 2.4715 | LR: 9.80e-05 | 1914.8ms/step | 31.3 samples/s
+Step 35960/50462 | Loss: 2.4834 | LR: 9.78e-05 | 1920.2ms/step | 31.2 samples/s
+Step 35970/50462 | Loss: 2.5085 | LR: 9.77e-05 | 1908.4ms/step | 31.4 samples/s
+Step 35980/50462 | Loss: 2.4668 | LR: 9.76e-05 | 1907.9ms/step | 31.4 samples/s
+Step 35990/50462 | Loss: 2.5036 | LR: 9.75e-05 | 1911.8ms/step | 31.4 samples/s
+Step 36000/50462 | Loss: 2.4970 | LR: 9.73e-05 | 1910.5ms/step | 31.4 samples/s
+Validation at step 36000: Loss = 2.5069, Perplexity = 12.27
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-36000
+Step 36010/50462 | Loss: 2.5395 | LR: 9.72e-05 | 3248.6ms/step | 18.5 samples/s
+Step 36020/50462 | Loss: 2.4988 | LR: 9.71e-05 | 1898.2ms/step | 31.6 samples/s
+Step 36030/50462 | Loss: 2.5221 | LR: 9.70e-05 | 1903.5ms/step | 31.5 samples/s
+Step 36040/50462 | Loss: 2.4976 | LR: 9.68e-05 | 1906.2ms/step | 31.5 samples/s
+Step 36050/50462 | Loss: 2.5068 | LR: 9.67e-05 | 1911.1ms/step | 31.4 samples/s
+Step 36060/50462 | Loss: 2.4272 | LR: 9.66e-05 | 1911.5ms/step | 31.4 samples/s
+Step 36070/50462 | Loss: 2.5186 | LR: 9.65e-05 | 1917.8ms/step | 31.3 samples/s
+Step 36080/50462 | Loss: 2.5109 | LR: 9.63e-05 | 1917.7ms/step | 31.3 samples/s
+Step 36090/50462 | Loss: 2.5127 | LR: 9.62e-05 | 1916.5ms/step | 31.3 samples/s
+Step 36100/50462 | Loss: 2.4825 | LR: 9.61e-05 | 1914.4ms/step | 31.3 samples/s
+Step 36110/50462 | Loss: 2.5098 | LR: 9.60e-05 | 1910.2ms/step | 31.4 samples/s
+Step 36120/50462 | Loss: 2.4785 | LR: 9.58e-05 | 1913.9ms/step | 31.4 samples/s
+Step 36130/50462 | Loss: 2.4871 | LR: 9.57e-05 | 1915.4ms/step | 31.3 samples/s
+Step 36140/50462 | Loss: 2.4980 | LR: 9.56e-05 | 1919.2ms/step | 31.3 samples/s
+Step 36150/50462 | Loss: 2.4715 | LR: 9.55e-05 | 1918.9ms/step | 31.3 samples/s
+Step 36160/50462 | Loss: 2.4628 | LR: 9.53e-05 | 1917.5ms/step | 31.3 samples/s
+Step 36170/50462 | Loss: 2.5018 | LR: 9.52e-05 | 1909.1ms/step | 31.4 samples/s
+Step 36180/50462 | Loss: 2.4930 | LR: 9.51e-05 | 1910.5ms/step | 31.4 samples/s
+Step 36190/50462 | Loss: 2.5179 | LR: 9.50e-05 | 1914.3ms/step | 31.3 samples/s
+Step 36200/50462 | Loss: 2.4832 | LR: 9.49e-05 | 1911.7ms/step | 31.4 samples/s
+Step 36210/50462 | Loss: 2.4520 | LR: 9.47e-05 | 1918.9ms/step | 31.3 samples/s
+Step 36220/50462 | Loss: 2.4946 | LR: 9.46e-05 | 1915.5ms/step | 31.3 samples/s
+Step 36230/50462 | Loss: 2.4994 | LR: 9.45e-05 | 1915.1ms/step | 31.3 samples/s
+Step 36240/50462 | Loss: 2.5369 | LR: 9.44e-05 | 1916.8ms/step | 31.3 samples/s
+Step 36250/50462 | Loss: 2.5093 | LR: 9.42e-05 | 1910.9ms/step | 31.4 samples/s
+Validation at step 36250: Loss = 2.5063, Perplexity = 12.26
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-36250
+Step 36260/50462 | Loss: 2.5003 | LR: 9.41e-05 | 3319.0ms/step | 18.1 samples/s
+Step 36270/50462 | Loss: 2.4798 | LR: 9.40e-05 | 1899.9ms/step | 31.6 samples/s
+Step 36280/50462 | Loss: 2.4779 | LR: 9.39e-05 | 1907.3ms/step | 31.5 samples/s
+Step 36290/50462 | Loss: 2.5110 | LR: 9.37e-05 | 1915.1ms/step | 31.3 samples/s
+Step 36300/50462 | Loss: 2.4877 | LR: 9.36e-05 | 1914.7ms/step | 31.3 samples/s
+Step 36310/50462 | Loss: 2.4679 | LR: 9.35e-05 | 1918.9ms/step | 31.3 samples/s
+Step 36320/50462 | Loss: 2.5204 | LR: 9.34e-05 | 1917.0ms/step | 31.3 samples/s
+Step 36330/50462 | Loss: 2.5257 | LR: 9.32e-05 | 1911.8ms/step | 31.4 samples/s
+Step 36340/50462 | Loss: 2.5120 | LR: 9.31e-05 | 1909.6ms/step | 31.4 samples/s
+Step 36350/50462 | Loss: 2.4890 | LR: 9.30e-05 | 1912.2ms/step | 31.4 samples/s
+Step 36360/50462 | Loss: 2.5029 | LR: 9.29e-05 | 1913.3ms/step | 31.4 samples/s
+Step 36370/50462 | Loss: 2.4702 | LR: 9.28e-05 | 1906.1ms/step | 31.5 samples/s
+Step 36380/50462 | Loss: 2.5024 | LR: 9.26e-05 | 1907.8ms/step | 31.4 samples/s
+Step 36390/50462 | Loss: 2.5278 | LR: 9.25e-05 | 1910.7ms/step | 31.4 samples/s
+Step 36400/50462 | Loss: 2.4794 | LR: 9.24e-05 | 1902.5ms/step | 31.5 samples/s
+Step 36410/50462 | Loss: 2.4486 | LR: 9.23e-05 | 1908.2ms/step | 31.4 samples/s
+Step 36420/50462 | Loss: 2.4931 | LR: 9.21e-05 | 1912.0ms/step | 31.4 samples/s
+Step 36430/50462 | Loss: 2.4850 | LR: 9.20e-05 | 1912.5ms/step | 31.4 samples/s
+Step 36440/50462 | Loss: 2.5045 | LR: 9.19e-05 | 1916.1ms/step | 31.3 samples/s
+Step 36450/50462 | Loss: 2.4805 | LR: 9.18e-05 | 1917.4ms/step | 31.3 samples/s
+Step 36460/50462 | Loss: 2.4529 | LR: 9.17e-05 | 1917.9ms/step | 31.3 samples/s
+Step 36470/50462 | Loss: 2.5024 | LR: 9.15e-05 | 1915.7ms/step | 31.3 samples/s
+Step 36480/50462 | Loss: 2.5058 | LR: 9.14e-05 | 1913.8ms/step | 31.4 samples/s
+Step 36490/50462 | Loss: 2.5241 | LR: 9.13e-05 | 1915.4ms/step | 31.3 samples/s
+Step 36500/50462 | Loss: 2.5154 | LR: 9.12e-05 | 1912.3ms/step | 31.4 samples/s
+Validation at step 36500: Loss = 2.5055, Perplexity = 12.25
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-36500
+Step 36510/50462 | Loss: 2.4969 | LR: 9.10e-05 | 3271.5ms/step | 18.3 samples/s
+Step 36520/50462 | Loss: 2.4981 | LR: 9.09e-05 | 1902.3ms/step | 31.5 samples/s
+Step 36530/50462 | Loss: 2.5404 | LR: 9.08e-05 | 1908.2ms/step | 31.4 samples/s
+Step 36540/50462 | Loss: 2.4950 | LR: 9.07e-05 | 1910.3ms/step | 31.4 samples/s
+Step 36550/50462 | Loss: 2.4931 | LR: 9.06e-05 | 1914.9ms/step | 31.3 samples/s
+Step 36560/50462 | Loss: 2.5049 | LR: 9.04e-05 | 1916.5ms/step | 31.3 samples/s
+Step 36570/50462 | Loss: 2.5463 | LR: 9.03e-05 | 1916.3ms/step | 31.3 samples/s
+Step 36580/50462 | Loss: 2.5118 | LR: 9.02e-05 | 1914.8ms/step | 31.3 samples/s
+Step 36590/50462 | Loss: 2.4822 | LR: 9.01e-05 | 1918.1ms/step | 31.3 samples/s
+Step 36600/50462 | Loss: 2.4988 | LR: 8.99e-05 | 1918.8ms/step | 31.3 samples/s
+Step 36610/50462 | Loss: 2.5313 | LR: 8.98e-05 | 1913.1ms/step | 31.4 samples/s
+Step 36620/50462 | Loss: 2.4485 | LR: 8.97e-05 | 1913.4ms/step | 31.4 samples/s
+Step 36630/50462 | Loss: 2.4852 | LR: 8.96e-05 | 1914.8ms/step | 31.3 samples/s
+Step 36640/50462 | Loss: 2.4604 | LR: 8.95e-05 | 1914.3ms/step | 31.3 samples/s
+Step 36650/50462 | Loss: 2.4901 | LR: 8.93e-05 | 1911.9ms/step | 31.4 samples/s
+Step 36660/50462 | Loss: 2.4532 | LR: 8.92e-05 | 1906.4ms/step | 31.5 samples/s
+Step 36670/50462 | Loss: 2.4870 | LR: 8.91e-05 | 1906.8ms/step | 31.5 samples/s
+Step 36680/50462 | Loss: 2.4976 | LR: 8.90e-05 | 1911.7ms/step | 31.4 samples/s
+Step 36690/50462 | Loss: 2.4430 | LR: 8.89e-05 | 1914.3ms/step | 31.3 samples/s
+Step 36700/50462 | Loss: 2.4809 | LR: 8.87e-05 | 1916.1ms/step | 31.3 samples/s
+Step 36710/50462 | Loss: 2.4622 | LR: 8.86e-05 | 1918.1ms/step | 31.3 samples/s
+Step 36720/50462 | Loss: 2.5381 | LR: 8.85e-05 | 1915.7ms/step | 31.3 samples/s
+Step 36730/50462 | Loss: 2.4593 | LR: 8.84e-05 | 1917.8ms/step | 31.3 samples/s
+Step 36740/50462 | Loss: 2.4952 | LR: 8.83e-05 | 1920.3ms/step | 31.2 samples/s
+Step 36750/50462 | Loss: 2.5218 | LR: 8.81e-05 | 1918.1ms/step | 31.3 samples/s
+Validation at step 36750: Loss = 2.5052, Perplexity = 12.25
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-36750
+Step 36760/50462 | Loss: 2.4895 | LR: 8.80e-05 | 3219.3ms/step | 18.6 samples/s
+Step 36770/50462 | Loss: 2.4900 | LR: 8.79e-05 | 1898.4ms/step | 31.6 samples/s
+Step 36780/50462 | Loss: 2.4407 | LR: 8.78e-05 | 1902.0ms/step | 31.5 samples/s
+Step 36790/50462 | Loss: 2.4694 | LR: 8.77e-05 | 1906.0ms/step | 31.5 samples/s
+Step 36800/50462 | Loss: 2.5387 | LR: 8.75e-05 | 1909.5ms/step | 31.4 samples/s
+Step 36810/50462 | Loss: 2.4934 | LR: 8.74e-05 | 1913.2ms/step | 31.4 samples/s
+Step 36820/50462 | Loss: 2.4705 | LR: 8.73e-05 | 1912.1ms/step | 31.4 samples/s
+Step 36830/50462 | Loss: 2.4749 | LR: 8.72e-05 | 1914.1ms/step | 31.3 samples/s
+Step 36840/50462 | Loss: 2.4848 | LR: 8.71e-05 | 1915.1ms/step | 31.3 samples/s
+Step 36850/50462 | Loss: 2.4981 | LR: 8.69e-05 | 1916.2ms/step | 31.3 samples/s
+Step 36860/50462 | Loss: 2.5250 | LR: 8.68e-05 | 1916.9ms/step | 31.3 samples/s
+Step 36870/50462 | Loss: 2.4698 | LR: 8.67e-05 | 1921.6ms/step | 31.2 samples/s
+Step 36880/50462 | Loss: 2.5026 | LR: 8.66e-05 | 1915.5ms/step | 31.3 samples/s
+Step 36890/50462 | Loss: 2.5003 | LR: 8.65e-05 | 1917.5ms/step | 31.3 samples/s
+Step 36900/50462 | Loss: 2.5101 | LR: 8.63e-05 | 1916.3ms/step | 31.3 samples/s
+Step 36910/50462 | Loss: 2.4833 | LR: 8.62e-05 | 1916.2ms/step | 31.3 samples/s
+Step 36920/50462 | Loss: 2.5346 | LR: 8.61e-05 | 1914.7ms/step | 31.3 samples/s
+Step 36930/50462 | Loss: 2.4820 | LR: 8.60e-05 | 1915.7ms/step | 31.3 samples/s
+Step 36940/50462 | Loss: 2.5242 | LR: 8.59e-05 | 1914.5ms/step | 31.3 samples/s
+Step 36950/50462 | Loss: 2.4873 | LR: 8.57e-05 | 1913.9ms/step | 31.4 samples/s
+Step 36960/50462 | Loss: 2.4774 | LR: 8.56e-05 | 1918.3ms/step | 31.3 samples/s
+Step 36970/50462 | Loss: 2.5293 | LR: 8.55e-05 | 1917.5ms/step | 31.3 samples/s
+Step 36980/50462 | Loss: 2.4873 | LR: 8.54e-05 | 1917.4ms/step | 31.3 samples/s
+Step 36990/50462 | Loss: 2.4967 | LR: 8.53e-05 | 1918.0ms/step | 31.3 samples/s
+Step 37000/50462 | Loss: 2.4924 | LR: 8.51e-05 | 1914.4ms/step | 31.3 samples/s
+Validation at step 37000: Loss = 2.5043, Perplexity = 12.24
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-37000
+Step 37010/50462 | Loss: 2.4209 | LR: 8.50e-05 | 3256.8ms/step | 18.4 samples/s
+Step 37020/50462 | Loss: 2.4980 | LR: 8.49e-05 | 1901.4ms/step | 31.6 samples/s
+Step 37030/50462 | Loss: 2.4732 | LR: 8.48e-05 | 1905.7ms/step | 31.5 samples/s
+Step 37040/50462 | Loss: 2.4640 | LR: 8.47e-05 | 1906.6ms/step | 31.5 samples/s
+Step 37050/50462 | Loss: 2.5106 | LR: 8.46e-05 | 1910.0ms/step | 31.4 samples/s
+Step 37060/50462 | Loss: 2.5001 | LR: 8.44e-05 | 1913.4ms/step | 31.4 samples/s
+Step 37070/50462 | Loss: 2.4775 | LR: 8.43e-05 | 1912.4ms/step | 31.4 samples/s
+Step 37080/50462 | Loss: 2.5455 | LR: 8.42e-05 | 1914.3ms/step | 31.3 samples/s
+Step 37090/50462 | Loss: 2.4927 | LR: 8.41e-05 | 1910.2ms/step | 31.4 samples/s
+Step 37100/50462 | Loss: 2.5025 | LR: 8.40e-05 | 1910.6ms/step | 31.4 samples/s
+Step 37110/50462 | Loss: 2.5176 | LR: 8.38e-05 | 1910.0ms/step | 31.4 samples/s
+Step 37120/50462 | Loss: 2.4841 | LR: 8.37e-05 | 1909.9ms/step | 31.4 samples/s
+Step 37130/50462 | Loss: 2.4609 | LR: 8.36e-05 | 1912.9ms/step | 31.4 samples/s
+Step 37140/50462 | Loss: 2.5067 | LR: 8.35e-05 | 1917.8ms/step | 31.3 samples/s
+Step 37150/50462 | Loss: 2.4832 | LR: 8.34e-05 | 1918.3ms/step | 31.3 samples/s
+Step 37160/50462 | Loss: 2.4770 | LR: 8.33e-05 | 1913.7ms/step | 31.4 samples/s
+Step 37170/50462 | Loss: 2.5160 | LR: 8.31e-05 | 1915.2ms/step | 31.3 samples/s
+Step 37180/50462 | Loss: 2.4643 | LR: 8.30e-05 | 1912.6ms/step | 31.4 samples/s
+Step 37190/50462 | Loss: 2.5082 | LR: 8.29e-05 | 1914.6ms/step | 31.3 samples/s
+Step 37200/50462 | Loss: 2.4941 | LR: 8.28e-05 | 1913.7ms/step | 31.4 samples/s
+Step 37210/50462 | Loss: 2.4908 | LR: 8.27e-05 | 1915.0ms/step | 31.3 samples/s
+Step 37220/50462 | Loss: 2.4991 | LR: 8.25e-05 | 1916.7ms/step | 31.3 samples/s
+Step 37230/50462 | Loss: 2.4885 | LR: 8.24e-05 | 1912.5ms/step | 31.4 samples/s
+Step 37240/50462 | Loss: 2.4869 | LR: 8.23e-05 | 1911.1ms/step | 31.4 samples/s
+Step 37250/50462 | Loss: 2.5236 | LR: 8.22e-05 | 1913.5ms/step | 31.4 samples/s
+Validation at step 37250: Loss = 2.5042, Perplexity = 12.23
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-37250
+Step 37260/50462 | Loss: 2.4648 | LR: 8.21e-05 | 3232.1ms/step | 18.6 samples/s
+Step 37270/50462 | Loss: 2.5040 | LR: 8.20e-05 | 1901.3ms/step | 31.6 samples/s
+Step 37280/50462 | Loss: 2.5210 | LR: 8.18e-05 | 1906.1ms/step | 31.5 samples/s
+Step 37290/50462 | Loss: 2.5204 | LR: 8.17e-05 | 1909.9ms/step | 31.4 samples/s
+Step 37300/50462 | Loss: 2.4694 | LR: 8.16e-05 | 1912.7ms/step | 31.4 samples/s
+Step 37310/50462 | Loss: 2.5182 | LR: 8.15e-05 | 1914.2ms/step | 31.3 samples/s
+Step 37320/50462 | Loss: 2.4857 | LR: 8.14e-05 | 1915.0ms/step | 31.3 samples/s
+Step 37330/50462 | Loss: 2.4962 | LR: 8.13e-05 | 1911.5ms/step | 31.4 samples/s
+Step 37340/50462 | Loss: 2.5260 | LR: 8.11e-05 | 1915.6ms/step | 31.3 samples/s
+Step 37350/50462 | Loss: 2.5239 | LR: 8.10e-05 | 1918.9ms/step | 31.3 samples/s
+Step 37360/50462 | Loss: 2.5150 | LR: 8.09e-05 | 1915.1ms/step | 31.3 samples/s
+Step 37370/50462 | Loss: 2.5062 | LR: 8.08e-05 | 1915.2ms/step | 31.3 samples/s
+Step 37380/50462 | Loss: 2.4522 | LR: 8.07e-05 | 1912.8ms/step | 31.4 samples/s
+Step 37390/50462 | Loss: 2.4689 | LR: 8.06e-05 | 1917.0ms/step | 31.3 samples/s
+Step 37400/50462 | Loss: 2.4961 | LR: 8.04e-05 | 1915.3ms/step | 31.3 samples/s
+Step 37410/50462 | Loss: 2.5061 | LR: 8.03e-05 | 1918.5ms/step | 31.3 samples/s
+Step 37420/50462 | Loss: 2.5010 | LR: 8.02e-05 | 1915.0ms/step | 31.3 samples/s
+Step 37430/50462 | Loss: 2.5070 | LR: 8.01e-05 | 1916.1ms/step | 31.3 samples/s
+Step 37440/50462 | Loss: 2.4826 | LR: 8.00e-05 | 1916.1ms/step | 31.3 samples/s
+Step 37450/50462 | Loss: 2.4965 | LR: 7.99e-05 | 1919.1ms/step | 31.3 samples/s
+Step 37460/50462 | Loss: 2.5015 | LR: 7.98e-05 | 1918.1ms/step | 31.3 samples/s
+Step 37470/50462 | Loss: 2.4803 | LR: 7.96e-05 | 1919.0ms/step | 31.3 samples/s
+Step 37480/50462 | Loss: 2.4591 | LR: 7.95e-05 | 1917.2ms/step | 31.3 samples/s
+Step 37490/50462 | Loss: 2.4755 | LR: 7.94e-05 | 1917.8ms/step | 31.3 samples/s
+Step 37500/50462 | Loss: 2.4302 | LR: 7.93e-05 | 1916.9ms/step | 31.3 samples/s
+Validation at step 37500: Loss = 2.5035, Perplexity = 12.23
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-37500
+Step 37510/50462 | Loss: 2.4933 | LR: 7.92e-05 | 3222.2ms/step | 18.6 samples/s
+Step 37520/50462 | Loss: 2.4861 | LR: 7.91e-05 | 1899.0ms/step | 31.6 samples/s
+Step 37530/50462 | Loss: 2.4549 | LR: 7.89e-05 | 1904.6ms/step | 31.5 samples/s
+Step 37540/50462 | Loss: 2.4372 | LR: 7.88e-05 | 1905.8ms/step | 31.5 samples/s
+Step 37550/50462 | Loss: 2.4803 | LR: 7.87e-05 | 1908.1ms/step | 31.4 samples/s
+Step 37560/50462 | Loss: 2.5232 | LR: 7.86e-05 | 1908.4ms/step | 31.4 samples/s
+Step 37570/50462 | Loss: 2.4755 | LR: 7.85e-05 | 1912.7ms/step | 31.4 samples/s
+Step 37580/50462 | Loss: 2.5029 | LR: 7.84e-05 | 1910.7ms/step | 31.4 samples/s
+Step 37590/50462 | Loss: 2.5202 | LR: 7.83e-05 | 1908.7ms/step | 31.4 samples/s
+Step 37600/50462 | Loss: 2.4905 | LR: 7.81e-05 | 1908.6ms/step | 31.4 samples/s
+Step 37610/50462 | Loss: 2.4892 | LR: 7.80e-05 | 1909.8ms/step | 31.4 samples/s
+Step 37620/50462 | Loss: 2.5195 | LR: 7.79e-05 | 1912.2ms/step | 31.4 samples/s
+Step 37630/50462 | Loss: 2.5068 | LR: 7.78e-05 | 1913.7ms/step | 31.4 samples/s
+Step 37640/50462 | Loss: 2.5121 | LR: 7.77e-05 | 1915.7ms/step | 31.3 samples/s
+Step 37650/50462 | Loss: 2.4986 | LR: 7.76e-05 | 1914.8ms/step | 31.3 samples/s
+Step 37660/50462 | Loss: 2.4897 | LR: 7.75e-05 | 1913.2ms/step | 31.4 samples/s
+Step 37670/50462 | Loss: 2.4722 | LR: 7.73e-05 | 1913.8ms/step | 31.4 samples/s
+Step 37680/50462 | Loss: 2.4942 | LR: 7.72e-05 | 1913.1ms/step | 31.4 samples/s
+Step 37690/50462 | Loss: 2.5142 | LR: 7.71e-05 | 1916.9ms/step | 31.3 samples/s
+Step 37700/50462 | Loss: 2.4717 | LR: 7.70e-05 | 1913.1ms/step | 31.4 samples/s
+Step 37710/50462 | Loss: 2.5051 | LR: 7.69e-05 | 1915.2ms/step | 31.3 samples/s
+Step 37720/50462 | Loss: 2.4899 | LR: 7.68e-05 | 1912.3ms/step | 31.4 samples/s
+Step 37730/50462 | Loss: 2.5075 | LR: 7.67e-05 | 1913.1ms/step | 31.4 samples/s
+Step 37740/50462 | Loss: 2.5137 | LR: 7.65e-05 | 1910.8ms/step | 31.4 samples/s
+Step 37750/50462 | Loss: 2.4850 | LR: 7.64e-05 | 1911.7ms/step | 31.4 samples/s
+Validation at step 37750: Loss = 2.5031, Perplexity = 12.22
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-37750
+Step 37760/50462 | Loss: 2.4698 | LR: 7.63e-05 | 3277.7ms/step | 18.3 samples/s
+Step 37770/50462 | Loss: 2.5251 | LR: 7.62e-05 | 1900.2ms/step | 31.6 samples/s
+Step 37780/50462 | Loss: 2.4860 | LR: 7.61e-05 | 1910.4ms/step | 31.4 samples/s
+Step 37790/50462 | Loss: 2.4662 | LR: 7.60e-05 | 1911.3ms/step | 31.4 samples/s
+Step 37800/50462 | Loss: 2.4280 | LR: 7.59e-05 | 1908.4ms/step | 31.4 samples/s
+Step 37810/50462 | Loss: 2.4986 | LR: 7.57e-05 | 1912.4ms/step | 31.4 samples/s
+Step 37820/50462 | Loss: 2.4835 | LR: 7.56e-05 | 1913.3ms/step | 31.4 samples/s
+Step 37830/50462 | Loss: 2.4961 | LR: 7.55e-05 | 1912.6ms/step | 31.4 samples/s
+Step 37840/50462 | Loss: 2.5270 | LR: 7.54e-05 | 1910.1ms/step | 31.4 samples/s
+Step 37850/50462 | Loss: 2.4732 | LR: 7.53e-05 | 1911.5ms/step | 31.4 samples/s
+Step 37860/50462 | Loss: 2.5476 | LR: 7.52e-05 | 1915.6ms/step | 31.3 samples/s
+Step 37870/50462 | Loss: 2.5253 | LR: 7.51e-05 | 1914.0ms/step | 31.3 samples/s
+Step 37880/50462 | Loss: 2.4975 | LR: 7.50e-05 | 1913.8ms/step | 31.4 samples/s
+Step 37890/50462 | Loss: 2.4717 | LR: 7.48e-05 | 1916.8ms/step | 31.3 samples/s
+Step 37900/50462 | Loss: 2.4347 | LR: 7.47e-05 | 1912.7ms/step | 31.4 samples/s
+Step 37910/50462 | Loss: 2.4776 | LR: 7.46e-05 | 1906.4ms/step | 31.5 samples/s
+Step 37920/50462 | Loss: 2.4940 | LR: 7.45e-05 | 1911.3ms/step | 31.4 samples/s
+Step 37930/50462 | Loss: 2.4999 | LR: 7.44e-05 | 1911.7ms/step | 31.4 samples/s
+Step 37940/50462 | Loss: 2.5382 | LR: 7.43e-05 | 1912.5ms/step | 31.4 samples/s
+Step 37950/50462 | Loss: 2.5078 | LR: 7.42e-05 | 1908.2ms/step | 31.4 samples/s
+Step 37960/50462 | Loss: 2.4972 | LR: 7.41e-05 | 1908.4ms/step | 31.4 samples/s
+Step 37970/50462 | Loss: 2.5354 | LR: 7.39e-05 | 1910.5ms/step | 31.4 samples/s
+Step 37980/50462 | Loss: 2.5073 | LR: 7.38e-05 | 1911.8ms/step | 31.4 samples/s
+Step 37990/50462 | Loss: 2.4999 | LR: 7.37e-05 | 1911.2ms/step | 31.4 samples/s
+Step 38000/50462 | Loss: 2.4788 | LR: 7.36e-05 | 1908.9ms/step | 31.4 samples/s
+Validation at step 38000: Loss = 2.5028, Perplexity = 12.22
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-38000
+Step 38010/50462 | Loss: 2.4934 | LR: 7.35e-05 | 3243.9ms/step | 18.5 samples/s
+Step 38020/50462 | Loss: 2.4907 | LR: 7.34e-05 | 1900.6ms/step | 31.6 samples/s
+Step 38030/50462 | Loss: 2.5166 | LR: 7.33e-05 | 1905.4ms/step | 31.5 samples/s
+Step 38040/50462 | Loss: 2.4852 | LR: 7.32e-05 | 1909.2ms/step | 31.4 samples/s
+Step 38050/50462 | Loss: 2.4741 | LR: 7.30e-05 | 1909.6ms/step | 31.4 samples/s
+Step 38060/50462 | Loss: 2.4731 | LR: 7.29e-05 | 1913.2ms/step | 31.4 samples/s
+Step 38070/50462 | Loss: 2.4844 | LR: 7.28e-05 | 1914.3ms/step | 31.3 samples/s
+Step 38080/50462 | Loss: 2.5107 | LR: 7.27e-05 | 1913.7ms/step | 31.4 samples/s
+Step 38090/50462 | Loss: 2.4803 | LR: 7.26e-05 | 1909.9ms/step | 31.4 samples/s
+Step 38100/50462 | Loss: 2.5166 | LR: 7.25e-05 | 1906.1ms/step | 31.5 samples/s
+Step 38110/50462 | Loss: 2.5122 | LR: 7.24e-05 | 1905.0ms/step | 31.5 samples/s
+Step 38120/50462 | Loss: 2.5078 | LR: 7.23e-05 | 1906.2ms/step | 31.5 samples/s
+Step 38130/50462 | Loss: 2.4915 | LR: 7.22e-05 | 1910.9ms/step | 31.4 samples/s
+Step 38140/50462 | Loss: 2.5109 | LR: 7.20e-05 | 1909.8ms/step | 31.4 samples/s
+Step 38150/50462 | Loss: 2.5031 | LR: 7.19e-05 | 1912.6ms/step | 31.4 samples/s
+Step 38160/50462 | Loss: 2.5020 | LR: 7.18e-05 | 1914.3ms/step | 31.3 samples/s
+Step 38170/50462 | Loss: 2.5450 | LR: 7.17e-05 | 1913.1ms/step | 31.4 samples/s
+Step 38180/50462 | Loss: 2.4759 | LR: 7.16e-05 | 1909.0ms/step | 31.4 samples/s
+Step 38190/50462 | Loss: 2.4822 | LR: 7.15e-05 | 1904.3ms/step | 31.5 samples/s
+Step 38200/50462 | Loss: 2.5139 | LR: 7.14e-05 | 1905.1ms/step | 31.5 samples/s
+Step 38210/50462 | Loss: 2.4812 | LR: 7.13e-05 | 1908.6ms/step | 31.4 samples/s
+Step 38220/50462 | Loss: 2.4836 | LR: 7.12e-05 | 1911.8ms/step | 31.4 samples/s
+Step 38230/50462 | Loss: 2.5066 | LR: 7.10e-05 | 1913.9ms/step | 31.3 samples/s
+Step 38240/50462 | Loss: 2.4901 | LR: 7.09e-05 | 1916.0ms/step | 31.3 samples/s
+Step 38250/50462 | Loss: 2.5418 | LR: 7.08e-05 | 1915.8ms/step | 31.3 samples/s
+Validation at step 38250: Loss = 2.5024, Perplexity = 12.21
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-38250
+Step 38260/50462 | Loss: 2.4272 | LR: 7.07e-05 | 3235.6ms/step | 18.5 samples/s
+Step 38270/50462 | Loss: 2.4546 | LR: 7.06e-05 | 1902.0ms/step | 31.5 samples/s
+Step 38280/50462 | Loss: 2.5088 | LR: 7.05e-05 | 1907.4ms/step | 31.5 samples/s
+Step 38290/50462 | Loss: 2.4620 | LR: 7.04e-05 | 1908.4ms/step | 31.4 samples/s
+Step 38300/50462 | Loss: 2.5093 | LR: 7.03e-05 | 1911.5ms/step | 31.4 samples/s
+Step 38310/50462 | Loss: 2.5016 | LR: 7.02e-05 | 1908.0ms/step | 31.4 samples/s
+Step 38320/50462 | Loss: 2.4870 | LR: 7.01e-05 | 1907.1ms/step | 31.5 samples/s
+Step 38330/50462 | Loss: 2.4832 | LR: 6.99e-05 | 1906.5ms/step | 31.5 samples/s
+Step 38340/50462 | Loss: 2.4758 | LR: 6.98e-05 | 1914.4ms/step | 31.3 samples/s
+Step 38350/50462 | Loss: 2.5057 | LR: 6.97e-05 | 1914.2ms/step | 31.3 samples/s
+Step 38360/50462 | Loss: 2.4924 | LR: 6.96e-05 | 1912.9ms/step | 31.4 samples/s
+Step 38370/50462 | Loss: 2.4918 | LR: 6.95e-05 | 1914.9ms/step | 31.3 samples/s
+Step 38380/50462 | Loss: 2.4599 | LR: 6.94e-05 | 1908.4ms/step | 31.4 samples/s
+Step 38390/50462 | Loss: 2.4817 | LR: 6.93e-05 | 1907.0ms/step | 31.5 samples/s
+Step 38400/50462 | Loss: 2.5070 | LR: 6.92e-05 | 1909.1ms/step | 31.4 samples/s
+Step 38410/50462 | Loss: 2.5230 | LR: 6.91e-05 | 1904.9ms/step | 31.5 samples/s
+Step 38420/50462 | Loss: 2.4646 | LR: 6.90e-05 | 1901.9ms/step | 31.5 samples/s
+Step 38430/50462 | Loss: 2.4610 | LR: 6.89e-05 | 1906.2ms/step | 31.5 samples/s
+Step 38440/50462 | Loss: 2.4784 | LR: 6.87e-05 | 1910.5ms/step | 31.4 samples/s
+Step 38450/50462 | Loss: 2.4733 | LR: 6.86e-05 | 1910.3ms/step | 31.4 samples/s
+Step 38460/50462 | Loss: 2.4964 | LR: 6.85e-05 | 1903.2ms/step | 31.5 samples/s
+Step 38470/50462 | Loss: 2.5062 | LR: 6.84e-05 | 1910.4ms/step | 31.4 samples/s
+Step 38480/50462 | Loss: 2.4451 | LR: 6.83e-05 | 1909.4ms/step | 31.4 samples/s
+Step 38490/50462 | Loss: 2.4833 | LR: 6.82e-05 | 1908.7ms/step | 31.4 samples/s
+Step 38500/50462 | Loss: 2.5297 | LR: 6.81e-05 | 1902.5ms/step | 31.5 samples/s
+Validation at step 38500: Loss = 2.5024, Perplexity = 12.21
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-38500
+Step 38510/50462 | Loss: 2.4974 | LR: 6.80e-05 | 3239.4ms/step | 18.5 samples/s
+Step 38520/50462 | Loss: 2.4710 | LR: 6.79e-05 | 1900.2ms/step | 31.6 samples/s
+Step 38530/50462 | Loss: 2.4709 | LR: 6.78e-05 | 1905.1ms/step | 31.5 samples/s
+Step 38540/50462 | Loss: 2.4826 | LR: 6.77e-05 | 1907.5ms/step | 31.5 samples/s
+Step 38550/50462 | Loss: 2.4993 | LR: 6.76e-05 | 1910.2ms/step | 31.4 samples/s
+Step 38560/50462 | Loss: 2.4430 | LR: 6.74e-05 | 1910.8ms/step | 31.4 samples/s
+Step 38570/50462 | Loss: 2.4775 | LR: 6.73e-05 | 1912.8ms/step | 31.4 samples/s
+Step 38580/50462 | Loss: 2.4603 | LR: 6.72e-05 | 1902.0ms/step | 31.5 samples/s
+Step 38590/50462 | Loss: 2.5197 | LR: 6.71e-05 | 1903.7ms/step | 31.5 samples/s
+Step 38600/50462 | Loss: 2.4655 | LR: 6.70e-05 | 1903.8ms/step | 31.5 samples/s
+Step 38610/50462 | Loss: 2.4720 | LR: 6.69e-05 | 1906.6ms/step | 31.5 samples/s
+Step 38620/50462 | Loss: 2.4757 | LR: 6.68e-05 | 1906.0ms/step | 31.5 samples/s
+Step 38630/50462 | Loss: 2.4726 | LR: 6.67e-05 | 1904.1ms/step | 31.5 samples/s
+Step 38640/50462 | Loss: 2.5201 | LR: 6.66e-05 | 1903.6ms/step | 31.5 samples/s
+Step 38650/50462 | Loss: 2.5114 | LR: 6.65e-05 | 1904.3ms/step | 31.5 samples/s
+Step 38660/50462 | Loss: 2.4834 | LR: 6.64e-05 | 1908.7ms/step | 31.4 samples/s
+Step 38670/50462 | Loss: 2.4914 | LR: 6.63e-05 | 1908.1ms/step | 31.4 samples/s
+Step 38680/50462 | Loss: 2.4941 | LR: 6.62e-05 | 1910.8ms/step | 31.4 samples/s
+Step 38690/50462 | Loss: 2.5292 | LR: 6.60e-05 | 1915.5ms/step | 31.3 samples/s
+Step 38700/50462 | Loss: 2.4601 | LR: 6.59e-05 | 1912.9ms/step | 31.4 samples/s
+Step 38710/50462 | Loss: 2.4879 | LR: 6.58e-05 | 1912.9ms/step | 31.4 samples/s
+Step 38720/50462 | Loss: 2.4540 | LR: 6.57e-05 | 1907.5ms/step | 31.5 samples/s
+Step 38730/50462 | Loss: 2.5259 | LR: 6.56e-05 | 1910.5ms/step | 31.4 samples/s
+Step 38740/50462 | Loss: 2.4564 | LR: 6.55e-05 | 1910.5ms/step | 31.4 samples/s
+Step 38750/50462 | Loss: 2.5120 | LR: 6.54e-05 | 1909.9ms/step | 31.4 samples/s
+Validation at step 38750: Loss = 2.5020, Perplexity = 12.21
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-38750
+Step 38760/50462 | Loss: 2.4821 | LR: 6.53e-05 | 3208.9ms/step | 18.7 samples/s
+Step 38770/50462 | Loss: 2.4912 | LR: 6.52e-05 | 1897.5ms/step | 31.6 samples/s
+Step 38780/50462 | Loss: 2.4909 | LR: 6.51e-05 | 1903.2ms/step | 31.5 samples/s
+Step 38790/50462 | Loss: 2.4943 | LR: 6.50e-05 | 1906.0ms/step | 31.5 samples/s
+Step 38800/50462 | Loss: 2.4502 | LR: 6.49e-05 | 1905.4ms/step | 31.5 samples/s
+Step 38810/50462 | Loss: 2.4472 | LR: 6.48e-05 | 1907.0ms/step | 31.5 samples/s
+Step 38820/50462 | Loss: 2.5037 | LR: 6.47e-05 | 1911.7ms/step | 31.4 samples/s
+Step 38830/50462 | Loss: 2.5056 | LR: 6.46e-05 | 1909.5ms/step | 31.4 samples/s
+Step 38840/50462 | Loss: 2.4758 | LR: 6.45e-05 | 1909.1ms/step | 31.4 samples/s
+Step 38850/50462 | Loss: 2.4859 | LR: 6.43e-05 | 1914.1ms/step | 31.3 samples/s
+Step 38860/50462 | Loss: 2.4829 | LR: 6.42e-05 | 1915.3ms/step | 31.3 samples/s
+Step 38870/50462 | Loss: 2.4387 | LR: 6.41e-05 | 1911.7ms/step | 31.4 samples/s
+Step 38880/50462 | Loss: 2.4860 | LR: 6.40e-05 | 1903.7ms/step | 31.5 samples/s
+Step 38890/50462 | Loss: 2.4709 | LR: 6.39e-05 | 1909.0ms/step | 31.4 samples/s
+Step 38900/50462 | Loss: 2.4866 | LR: 6.38e-05 | 1914.4ms/step | 31.3 samples/s
+Step 38910/50462 | Loss: 2.4845 | LR: 6.37e-05 | 1913.0ms/step | 31.4 samples/s
+Step 38920/50462 | Loss: 2.5273 | LR: 6.36e-05 | 1913.2ms/step | 31.4 samples/s
+Step 38930/50462 | Loss: 2.4832 | LR: 6.35e-05 | 1911.4ms/step | 31.4 samples/s
+Step 38940/50462 | Loss: 2.4879 | LR: 6.34e-05 | 1914.3ms/step | 31.3 samples/s
+Step 38950/50462 | Loss: 2.5084 | LR: 6.33e-05 | 1914.7ms/step | 31.3 samples/s
+Step 38960/50462 | Loss: 2.4591 | LR: 6.32e-05 | 1911.5ms/step | 31.4 samples/s
+Step 38970/50462 | Loss: 2.4940 | LR: 6.31e-05 | 1913.6ms/step | 31.4 samples/s
+Step 38980/50462 | Loss: 2.4690 | LR: 6.30e-05 | 1911.8ms/step | 31.4 samples/s
+Step 38990/50462 | Loss: 2.4968 | LR: 6.29e-05 | 1911.7ms/step | 31.4 samples/s
+Step 39000/50462 | Loss: 2.4612 | LR: 6.28e-05 | 1913.6ms/step | 31.4 samples/s
+Validation at step 39000: Loss = 2.5016, Perplexity = 12.20
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-39000
+Step 39010/50462 | Loss: 2.4515 | LR: 6.27e-05 | 3224.1ms/step | 18.6 samples/s
+Step 39020/50462 | Loss: 2.5099 | LR: 6.26e-05 | 1897.6ms/step | 31.6 samples/s
+Step 39030/50462 | Loss: 2.5208 | LR: 6.25e-05 | 1904.8ms/step | 31.5 samples/s
+Step 39040/50462 | Loss: 2.4325 | LR: 6.23e-05 | 1906.8ms/step | 31.5 samples/s
+Step 39050/50462 | Loss: 2.5027 | LR: 6.22e-05 | 1905.7ms/step | 31.5 samples/s
+Step 39060/50462 | Loss: 2.4958 | LR: 6.21e-05 | 1906.1ms/step | 31.5 samples/s
+Step 39070/50462 | Loss: 2.4695 | LR: 6.20e-05 | 1905.2ms/step | 31.5 samples/s
+Step 39080/50462 | Loss: 2.4896 | LR: 6.19e-05 | 1907.9ms/step | 31.4 samples/s
+Step 39090/50462 | Loss: 2.4618 | LR: 6.18e-05 | 1910.6ms/step | 31.4 samples/s
+Step 39100/50462 | Loss: 2.5075 | LR: 6.17e-05 | 1911.6ms/step | 31.4 samples/s
+Step 39110/50462 | Loss: 2.5073 | LR: 6.16e-05 | 1910.8ms/step | 31.4 samples/s
+Step 39120/50462 | Loss: 2.4912 | LR: 6.15e-05 | 1909.3ms/step | 31.4 samples/s
+Step 39130/50462 | Loss: 2.5231 | LR: 6.14e-05 | 1903.4ms/step | 31.5 samples/s
+Step 39140/50462 | Loss: 2.4319 | LR: 6.13e-05 | 1908.2ms/step | 31.4 samples/s
+Step 39150/50462 | Loss: 2.4830 | LR: 6.12e-05 | 1911.8ms/step | 31.4 samples/s
+Step 39160/50462 | Loss: 2.4999 | LR: 6.11e-05 | 1908.3ms/step | 31.4 samples/s
+Step 39170/50462 | Loss: 2.5056 | LR: 6.10e-05 | 1911.7ms/step | 31.4 samples/s
+Step 39180/50462 | Loss: 2.4659 | LR: 6.09e-05 | 1913.9ms/step | 31.4 samples/s
+Step 39190/50462 | Loss: 2.4421 | LR: 6.08e-05 | 1912.4ms/step | 31.4 samples/s
+Step 39200/50462 | Loss: 2.4794 | LR: 6.07e-05 | 1909.2ms/step | 31.4 samples/s
+Step 39210/50462 | Loss: 2.4959 | LR: 6.06e-05 | 1910.5ms/step | 31.4 samples/s
+Step 39220/50462 | Loss: 2.5357 | LR: 6.05e-05 | 1905.9ms/step | 31.5 samples/s
+Step 39230/50462 | Loss: 2.4372 | LR: 6.04e-05 | 1905.2ms/step | 31.5 samples/s
+Step 39240/50462 | Loss: 2.4938 | LR: 6.03e-05 | 1908.7ms/step | 31.4 samples/s
+Step 39250/50462 | Loss: 2.5061 | LR: 6.02e-05 | 1910.5ms/step | 31.4 samples/s
+Validation at step 39250: Loss = 2.5014, Perplexity = 12.20
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-39250
+Step 39260/50462 | Loss: 2.4538 | LR: 6.01e-05 | 3247.1ms/step | 18.5 samples/s
+Step 39270/50462 | Loss: 2.4858 | LR: 6.00e-05 | 1896.4ms/step | 31.6 samples/s
+Step 39280/50462 | Loss: 2.4720 | LR: 5.99e-05 | 1902.2ms/step | 31.5 samples/s
+Step 39290/50462 | Loss: 2.4783 | LR: 5.98e-05 | 1907.9ms/step | 31.4 samples/s
+Step 39300/50462 | Loss: 2.5019 | LR: 5.97e-05 | 1911.0ms/step | 31.4 samples/s
+Step 39310/50462 | Loss: 2.4715 | LR: 5.96e-05 | 1909.9ms/step | 31.4 samples/s
+Step 39320/50462 | Loss: 2.4412 | LR: 5.95e-05 | 1911.9ms/step | 31.4 samples/s
+Step 39330/50462 | Loss: 2.4685 | LR: 5.94e-05 | 1911.5ms/step | 31.4 samples/s
+Step 39340/50462 | Loss: 2.4567 | LR: 5.93e-05 | 1914.1ms/step | 31.3 samples/s
+Step 39350/50462 | Loss: 2.4819 | LR: 5.91e-05 | 1914.9ms/step | 31.3 samples/s
+Step 39360/50462 | Loss: 2.4862 | LR: 5.90e-05 | 1912.3ms/step | 31.4 samples/s
+Step 39370/50462 | Loss: 2.4971 | LR: 5.89e-05 | 1910.4ms/step | 31.4 samples/s
+Step 39380/50462 | Loss: 2.5012 | LR: 5.88e-05 | 1909.6ms/step | 31.4 samples/s
+Step 39390/50462 | Loss: 2.4657 | LR: 5.87e-05 | 1910.5ms/step | 31.4 samples/s
+Step 39400/50462 | Loss: 2.4930 | LR: 5.86e-05 | 1914.3ms/step | 31.3 samples/s
+Step 39410/50462 | Loss: 2.4845 | LR: 5.85e-05 | 1911.7ms/step | 31.4 samples/s
+Step 39420/50462 | Loss: 2.4328 | LR: 5.84e-05 | 1915.1ms/step | 31.3 samples/s
+Step 39430/50462 | Loss: 2.4467 | LR: 5.83e-05 | 1915.7ms/step | 31.3 samples/s
+Step 39440/50462 | Loss: 2.4943 | LR: 5.82e-05 | 1914.8ms/step | 31.3 samples/s
+Step 39450/50462 | Loss: 2.4871 | LR: 5.81e-05 | 1911.5ms/step | 31.4 samples/s
+Step 39460/50462 | Loss: 2.4951 | LR: 5.80e-05 | 1912.2ms/step | 31.4 samples/s
+Step 39470/50462 | Loss: 2.5121 | LR: 5.79e-05 | 1912.2ms/step | 31.4 samples/s
+Step 39480/50462 | Loss: 2.4723 | LR: 5.78e-05 | 1915.9ms/step | 31.3 samples/s
+Step 39490/50462 | Loss: 2.5062 | LR: 5.77e-05 | 1913.2ms/step | 31.4 samples/s
+Step 39500/50462 | Loss: 2.4814 | LR: 5.76e-05 | 1910.8ms/step | 31.4 samples/s
+Validation at step 39500: Loss = 2.5012, Perplexity = 12.20
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-39500
+Step 39510/50462 | Loss: 2.4763 | LR: 5.75e-05 | 3263.4ms/step | 18.4 samples/s
+Step 39520/50462 | Loss: 2.4645 | LR: 5.74e-05 | 1903.5ms/step | 31.5 samples/s
+Step 39530/50462 | Loss: 2.4750 | LR: 5.73e-05 | 1908.4ms/step | 31.4 samples/s
+Step 39540/50462 | Loss: 2.4553 | LR: 5.72e-05 | 1911.8ms/step | 31.4 samples/s
+Step 39550/50462 | Loss: 2.4819 | LR: 5.71e-05 | 1906.4ms/step | 31.5 samples/s
+Step 39560/50462 | Loss: 2.5036 | LR: 5.70e-05 | 1907.8ms/step | 31.4 samples/s
+Step 39570/50462 | Loss: 2.4583 | LR: 5.69e-05 | 1915.9ms/step | 31.3 samples/s
+Step 39580/50462 | Loss: 2.4344 | LR: 5.68e-05 | 1915.2ms/step | 31.3 samples/s
+Step 39590/50462 | Loss: 2.4984 | LR: 5.67e-05 | 1913.5ms/step | 31.4 samples/s
+Step 39600/50462 | Loss: 2.4601 | LR: 5.66e-05 | 1914.9ms/step | 31.3 samples/s
+Step 39610/50462 | Loss: 2.4766 | LR: 5.65e-05 | 1910.0ms/step | 31.4 samples/s
+Step 39620/50462 | Loss: 2.4694 | LR: 5.64e-05 | 1914.9ms/step | 31.3 samples/s
+Step 39630/50462 | Loss: 2.4847 | LR: 5.63e-05 | 1916.7ms/step | 31.3 samples/s
+Step 39640/50462 | Loss: 2.5145 | LR: 5.62e-05 | 1920.6ms/step | 31.2 samples/s
+Step 39650/50462 | Loss: 2.4727 | LR: 5.61e-05 | 1917.1ms/step | 31.3 samples/s
+Step 39660/50462 | Loss: 2.4678 | LR: 5.60e-05 | 1910.2ms/step | 31.4 samples/s
+Step 39670/50462 | Loss: 2.4771 | LR: 5.59e-05 | 1906.5ms/step | 31.5 samples/s
+Step 39680/50462 | Loss: 2.5100 | LR: 5.58e-05 | 1911.1ms/step | 31.4 samples/s
+Step 39690/50462 | Loss: 2.5413 | LR: 5.57e-05 | 1910.9ms/step | 31.4 samples/s
+Step 39700/50462 | Loss: 2.5118 | LR: 5.56e-05 | 1910.7ms/step | 31.4 samples/s
+Step 39710/50462 | Loss: 2.5255 | LR: 5.55e-05 | 1908.0ms/step | 31.4 samples/s
+Step 39720/50462 | Loss: 2.4972 | LR: 5.54e-05 | 1906.4ms/step | 31.5 samples/s
+Step 39730/50462 | Loss: 2.5244 | LR: 5.53e-05 | 1910.0ms/step | 31.4 samples/s
+Step 39740/50462 | Loss: 2.4707 | LR: 5.52e-05 | 1908.0ms/step | 31.4 samples/s
+Step 39750/50462 | Loss: 2.4595 | LR: 5.51e-05 | 1912.8ms/step | 31.4 samples/s
+Validation at step 39750: Loss = 2.5009, Perplexity = 12.19
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-39750
+Step 39760/50462 | Loss: 2.4977 | LR: 5.50e-05 | 3226.8ms/step | 18.6 samples/s
+Step 39770/50462 | Loss: 2.4665 | LR: 5.49e-05 | 1901.4ms/step | 31.6 samples/s
+Step 39780/50462 | Loss: 2.4693 | LR: 5.48e-05 | 1903.5ms/step | 31.5 samples/s
+Step 39790/50462 | Loss: 2.4519 | LR: 5.47e-05 | 1903.6ms/step | 31.5 samples/s
+Step 39800/50462 | Loss: 2.4929 | LR: 5.46e-05 | 1909.2ms/step | 31.4 samples/s
+Step 39810/50462 | Loss: 2.5056 | LR: 5.45e-05 | 1913.6ms/step | 31.4 samples/s
+Step 39820/50462 | Loss: 2.4870 | LR: 5.44e-05 | 1916.2ms/step | 31.3 samples/s
+Step 39830/50462 | Loss: 2.4924 | LR: 5.43e-05 | 1913.8ms/step | 31.4 samples/s
+Step 39840/50462 | Loss: 2.5085 | LR: 5.42e-05 | 1915.0ms/step | 31.3 samples/s
+Step 39850/50462 | Loss: 2.5134 | LR: 5.41e-05 | 1904.3ms/step | 31.5 samples/s
+Step 39860/50462 | Loss: 2.5015 | LR: 5.40e-05 | 1906.5ms/step | 31.5 samples/s
+Step 39870/50462 | Loss: 2.4831 | LR: 5.39e-05 | 1906.3ms/step | 31.5 samples/s
+Step 39880/50462 | Loss: 2.4957 | LR: 5.38e-05 | 1911.0ms/step | 31.4 samples/s
+Step 39890/50462 | Loss: 2.4954 | LR: 5.38e-05 | 1913.3ms/step | 31.4 samples/s
+Step 39900/50462 | Loss: 2.5022 | LR: 5.37e-05 | 1916.7ms/step | 31.3 samples/s
+Step 39910/50462 | Loss: 2.4859 | LR: 5.36e-05 | 1915.3ms/step | 31.3 samples/s
+Step 39920/50462 | Loss: 2.4717 | LR: 5.35e-05 | 1915.2ms/step | 31.3 samples/s
+Step 39930/50462 | Loss: 2.4417 | LR: 5.34e-05 | 1918.0ms/step | 31.3 samples/s
+Step 39940/50462 | Loss: 2.4867 | LR: 5.33e-05 | 1918.0ms/step | 31.3 samples/s
+Step 39950/50462 | Loss: 2.4999 | LR: 5.32e-05 | 1917.2ms/step | 31.3 samples/s
+Step 39960/50462 | Loss: 2.4942 | LR: 5.31e-05 | 1902.0ms/step | 31.5 samples/s
+Step 39970/50462 | Loss: 2.4774 | LR: 5.30e-05 | 1904.0ms/step | 31.5 samples/s
+Step 39980/50462 | Loss: 2.5024 | LR: 5.29e-05 | 1902.8ms/step | 31.5 samples/s
+Step 39990/50462 | Loss: 2.4945 | LR: 5.28e-05 | 1905.3ms/step | 31.5 samples/s
+Step 40000/50462 | Loss: 2.5008 | LR: 5.27e-05 | 1911.7ms/step | 31.4 samples/s
+Validation at step 40000: Loss = 2.5006, Perplexity = 12.19
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-40000
+Step 40010/50462 | Loss: 2.4687 | LR: 5.26e-05 | 3260.7ms/step | 18.4 samples/s
+Step 40020/50462 | Loss: 2.5041 | LR: 5.25e-05 | 1899.1ms/step | 31.6 samples/s
+Step 40030/50462 | Loss: 2.4762 | LR: 5.24e-05 | 1903.6ms/step | 31.5 samples/s
+Step 40040/50462 | Loss: 2.4697 | LR: 5.23e-05 | 1902.0ms/step | 31.5 samples/s
+Step 40050/50462 | Loss: 2.4677 | LR: 5.22e-05 | 1905.3ms/step | 31.5 samples/s
+Step 40060/50462 | Loss: 2.5143 | LR: 5.21e-05 | 1912.8ms/step | 31.4 samples/s
+Step 40070/50462 | Loss: 2.5047 | LR: 5.20e-05 | 1914.7ms/step | 31.3 samples/s
+Step 40080/50462 | Loss: 2.4890 | LR: 5.19e-05 | 1913.1ms/step | 31.4 samples/s
+Step 40090/50462 | Loss: 2.4571 | LR: 5.18e-05 | 1914.0ms/step | 31.3 samples/s
+Step 40100/50462 | Loss: 2.4327 | LR: 5.17e-05 | 1911.6ms/step | 31.4 samples/s
+Step 40110/50462 | Loss: 2.5371 | LR: 5.16e-05 | 1905.9ms/step | 31.5 samples/s
+Step 40120/50462 | Loss: 2.5007 | LR: 5.15e-05 | 1901.7ms/step | 31.6 samples/s
+Step 40130/50462 | Loss: 2.4701 | LR: 5.14e-05 | 1905.1ms/step | 31.5 samples/s
+Step 40140/50462 | Loss: 2.4891 | LR: 5.13e-05 | 1908.7ms/step | 31.4 samples/s
+Step 40150/50462 | Loss: 2.5014 | LR: 5.12e-05 | 1911.7ms/step | 31.4 samples/s
+Step 40160/50462 | Loss: 2.4826 | LR: 5.11e-05 | 1911.2ms/step | 31.4 samples/s
+Step 40170/50462 | Loss: 2.4291 | LR: 5.10e-05 | 1913.5ms/step | 31.4 samples/s
+Step 40180/50462 | Loss: 2.4815 | LR: 5.09e-05 | 1915.0ms/step | 31.3 samples/s
+Step 40190/50462 | Loss: 2.5165 | LR: 5.09e-05 | 1912.2ms/step | 31.4 samples/s
+Step 40200/50462 | Loss: 2.4453 | LR: 5.08e-05 | 1908.8ms/step | 31.4 samples/s
+Step 40210/50462 | Loss: 2.4965 | LR: 5.07e-05 | 1909.1ms/step | 31.4 samples/s
+Step 40220/50462 | Loss: 2.5077 | LR: 5.06e-05 | 1911.0ms/step | 31.4 samples/s
+Step 40230/50462 | Loss: 2.4971 | LR: 5.05e-05 | 1913.1ms/step | 31.4 samples/s
+Step 40240/50462 | Loss: 2.4656 | LR: 5.04e-05 | 1918.3ms/step | 31.3 samples/s
+Step 40250/50462 | Loss: 2.5047 | LR: 5.03e-05 | 1914.9ms/step | 31.3 samples/s
+Validation at step 40250: Loss = 2.5002, Perplexity = 12.18
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-40250
+Step 40260/50462 | Loss: 2.4563 | LR: 5.02e-05 | 3240.6ms/step | 18.5 samples/s
+Step 40270/50462 | Loss: 2.5183 | LR: 5.01e-05 | 1901.5ms/step | 31.6 samples/s
+Step 40280/50462 | Loss: 2.4729 | LR: 5.00e-05 | 1905.1ms/step | 31.5 samples/s
+Step 40290/50462 | Loss: 2.4348 | LR: 4.99e-05 | 1910.7ms/step | 31.4 samples/s
+Step 40300/50462 | Loss: 2.4898 | LR: 4.98e-05 | 1911.3ms/step | 31.4 samples/s
+Step 40310/50462 | Loss: 2.4858 | LR: 4.97e-05 | 1913.7ms/step | 31.4 samples/s
+Step 40320/50462 | Loss: 2.5080 | LR: 4.96e-05 | 1912.1ms/step | 31.4 samples/s
+Step 40330/50462 | Loss: 2.4797 | LR: 4.95e-05 | 1916.3ms/step | 31.3 samples/s
+Step 40340/50462 | Loss: 2.4769 | LR: 4.94e-05 | 1916.1ms/step | 31.3 samples/s
+Step 40350/50462 | Loss: 2.4420 | LR: 4.93e-05 | 1917.7ms/step | 31.3 samples/s
+Step 40360/50462 | Loss: 2.5301 | LR: 4.92e-05 | 1912.7ms/step | 31.4 samples/s
+Step 40370/50462 | Loss: 2.4907 | LR: 4.91e-05 | 1917.3ms/step | 31.3 samples/s
+Step 40380/50462 | Loss: 2.4765 | LR: 4.90e-05 | 1916.6ms/step | 31.3 samples/s
+Step 40390/50462 | Loss: 2.4579 | LR: 4.90e-05 | 1914.0ms/step | 31.3 samples/s
+Step 40400/50462 | Loss: 2.4924 | LR: 4.89e-05 | 1909.0ms/step | 31.4 samples/s
+Step 40410/50462 | Loss: 2.5231 | LR: 4.88e-05 | 1904.6ms/step | 31.5 samples/s
+Step 40420/50462 | Loss: 2.4860 | LR: 4.87e-05 | 1903.1ms/step | 31.5 samples/s
+Step 40430/50462 | Loss: 2.4883 | LR: 4.86e-05 | 1905.1ms/step | 31.5 samples/s
+Step 40440/50462 | Loss: 2.4776 | LR: 4.85e-05 | 1912.6ms/step | 31.4 samples/s
+Step 40450/50462 | Loss: 2.4760 | LR: 4.84e-05 | 1916.2ms/step | 31.3 samples/s
+Step 40460/50462 | Loss: 2.4932 | LR: 4.83e-05 | 1915.8ms/step | 31.3 samples/s
+Step 40470/50462 | Loss: 2.4538 | LR: 4.82e-05 | 1901.9ms/step | 31.5 samples/s
+Step 40480/50462 | Loss: 2.4878 | LR: 4.81e-05 | 1906.4ms/step | 31.5 samples/s
+Step 40490/50462 | Loss: 2.4864 | LR: 4.80e-05 | 1910.0ms/step | 31.4 samples/s
+Step 40500/50462 | Loss: 2.4189 | LR: 4.79e-05 | 1912.9ms/step | 31.4 samples/s
+Validation at step 40500: Loss = 2.5000, Perplexity = 12.18
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-40500
+Step 40510/50462 | Loss: 2.5056 | LR: 4.78e-05 | 3230.3ms/step | 18.6 samples/s
+Step 40520/50462 | Loss: 2.4979 | LR: 4.77e-05 | 1898.3ms/step | 31.6 samples/s
+Step 40530/50462 | Loss: 2.4717 | LR: 4.76e-05 | 1906.2ms/step | 31.5 samples/s
+Step 40540/50462 | Loss: 2.4439 | LR: 4.76e-05 | 1905.6ms/step | 31.5 samples/s
+Step 40550/50462 | Loss: 2.4692 | LR: 4.75e-05 | 1908.4ms/step | 31.4 samples/s
+Step 40560/50462 | Loss: 2.4978 | LR: 4.74e-05 | 1910.7ms/step | 31.4 samples/s
+Step 40570/50462 | Loss: 2.5075 | LR: 4.73e-05 | 1914.0ms/step | 31.3 samples/s
+Step 40580/50462 | Loss: 2.4818 | LR: 4.72e-05 | 1905.4ms/step | 31.5 samples/s
+Step 40590/50462 | Loss: 2.4869 | LR: 4.71e-05 | 1911.7ms/step | 31.4 samples/s
+Step 40600/50462 | Loss: 2.4530 | LR: 4.70e-05 | 1911.7ms/step | 31.4 samples/s
+Step 40610/50462 | Loss: 2.4656 | LR: 4.69e-05 | 1913.0ms/step | 31.4 samples/s
+Step 40620/50462 | Loss: 2.5198 | LR: 4.68e-05 | 1911.6ms/step | 31.4 samples/s
+Step 40630/50462 | Loss: 2.4830 | LR: 4.67e-05 | 1906.9ms/step | 31.5 samples/s
+Step 40640/50462 | Loss: 2.4674 | LR: 4.66e-05 | 1911.3ms/step | 31.4 samples/s
+Step 40650/50462 | Loss: 2.4968 | LR: 4.65e-05 | 1906.6ms/step | 31.5 samples/s
+Step 40660/50462 | Loss: 2.5126 | LR: 4.64e-05 | 1907.4ms/step | 31.5 samples/s
+Step 40670/50462 | Loss: 2.4803 | LR: 4.64e-05 | 1908.0ms/step | 31.4 samples/s
+Step 40680/50462 | Loss: 2.4766 | LR: 4.63e-05 | 1909.2ms/step | 31.4 samples/s
+Step 40690/50462 | Loss: 2.4978 | LR: 4.62e-05 | 1912.3ms/step | 31.4 samples/s
+Step 40700/50462 | Loss: 2.5106 | LR: 4.61e-05 | 1911.1ms/step | 31.4 samples/s
+Step 40710/50462 | Loss: 2.4478 | LR: 4.60e-05 | 1908.9ms/step | 31.4 samples/s
+Step 40720/50462 | Loss: 2.5281 | LR: 4.59e-05 | 1912.8ms/step | 31.4 samples/s
+Step 40730/50462 | Loss: 2.4716 | LR: 4.58e-05 | 1913.2ms/step | 31.4 samples/s
+Step 40740/50462 | Loss: 2.4810 | LR: 4.57e-05 | 1914.0ms/step | 31.3 samples/s
+Step 40750/50462 | Loss: 2.4979 | LR: 4.56e-05 | 1916.9ms/step | 31.3 samples/s
+Validation at step 40750: Loss = 2.4997, Perplexity = 12.18
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-40750
+Step 40760/50462 | Loss: 2.5134 | LR: 4.55e-05 | 3211.0ms/step | 18.7 samples/s
+Step 40770/50462 | Loss: 2.4836 | LR: 4.54e-05 | 1901.0ms/step | 31.6 samples/s
+Step 40780/50462 | Loss: 2.5233 | LR: 4.54e-05 | 1906.4ms/step | 31.5 samples/s
+Step 40790/50462 | Loss: 2.4604 | LR: 4.53e-05 | 1912.5ms/step | 31.4 samples/s
+Step 40800/50462 | Loss: 2.4747 | LR: 4.52e-05 | 1914.2ms/step | 31.3 samples/s
+Step 40810/50462 | Loss: 2.4702 | LR: 4.51e-05 | 1915.8ms/step | 31.3 samples/s
+Step 40820/50462 | Loss: 2.4585 | LR: 4.50e-05 | 1912.4ms/step | 31.4 samples/s
+Step 40830/50462 | Loss: 2.4644 | LR: 4.49e-05 | 1912.6ms/step | 31.4 samples/s
+Step 40840/50462 | Loss: 2.4614 | LR: 4.48e-05 | 1902.5ms/step | 31.5 samples/s
+Step 40850/50462 | Loss: 2.4527 | LR: 4.47e-05 | 1906.5ms/step | 31.5 samples/s
+Step 40860/50462 | Loss: 2.5000 | LR: 4.46e-05 | 1908.6ms/step | 31.4 samples/s
+Step 40870/50462 | Loss: 2.4343 | LR: 4.45e-05 | 1912.9ms/step | 31.4 samples/s
+Step 40880/50462 | Loss: 2.4310 | LR: 4.45e-05 | 1912.9ms/step | 31.4 samples/s
+Step 40890/50462 | Loss: 2.4309 | LR: 4.44e-05 | 1906.6ms/step | 31.5 samples/s
+Step 40900/50462 | Loss: 2.5144 | LR: 4.43e-05 | 1908.3ms/step | 31.4 samples/s
+Step 40910/50462 | Loss: 2.4829 | LR: 4.42e-05 | 1910.8ms/step | 31.4 samples/s
+Step 40920/50462 | Loss: 2.4753 | LR: 4.41e-05 | 1912.8ms/step | 31.4 samples/s
+Step 40930/50462 | Loss: 2.4875 | LR: 4.40e-05 | 1911.6ms/step | 31.4 samples/s
+Step 40940/50462 | Loss: 2.4693 | LR: 4.39e-05 | 1904.5ms/step | 31.5 samples/s
+Step 40950/50462 | Loss: 2.4746 | LR: 4.38e-05 | 1909.0ms/step | 31.4 samples/s
+Step 40960/50462 | Loss: 2.4727 | LR: 4.37e-05 | 1909.7ms/step | 31.4 samples/s
+Step 40970/50462 | Loss: 2.4505 | LR: 4.36e-05 | 1913.4ms/step | 31.4 samples/s
+Step 40980/50462 | Loss: 2.4781 | LR: 4.36e-05 | 1914.6ms/step | 31.3 samples/s
+Step 40990/50462 | Loss: 2.5150 | LR: 4.35e-05 | 1913.9ms/step | 31.3 samples/s
+Step 41000/50462 | Loss: 2.4566 | LR: 4.34e-05 | 1914.1ms/step | 31.3 samples/s
+Validation at step 41000: Loss = 2.4996, Perplexity = 12.18
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-41000
+Step 41010/50462 | Loss: 2.4533 | LR: 4.33e-05 | 3235.5ms/step | 18.5 samples/s
+Step 41020/50462 | Loss: 2.4536 | LR: 4.32e-05 | 1898.3ms/step | 31.6 samples/s
+Step 41030/50462 | Loss: 2.4607 | LR: 4.31e-05 | 1901.1ms/step | 31.6 samples/s
+Step 41040/50462 | Loss: 2.4620 | LR: 4.30e-05 | 1907.3ms/step | 31.5 samples/s
+Step 41050/50462 | Loss: 2.4606 | LR: 4.29e-05 | 1906.4ms/step | 31.5 samples/s
+Step 41060/50462 | Loss: 2.4950 | LR: 4.28e-05 | 1908.7ms/step | 31.4 samples/s
+Step 41070/50462 | Loss: 2.5082 | LR: 4.28e-05 | 1908.5ms/step | 31.4 samples/s
+Step 41080/50462 | Loss: 2.5029 | LR: 4.27e-05 | 1911.6ms/step | 31.4 samples/s
+Step 41090/50462 | Loss: 2.5107 | LR: 4.26e-05 | 1910.3ms/step | 31.4 samples/s
+Step 41100/50462 | Loss: 2.4698 | LR: 4.25e-05 | 1910.4ms/step | 31.4 samples/s
+Step 41110/50462 | Loss: 2.5057 | LR: 4.24e-05 | 1909.2ms/step | 31.4 samples/s
+Step 41120/50462 | Loss: 2.5104 | LR: 4.23e-05 | 1908.2ms/step | 31.4 samples/s
+Step 41130/50462 | Loss: 2.4433 | LR: 4.22e-05 | 1901.5ms/step | 31.6 samples/s
+Step 41140/50462 | Loss: 2.5001 | LR: 4.21e-05 | 1903.7ms/step | 31.5 samples/s
+Step 41150/50462 | Loss: 2.4943 | LR: 4.21e-05 | 1908.0ms/step | 31.4 samples/s
+Step 41160/50462 | Loss: 2.4574 | LR: 4.20e-05 | 1910.6ms/step | 31.4 samples/s
+Step 41170/50462 | Loss: 2.4784 | LR: 4.19e-05 | 1905.9ms/step | 31.5 samples/s
+Step 41180/50462 | Loss: 2.5213 | LR: 4.18e-05 | 1904.8ms/step | 31.5 samples/s
+Step 41190/50462 | Loss: 2.5099 | LR: 4.17e-05 | 1906.8ms/step | 31.5 samples/s
+Step 41200/50462 | Loss: 2.5087 | LR: 4.16e-05 | 1905.5ms/step | 31.5 samples/s
+Step 41210/50462 | Loss: 2.4915 | LR: 4.15e-05 | 1912.3ms/step | 31.4 samples/s
+Step 41220/50462 | Loss: 2.4834 | LR: 4.14e-05 | 1909.6ms/step | 31.4 samples/s
+Step 41230/50462 | Loss: 2.4764 | LR: 4.14e-05 | 1905.9ms/step | 31.5 samples/s
+Step 41240/50462 | Loss: 2.4966 | LR: 4.13e-05 | 1905.8ms/step | 31.5 samples/s
+Step 41250/50462 | Loss: 2.5131 | LR: 4.12e-05 | 1908.2ms/step | 31.4 samples/s
+Validation at step 41250: Loss = 2.4995, Perplexity = 12.18
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-41250
+Step 41260/50462 | Loss: 2.4909 | LR: 4.11e-05 | 3249.2ms/step | 18.5 samples/s
+Step 41270/50462 | Loss: 2.4871 | LR: 4.10e-05 | 1899.0ms/step | 31.6 samples/s
+Step 41280/50462 | Loss: 2.4565 | LR: 4.09e-05 | 1905.2ms/step | 31.5 samples/s
+Step 41290/50462 | Loss: 2.4331 | LR: 4.08e-05 | 1907.6ms/step | 31.5 samples/s
+Step 41300/50462 | Loss: 2.4720 | LR: 4.07e-05 | 1911.3ms/step | 31.4 samples/s
+Step 41310/50462 | Loss: 2.4622 | LR: 4.07e-05 | 1910.8ms/step | 31.4 samples/s
+Step 41320/50462 | Loss: 2.4223 | LR: 4.06e-05 | 1910.5ms/step | 31.4 samples/s
+Step 41330/50462 | Loss: 2.4394 | LR: 4.05e-05 | 1912.8ms/step | 31.4 samples/s
+Step 41340/50462 | Loss: 2.4939 | LR: 4.04e-05 | 1911.6ms/step | 31.4 samples/s
+Step 41350/50462 | Loss: 2.4919 | LR: 4.03e-05 | 1904.7ms/step | 31.5 samples/s
+Step 41360/50462 | Loss: 2.4293 | LR: 4.02e-05 | 1909.0ms/step | 31.4 samples/s
+Step 41370/50462 | Loss: 2.4568 | LR: 4.01e-05 | 1913.9ms/step | 31.3 samples/s
+Step 41380/50462 | Loss: 2.4334 | LR: 4.01e-05 | 1908.8ms/step | 31.4 samples/s
+Step 41390/50462 | Loss: 2.4303 | LR: 4.00e-05 | 1911.6ms/step | 31.4 samples/s
+Step 41400/50462 | Loss: 2.4492 | LR: 3.99e-05 | 1916.7ms/step | 31.3 samples/s
+Step 41410/50462 | Loss: 2.4324 | LR: 3.98e-05 | 1909.5ms/step | 31.4 samples/s
+Step 41420/50462 | Loss: 2.4122 | LR: 3.97e-05 | 1911.5ms/step | 31.4 samples/s
+Step 41430/50462 | Loss: 2.4613 | LR: 3.96e-05 | 1909.9ms/step | 31.4 samples/s
+Step 41440/50462 | Loss: 2.4220 | LR: 3.95e-05 | 1911.3ms/step | 31.4 samples/s
+Step 41450/50462 | Loss: 2.4919 | LR: 3.95e-05 | 1912.9ms/step | 31.4 samples/s
+Step 41460/50462 | Loss: 2.4326 | LR: 3.94e-05 | 1916.9ms/step | 31.3 samples/s
+Step 41470/50462 | Loss: 2.4562 | LR: 3.93e-05 | 1920.4ms/step | 31.2 samples/s
+Step 41480/50462 | Loss: 2.4470 | LR: 3.92e-05 | 1916.4ms/step | 31.3 samples/s
+Step 41490/50462 | Loss: 2.4286 | LR: 3.91e-05 | 1920.2ms/step | 31.2 samples/s
+Step 41500/50462 | Loss: 2.5120 | LR: 3.90e-05 | 1911.6ms/step | 31.4 samples/s
+Validation at step 41500: Loss = 2.4999, Perplexity = 12.18
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-41500
+Step 41510/50462 | Loss: 2.4855 | LR: 3.90e-05 | 3267.7ms/step | 18.4 samples/s
+Step 41520/50462 | Loss: 2.4791 | LR: 3.89e-05 | 1900.7ms/step | 31.6 samples/s
+Step 41530/50462 | Loss: 2.4304 | LR: 3.88e-05 | 1909.6ms/step | 31.4 samples/s
+Step 41540/50462 | Loss: 2.4501 | LR: 3.87e-05 | 1909.3ms/step | 31.4 samples/s
+Step 41550/50462 | Loss: 2.4246 | LR: 3.86e-05 | 1907.8ms/step | 31.5 samples/s
+Step 41560/50462 | Loss: 2.4521 | LR: 3.85e-05 | 1909.5ms/step | 31.4 samples/s
+Step 41570/50462 | Loss: 2.4463 | LR: 3.84e-05 | 1914.7ms/step | 31.3 samples/s
+Step 41580/50462 | Loss: 2.4679 | LR: 3.84e-05 | 1916.1ms/step | 31.3 samples/s
+Step 41590/50462 | Loss: 2.4522 | LR: 3.83e-05 | 1910.7ms/step | 31.4 samples/s
+Step 41600/50462 | Loss: 2.4365 | LR: 3.82e-05 | 1914.1ms/step | 31.3 samples/s
+Step 41610/50462 | Loss: 2.4415 | LR: 3.81e-05 | 1912.4ms/step | 31.4 samples/s
+Step 41620/50462 | Loss: 2.4759 | LR: 3.80e-05 | 1917.6ms/step | 31.3 samples/s
+Step 41630/50462 | Loss: 2.4692 | LR: 3.79e-05 | 1915.0ms/step | 31.3 samples/s
+Step 41640/50462 | Loss: 2.4616 | LR: 3.79e-05 | 1909.4ms/step | 31.4 samples/s
+Step 41650/50462 | Loss: 2.4704 | LR: 3.78e-05 | 1913.8ms/step | 31.4 samples/s
+Step 41660/50462 | Loss: 2.4468 | LR: 3.77e-05 | 1910.4ms/step | 31.4 samples/s
+Step 41670/50462 | Loss: 2.4982 | LR: 3.76e-05 | 1915.5ms/step | 31.3 samples/s
+Step 41680/50462 | Loss: 2.4105 | LR: 3.75e-05 | 1915.6ms/step | 31.3 samples/s
+Step 41690/50462 | Loss: 2.4330 | LR: 3.74e-05 | 1911.7ms/step | 31.4 samples/s
+Step 41700/50462 | Loss: 2.4075 | LR: 3.74e-05 | 1908.8ms/step | 31.4 samples/s
+Step 41710/50462 | Loss: 2.4750 | LR: 3.73e-05 | 1913.3ms/step | 31.4 samples/s
+Step 41720/50462 | Loss: 2.4661 | LR: 3.72e-05 | 1914.7ms/step | 31.3 samples/s
+Step 41730/50462 | Loss: 2.4525 | LR: 3.71e-05 | 1916.1ms/step | 31.3 samples/s
+Step 41740/50462 | Loss: 2.4385 | LR: 3.70e-05 | 1917.0ms/step | 31.3 samples/s
+Step 41750/50462 | Loss: 2.4660 | LR: 3.69e-05 | 1914.6ms/step | 31.3 samples/s
+Validation at step 41750: Loss = 2.4998, Perplexity = 12.18
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-41750
+Step 41760/50462 | Loss: 2.4765 | LR: 3.69e-05 | 3279.4ms/step | 18.3 samples/s
+Step 41770/50462 | Loss: 2.4496 | LR: 3.68e-05 | 1898.5ms/step | 31.6 samples/s
+Step 41780/50462 | Loss: 2.4236 | LR: 3.67e-05 | 1906.4ms/step | 31.5 samples/s
+Step 41790/50462 | Loss: 2.4384 | LR: 3.66e-05 | 1906.8ms/step | 31.5 samples/s
+Step 41800/50462 | Loss: 2.4967 | LR: 3.65e-05 | 1907.0ms/step | 31.5 samples/s
+Step 41810/50462 | Loss: 2.4740 | LR: 3.64e-05 | 1911.2ms/step | 31.4 samples/s
+Step 41820/50462 | Loss: 2.4658 | LR: 3.64e-05 | 1912.3ms/step | 31.4 samples/s
+Step 41830/50462 | Loss: 2.4844 | LR: 3.63e-05 | 1913.2ms/step | 31.4 samples/s
+Step 41840/50462 | Loss: 2.4547 | LR: 3.62e-05 | 1913.2ms/step | 31.4 samples/s
+Step 41850/50462 | Loss: 2.4263 | LR: 3.61e-05 | 1908.4ms/step | 31.4 samples/s
+Step 41860/50462 | Loss: 2.4356 | LR: 3.60e-05 | 1909.9ms/step | 31.4 samples/s
+Step 41870/50462 | Loss: 2.4848 | LR: 3.60e-05 | 1912.4ms/step | 31.4 samples/s

epfl_final/SparseAtt/wandb/run-20260826_173107-rgmkfpnl/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260826_173107-rgmkfpnl/files/wandb-metadata.json

@@ -0,0 +1,79 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-08-26T17:31:07.762607Z",
+  "args":  [
+    "--config",
+    "configs/p2_1p2b_dense_seed70_6gpu_switch.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/SparseAtt.git",
+    "commit":  "8f862f749eb4d5c17f688eb1562235e3fe5b2550"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-train-6-gpu-0-39",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  6,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "4878134013952"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-99f99f92-e501-0b15-5782-6beb2c34fa1a"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-3f5bace4-0066-71eb-863d-73a3a6d79e3b"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-1aef7113-cfe5-3f49-53bb-a84f3c5abfa0"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-5e17ddff-2e95-8c68-b074-b0189417e834"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-61e64705-1798-f2b5-568f-877549238703"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-9cd8a6e5-05e9-cfde-0a28-d8058797b3bf"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "6rku4ldirjp9pwnnvbcrvnvhh39m6tl0"
+}

epfl_final/SparseAtt/wandb/run-20260826_173107-rgmkfpnl/logs/debug.log

@@ -0,0 +1,20 @@
+2026-08-26 17:31:07,781 INFO    MainThread:18360 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-08-26 17:31:07,781 INFO    MainThread:18360 [wandb_setup.py:_flush():81] Configure stats pid to 18360
+2026-08-26 17:31:07,782 INFO    MainThread:18360 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-26 17:31:07,782 INFO    MainThread:18360 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260826_173107-rgmkfpnl/logs/debug.log
+2026-08-26 17:31:07,782 INFO    MainThread:18360 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260826_173107-rgmkfpnl/logs/debug-internal.log
+2026-08-26 17:31:07,782 INFO    MainThread:18360 [wandb_init.py:init():850] calling init triggers
+2026-08-26 17:31:07,782 INFO    MainThread:18360 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-33250', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 70, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b dense 8k seed70 6gpu switch', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'dense', 'seed71'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
+2026-08-26 17:31:07,782 INFO    MainThread:18360 [wandb_init.py:init():898] starting backend
+2026-08-26 17:31:08,047 INFO    MainThread:18360 [wandb_init.py:init():913] sending inform_init request
+2026-08-26 17:31:08,390 INFO    MainThread:18360 [wandb_init.py:init():918] backend started and connected
+2026-08-26 17:31:08,393 INFO    MainThread:18360 [wandb_init.py:init():988] updated telemetry
+2026-08-26 17:31:08,421 INFO    MainThread:18360 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-08-26 17:31:08,909 INFO    MainThread:18360 [wandb_init.py:init():1051] run resumed
+2026-08-26 17:31:08,910 INFO    MainThread:18360 [wandb_init.py:init():1056] starting run threads in backend
+2026-08-26 17:31:09,143 INFO    MainThread:18360 [wandb_run.py:_console_start():2554] atexit reg
+2026-08-26 17:31:09,143 INFO    MainThread:18360 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-08-26 17:31:09,143 INFO    MainThread:18360 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-08-26 17:31:09,143 INFO    MainThread:18360 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-08-26 17:31:09,180 INFO    MainThread:18360 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/run-20260826_173107-rgmkfpnl/run-rgmkfpnl.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce685e064b32bda15b3d7de45825fe09464d4b631f4dce54c3a160362d5cada0
+size 8093696

epfl_final/SparseAtt/wandb/run-20260826_175546-8ynn1we6/files/output.log

@@ -0,0 +1,979 @@
+Starting training...
+Switching attention mode for step 1: sparse_stochastic (configured sparse attention)
+Step 10/50462 | Loss: 12.1735 | LR: 6.67e-06 | 10914.5ms/step | 5.5 samples/s
+Step 20/50462 | Loss: 11.8523 | LR: 1.33e-05 | 4043.1ms/step | 14.8 samples/s
+Step 30/50462 | Loss: 10.8619 | LR: 2.00e-05 | 4052.1ms/step | 14.8 samples/s
+Step 40/50462 | Loss: 10.0261 | LR: 2.67e-05 | 4038.0ms/step | 14.9 samples/s
+Step 50/50462 | Loss: 9.6674 | LR: 3.33e-05 | 4028.9ms/step | 14.9 samples/s
+Step 60/50462 | Loss: 9.2177 | LR: 4.00e-05 | 4032.8ms/step | 14.9 samples/s
+Step 70/50462 | Loss: 8.6984 | LR: 4.67e-05 | 4031.8ms/step | 14.9 samples/s
+Step 80/50462 | Loss: 8.2041 | LR: 5.33e-05 | 4051.0ms/step | 14.8 samples/s
+Step 90/50462 | Loss: 7.8170 | LR: 6.00e-05 | 4032.9ms/step | 14.9 samples/s
+Step 100/50462 | Loss: 7.4934 | LR: 6.67e-05 | 4033.4ms/step | 14.9 samples/s
+Step 110/50462 | Loss: 7.2614 | LR: 7.33e-05 | 4036.6ms/step | 14.9 samples/s
+Step 120/50462 | Loss: 7.0915 | LR: 8.00e-05 | 4027.7ms/step | 14.9 samples/s
+Step 130/50462 | Loss: 6.9530 | LR: 8.67e-05 | 4026.1ms/step | 14.9 samples/s
+Step 140/50462 | Loss: 6.8710 | LR: 9.33e-05 | 4021.3ms/step | 14.9 samples/s
+Step 150/50462 | Loss: 6.7438 | LR: 1.00e-04 | 4034.9ms/step | 14.9 samples/s
+Step 160/50462 | Loss: 6.6806 | LR: 1.07e-04 | 4034.6ms/step | 14.9 samples/s
+Step 170/50462 | Loss: 6.5647 | LR: 1.13e-04 | 4036.0ms/step | 14.9 samples/s
+Step 180/50462 | Loss: 6.4892 | LR: 1.20e-04 | 4051.5ms/step | 14.8 samples/s
+Step 190/50462 | Loss: 6.3867 | LR: 1.27e-04 | 4031.4ms/step | 14.9 samples/s
+Step 200/50462 | Loss: 6.3321 | LR: 1.33e-04 | 4043.4ms/step | 14.8 samples/s
+Step 210/50462 | Loss: 6.2420 | LR: 1.40e-04 | 4024.3ms/step | 14.9 samples/s
+Step 220/50462 | Loss: 6.2222 | LR: 1.47e-04 | 4006.2ms/step | 15.0 samples/s
+Step 230/50462 | Loss: 6.0969 | LR: 1.53e-04 | 4037.2ms/step | 14.9 samples/s
+Step 240/50462 | Loss: 6.0437 | LR: 1.60e-04 | 4031.9ms/step | 14.9 samples/s
+Step 250/50462 | Loss: 5.9955 | LR: 1.67e-04 | 4037.8ms/step | 14.9 samples/s
+Validation at step 250: Loss = 6.0075, Perplexity = 406.45
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-250
+Step 260/50462 | Loss: 5.9751 | LR: 1.73e-04 | 9843.3ms/step | 6.1 samples/s
+Step 270/50462 | Loss: 5.9334 | LR: 1.80e-04 | 4045.9ms/step | 14.8 samples/s
+Step 280/50462 | Loss: 5.8426 | LR: 1.87e-04 | 4050.0ms/step | 14.8 samples/s
+Step 290/50462 | Loss: 5.7715 | LR: 1.93e-04 | 4055.3ms/step | 14.8 samples/s
+Step 300/50462 | Loss: 5.7200 | LR: 2.00e-04 | 4033.8ms/step | 14.9 samples/s
+Step 310/50462 | Loss: 5.7046 | LR: 2.07e-04 | 4047.1ms/step | 14.8 samples/s
+Step 320/50462 | Loss: 5.6271 | LR: 2.13e-04 | 4044.0ms/step | 14.8 samples/s
+Step 330/50462 | Loss: 5.5690 | LR: 2.20e-04 | 4048.6ms/step | 14.8 samples/s
+Step 340/50462 | Loss: 5.4880 | LR: 2.27e-04 | 4064.3ms/step | 14.8 samples/s
+Step 350/50462 | Loss: 5.4937 | LR: 2.33e-04 | 4031.6ms/step | 14.9 samples/s
+Step 360/50462 | Loss: 5.4076 | LR: 2.40e-04 | 4059.5ms/step | 14.8 samples/s
+Step 370/50462 | Loss: 5.3944 | LR: 2.47e-04 | 4064.5ms/step | 14.8 samples/s
+Step 380/50462 | Loss: 5.3287 | LR: 2.53e-04 | 4043.6ms/step | 14.8 samples/s
+Step 390/50462 | Loss: 5.2801 | LR: 2.60e-04 | 4061.2ms/step | 14.8 samples/s
+Step 400/50462 | Loss: 5.2332 | LR: 2.67e-04 | 4035.4ms/step | 14.9 samples/s
+Step 410/50462 | Loss: 5.2196 | LR: 2.73e-04 | 4034.6ms/step | 14.9 samples/s
+Step 420/50462 | Loss: 5.1525 | LR: 2.80e-04 | 4035.1ms/step | 14.9 samples/s
+Step 430/50462 | Loss: 5.0929 | LR: 2.87e-04 | 4039.0ms/step | 14.9 samples/s
+Step 440/50462 | Loss: 5.1199 | LR: 2.93e-04 | 4045.9ms/step | 14.8 samples/s
+Step 450/50462 | Loss: 5.0666 | LR: 3.00e-04 | 4035.7ms/step | 14.9 samples/s
+Step 460/50462 | Loss: 5.0102 | LR: 3.07e-04 | 4029.2ms/step | 14.9 samples/s
+Step 470/50462 | Loss: 4.9350 | LR: 3.13e-04 | 4058.3ms/step | 14.8 samples/s
+Step 480/50462 | Loss: 4.9730 | LR: 3.20e-04 | 4041.4ms/step | 14.8 samples/s
+Step 490/50462 | Loss: 4.8980 | LR: 3.27e-04 | 4039.8ms/step | 14.9 samples/s
+Step 500/50462 | Loss: 4.8829 | LR: 3.33e-04 | 4061.7ms/step | 14.8 samples/s
+Validation at step 500: Loss = 4.8204, Perplexity = 124.02
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-500
+Step 510/50462 | Loss: 4.8219 | LR: 3.40e-04 | 6057.2ms/step | 9.9 samples/s
+Step 520/50462 | Loss: 4.8302 | LR: 3.47e-04 | 4050.4ms/step | 14.8 samples/s
+Step 530/50462 | Loss: 4.8385 | LR: 3.53e-04 | 4064.3ms/step | 14.8 samples/s
+Step 540/50462 | Loss: 4.7450 | LR: 3.60e-04 | 4049.0ms/step | 14.8 samples/s
+Step 550/50462 | Loss: 4.7148 | LR: 3.67e-04 | 4062.1ms/step | 14.8 samples/s
+Step 560/50462 | Loss: 4.7261 | LR: 3.73e-04 | 4040.8ms/step | 14.8 samples/s
+Step 570/50462 | Loss: 4.6851 | LR: 3.80e-04 | 4054.8ms/step | 14.8 samples/s
+Step 580/50462 | Loss: 4.6015 | LR: 3.87e-04 | 4055.9ms/step | 14.8 samples/s
+Step 590/50462 | Loss: 4.5734 | LR: 3.93e-04 | 4059.7ms/step | 14.8 samples/s
+Step 600/50462 | Loss: 4.6757 | LR: 4.00e-04 | 4007.8ms/step | 15.0 samples/s
+Step 610/50462 | Loss: 4.6519 | LR: 4.07e-04 | 4002.3ms/step | 15.0 samples/s
+Step 620/50462 | Loss: 4.5430 | LR: 4.13e-04 | 4024.8ms/step | 14.9 samples/s
+Step 630/50462 | Loss: 4.4615 | LR: 4.20e-04 | 4044.7ms/step | 14.8 samples/s
+Step 640/50462 | Loss: 4.4721 | LR: 4.27e-04 | 4041.8ms/step | 14.8 samples/s
+Step 650/50462 | Loss: 4.4471 | LR: 4.33e-04 | 4047.8ms/step | 14.8 samples/s
+Step 660/50462 | Loss: 4.4082 | LR: 4.40e-04 | 4037.2ms/step | 14.9 samples/s
+Step 670/50462 | Loss: 4.3286 | LR: 4.47e-04 | 4042.3ms/step | 14.8 samples/s
+Step 680/50462 | Loss: 4.3621 | LR: 4.53e-04 | 4044.8ms/step | 14.8 samples/s
+Step 690/50462 | Loss: 4.3784 | LR: 4.60e-04 | 4033.8ms/step | 14.9 samples/s
+Step 700/50462 | Loss: 4.3101 | LR: 4.67e-04 | 4039.5ms/step | 14.9 samples/s
+Step 710/50462 | Loss: 4.2408 | LR: 4.73e-04 | 4049.4ms/step | 14.8 samples/s
+Step 720/50462 | Loss: 4.2553 | LR: 4.80e-04 | 4035.0ms/step | 14.9 samples/s
+Step 730/50462 | Loss: 4.2802 | LR: 4.87e-04 | 4057.5ms/step | 14.8 samples/s
+Step 740/50462 | Loss: 4.2105 | LR: 4.93e-04 | 4046.9ms/step | 14.8 samples/s
+Step 750/50462 | Loss: 4.1277 | LR: 5.00e-04 | 4043.5ms/step | 14.8 samples/s
+Validation at step 750: Loss = 4.1305, Perplexity = 62.21
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-750
+Step 760/50462 | Loss: 4.1138 | LR: 5.00e-04 | 6180.2ms/step | 9.7 samples/s
+Step 770/50462 | Loss: 4.1534 | LR: 5.00e-04 | 4028.5ms/step | 14.9 samples/s
+Step 780/50462 | Loss: 4.0919 | LR: 5.00e-04 | 4036.7ms/step | 14.9 samples/s
+Step 790/50462 | Loss: 4.0636 | LR: 5.00e-04 | 4049.9ms/step | 14.8 samples/s
+Step 800/50462 | Loss: 4.0680 | LR: 5.00e-04 | 4050.8ms/step | 14.8 samples/s
+Step 810/50462 | Loss: 4.0333 | LR: 5.00e-04 | 4055.9ms/step | 14.8 samples/s
+Step 820/50462 | Loss: 4.0445 | LR: 5.00e-04 | 4031.4ms/step | 14.9 samples/s
+Step 830/50462 | Loss: 3.9848 | LR: 5.00e-04 | 4049.4ms/step | 14.8 samples/s
+Step 840/50462 | Loss: 3.9722 | LR: 5.00e-04 | 4030.4ms/step | 14.9 samples/s
+Step 850/50462 | Loss: 3.9518 | LR: 5.00e-04 | 4048.5ms/step | 14.8 samples/s
+Step 860/50462 | Loss: 3.9538 | LR: 5.00e-04 | 4041.9ms/step | 14.8 samples/s
+Step 870/50462 | Loss: 3.9220 | LR: 5.00e-04 | 4027.8ms/step | 14.9 samples/s
+Step 880/50462 | Loss: 3.9140 | LR: 5.00e-04 | 4062.3ms/step | 14.8 samples/s
+Step 890/50462 | Loss: 3.9015 | LR: 5.00e-04 | 4050.0ms/step | 14.8 samples/s
+Step 900/50462 | Loss: 3.8853 | LR: 5.00e-04 | 4043.9ms/step | 14.8 samples/s
+Step 910/50462 | Loss: 3.8627 | LR: 5.00e-04 | 4051.2ms/step | 14.8 samples/s
+Step 920/50462 | Loss: 3.8530 | LR: 5.00e-04 | 4039.9ms/step | 14.9 samples/s
+Step 930/50462 | Loss: 3.8373 | LR: 5.00e-04 | 4045.4ms/step | 14.8 samples/s
+Step 940/50462 | Loss: 3.8596 | LR: 5.00e-04 | 4041.1ms/step | 14.8 samples/s
+Step 950/50462 | Loss: 3.7965 | LR: 5.00e-04 | 4038.6ms/step | 14.9 samples/s
+Step 960/50462 | Loss: 3.7957 | LR: 5.00e-04 | 4052.3ms/step | 14.8 samples/s
+Step 970/50462 | Loss: 3.7822 | LR: 5.00e-04 | 4027.6ms/step | 14.9 samples/s
+Step 980/50462 | Loss: 3.7720 | LR: 5.00e-04 | 4053.0ms/step | 14.8 samples/s
+Step 990/50462 | Loss: 3.7532 | LR: 5.00e-04 | 4042.5ms/step | 14.8 samples/s
+Step 1000/50462 | Loss: 3.7186 | LR: 5.00e-04 | 4027.7ms/step | 14.9 samples/s
+Validation at step 1000: Loss = 3.7348, Perplexity = 41.88
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-1000
+Step 1010/50462 | Loss: 3.7375 | LR: 5.00e-04 | 6097.1ms/step | 9.8 samples/s
+Step 1020/50462 | Loss: 3.7275 | LR: 5.00e-04 | 4049.1ms/step | 14.8 samples/s
+Step 1030/50462 | Loss: 3.7092 | LR: 5.00e-04 | 4038.4ms/step | 14.9 samples/s
+Step 1040/50462 | Loss: 3.6871 | LR: 5.00e-04 | 4052.2ms/step | 14.8 samples/s
+Step 1050/50462 | Loss: 3.6919 | LR: 5.00e-04 | 4042.2ms/step | 14.8 samples/s
+Step 1060/50462 | Loss: 3.7177 | LR: 5.00e-04 | 4038.7ms/step | 14.9 samples/s
+Step 1070/50462 | Loss: 3.6661 | LR: 5.00e-04 | 4038.2ms/step | 14.9 samples/s
+Step 1080/50462 | Loss: 3.6734 | LR: 5.00e-04 | 4020.2ms/step | 14.9 samples/s
+Step 1090/50462 | Loss: 3.6472 | LR: 5.00e-04 | 4036.8ms/step | 14.9 samples/s
+Step 1100/50462 | Loss: 3.6051 | LR: 5.00e-04 | 4047.9ms/step | 14.8 samples/s
+Step 1110/50462 | Loss: 3.6151 | LR: 5.00e-04 | 4045.6ms/step | 14.8 samples/s
+Step 1120/50462 | Loss: 3.6332 | LR: 5.00e-04 | 4047.2ms/step | 14.8 samples/s
+Step 1130/50462 | Loss: 3.6328 | LR: 5.00e-04 | 4044.2ms/step | 14.8 samples/s
+Step 1140/50462 | Loss: 3.6429 | LR: 5.00e-04 | 4031.2ms/step | 14.9 samples/s
+Step 1150/50462 | Loss: 3.6229 | LR: 5.00e-04 | 4041.4ms/step | 14.8 samples/s
+Step 1160/50462 | Loss: 3.5861 | LR: 5.00e-04 | 4036.5ms/step | 14.9 samples/s
+Step 1170/50462 | Loss: 3.5737 | LR: 5.00e-04 | 4043.1ms/step | 14.8 samples/s
+Step 1180/50462 | Loss: 3.6065 | LR: 5.00e-04 | 4040.0ms/step | 14.9 samples/s
+Step 1190/50462 | Loss: 3.6234 | LR: 5.00e-04 | 4044.0ms/step | 14.8 samples/s
+Step 1200/50462 | Loss: 3.5905 | LR: 5.00e-04 | 4047.0ms/step | 14.8 samples/s
+Step 1210/50462 | Loss: 3.5825 | LR: 5.00e-04 | 4049.3ms/step | 14.8 samples/s
+Step 1220/50462 | Loss: 3.5604 | LR: 5.00e-04 | 4036.3ms/step | 14.9 samples/s
+Step 1230/50462 | Loss: 3.5797 | LR: 5.00e-04 | 4036.8ms/step | 14.9 samples/s
+Step 1240/50462 | Loss: 3.5495 | LR: 5.00e-04 | 4047.2ms/step | 14.8 samples/s
+Step 1250/50462 | Loss: 3.5139 | LR: 5.00e-04 | 4031.5ms/step | 14.9 samples/s
+Validation at step 1250: Loss = 3.5347, Perplexity = 34.28
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-1250
+Step 1260/50462 | Loss: 3.5406 | LR: 5.00e-04 | 6079.3ms/step | 9.9 samples/s
+Step 1270/50462 | Loss: 3.5224 | LR: 5.00e-04 | 4032.7ms/step | 14.9 samples/s
+Step 1280/50462 | Loss: 3.5111 | LR: 5.00e-04 | 4028.8ms/step | 14.9 samples/s
+Step 1290/50462 | Loss: 3.4851 | LR: 5.00e-04 | 4033.6ms/step | 14.9 samples/s
+Step 1300/50462 | Loss: 3.5624 | LR: 5.00e-04 | 4027.6ms/step | 14.9 samples/s
+Step 1310/50462 | Loss: 3.5096 | LR: 5.00e-04 | 4042.0ms/step | 14.8 samples/s
+Step 1320/50462 | Loss: 3.5003 | LR: 5.00e-04 | 4028.1ms/step | 14.9 samples/s
+Step 1330/50462 | Loss: 3.4819 | LR: 5.00e-04 | 4031.0ms/step | 14.9 samples/s
+Step 1340/50462 | Loss: 3.4801 | LR: 5.00e-04 | 4035.6ms/step | 14.9 samples/s
+Step 1350/50462 | Loss: 3.5217 | LR: 5.00e-04 | 4036.5ms/step | 14.9 samples/s
+Step 1360/50462 | Loss: 3.5027 | LR: 5.00e-04 | 4033.9ms/step | 14.9 samples/s
+Step 1370/50462 | Loss: 3.4730 | LR: 5.00e-04 | 4036.1ms/step | 14.9 samples/s
+Step 1380/50462 | Loss: 3.4499 | LR: 5.00e-04 | 4038.4ms/step | 14.9 samples/s
+Step 1390/50462 | Loss: 3.4604 | LR: 5.00e-04 | 4032.1ms/step | 14.9 samples/s
+Step 1400/50462 | Loss: 3.4714 | LR: 5.00e-04 | 4030.0ms/step | 14.9 samples/s
+Step 1410/50462 | Loss: 3.4416 | LR: 5.00e-04 | 4034.9ms/step | 14.9 samples/s
+Step 1420/50462 | Loss: 3.4309 | LR: 5.00e-04 | 4024.7ms/step | 14.9 samples/s
+Step 1430/50462 | Loss: 3.4354 | LR: 5.00e-04 | 4027.3ms/step | 14.9 samples/s
+Step 1440/50462 | Loss: 3.4276 | LR: 5.00e-04 | 4030.1ms/step | 14.9 samples/s
+Step 1450/50462 | Loss: 3.4406 | LR: 5.00e-04 | 4046.4ms/step | 14.8 samples/s
+Step 1460/50462 | Loss: 3.4301 | LR: 5.00e-04 | 4037.9ms/step | 14.9 samples/s
+Step 1470/50462 | Loss: 3.4453 | LR: 5.00e-04 | 4043.3ms/step | 14.8 samples/s
+Step 1480/50462 | Loss: 3.4068 | LR: 5.00e-04 | 4018.8ms/step | 14.9 samples/s
+Step 1490/50462 | Loss: 3.4032 | LR: 5.00e-04 | 4027.8ms/step | 14.9 samples/s
+Step 1500/50462 | Loss: 3.4007 | LR: 5.00e-04 | 4010.5ms/step | 15.0 samples/s
+Validation at step 1500: Loss = 3.4040, Perplexity = 30.08
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-1500
+Step 1510/50462 | Loss: 3.3693 | LR: 5.00e-04 | 6131.6ms/step | 9.8 samples/s
+Step 1520/50462 | Loss: 3.4414 | LR: 5.00e-04 | 4029.7ms/step | 14.9 samples/s
+Step 1530/50462 | Loss: 3.3694 | LR: 5.00e-04 | 4040.9ms/step | 14.8 samples/s
+Step 1540/50462 | Loss: 3.4128 | LR: 5.00e-04 | 4020.1ms/step | 14.9 samples/s
+Step 1550/50462 | Loss: 3.4069 | LR: 5.00e-04 | 4026.1ms/step | 14.9 samples/s
+Step 1560/50462 | Loss: 3.3885 | LR: 5.00e-04 | 4028.1ms/step | 14.9 samples/s
+Step 1570/50462 | Loss: 3.3510 | LR: 5.00e-04 | 4023.2ms/step | 14.9 samples/s
+Step 1580/50462 | Loss: 3.3978 | LR: 5.00e-04 | 4016.6ms/step | 14.9 samples/s
+Step 1590/50462 | Loss: 3.4003 | LR: 5.00e-04 | 4020.8ms/step | 14.9 samples/s
+Step 1600/50462 | Loss: 3.3852 | LR: 5.00e-04 | 4015.5ms/step | 14.9 samples/s
+Step 1610/50462 | Loss: 3.3786 | LR: 5.00e-04 | 4018.6ms/step | 14.9 samples/s
+Step 1620/50462 | Loss: 3.3755 | LR: 5.00e-04 | 4019.0ms/step | 14.9 samples/s
+Step 1630/50462 | Loss: 3.4199 | LR: 5.00e-04 | 4015.0ms/step | 14.9 samples/s
+Step 1640/50462 | Loss: 3.3860 | LR: 5.00e-04 | 4025.7ms/step | 14.9 samples/s
+Step 1650/50462 | Loss: 3.3769 | LR: 5.00e-04 | 4019.1ms/step | 14.9 samples/s
+Step 1660/50462 | Loss: 3.3397 | LR: 5.00e-04 | 4002.9ms/step | 15.0 samples/s
+Step 1670/50462 | Loss: 3.3826 | LR: 5.00e-04 | 4023.2ms/step | 14.9 samples/s
+Step 1680/50462 | Loss: 3.3352 | LR: 5.00e-04 | 4014.7ms/step | 14.9 samples/s
+Step 1690/50462 | Loss: 3.3529 | LR: 5.00e-04 | 4020.1ms/step | 14.9 samples/s
+Step 1700/50462 | Loss: 3.3209 | LR: 5.00e-04 | 4046.0ms/step | 14.8 samples/s
+Step 1710/50462 | Loss: 3.3333 | LR: 5.00e-04 | 4022.4ms/step | 14.9 samples/s
+Step 1720/50462 | Loss: 3.3196 | LR: 5.00e-04 | 4015.4ms/step | 14.9 samples/s
+Step 1730/50462 | Loss: 3.3285 | LR: 5.00e-04 | 4021.2ms/step | 14.9 samples/s
+Step 1740/50462 | Loss: 3.3809 | LR: 5.00e-04 | 4011.3ms/step | 15.0 samples/s
+Step 1750/50462 | Loss: 3.3392 | LR: 5.00e-04 | 4022.6ms/step | 14.9 samples/s
+Validation at step 1750: Loss = 3.3174, Perplexity = 27.59
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-1750
+Step 1760/50462 | Loss: 3.3226 | LR: 4.99e-04 | 6096.3ms/step | 9.8 samples/s
+Step 1770/50462 | Loss: 3.3228 | LR: 4.99e-04 | 4011.9ms/step | 15.0 samples/s
+Step 1780/50462 | Loss: 3.3117 | LR: 4.99e-04 | 4008.0ms/step | 15.0 samples/s
+Step 1790/50462 | Loss: 3.3102 | LR: 4.99e-04 | 4020.2ms/step | 14.9 samples/s
+Step 1800/50462 | Loss: 3.3048 | LR: 4.99e-04 | 4013.8ms/step | 14.9 samples/s
+Step 1810/50462 | Loss: 3.3055 | LR: 4.99e-04 | 4006.4ms/step | 15.0 samples/s
+Step 1820/50462 | Loss: 3.3028 | LR: 4.99e-04 | 4009.3ms/step | 15.0 samples/s
+Step 1830/50462 | Loss: 3.2910 | LR: 4.99e-04 | 4021.1ms/step | 14.9 samples/s
+Step 1840/50462 | Loss: 3.2755 | LR: 4.99e-04 | 4008.1ms/step | 15.0 samples/s
+Step 1850/50462 | Loss: 3.2821 | LR: 4.99e-04 | 4012.1ms/step | 15.0 samples/s
+Step 1860/50462 | Loss: 3.2899 | LR: 4.99e-04 | 4020.0ms/step | 14.9 samples/s
+Step 1870/50462 | Loss: 3.2832 | LR: 4.99e-04 | 4032.3ms/step | 14.9 samples/s
+Step 1880/50462 | Loss: 3.2610 | LR: 4.99e-04 | 4008.4ms/step | 15.0 samples/s
+Step 1890/50462 | Loss: 3.2963 | LR: 4.99e-04 | 4016.0ms/step | 14.9 samples/s
+Step 1900/50462 | Loss: 3.3237 | LR: 4.99e-04 | 4007.6ms/step | 15.0 samples/s
+Step 1910/50462 | Loss: 3.2761 | LR: 4.99e-04 | 4010.9ms/step | 15.0 samples/s
+Step 1920/50462 | Loss: 3.2940 | LR: 4.99e-04 | 4022.4ms/step | 14.9 samples/s
+Step 1930/50462 | Loss: 3.2614 | LR: 4.99e-04 | 4027.3ms/step | 14.9 samples/s
+Step 1940/50462 | Loss: 3.2820 | LR: 4.99e-04 | 4013.1ms/step | 15.0 samples/s
+Step 1950/50462 | Loss: 3.2723 | LR: 4.99e-04 | 3993.3ms/step | 15.0 samples/s
+Step 1960/50462 | Loss: 3.2429 | LR: 4.99e-04 | 3979.0ms/step | 15.1 samples/s
+Step 1970/50462 | Loss: 3.2675 | LR: 4.99e-04 | 3984.9ms/step | 15.1 samples/s
+Step 1980/50462 | Loss: 3.2864 | LR: 4.99e-04 | 3985.3ms/step | 15.1 samples/s
+Step 1990/50462 | Loss: 3.2743 | LR: 4.99e-04 | 3989.3ms/step | 15.0 samples/s
+Step 2000/50462 | Loss: 3.2747 | LR: 4.99e-04 | 3994.3ms/step | 15.0 samples/s
+Validation at step 2000: Loss = 3.2505, Perplexity = 25.80
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-2000
+Step 2010/50462 | Loss: 3.2765 | LR: 4.99e-04 | 6057.1ms/step | 9.9 samples/s
+Step 2020/50462 | Loss: 3.2408 | LR: 4.99e-04 | 3999.8ms/step | 15.0 samples/s
+Step 2030/50462 | Loss: 3.2790 | LR: 4.99e-04 | 3997.0ms/step | 15.0 samples/s
+Step 2040/50462 | Loss: 3.2488 | LR: 4.99e-04 | 3987.9ms/step | 15.0 samples/s
+Step 2050/50462 | Loss: 3.2438 | LR: 4.99e-04 | 3992.1ms/step | 15.0 samples/s
+Step 2060/50462 | Loss: 3.2687 | LR: 4.99e-04 | 3987.9ms/step | 15.0 samples/s
+Step 2070/50462 | Loss: 3.2609 | LR: 4.99e-04 | 3989.4ms/step | 15.0 samples/s
+Step 2080/50462 | Loss: 3.2888 | LR: 4.99e-04 | 3990.9ms/step | 15.0 samples/s
+Step 2090/50462 | Loss: 3.2493 | LR: 4.99e-04 | 3980.8ms/step | 15.1 samples/s
+Step 2100/50462 | Loss: 3.2734 | LR: 4.99e-04 | 3989.8ms/step | 15.0 samples/s
+Step 2110/50462 | Loss: 3.2546 | LR: 4.99e-04 | 3984.5ms/step | 15.1 samples/s
+Step 2120/50462 | Loss: 3.2458 | LR: 4.99e-04 | 3981.5ms/step | 15.1 samples/s
+Step 2130/50462 | Loss: 3.2234 | LR: 4.99e-04 | 3979.7ms/step | 15.1 samples/s
+Step 2140/50462 | Loss: 3.1967 | LR: 4.99e-04 | 3991.3ms/step | 15.0 samples/s
+Step 2150/50462 | Loss: 3.2337 | LR: 4.99e-04 | 3976.9ms/step | 15.1 samples/s
+Step 2160/50462 | Loss: 3.2203 | LR: 4.99e-04 | 3982.7ms/step | 15.1 samples/s
+Step 2170/50462 | Loss: 3.2035 | LR: 4.99e-04 | 3985.8ms/step | 15.1 samples/s
+Step 2180/50462 | Loss: 3.2200 | LR: 4.99e-04 | 3990.9ms/step | 15.0 samples/s
+Step 2190/50462 | Loss: 3.2185 | LR: 4.99e-04 | 3987.6ms/step | 15.0 samples/s
+Step 2200/50462 | Loss: 3.2093 | LR: 4.99e-04 | 3989.9ms/step | 15.0 samples/s
+Step 2210/50462 | Loss: 3.2388 | LR: 4.99e-04 | 3988.8ms/step | 15.0 samples/s
+Step 2220/50462 | Loss: 3.2118 | LR: 4.99e-04 | 3990.3ms/step | 15.0 samples/s
+Step 2230/50462 | Loss: 3.2195 | LR: 4.99e-04 | 3990.7ms/step | 15.0 samples/s
+Step 2240/50462 | Loss: 3.2187 | LR: 4.99e-04 | 3999.7ms/step | 15.0 samples/s
+Step 2250/50462 | Loss: 3.2305 | LR: 4.99e-04 | 3987.1ms/step | 15.0 samples/s
+Validation at step 2250: Loss = 3.1983, Perplexity = 24.49
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-2250
+Step 2260/50462 | Loss: 3.2231 | LR: 4.99e-04 | 5999.0ms/step | 10.0 samples/s
+Step 2270/50462 | Loss: 3.2209 | LR: 4.99e-04 | 3991.1ms/step | 15.0 samples/s
+Step 2280/50462 | Loss: 3.1863 | LR: 4.99e-04 | 3982.2ms/step | 15.1 samples/s
+Step 2290/50462 | Loss: 3.2290 | LR: 4.99e-04 | 3988.6ms/step | 15.0 samples/s
+Step 2300/50462 | Loss: 3.2003 | LR: 4.99e-04 | 4001.6ms/step | 15.0 samples/s
+Step 2310/50462 | Loss: 3.1760 | LR: 4.99e-04 | 3987.1ms/step | 15.0 samples/s
+Step 2320/50462 | Loss: 3.1814 | LR: 4.99e-04 | 3985.3ms/step | 15.1 samples/s
+Step 2330/50462 | Loss: 3.2158 | LR: 4.99e-04 | 3980.2ms/step | 15.1 samples/s
+Step 2340/50462 | Loss: 3.1555 | LR: 4.99e-04 | 3994.9ms/step | 15.0 samples/s
+Step 2350/50462 | Loss: 3.1332 | LR: 4.99e-04 | 4003.0ms/step | 15.0 samples/s
+Step 2360/50462 | Loss: 3.2012 | LR: 4.99e-04 | 3983.8ms/step | 15.1 samples/s
+Step 2370/50462 | Loss: 3.2130 | LR: 4.99e-04 | 3991.9ms/step | 15.0 samples/s
+Step 2380/50462 | Loss: 3.2279 | LR: 4.99e-04 | 3989.4ms/step | 15.0 samples/s
+Step 2390/50462 | Loss: 3.1443 | LR: 4.99e-04 | 3983.3ms/step | 15.1 samples/s
+Step 2400/50462 | Loss: 3.1873 | LR: 4.99e-04 | 3982.5ms/step | 15.1 samples/s
+Step 2410/50462 | Loss: 3.1734 | LR: 4.99e-04 | 3995.2ms/step | 15.0 samples/s
+Step 2420/50462 | Loss: 3.1983 | LR: 4.99e-04 | 3991.1ms/step | 15.0 samples/s
+Step 2430/50462 | Loss: 3.1771 | LR: 4.99e-04 | 3983.0ms/step | 15.1 samples/s
+Step 2440/50462 | Loss: 3.1816 | LR: 4.99e-04 | 3987.5ms/step | 15.0 samples/s
+Step 2450/50462 | Loss: 3.1307 | LR: 4.99e-04 | 3982.4ms/step | 15.1 samples/s
+Step 2460/50462 | Loss: 3.1900 | LR: 4.99e-04 | 3985.9ms/step | 15.1 samples/s
+Step 2470/50462 | Loss: 3.1579 | LR: 4.99e-04 | 3991.3ms/step | 15.0 samples/s
+Step 2480/50462 | Loss: 3.1513 | LR: 4.99e-04 | 3981.8ms/step | 15.1 samples/s
+Step 2490/50462 | Loss: 3.1298 | LR: 4.98e-04 | 3994.1ms/step | 15.0 samples/s
+Step 2500/50462 | Loss: 3.1713 | LR: 4.98e-04 | 3993.0ms/step | 15.0 samples/s
+Validation at step 2500: Loss = 3.1546, Perplexity = 23.44
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-2500
+Step 2510/50462 | Loss: 3.1459 | LR: 4.98e-04 | 5993.0ms/step | 10.0 samples/s
+Step 2520/50462 | Loss: 3.1609 | LR: 4.98e-04 | 3984.7ms/step | 15.1 samples/s
+Step 2530/50462 | Loss: 3.1561 | LR: 4.98e-04 | 3983.9ms/step | 15.1 samples/s
+Step 2540/50462 | Loss: 3.1871 | LR: 4.98e-04 | 3992.1ms/step | 15.0 samples/s
+Step 2550/50462 | Loss: 3.1609 | LR: 4.98e-04 | 3979.9ms/step | 15.1 samples/s
+Step 2560/50462 | Loss: 3.1291 | LR: 4.98e-04 | 3990.0ms/step | 15.0 samples/s
+Step 2570/50462 | Loss: 3.1458 | LR: 4.98e-04 | 3974.2ms/step | 15.1 samples/s
+Step 2580/50462 | Loss: 3.1359 | LR: 4.98e-04 | 3981.6ms/step | 15.1 samples/s
+Step 2590/50462 | Loss: 3.1367 | LR: 4.98e-04 | 3982.1ms/step | 15.1 samples/s
+Step 2600/50462 | Loss: 3.1431 | LR: 4.98e-04 | 3983.7ms/step | 15.1 samples/s
+Step 2610/50462 | Loss: 3.1733 | LR: 4.98e-04 | 3996.3ms/step | 15.0 samples/s
+Step 2620/50462 | Loss: 3.1513 | LR: 4.98e-04 | 4028.6ms/step | 14.9 samples/s
+Step 2630/50462 | Loss: 3.1972 | LR: 4.98e-04 | 3983.9ms/step | 15.1 samples/s
+Step 2640/50462 | Loss: 3.1369 | LR: 4.98e-04 | 4004.2ms/step | 15.0 samples/s
+Step 2650/50462 | Loss: 3.1357 | LR: 4.98e-04 | 4011.6ms/step | 15.0 samples/s
+Step 2660/50462 | Loss: 3.2074 | LR: 4.98e-04 | 4007.9ms/step | 15.0 samples/s
+Step 2670/50462 | Loss: 3.1318 | LR: 4.98e-04 | 4028.4ms/step | 14.9 samples/s
+Step 2680/50462 | Loss: 3.1457 | LR: 4.98e-04 | 4045.7ms/step | 14.8 samples/s
+Step 2690/50462 | Loss: 3.0889 | LR: 4.98e-04 | 3998.3ms/step | 15.0 samples/s
+Step 2700/50462 | Loss: 3.1390 | LR: 4.98e-04 | 3985.3ms/step | 15.1 samples/s
+Step 2710/50462 | Loss: 3.1177 | LR: 4.98e-04 | 3991.4ms/step | 15.0 samples/s
+Step 2720/50462 | Loss: 3.1123 | LR: 4.98e-04 | 3991.8ms/step | 15.0 samples/s
+Step 2730/50462 | Loss: 3.0990 | LR: 4.98e-04 | 3985.1ms/step | 15.1 samples/s
+Step 2740/50462 | Loss: 3.1374 | LR: 4.98e-04 | 3990.4ms/step | 15.0 samples/s
+Step 2750/50462 | Loss: 3.1167 | LR: 4.98e-04 | 3990.5ms/step | 15.0 samples/s
+Validation at step 2750: Loss = 3.1187, Perplexity = 22.62
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-2750
+Step 2760/50462 | Loss: 3.1538 | LR: 4.98e-04 | 6100.6ms/step | 9.8 samples/s
+Step 2770/50462 | Loss: 3.1501 | LR: 4.98e-04 | 3976.1ms/step | 15.1 samples/s
+Step 2780/50462 | Loss: 3.1192 | LR: 4.98e-04 | 3988.3ms/step | 15.0 samples/s
+Step 2790/50462 | Loss: 3.1541 | LR: 4.98e-04 | 3979.4ms/step | 15.1 samples/s
+Step 2800/50462 | Loss: 3.1134 | LR: 4.98e-04 | 3991.5ms/step | 15.0 samples/s
+Step 2810/50462 | Loss: 3.1375 | LR: 4.98e-04 | 3983.6ms/step | 15.1 samples/s
+Step 2820/50462 | Loss: 3.1222 | LR: 4.98e-04 | 3987.7ms/step | 15.0 samples/s
+Step 2830/50462 | Loss: 3.1268 | LR: 4.98e-04 | 3983.1ms/step | 15.1 samples/s
+Step 2840/50462 | Loss: 3.1126 | LR: 4.98e-04 | 3993.3ms/step | 15.0 samples/s
+Step 2850/50462 | Loss: 3.1511 | LR: 4.98e-04 | 3988.6ms/step | 15.0 samples/s
+Step 2860/50462 | Loss: 3.0980 | LR: 4.98e-04 | 3998.3ms/step | 15.0 samples/s
+Step 2870/50462 | Loss: 3.0944 | LR: 4.98e-04 | 3986.2ms/step | 15.1 samples/s
+Step 2880/50462 | Loss: 3.1326 | LR: 4.98e-04 | 3983.3ms/step | 15.1 samples/s
+Step 2890/50462 | Loss: 3.0971 | LR: 4.98e-04 | 3978.8ms/step | 15.1 samples/s
+Step 2900/50462 | Loss: 3.1530 | LR: 4.98e-04 | 3991.4ms/step | 15.0 samples/s
+Step 2910/50462 | Loss: 3.0981 | LR: 4.98e-04 | 3980.2ms/step | 15.1 samples/s
+Step 2920/50462 | Loss: 3.1063 | LR: 4.98e-04 | 3987.8ms/step | 15.0 samples/s
+Step 2930/50462 | Loss: 3.1022 | LR: 4.98e-04 | 3988.3ms/step | 15.0 samples/s
+Step 2940/50462 | Loss: 3.1334 | LR: 4.98e-04 | 3986.0ms/step | 15.1 samples/s
+Step 2950/50462 | Loss: 3.0795 | LR: 4.98e-04 | 3993.0ms/step | 15.0 samples/s
+Step 2960/50462 | Loss: 3.0865 | LR: 4.98e-04 | 3990.5ms/step | 15.0 samples/s
+Step 2970/50462 | Loss: 3.0882 | LR: 4.98e-04 | 3984.0ms/step | 15.1 samples/s
+Step 2980/50462 | Loss: 3.1102 | LR: 4.98e-04 | 3982.6ms/step | 15.1 samples/s
+Step 2990/50462 | Loss: 3.1112 | LR: 4.97e-04 | 3992.2ms/step | 15.0 samples/s
+Step 3000/50462 | Loss: 3.0782 | LR: 4.97e-04 | 3993.8ms/step | 15.0 samples/s
+Validation at step 3000: Loss = 3.0882, Perplexity = 21.94
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-3000
+Step 3010/50462 | Loss: 3.0818 | LR: 4.97e-04 | 6123.4ms/step | 9.8 samples/s
+Step 3020/50462 | Loss: 3.0965 | LR: 4.97e-04 | 3991.2ms/step | 15.0 samples/s
+Step 3030/50462 | Loss: 3.0954 | LR: 4.97e-04 | 3989.3ms/step | 15.0 samples/s
+Step 3040/50462 | Loss: 3.0663 | LR: 4.97e-04 | 3989.3ms/step | 15.0 samples/s
+Step 3050/50462 | Loss: 3.1164 | LR: 4.97e-04 | 3984.3ms/step | 15.1 samples/s
+Step 3060/50462 | Loss: 3.0906 | LR: 4.97e-04 | 3985.1ms/step | 15.1 samples/s
+Step 3070/50462 | Loss: 3.0994 | LR: 4.97e-04 | 3992.8ms/step | 15.0 samples/s
+Step 3080/50462 | Loss: 3.0867 | LR: 4.97e-04 | 3982.5ms/step | 15.1 samples/s
+Step 3090/50462 | Loss: 3.0992 | LR: 4.97e-04 | 3993.5ms/step | 15.0 samples/s
+Step 3100/50462 | Loss: 3.0870 | LR: 4.97e-04 | 3986.4ms/step | 15.1 samples/s
+Step 3110/50462 | Loss: 3.1067 | LR: 4.97e-04 | 3992.3ms/step | 15.0 samples/s
+Step 3120/50462 | Loss: 3.0678 | LR: 4.97e-04 | 3967.7ms/step | 15.1 samples/s
+Step 3130/50462 | Loss: 3.0697 | LR: 4.97e-04 | 3985.3ms/step | 15.1 samples/s
+Step 3140/50462 | Loss: 3.1153 | LR: 4.97e-04 | 3978.4ms/step | 15.1 samples/s
+Step 3150/50462 | Loss: 3.0737 | LR: 4.97e-04 | 3997.2ms/step | 15.0 samples/s
+Step 3160/50462 | Loss: 3.0766 | LR: 4.97e-04 | 4003.3ms/step | 15.0 samples/s
+Step 3170/50462 | Loss: 3.0800 | LR: 4.97e-04 | 3998.7ms/step | 15.0 samples/s
+Step 3180/50462 | Loss: 3.0478 | LR: 4.97e-04 | 3997.0ms/step | 15.0 samples/s
+Step 3190/50462 | Loss: 3.0715 | LR: 4.97e-04 | 3997.7ms/step | 15.0 samples/s
+Step 3200/50462 | Loss: 3.1076 | LR: 4.97e-04 | 3982.9ms/step | 15.1 samples/s
+Step 3210/50462 | Loss: 3.0598 | LR: 4.97e-04 | 3975.8ms/step | 15.1 samples/s
+Step 3220/50462 | Loss: 3.0471 | LR: 4.97e-04 | 3987.2ms/step | 15.0 samples/s
+Step 3230/50462 | Loss: 3.0512 | LR: 4.97e-04 | 3995.0ms/step | 15.0 samples/s
+Step 3240/50462 | Loss: 3.0687 | LR: 4.97e-04 | 3986.3ms/step | 15.1 samples/s
+Step 3250/50462 | Loss: 3.0758 | LR: 4.97e-04 | 3992.8ms/step | 15.0 samples/s
+Validation at step 3250: Loss = 3.0554, Perplexity = 21.23
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-3250
+Step 3260/50462 | Loss: 3.0806 | LR: 4.97e-04 | 5999.0ms/step | 10.0 samples/s
+Step 3270/50462 | Loss: 3.0417 | LR: 4.97e-04 | 3990.4ms/step | 15.0 samples/s
+Step 3280/50462 | Loss: 3.1087 | LR: 4.97e-04 | 3977.0ms/step | 15.1 samples/s
+Step 3290/50462 | Loss: 3.0814 | LR: 4.97e-04 | 3991.0ms/step | 15.0 samples/s
+Step 3300/50462 | Loss: 3.0491 | LR: 4.97e-04 | 3991.6ms/step | 15.0 samples/s
+Step 3310/50462 | Loss: 3.0477 | LR: 4.97e-04 | 3994.2ms/step | 15.0 samples/s
+Step 3320/50462 | Loss: 3.0766 | LR: 4.97e-04 | 3993.7ms/step | 15.0 samples/s
+Step 3330/50462 | Loss: 3.0506 | LR: 4.97e-04 | 3998.5ms/step | 15.0 samples/s
+Step 3340/50462 | Loss: 3.0455 | LR: 4.97e-04 | 3999.1ms/step | 15.0 samples/s
+Step 3350/50462 | Loss: 3.0469 | LR: 4.97e-04 | 3987.4ms/step | 15.0 samples/s
+Step 3360/50462 | Loss: 3.1033 | LR: 4.97e-04 | 3997.4ms/step | 15.0 samples/s
+Step 3370/50462 | Loss: 3.0543 | LR: 4.97e-04 | 3998.7ms/step | 15.0 samples/s
+Step 3380/50462 | Loss: 3.0528 | LR: 4.97e-04 | 3985.6ms/step | 15.1 samples/s
+Step 3390/50462 | Loss: 3.0622 | LR: 4.97e-04 | 3999.5ms/step | 15.0 samples/s
+Step 3400/50462 | Loss: 3.0556 | LR: 4.97e-04 | 3989.8ms/step | 15.0 samples/s
+Step 3410/50462 | Loss: 3.0701 | LR: 4.96e-04 | 4009.0ms/step | 15.0 samples/s
+Step 3420/50462 | Loss: 3.0529 | LR: 4.96e-04 | 3999.3ms/step | 15.0 samples/s
+Step 3430/50462 | Loss: 3.0166 | LR: 4.96e-04 | 4009.7ms/step | 15.0 samples/s
+Step 3440/50462 | Loss: 3.0465 | LR: 4.96e-04 | 3995.5ms/step | 15.0 samples/s
+Step 3450/50462 | Loss: 3.0772 | LR: 4.96e-04 | 3982.9ms/step | 15.1 samples/s
+Step 3460/50462 | Loss: 3.0613 | LR: 4.96e-04 | 3998.2ms/step | 15.0 samples/s
+Step 3470/50462 | Loss: 3.0546 | LR: 4.96e-04 | 3995.2ms/step | 15.0 samples/s
+Step 3480/50462 | Loss: 3.0614 | LR: 4.96e-04 | 3993.7ms/step | 15.0 samples/s
+Step 3490/50462 | Loss: 3.0375 | LR: 4.96e-04 | 3995.9ms/step | 15.0 samples/s
+Step 3500/50462 | Loss: 3.0413 | LR: 4.96e-04 | 3985.4ms/step | 15.1 samples/s
+Validation at step 3500: Loss = 3.0330, Perplexity = 20.76
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-3500
+Step 3510/50462 | Loss: 3.0623 | LR: 4.96e-04 | 6012.7ms/step | 10.0 samples/s
+Step 3520/50462 | Loss: 3.0407 | LR: 4.96e-04 | 3991.3ms/step | 15.0 samples/s
+Step 3530/50462 | Loss: 3.0732 | LR: 4.96e-04 | 3989.5ms/step | 15.0 samples/s
+Step 3540/50462 | Loss: 3.0360 | LR: 4.96e-04 | 3988.9ms/step | 15.0 samples/s
+Step 3550/50462 | Loss: 3.0210 | LR: 4.96e-04 | 3980.2ms/step | 15.1 samples/s
+Step 3560/50462 | Loss: 3.0425 | LR: 4.96e-04 | 3989.3ms/step | 15.0 samples/s
+Step 3570/50462 | Loss: 3.0507 | LR: 4.96e-04 | 3979.4ms/step | 15.1 samples/s
+Step 3580/50462 | Loss: 3.0449 | LR: 4.96e-04 | 3995.1ms/step | 15.0 samples/s
+Step 3590/50462 | Loss: 3.0183 | LR: 4.96e-04 | 3993.7ms/step | 15.0 samples/s
+Step 3600/50462 | Loss: 3.0273 | LR: 4.96e-04 | 3985.1ms/step | 15.1 samples/s
+Step 3610/50462 | Loss: 3.0464 | LR: 4.96e-04 | 3991.0ms/step | 15.0 samples/s
+Step 3620/50462 | Loss: 3.0376 | LR: 4.96e-04 | 3983.9ms/step | 15.1 samples/s
+Step 3630/50462 | Loss: 3.0378 | LR: 4.96e-04 | 3988.5ms/step | 15.0 samples/s
+Step 3640/50462 | Loss: 3.0345 | LR: 4.96e-04 | 3978.0ms/step | 15.1 samples/s
+Step 3650/50462 | Loss: 3.0864 | LR: 4.96e-04 | 3982.7ms/step | 15.1 samples/s
+Step 3660/50462 | Loss: 3.0629 | LR: 4.96e-04 | 3992.8ms/step | 15.0 samples/s
+Step 3670/50462 | Loss: 3.0506 | LR: 4.96e-04 | 3989.1ms/step | 15.0 samples/s
+Step 3680/50462 | Loss: 3.0342 | LR: 4.96e-04 | 3999.7ms/step | 15.0 samples/s
+Step 3690/50462 | Loss: 3.0155 | LR: 4.96e-04 | 3986.5ms/step | 15.1 samples/s
+Step 3700/50462 | Loss: 3.0224 | LR: 4.96e-04 | 3990.2ms/step | 15.0 samples/s
+Step 3710/50462 | Loss: 3.0501 | LR: 4.96e-04 | 3985.3ms/step | 15.1 samples/s
+Step 3720/50462 | Loss: 3.0234 | LR: 4.96e-04 | 3977.1ms/step | 15.1 samples/s
+Step 3730/50462 | Loss: 2.9859 | LR: 4.96e-04 | 3995.4ms/step | 15.0 samples/s
+Step 3740/50462 | Loss: 3.0114 | LR: 4.96e-04 | 3999.4ms/step | 15.0 samples/s
+Step 3750/50462 | Loss: 2.9975 | LR: 4.96e-04 | 3983.5ms/step | 15.1 samples/s
+Validation at step 3750: Loss = 3.0106, Perplexity = 20.30
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-3750
+Step 3760/50462 | Loss: 3.0581 | LR: 4.95e-04 | 6009.7ms/step | 10.0 samples/s
+Step 3770/50462 | Loss: 3.0387 | LR: 4.95e-04 | 3992.9ms/step | 15.0 samples/s
+Step 3780/50462 | Loss: 3.0441 | LR: 4.95e-04 | 3974.2ms/step | 15.1 samples/s
+Step 3790/50462 | Loss: 3.0143 | LR: 4.95e-04 | 3982.4ms/step | 15.1 samples/s
+Step 3800/50462 | Loss: 2.9959 | LR: 4.95e-04 | 3993.9ms/step | 15.0 samples/s
+Step 3810/50462 | Loss: 3.0031 | LR: 4.95e-04 | 3986.4ms/step | 15.1 samples/s
+Step 3820/50462 | Loss: 3.0166 | LR: 4.95e-04 | 3997.3ms/step | 15.0 samples/s
+Step 3830/50462 | Loss: 3.0050 | LR: 4.95e-04 | 4018.2ms/step | 14.9 samples/s
+Step 3840/50462 | Loss: 3.0229 | LR: 4.95e-04 | 3995.8ms/step | 15.0 samples/s
+Step 3850/50462 | Loss: 2.9943 | LR: 4.95e-04 | 3989.1ms/step | 15.0 samples/s
+Step 3860/50462 | Loss: 2.9873 | LR: 4.95e-04 | 3982.6ms/step | 15.1 samples/s
+Step 3870/50462 | Loss: 3.0147 | LR: 4.95e-04 | 3992.9ms/step | 15.0 samples/s
+Step 3880/50462 | Loss: 3.0289 | LR: 4.95e-04 | 3990.0ms/step | 15.0 samples/s
+Step 3890/50462 | Loss: 2.9880 | LR: 4.95e-04 | 3979.2ms/step | 15.1 samples/s
+Step 3900/50462 | Loss: 2.9929 | LR: 4.95e-04 | 3983.1ms/step | 15.1 samples/s
+Step 3910/50462 | Loss: 3.0313 | LR: 4.95e-04 | 3993.0ms/step | 15.0 samples/s
+Step 3920/50462 | Loss: 3.0375 | LR: 4.95e-04 | 3997.1ms/step | 15.0 samples/s
+Step 3930/50462 | Loss: 3.0633 | LR: 4.95e-04 | 3988.7ms/step | 15.0 samples/s
+Step 3940/50462 | Loss: 3.0146 | LR: 4.95e-04 | 3987.2ms/step | 15.0 samples/s
+Step 3950/50462 | Loss: 3.0253 | LR: 4.95e-04 | 3986.9ms/step | 15.0 samples/s
+Step 3960/50462 | Loss: 3.0296 | LR: 4.95e-04 | 3994.0ms/step | 15.0 samples/s
+Step 3970/50462 | Loss: 3.0175 | LR: 4.95e-04 | 4006.5ms/step | 15.0 samples/s
+Step 3980/50462 | Loss: 2.9935 | LR: 4.95e-04 | 4013.1ms/step | 15.0 samples/s
+Step 3990/50462 | Loss: 3.0153 | LR: 4.95e-04 | 4007.2ms/step | 15.0 samples/s
+Step 4000/50462 | Loss: 3.0265 | LR: 4.95e-04 | 4003.2ms/step | 15.0 samples/s
+Validation at step 4000: Loss = 2.9910, Perplexity = 19.91
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-4000
+Step 4010/50462 | Loss: 3.0411 | LR: 4.95e-04 | 6043.6ms/step | 9.9 samples/s
+Step 4020/50462 | Loss: 3.0378 | LR: 4.95e-04 | 3993.6ms/step | 15.0 samples/s
+Step 4030/50462 | Loss: 2.9892 | LR: 4.95e-04 | 3998.6ms/step | 15.0 samples/s
+Step 4040/50462 | Loss: 3.0110 | LR: 4.95e-04 | 3993.9ms/step | 15.0 samples/s
+Step 4050/50462 | Loss: 2.9983 | LR: 4.95e-04 | 4003.7ms/step | 15.0 samples/s
+Step 4060/50462 | Loss: 2.9678 | LR: 4.95e-04 | 3998.7ms/step | 15.0 samples/s
+Step 4070/50462 | Loss: 3.0065 | LR: 4.95e-04 | 3987.1ms/step | 15.0 samples/s
+Step 4080/50462 | Loss: 3.0095 | LR: 4.94e-04 | 3984.6ms/step | 15.1 samples/s
+Step 4090/50462 | Loss: 2.9931 | LR: 4.94e-04 | 3984.7ms/step | 15.1 samples/s
+Step 4100/50462 | Loss: 2.9956 | LR: 4.94e-04 | 3986.9ms/step | 15.0 samples/s
+Step 4110/50462 | Loss: 2.9643 | LR: 4.94e-04 | 3992.8ms/step | 15.0 samples/s
+Step 4120/50462 | Loss: 2.9810 | LR: 4.94e-04 | 3995.5ms/step | 15.0 samples/s
+Step 4130/50462 | Loss: 3.0015 | LR: 4.94e-04 | 3983.9ms/step | 15.1 samples/s
+Step 4140/50462 | Loss: 2.9456 | LR: 4.94e-04 | 3992.9ms/step | 15.0 samples/s
+Step 4150/50462 | Loss: 2.9780 | LR: 4.94e-04 | 3976.6ms/step | 15.1 samples/s
+Step 4160/50462 | Loss: 3.0106 | LR: 4.94e-04 | 3994.4ms/step | 15.0 samples/s
+Step 4170/50462 | Loss: 2.9951 | LR: 4.94e-04 | 4004.2ms/step | 15.0 samples/s
+Step 4180/50462 | Loss: 3.0016 | LR: 4.94e-04 | 4005.1ms/step | 15.0 samples/s
+Step 4190/50462 | Loss: 3.0069 | LR: 4.94e-04 | 3993.8ms/step | 15.0 samples/s
+Step 4200/50462 | Loss: 2.9877 | LR: 4.94e-04 | 4009.9ms/step | 15.0 samples/s
+Step 4210/50462 | Loss: 2.9953 | LR: 4.94e-04 | 4008.6ms/step | 15.0 samples/s
+Step 4220/50462 | Loss: 2.9786 | LR: 4.94e-04 | 4017.4ms/step | 14.9 samples/s
+Step 4230/50462 | Loss: 2.9787 | LR: 4.94e-04 | 3999.1ms/step | 15.0 samples/s
+Step 4240/50462 | Loss: 2.9712 | LR: 4.94e-04 | 4008.1ms/step | 15.0 samples/s
+Step 4250/50462 | Loss: 3.0066 | LR: 4.94e-04 | 4010.3ms/step | 15.0 samples/s
+Validation at step 4250: Loss = 2.9732, Perplexity = 19.56
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-4250
+Step 4260/50462 | Loss: 2.9733 | LR: 4.94e-04 | 6008.2ms/step | 10.0 samples/s
+Step 4270/50462 | Loss: 3.0194 | LR: 4.94e-04 | 4008.4ms/step | 15.0 samples/s
+Step 4280/50462 | Loss: 3.0221 | LR: 4.94e-04 | 4001.4ms/step | 15.0 samples/s
+Step 4290/50462 | Loss: 2.9857 | LR: 4.94e-04 | 4011.5ms/step | 15.0 samples/s
+Step 4300/50462 | Loss: 2.9612 | LR: 4.94e-04 | 3997.0ms/step | 15.0 samples/s
+Step 4310/50462 | Loss: 2.9813 | LR: 4.94e-04 | 3996.7ms/step | 15.0 samples/s
+Step 4320/50462 | Loss: 2.9991 | LR: 4.94e-04 | 3990.0ms/step | 15.0 samples/s
+Step 4330/50462 | Loss: 3.0268 | LR: 4.94e-04 | 4003.1ms/step | 15.0 samples/s
+Step 4340/50462 | Loss: 2.9888 | LR: 4.94e-04 | 3992.0ms/step | 15.0 samples/s
+Step 4350/50462 | Loss: 3.0124 | LR: 4.94e-04 | 3980.6ms/step | 15.1 samples/s
+Step 4360/50462 | Loss: 3.0025 | LR: 4.94e-04 | 4000.9ms/step | 15.0 samples/s
+Step 4370/50462 | Loss: 2.9873 | LR: 4.93e-04 | 4007.1ms/step | 15.0 samples/s
+Step 4380/50462 | Loss: 2.9763 | LR: 4.93e-04 | 4006.2ms/step | 15.0 samples/s
+Step 4390/50462 | Loss: 2.9502 | LR: 4.93e-04 | 3998.9ms/step | 15.0 samples/s
+Step 4400/50462 | Loss: 2.9783 | LR: 4.93e-04 | 4003.0ms/step | 15.0 samples/s
+Step 4410/50462 | Loss: 2.9989 | LR: 4.93e-04 | 4003.8ms/step | 15.0 samples/s
+Step 4420/50462 | Loss: 2.9809 | LR: 4.93e-04 | 4014.1ms/step | 14.9 samples/s
+Step 4430/50462 | Loss: 2.9535 | LR: 4.93e-04 | 3998.0ms/step | 15.0 samples/s
+Step 4440/50462 | Loss: 2.9645 | LR: 4.93e-04 | 4005.9ms/step | 15.0 samples/s
+Step 4450/50462 | Loss: 2.9722 | LR: 4.93e-04 | 3998.0ms/step | 15.0 samples/s
+Step 4460/50462 | Loss: 3.0133 | LR: 4.93e-04 | 4000.6ms/step | 15.0 samples/s
+Step 4470/50462 | Loss: 3.0149 | LR: 4.93e-04 | 3989.7ms/step | 15.0 samples/s
+Step 4480/50462 | Loss: 2.9755 | LR: 4.93e-04 | 3994.4ms/step | 15.0 samples/s
+Step 4490/50462 | Loss: 2.9749 | LR: 4.93e-04 | 3988.1ms/step | 15.0 samples/s
+Step 4500/50462 | Loss: 2.9707 | LR: 4.93e-04 | 3994.2ms/step | 15.0 samples/s
+Validation at step 4500: Loss = 2.9560, Perplexity = 19.22
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-4500
+Step 4510/50462 | Loss: 2.9659 | LR: 4.93e-04 | 6010.0ms/step | 10.0 samples/s
+Step 4520/50462 | Loss: 2.9517 | LR: 4.93e-04 | 3995.2ms/step | 15.0 samples/s
+Step 4530/50462 | Loss: 2.9667 | LR: 4.93e-04 | 3988.7ms/step | 15.0 samples/s
+Step 4540/50462 | Loss: 2.9517 | LR: 4.93e-04 | 3991.5ms/step | 15.0 samples/s
+Step 4550/50462 | Loss: 2.9767 | LR: 4.93e-04 | 3999.5ms/step | 15.0 samples/s
+Step 4560/50462 | Loss: 2.9688 | LR: 4.93e-04 | 3994.8ms/step | 15.0 samples/s
+Step 4570/50462 | Loss: 2.9816 | LR: 4.93e-04 | 4003.7ms/step | 15.0 samples/s
+Step 4580/50462 | Loss: 2.9599 | LR: 4.93e-04 | 3992.4ms/step | 15.0 samples/s
+Step 4590/50462 | Loss: 2.9828 | LR: 4.93e-04 | 3995.2ms/step | 15.0 samples/s
+Step 4600/50462 | Loss: 2.9727 | LR: 4.93e-04 | 4000.3ms/step | 15.0 samples/s
+Step 4610/50462 | Loss: 2.9293 | LR: 4.93e-04 | 3999.7ms/step | 15.0 samples/s
+Step 4620/50462 | Loss: 2.9533 | LR: 4.93e-04 | 3989.2ms/step | 15.0 samples/s
+Step 4630/50462 | Loss: 2.9428 | LR: 4.93e-04 | 3995.8ms/step | 15.0 samples/s
+Step 4640/50462 | Loss: 3.0061 | LR: 4.92e-04 | 4001.8ms/step | 15.0 samples/s
+Step 4650/50462 | Loss: 2.9444 | LR: 4.92e-04 | 4006.6ms/step | 15.0 samples/s
+Step 4660/50462 | Loss: 2.9510 | LR: 4.92e-04 | 3989.6ms/step | 15.0 samples/s
+Step 4670/50462 | Loss: 2.9671 | LR: 4.92e-04 | 3996.8ms/step | 15.0 samples/s
+Step 4680/50462 | Loss: 2.9347 | LR: 4.92e-04 | 3983.7ms/step | 15.1 samples/s
+Step 4690/50462 | Loss: 2.9624 | LR: 4.92e-04 | 4000.6ms/step | 15.0 samples/s
+Step 4700/50462 | Loss: 2.9583 | LR: 4.92e-04 | 3990.2ms/step | 15.0 samples/s
+Step 4710/50462 | Loss: 2.9622 | LR: 4.92e-04 | 4003.0ms/step | 15.0 samples/s
+Step 4720/50462 | Loss: 2.9400 | LR: 4.92e-04 | 3993.4ms/step | 15.0 samples/s
+Step 4730/50462 | Loss: 2.9385 | LR: 4.92e-04 | 4069.5ms/step | 14.7 samples/s
+Step 4740/50462 | Loss: 2.9569 | LR: 4.92e-04 | 4047.9ms/step | 14.8 samples/s
+Step 4750/50462 | Loss: 2.9578 | LR: 4.92e-04 | 4014.7ms/step | 14.9 samples/s
+Validation at step 4750: Loss = 2.9431, Perplexity = 18.98
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-4750
+Step 4760/50462 | Loss: 2.9421 | LR: 4.92e-04 | 6107.8ms/step | 9.8 samples/s
+Step 4770/50462 | Loss: 2.9415 | LR: 4.92e-04 | 3990.3ms/step | 15.0 samples/s
+Step 4780/50462 | Loss: 2.9117 | LR: 4.92e-04 | 3991.7ms/step | 15.0 samples/s
+Step 4790/50462 | Loss: 2.9611 | LR: 4.92e-04 | 3986.9ms/step | 15.0 samples/s
+Step 4800/50462 | Loss: 2.9396 | LR: 4.92e-04 | 3992.1ms/step | 15.0 samples/s
+Step 4810/50462 | Loss: 2.9931 | LR: 4.92e-04 | 3993.3ms/step | 15.0 samples/s
+Step 4820/50462 | Loss: 3.0074 | LR: 4.92e-04 | 3996.7ms/step | 15.0 samples/s
+Step 4830/50462 | Loss: 2.9687 | LR: 4.92e-04 | 3988.9ms/step | 15.0 samples/s
+Step 4840/50462 | Loss: 2.9313 | LR: 4.92e-04 | 3989.5ms/step | 15.0 samples/s
+Step 4850/50462 | Loss: 2.9501 | LR: 4.92e-04 | 3990.9ms/step | 15.0 samples/s
+Step 4860/50462 | Loss: 2.9635 | LR: 4.92e-04 | 3997.5ms/step | 15.0 samples/s
+Step 4870/50462 | Loss: 2.9523 | LR: 4.92e-04 | 4000.8ms/step | 15.0 samples/s
+Step 4880/50462 | Loss: 2.9503 | LR: 4.92e-04 | 3988.6ms/step | 15.0 samples/s
+Step 4890/50462 | Loss: 2.9399 | LR: 4.91e-04 | 3990.7ms/step | 15.0 samples/s
+Step 4900/50462 | Loss: 2.9332 | LR: 4.91e-04 | 3984.4ms/step | 15.1 samples/s
+Step 4910/50462 | Loss: 2.9570 | LR: 4.91e-04 | 3988.0ms/step | 15.0 samples/s
+Step 4920/50462 | Loss: 2.9496 | LR: 4.91e-04 | 3979.6ms/step | 15.1 samples/s
+Step 4930/50462 | Loss: 2.9585 | LR: 4.91e-04 | 3991.7ms/step | 15.0 samples/s
+Step 4940/50462 | Loss: 2.9479 | LR: 4.91e-04 | 4001.3ms/step | 15.0 samples/s
+Step 4950/50462 | Loss: 2.9230 | LR: 4.91e-04 | 3993.7ms/step | 15.0 samples/s
+Step 4960/50462 | Loss: 2.9649 | LR: 4.91e-04 | 3986.9ms/step | 15.0 samples/s
+Step 4970/50462 | Loss: 2.9156 | LR: 4.91e-04 | 3994.3ms/step | 15.0 samples/s
+Step 4980/50462 | Loss: 2.9427 | LR: 4.91e-04 | 3999.2ms/step | 15.0 samples/s
+Step 4990/50462 | Loss: 2.9468 | LR: 4.91e-04 | 3996.6ms/step | 15.0 samples/s
+Step 5000/50462 | Loss: 2.9317 | LR: 4.91e-04 | 3984.3ms/step | 15.1 samples/s
+Validation at step 5000: Loss = 2.9257, Perplexity = 18.65
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-5000
+Step 5010/50462 | Loss: 2.9447 | LR: 4.91e-04 | 5973.7ms/step | 10.0 samples/s
+Step 5020/50462 | Loss: 2.9276 | LR: 4.91e-04 | 3983.3ms/step | 15.1 samples/s
+Step 5030/50462 | Loss: 2.8910 | LR: 4.91e-04 | 3984.6ms/step | 15.1 samples/s
+Step 5040/50462 | Loss: 2.9365 | LR: 4.91e-04 | 3985.3ms/step | 15.1 samples/s
+Step 5050/50462 | Loss: 2.9473 | LR: 4.91e-04 | 3986.6ms/step | 15.1 samples/s
+Step 5060/50462 | Loss: 2.9685 | LR: 4.91e-04 | 3992.8ms/step | 15.0 samples/s
+Step 5070/50462 | Loss: 2.9517 | LR: 4.91e-04 | 3991.5ms/step | 15.0 samples/s
+Step 5080/50462 | Loss: 2.9092 | LR: 4.91e-04 | 4012.3ms/step | 15.0 samples/s
+Step 5090/50462 | Loss: 2.8969 | LR: 4.91e-04 | 4009.8ms/step | 15.0 samples/s
+Step 5100/50462 | Loss: 2.9286 | LR: 4.91e-04 | 4002.0ms/step | 15.0 samples/s
+Step 5110/50462 | Loss: 2.9720 | LR: 4.91e-04 | 4005.4ms/step | 15.0 samples/s
+Step 5120/50462 | Loss: 2.9403 | LR: 4.91e-04 | 3998.1ms/step | 15.0 samples/s
+Step 5130/50462 | Loss: 2.9276 | LR: 4.90e-04 | 3990.4ms/step | 15.0 samples/s
+Step 5140/50462 | Loss: 2.9253 | LR: 4.90e-04 | 3988.8ms/step | 15.0 samples/s
+Step 5150/50462 | Loss: 2.9352 | LR: 4.90e-04 | 3998.1ms/step | 15.0 samples/s
+Step 5160/50462 | Loss: 2.9306 | LR: 4.90e-04 | 3991.5ms/step | 15.0 samples/s
+Step 5170/50462 | Loss: 2.9297 | LR: 4.90e-04 | 3993.8ms/step | 15.0 samples/s
+Step 5180/50462 | Loss: 2.9470 | LR: 4.90e-04 | 3993.2ms/step | 15.0 samples/s
+Step 5190/50462 | Loss: 2.9442 | LR: 4.90e-04 | 4000.7ms/step | 15.0 samples/s
+Step 5200/50462 | Loss: 2.9267 | LR: 4.90e-04 | 4002.6ms/step | 15.0 samples/s
+Step 5210/50462 | Loss: 2.8954 | LR: 4.90e-04 | 3991.1ms/step | 15.0 samples/s
+Step 5220/50462 | Loss: 2.9127 | LR: 4.90e-04 | 3982.6ms/step | 15.1 samples/s
+Step 5230/50462 | Loss: 2.9509 | LR: 4.90e-04 | 3990.5ms/step | 15.0 samples/s
+Step 5240/50462 | Loss: 2.9200 | LR: 4.90e-04 | 3991.6ms/step | 15.0 samples/s
+Step 5250/50462 | Loss: 2.9357 | LR: 4.90e-04 | 3982.7ms/step | 15.1 samples/s
+Validation at step 5250: Loss = 2.9127, Perplexity = 18.41
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-5250
+Step 5260/50462 | Loss: 2.9257 | LR: 4.90e-04 | 5997.5ms/step | 10.0 samples/s
+Step 5270/50462 | Loss: 2.9084 | LR: 4.90e-04 | 3991.8ms/step | 15.0 samples/s
+Step 5280/50462 | Loss: 2.9156 | LR: 4.90e-04 | 3996.8ms/step | 15.0 samples/s
+Step 5290/50462 | Loss: 2.9273 | LR: 4.90e-04 | 3992.1ms/step | 15.0 samples/s
+Step 5300/50462 | Loss: 2.8926 | LR: 4.90e-04 | 3995.1ms/step | 15.0 samples/s
+Step 5310/50462 | Loss: 2.9101 | LR: 4.90e-04 | 3992.4ms/step | 15.0 samples/s
+Step 5320/50462 | Loss: 2.8965 | LR: 4.90e-04 | 3995.2ms/step | 15.0 samples/s
+Step 5330/50462 | Loss: 2.9197 | LR: 4.90e-04 | 3984.3ms/step | 15.1 samples/s
+Step 5340/50462 | Loss: 2.9021 | LR: 4.90e-04 | 3996.2ms/step | 15.0 samples/s
+Step 5350/50462 | Loss: 2.8940 | LR: 4.90e-04 | 3989.7ms/step | 15.0 samples/s
+Step 5360/50462 | Loss: 2.9121 | LR: 4.89e-04 | 3989.7ms/step | 15.0 samples/s
+Step 5370/50462 | Loss: 2.9144 | LR: 4.89e-04 | 3987.6ms/step | 15.0 samples/s
+Step 5380/50462 | Loss: 2.8899 | LR: 4.89e-04 | 3998.6ms/step | 15.0 samples/s
+Step 5390/50462 | Loss: 2.9304 | LR: 4.89e-04 | 3996.2ms/step | 15.0 samples/s
+Step 5400/50462 | Loss: 2.9357 | LR: 4.89e-04 | 3990.8ms/step | 15.0 samples/s
+Step 5410/50462 | Loss: 2.9760 | LR: 4.89e-04 | 3984.6ms/step | 15.1 samples/s
+Step 5420/50462 | Loss: 2.9028 | LR: 4.89e-04 | 3994.0ms/step | 15.0 samples/s
+Step 5430/50462 | Loss: 2.9555 | LR: 4.89e-04 | 3983.2ms/step | 15.1 samples/s
+Step 5440/50462 | Loss: 2.9203 | LR: 4.89e-04 | 3990.7ms/step | 15.0 samples/s
+Step 5450/50462 | Loss: 2.9186 | LR: 4.89e-04 | 3976.9ms/step | 15.1 samples/s
+Step 5460/50462 | Loss: 2.9068 | LR: 4.89e-04 | 3994.5ms/step | 15.0 samples/s
+Step 5470/50462 | Loss: 2.9363 | LR: 4.89e-04 | 3999.2ms/step | 15.0 samples/s
+Step 5480/50462 | Loss: 2.9324 | LR: 4.89e-04 | 3985.5ms/step | 15.1 samples/s
+Step 5490/50462 | Loss: 2.9109 | LR: 4.89e-04 | 3988.4ms/step | 15.0 samples/s
+Step 5500/50462 | Loss: 2.9602 | LR: 4.89e-04 | 4002.6ms/step | 15.0 samples/s
+Validation at step 5500: Loss = 2.8996, Perplexity = 18.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-5500
+Step 5510/50462 | Loss: 2.9205 | LR: 4.89e-04 | 6011.7ms/step | 10.0 samples/s
+Step 5520/50462 | Loss: 2.9043 | LR: 4.89e-04 | 3995.3ms/step | 15.0 samples/s
+Step 5530/50462 | Loss: 2.9359 | LR: 4.89e-04 | 3993.9ms/step | 15.0 samples/s
+Step 5540/50462 | Loss: 2.8919 | LR: 4.89e-04 | 3989.6ms/step | 15.0 samples/s
+Step 5550/50462 | Loss: 2.9358 | LR: 4.89e-04 | 4002.9ms/step | 15.0 samples/s
+Step 5560/50462 | Loss: 2.9147 | LR: 4.89e-04 | 4011.4ms/step | 15.0 samples/s
+Step 5570/50462 | Loss: 2.9171 | LR: 4.88e-04 | 4008.3ms/step | 15.0 samples/s
+Step 5580/50462 | Loss: 2.9334 | LR: 4.88e-04 | 3981.6ms/step | 15.1 samples/s
+Step 5590/50462 | Loss: 2.8927 | LR: 4.88e-04 | 3998.7ms/step | 15.0 samples/s
+Step 5600/50462 | Loss: 2.9366 | LR: 4.88e-04 | 3989.2ms/step | 15.0 samples/s
+Step 5610/50462 | Loss: 2.9151 | LR: 4.88e-04 | 3994.7ms/step | 15.0 samples/s
+Step 5620/50462 | Loss: 2.9004 | LR: 4.88e-04 | 3988.7ms/step | 15.0 samples/s
+Step 5630/50462 | Loss: 2.9035 | LR: 4.88e-04 | 3995.2ms/step | 15.0 samples/s
+Step 5640/50462 | Loss: 2.8853 | LR: 4.88e-04 | 3988.6ms/step | 15.0 samples/s
+Step 5650/50462 | Loss: 2.9257 | LR: 4.88e-04 | 3992.4ms/step | 15.0 samples/s
+Step 5660/50462 | Loss: 2.9348 | LR: 4.88e-04 | 3986.9ms/step | 15.0 samples/s
+Step 5670/50462 | Loss: 2.9003 | LR: 4.88e-04 | 3997.7ms/step | 15.0 samples/s
+Step 5680/50462 | Loss: 2.9290 | LR: 4.88e-04 | 4014.3ms/step | 14.9 samples/s
+Step 5690/50462 | Loss: 2.9020 | LR: 4.88e-04 | 4012.3ms/step | 15.0 samples/s
+Step 5700/50462 | Loss: 2.8653 | LR: 4.88e-04 | 3999.1ms/step | 15.0 samples/s
+Step 5710/50462 | Loss: 2.8876 | LR: 4.88e-04 | 3994.8ms/step | 15.0 samples/s
+Step 5720/50462 | Loss: 2.8971 | LR: 4.88e-04 | 4006.1ms/step | 15.0 samples/s
+Step 5730/50462 | Loss: 2.8780 | LR: 4.88e-04 | 3997.3ms/step | 15.0 samples/s
+Step 5740/50462 | Loss: 2.9141 | LR: 4.88e-04 | 3998.9ms/step | 15.0 samples/s
+Step 5750/50462 | Loss: 2.8787 | LR: 4.88e-04 | 3989.4ms/step | 15.0 samples/s
+Validation at step 5750: Loss = 2.8896, Perplexity = 17.99
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-5750
+Step 5760/50462 | Loss: 2.8969 | LR: 4.88e-04 | 5994.8ms/step | 10.0 samples/s
+Step 5770/50462 | Loss: 2.8653 | LR: 4.88e-04 | 4006.2ms/step | 15.0 samples/s
+Step 5780/50462 | Loss: 2.8977 | LR: 4.87e-04 | 4012.1ms/step | 15.0 samples/s
+Step 5790/50462 | Loss: 2.9166 | LR: 4.87e-04 | 3994.8ms/step | 15.0 samples/s
+Step 5800/50462 | Loss: 2.9178 | LR: 4.87e-04 | 3992.4ms/step | 15.0 samples/s
+Step 5810/50462 | Loss: 2.8948 | LR: 4.87e-04 | 3980.7ms/step | 15.1 samples/s
+Step 5820/50462 | Loss: 2.8983 | LR: 4.87e-04 | 3986.4ms/step | 15.1 samples/s
+Step 5830/50462 | Loss: 2.9053 | LR: 4.87e-04 | 3992.3ms/step | 15.0 samples/s
+Step 5840/50462 | Loss: 2.8874 | LR: 4.87e-04 | 3996.1ms/step | 15.0 samples/s
+Step 5850/50462 | Loss: 2.8796 | LR: 4.87e-04 | 3998.1ms/step | 15.0 samples/s
+Step 5860/50462 | Loss: 2.9341 | LR: 4.87e-04 | 3982.3ms/step | 15.1 samples/s
+Step 5870/50462 | Loss: 2.8984 | LR: 4.87e-04 | 3999.4ms/step | 15.0 samples/s
+Step 5880/50462 | Loss: 2.8723 | LR: 4.87e-04 | 3996.4ms/step | 15.0 samples/s
+Step 5890/50462 | Loss: 2.8854 | LR: 4.87e-04 | 3986.9ms/step | 15.0 samples/s
+Step 5900/50462 | Loss: 2.8809 | LR: 4.87e-04 | 3972.7ms/step | 15.1 samples/s
+Step 5910/50462 | Loss: 2.9373 | LR: 4.87e-04 | 3973.5ms/step | 15.1 samples/s
+Step 5920/50462 | Loss: 2.8635 | LR: 4.87e-04 | 3984.7ms/step | 15.1 samples/s
+Step 5930/50462 | Loss: 2.8845 | LR: 4.87e-04 | 3992.4ms/step | 15.0 samples/s
+Step 5940/50462 | Loss: 2.9102 | LR: 4.87e-04 | 3985.6ms/step | 15.1 samples/s
+Step 5950/50462 | Loss: 2.8786 | LR: 4.87e-04 | 3980.7ms/step | 15.1 samples/s
+Step 5960/50462 | Loss: 2.8893 | LR: 4.87e-04 | 3985.1ms/step | 15.1 samples/s
+Step 5970/50462 | Loss: 2.8962 | LR: 4.87e-04 | 3995.0ms/step | 15.0 samples/s
+Step 5980/50462 | Loss: 2.8859 | LR: 4.86e-04 | 3987.8ms/step | 15.0 samples/s
+Step 5990/50462 | Loss: 2.8780 | LR: 4.86e-04 | 3989.8ms/step | 15.0 samples/s
+Step 6000/50462 | Loss: 2.8776 | LR: 4.86e-04 | 3988.8ms/step | 15.0 samples/s
+Validation at step 6000: Loss = 2.8804, Perplexity = 17.82
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-6000
+Step 6010/50462 | Loss: 2.9013 | LR: 4.86e-04 | 5952.9ms/step | 10.1 samples/s
+Step 6020/50462 | Loss: 2.8886 | LR: 4.86e-04 | 3994.7ms/step | 15.0 samples/s
+Step 6030/50462 | Loss: 2.9153 | LR: 4.86e-04 | 3979.5ms/step | 15.1 samples/s
+Step 6040/50462 | Loss: 2.8790 | LR: 4.86e-04 | 4004.4ms/step | 15.0 samples/s
+Step 6050/50462 | Loss: 2.8708 | LR: 4.86e-04 | 3992.5ms/step | 15.0 samples/s
+Step 6060/50462 | Loss: 2.8571 | LR: 4.86e-04 | 3985.5ms/step | 15.1 samples/s
+Step 6070/50462 | Loss: 2.8786 | LR: 4.86e-04 | 3982.8ms/step | 15.1 samples/s
+Step 6080/50462 | Loss: 2.8614 | LR: 4.86e-04 | 3988.0ms/step | 15.0 samples/s
+Step 6090/50462 | Loss: 2.9012 | LR: 4.86e-04 | 3986.9ms/step | 15.0 samples/s
+Step 6100/50462 | Loss: 2.9043 | LR: 4.86e-04 | 3987.6ms/step | 15.0 samples/s
+Step 6110/50462 | Loss: 2.8635 | LR: 4.86e-04 | 3987.2ms/step | 15.0 samples/s
+Step 6120/50462 | Loss: 2.8498 | LR: 4.86e-04 | 3992.3ms/step | 15.0 samples/s
+Step 6130/50462 | Loss: 2.8763 | LR: 4.86e-04 | 3986.3ms/step | 15.1 samples/s
+Step 6140/50462 | Loss: 2.8952 | LR: 4.86e-04 | 4006.0ms/step | 15.0 samples/s
+Step 6150/50462 | Loss: 2.9106 | LR: 4.86e-04 | 3984.0ms/step | 15.1 samples/s
+Step 6160/50462 | Loss: 2.8710 | LR: 4.86e-04 | 3999.5ms/step | 15.0 samples/s
+Step 6170/50462 | Loss: 2.8991 | LR: 4.85e-04 | 3995.1ms/step | 15.0 samples/s
+Step 6180/50462 | Loss: 2.8730 | LR: 4.85e-04 | 3990.2ms/step | 15.0 samples/s
+Step 6190/50462 | Loss: 2.8706 | LR: 4.85e-04 | 3974.2ms/step | 15.1 samples/s
+Step 6200/50462 | Loss: 2.9176 | LR: 4.85e-04 | 3982.1ms/step | 15.1 samples/s
+Step 6210/50462 | Loss: 2.8935 | LR: 4.85e-04 | 3999.5ms/step | 15.0 samples/s
+Step 6220/50462 | Loss: 2.8980 | LR: 4.85e-04 | 3994.3ms/step | 15.0 samples/s
+Step 6230/50462 | Loss: 2.8748 | LR: 4.85e-04 | 3984.3ms/step | 15.1 samples/s
+Step 6240/50462 | Loss: 2.8833 | LR: 4.85e-04 | 3988.2ms/step | 15.0 samples/s
+Step 6250/50462 | Loss: 2.8769 | LR: 4.85e-04 | 3982.3ms/step | 15.1 samples/s
+Validation at step 6250: Loss = 2.8691, Perplexity = 17.62
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-6250
+Step 6260/50462 | Loss: 2.8870 | LR: 4.85e-04 | 5965.5ms/step | 10.1 samples/s
+Step 6270/50462 | Loss: 2.8985 | LR: 4.85e-04 | 4000.0ms/step | 15.0 samples/s
+Step 6280/50462 | Loss: 2.9142 | LR: 4.85e-04 | 3991.5ms/step | 15.0 samples/s
+Step 6290/50462 | Loss: 2.8947 | LR: 4.85e-04 | 3986.1ms/step | 15.1 samples/s
+Step 6300/50462 | Loss: 2.9049 | LR: 4.85e-04 | 3990.5ms/step | 15.0 samples/s
+Step 6310/50462 | Loss: 2.8851 | LR: 4.85e-04 | 3985.4ms/step | 15.1 samples/s
+Step 6320/50462 | Loss: 2.8761 | LR: 4.85e-04 | 3997.3ms/step | 15.0 samples/s
+Step 6330/50462 | Loss: 2.8678 | LR: 4.85e-04 | 3983.6ms/step | 15.1 samples/s
+Step 6340/50462 | Loss: 2.8764 | LR: 4.85e-04 | 3995.9ms/step | 15.0 samples/s
+Step 6350/50462 | Loss: 2.8818 | LR: 4.85e-04 | 3990.7ms/step | 15.0 samples/s
+Step 6360/50462 | Loss: 2.9204 | LR: 4.84e-04 | 3991.7ms/step | 15.0 samples/s
+Step 6370/50462 | Loss: 2.8934 | LR: 4.84e-04 | 3984.2ms/step | 15.1 samples/s
+Step 6380/50462 | Loss: 2.8762 | LR: 4.84e-04 | 3975.6ms/step | 15.1 samples/s
+Step 6390/50462 | Loss: 2.8611 | LR: 4.84e-04 | 3994.9ms/step | 15.0 samples/s
+Step 6400/50462 | Loss: 2.8888 | LR: 4.84e-04 | 3989.7ms/step | 15.0 samples/s
+Step 6410/50462 | Loss: 2.8523 | LR: 4.84e-04 | 3993.5ms/step | 15.0 samples/s
+Step 6420/50462 | Loss: 2.8674 | LR: 4.84e-04 | 3989.8ms/step | 15.0 samples/s
+Step 6430/50462 | Loss: 2.8689 | LR: 4.84e-04 | 4001.9ms/step | 15.0 samples/s
+Step 6440/50462 | Loss: 2.8863 | LR: 4.84e-04 | 3984.6ms/step | 15.1 samples/s
+Step 6450/50462 | Loss: 2.8787 | LR: 4.84e-04 | 3992.7ms/step | 15.0 samples/s
+Step 6460/50462 | Loss: 2.8960 | LR: 4.84e-04 | 3988.4ms/step | 15.0 samples/s
+Step 6470/50462 | Loss: 2.9069 | LR: 4.84e-04 | 3983.1ms/step | 15.1 samples/s
+Step 6480/50462 | Loss: 2.8914 | LR: 4.84e-04 | 3996.5ms/step | 15.0 samples/s
+Step 6490/50462 | Loss: 2.8399 | LR: 4.84e-04 | 3987.5ms/step | 15.0 samples/s
+Step 6500/50462 | Loss: 2.8945 | LR: 4.84e-04 | 3990.3ms/step | 15.0 samples/s
+Validation at step 6500: Loss = 2.8610, Perplexity = 17.48
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-6500
+Step 6510/50462 | Loss: 2.8794 | LR: 4.84e-04 | 5946.9ms/step | 10.1 samples/s
+Step 6520/50462 | Loss: 2.8735 | LR: 4.84e-04 | 3987.7ms/step | 15.0 samples/s
+Step 6530/50462 | Loss: 2.8510 | LR: 4.84e-04 | 3982.5ms/step | 15.1 samples/s
+Step 6540/50462 | Loss: 2.8713 | LR: 4.83e-04 | 3987.9ms/step | 15.0 samples/s
+Step 6550/50462 | Loss: 2.8857 | LR: 4.83e-04 | 3991.2ms/step | 15.0 samples/s
+Step 6560/50462 | Loss: 2.8432 | LR: 4.83e-04 | 3983.8ms/step | 15.1 samples/s
+Step 6570/50462 | Loss: 2.8939 | LR: 4.83e-04 | 3993.7ms/step | 15.0 samples/s
+Step 6580/50462 | Loss: 2.8644 | LR: 4.83e-04 | 3997.7ms/step | 15.0 samples/s
+Step 6590/50462 | Loss: 2.8766 | LR: 4.83e-04 | 3994.1ms/step | 15.0 samples/s
+Step 6600/50462 | Loss: 2.8644 | LR: 4.83e-04 | 3988.3ms/step | 15.0 samples/s
+Step 6610/50462 | Loss: 2.8820 | LR: 4.83e-04 | 3995.2ms/step | 15.0 samples/s
+Step 6620/50462 | Loss: 2.8750 | LR: 4.83e-04 | 3986.3ms/step | 15.1 samples/s
+Step 6630/50462 | Loss: 2.8852 | LR: 4.83e-04 | 3986.3ms/step | 15.1 samples/s
+Step 6640/50462 | Loss: 2.8826 | LR: 4.83e-04 | 3985.1ms/step | 15.1 samples/s
+Step 6650/50462 | Loss: 2.8387 | LR: 4.83e-04 | 4007.3ms/step | 15.0 samples/s
+Step 6660/50462 | Loss: 2.8703 | LR: 4.83e-04 | 3987.4ms/step | 15.0 samples/s
+Step 6670/50462 | Loss: 2.8756 | LR: 4.83e-04 | 3991.4ms/step | 15.0 samples/s
+Step 6680/50462 | Loss: 2.8170 | LR: 4.83e-04 | 3993.2ms/step | 15.0 samples/s
+Step 6690/50462 | Loss: 2.8597 | LR: 4.83e-04 | 4007.6ms/step | 15.0 samples/s
+Step 6700/50462 | Loss: 2.8211 | LR: 4.83e-04 | 4000.2ms/step | 15.0 samples/s
+Step 6710/50462 | Loss: 2.8495 | LR: 4.82e-04 | 3996.7ms/step | 15.0 samples/s
+Step 6720/50462 | Loss: 2.8756 | LR: 4.82e-04 | 3986.7ms/step | 15.0 samples/s
+Step 6730/50462 | Loss: 2.8878 | LR: 4.82e-04 | 3995.4ms/step | 15.0 samples/s
+Step 6740/50462 | Loss: 2.8462 | LR: 4.82e-04 | 3985.3ms/step | 15.1 samples/s
+Step 6750/50462 | Loss: 2.8417 | LR: 4.82e-04 | 3990.3ms/step | 15.0 samples/s
+Validation at step 6750: Loss = 2.8506, Perplexity = 17.30
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-6750
+Step 6760/50462 | Loss: 2.8728 | LR: 4.82e-04 | 5962.5ms/step | 10.1 samples/s
+Step 6770/50462 | Loss: 2.8898 | LR: 4.82e-04 | 3986.9ms/step | 15.0 samples/s
+Step 6780/50462 | Loss: 2.8978 | LR: 4.82e-04 | 3983.6ms/step | 15.1 samples/s
+Step 6790/50462 | Loss: 2.8648 | LR: 4.82e-04 | 3989.6ms/step | 15.0 samples/s
+Step 6800/50462 | Loss: 2.8366 | LR: 4.82e-04 | 3993.4ms/step | 15.0 samples/s
+Step 6810/50462 | Loss: 2.8906 | LR: 4.82e-04 | 3995.4ms/step | 15.0 samples/s
+Step 6820/50462 | Loss: 2.9028 | LR: 4.82e-04 | 3989.3ms/step | 15.0 samples/s
+Step 6830/50462 | Loss: 2.8177 | LR: 4.82e-04 | 3990.3ms/step | 15.0 samples/s
+Step 6840/50462 | Loss: 2.8562 | LR: 4.82e-04 | 3988.9ms/step | 15.0 samples/s
+Step 6850/50462 | Loss: 2.8484 | LR: 4.82e-04 | 3988.2ms/step | 15.0 samples/s
+Step 6860/50462 | Loss: 2.8783 | LR: 4.82e-04 | 3988.6ms/step | 15.0 samples/s
+Step 6870/50462 | Loss: 2.8613 | LR: 4.82e-04 | 3987.9ms/step | 15.0 samples/s
+Step 6880/50462 | Loss: 2.9112 | LR: 4.81e-04 | 3987.1ms/step | 15.0 samples/s
+Step 6890/50462 | Loss: 2.8660 | LR: 4.81e-04 | 3986.8ms/step | 15.0 samples/s
+Step 6900/50462 | Loss: 2.8635 | LR: 4.81e-04 | 3984.1ms/step | 15.1 samples/s
+Step 6910/50462 | Loss: 2.8328 | LR: 4.81e-04 | 4001.3ms/step | 15.0 samples/s
+Step 6920/50462 | Loss: 2.8597 | LR: 4.81e-04 | 4000.4ms/step | 15.0 samples/s
+Step 6930/50462 | Loss: 2.8544 | LR: 4.81e-04 | 3994.3ms/step | 15.0 samples/s
+Step 6940/50462 | Loss: 2.8624 | LR: 4.81e-04 | 3983.5ms/step | 15.1 samples/s
+Step 6950/50462 | Loss: 2.8439 | LR: 4.81e-04 | 3989.2ms/step | 15.0 samples/s
+Step 6960/50462 | Loss: 2.8431 | LR: 4.81e-04 | 3994.0ms/step | 15.0 samples/s
+Step 6970/50462 | Loss: 2.8725 | LR: 4.81e-04 | 3988.7ms/step | 15.0 samples/s
+Step 6980/50462 | Loss: 2.8759 | LR: 4.81e-04 | 4007.6ms/step | 15.0 samples/s
+Step 6990/50462 | Loss: 2.8470 | LR: 4.81e-04 | 3992.5ms/step | 15.0 samples/s
+Step 7000/50462 | Loss: 2.8479 | LR: 4.81e-04 | 3986.0ms/step | 15.1 samples/s
+Validation at step 7000: Loss = 2.8399, Perplexity = 17.11
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-7000
+Step 7010/50462 | Loss: 2.8623 | LR: 4.81e-04 | 6093.3ms/step | 9.8 samples/s
+Step 7020/50462 | Loss: 2.8466 | LR: 4.81e-04 | 3986.8ms/step | 15.0 samples/s
+Step 7030/50462 | Loss: 2.8529 | LR: 4.81e-04 | 3991.9ms/step | 15.0 samples/s
+Step 7040/50462 | Loss: 2.8591 | LR: 4.81e-04 | 3984.0ms/step | 15.1 samples/s
+Step 7050/50462 | Loss: 2.8442 | LR: 4.80e-04 | 3997.1ms/step | 15.0 samples/s
+Step 7060/50462 | Loss: 2.8518 | LR: 4.80e-04 | 3993.7ms/step | 15.0 samples/s
+Step 7070/50462 | Loss: 2.8505 | LR: 4.80e-04 | 3975.3ms/step | 15.1 samples/s
+Step 7080/50462 | Loss: 2.8231 | LR: 4.80e-04 | 3983.0ms/step | 15.1 samples/s
+Step 7090/50462 | Loss: 2.8237 | LR: 4.80e-04 | 3988.9ms/step | 15.0 samples/s
+Step 7100/50462 | Loss: 2.8186 | LR: 4.80e-04 | 3992.1ms/step | 15.0 samples/s
+Step 7110/50462 | Loss: 2.8395 | LR: 4.80e-04 | 3975.9ms/step | 15.1 samples/s
+Step 7120/50462 | Loss: 2.8450 | LR: 4.80e-04 | 3992.2ms/step | 15.0 samples/s
+Step 7130/50462 | Loss: 2.8577 | LR: 4.80e-04 | 3993.7ms/step | 15.0 samples/s
+Step 7140/50462 | Loss: 2.8574 | LR: 4.80e-04 | 3987.7ms/step | 15.0 samples/s
+Step 7150/50462 | Loss: 2.8311 | LR: 4.80e-04 | 3982.6ms/step | 15.1 samples/s
+Step 7160/50462 | Loss: 2.8738 | LR: 4.80e-04 | 3979.1ms/step | 15.1 samples/s
+Step 7170/50462 | Loss: 2.8583 | LR: 4.80e-04 | 3985.8ms/step | 15.1 samples/s
+Step 7180/50462 | Loss: 2.8457 | LR: 4.80e-04 | 4002.1ms/step | 15.0 samples/s
+Step 7190/50462 | Loss: 2.8576 | LR: 4.80e-04 | 3991.7ms/step | 15.0 samples/s
+Step 7200/50462 | Loss: 2.8909 | LR: 4.80e-04 | 3988.1ms/step | 15.0 samples/s
+Step 7210/50462 | Loss: 2.8437 | LR: 4.79e-04 | 3981.2ms/step | 15.1 samples/s
+Step 7220/50462 | Loss: 2.8743 | LR: 4.79e-04 | 3986.8ms/step | 15.0 samples/s
+Step 7230/50462 | Loss: 2.8322 | LR: 4.79e-04 | 3990.7ms/step | 15.0 samples/s
+Step 7240/50462 | Loss: 2.8249 | LR: 4.79e-04 | 3978.5ms/step | 15.1 samples/s
+Step 7250/50462 | Loss: 2.8418 | LR: 4.79e-04 | 3993.2ms/step | 15.0 samples/s
+Validation at step 7250: Loss = 2.8335, Perplexity = 17.00
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-7250
+Step 7260/50462 | Loss: 2.8316 | LR: 4.79e-04 | 5957.2ms/step | 10.1 samples/s
+Step 7270/50462 | Loss: 2.8407 | LR: 4.79e-04 | 3992.4ms/step | 15.0 samples/s
+Step 7280/50462 | Loss: 2.8223 | LR: 4.79e-04 | 3984.3ms/step | 15.1 samples/s
+Step 7290/50462 | Loss: 2.8543 | LR: 4.79e-04 | 3994.7ms/step | 15.0 samples/s
+Step 7300/50462 | Loss: 2.8391 | LR: 4.79e-04 | 3975.4ms/step | 15.1 samples/s
+Step 7310/50462 | Loss: 2.8516 | LR: 4.79e-04 | 3997.0ms/step | 15.0 samples/s
+Step 7320/50462 | Loss: 2.8645 | LR: 4.79e-04 | 4005.2ms/step | 15.0 samples/s
+Step 7330/50462 | Loss: 2.8290 | LR: 4.79e-04 | 3997.0ms/step | 15.0 samples/s
+Step 7340/50462 | Loss: 2.8552 | LR: 4.79e-04 | 3977.8ms/step | 15.1 samples/s
+Step 7350/50462 | Loss: 2.8284 | LR: 4.79e-04 | 3993.6ms/step | 15.0 samples/s
+Step 7360/50462 | Loss: 2.8674 | LR: 4.79e-04 | 3997.7ms/step | 15.0 samples/s
+Step 7370/50462 | Loss: 2.8721 | LR: 4.78e-04 | 3990.6ms/step | 15.0 samples/s
+Step 7380/50462 | Loss: 2.8423 | LR: 4.78e-04 | 3990.1ms/step | 15.0 samples/s
+Step 7390/50462 | Loss: 2.7893 | LR: 4.78e-04 | 3989.9ms/step | 15.0 samples/s
+Step 7400/50462 | Loss: 2.8188 | LR: 4.78e-04 | 3985.3ms/step | 15.1 samples/s
+Step 7410/50462 | Loss: 2.8570 | LR: 4.78e-04 | 3984.3ms/step | 15.1 samples/s
+Step 7420/50462 | Loss: 2.8868 | LR: 4.78e-04 | 3991.8ms/step | 15.0 samples/s
+Step 7430/50462 | Loss: 2.8462 | LR: 4.78e-04 | 3999.8ms/step | 15.0 samples/s
+Step 7440/50462 | Loss: 2.8698 | LR: 4.78e-04 | 3997.3ms/step | 15.0 samples/s
+Step 7450/50462 | Loss: 2.8474 | LR: 4.78e-04 | 3983.8ms/step | 15.1 samples/s
+Step 7460/50462 | Loss: 2.7972 | LR: 4.78e-04 | 3991.9ms/step | 15.0 samples/s
+Step 7470/50462 | Loss: 2.8769 | LR: 4.78e-04 | 3997.7ms/step | 15.0 samples/s
+Step 7480/50462 | Loss: 2.8945 | LR: 4.78e-04 | 3999.8ms/step | 15.0 samples/s
+Step 7490/50462 | Loss: 2.8591 | LR: 4.78e-04 | 3993.5ms/step | 15.0 samples/s
+Step 7500/50462 | Loss: 2.8239 | LR: 4.78e-04 | 3995.5ms/step | 15.0 samples/s
+Validation at step 7500: Loss = 2.8249, Perplexity = 16.86
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-7500
+Step 7510/50462 | Loss: 2.8597 | LR: 4.78e-04 | 5991.9ms/step | 10.0 samples/s
+Step 7520/50462 | Loss: 2.8394 | LR: 4.77e-04 | 3987.7ms/step | 15.0 samples/s
+Step 7530/50462 | Loss: 2.8442 | LR: 4.77e-04 | 3991.1ms/step | 15.0 samples/s
+Step 7540/50462 | Loss: 2.7883 | LR: 4.77e-04 | 3994.4ms/step | 15.0 samples/s
+Step 7550/50462 | Loss: 2.8260 | LR: 4.77e-04 | 3993.5ms/step | 15.0 samples/s
+Step 7560/50462 | Loss: 2.8084 | LR: 4.77e-04 | 3985.0ms/step | 15.1 samples/s
+Step 7570/50462 | Loss: 2.8399 | LR: 4.77e-04 | 4001.5ms/step | 15.0 samples/s
+Step 7580/50462 | Loss: 2.8319 | LR: 4.77e-04 | 3984.0ms/step | 15.1 samples/s
+Step 7590/50462 | Loss: 2.7916 | LR: 4.77e-04 | 3992.4ms/step | 15.0 samples/s
+Step 7600/50462 | Loss: 2.8154 | LR: 4.77e-04 | 3992.3ms/step | 15.0 samples/s
+Step 7610/50462 | Loss: 2.8398 | LR: 4.77e-04 | 3994.6ms/step | 15.0 samples/s
+Step 7620/50462 | Loss: 2.8016 | LR: 4.77e-04 | 3988.1ms/step | 15.0 samples/s
+Step 7630/50462 | Loss: 2.8287 | LR: 4.77e-04 | 3989.0ms/step | 15.0 samples/s
+Step 7640/50462 | Loss: 2.8356 | LR: 4.77e-04 | 3991.3ms/step | 15.0 samples/s
+Step 7650/50462 | Loss: 2.8173 | LR: 4.77e-04 | 4000.4ms/step | 15.0 samples/s
+Step 7660/50462 | Loss: 2.8803 | LR: 4.77e-04 | 3991.0ms/step | 15.0 samples/s
+Step 7670/50462 | Loss: 2.7741 | LR: 4.76e-04 | 3994.1ms/step | 15.0 samples/s
+Step 7680/50462 | Loss: 2.8322 | LR: 4.76e-04 | 3986.2ms/step | 15.1 samples/s
+Step 7690/50462 | Loss: 2.7868 | LR: 4.76e-04 | 4002.7ms/step | 15.0 samples/s
+Step 7700/50462 | Loss: 2.8214 | LR: 4.76e-04 | 3994.8ms/step | 15.0 samples/s
+Step 7710/50462 | Loss: 2.8286 | LR: 4.76e-04 | 3983.9ms/step | 15.1 samples/s
+Step 7720/50462 | Loss: 2.8349 | LR: 4.76e-04 | 3994.2ms/step | 15.0 samples/s
+Step 7730/50462 | Loss: 2.8512 | LR: 4.76e-04 | 4007.6ms/step | 15.0 samples/s
+Step 7740/50462 | Loss: 2.8034 | LR: 4.76e-04 | 3992.8ms/step | 15.0 samples/s
+Step 7750/50462 | Loss: 2.8335 | LR: 4.76e-04 | 3991.6ms/step | 15.0 samples/s
+Validation at step 7750: Loss = 2.8180, Perplexity = 16.74
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-7750
+Step 7760/50462 | Loss: 2.8486 | LR: 4.76e-04 | 5965.3ms/step | 10.1 samples/s
+Step 7770/50462 | Loss: 2.8426 | LR: 4.76e-04 | 3983.8ms/step | 15.1 samples/s
+Step 7780/50462 | Loss: 2.8464 | LR: 4.76e-04 | 4001.3ms/step | 15.0 samples/s
+Step 7790/50462 | Loss: 2.8227 | LR: 4.76e-04 | 3988.5ms/step | 15.0 samples/s
+Step 7800/50462 | Loss: 2.8016 | LR: 4.76e-04 | 3991.0ms/step | 15.0 samples/s
+Step 7810/50462 | Loss: 2.8297 | LR: 4.76e-04 | 3995.0ms/step | 15.0 samples/s
+Step 7820/50462 | Loss: 2.8308 | LR: 4.75e-04 | 4003.3ms/step | 15.0 samples/s
+Step 7830/50462 | Loss: 2.8330 | LR: 4.75e-04 | 3987.8ms/step | 15.0 samples/s
+Step 7840/50462 | Loss: 2.8084 | LR: 4.75e-04 | 4007.9ms/step | 15.0 samples/s
+Step 7850/50462 | Loss: 2.8396 | LR: 4.75e-04 | 4005.1ms/step | 15.0 samples/s
+Step 7860/50462 | Loss: 2.8252 | LR: 4.75e-04 | 4011.3ms/step | 15.0 samples/s
+Step 7870/50462 | Loss: 2.8420 | LR: 4.75e-04 | 3998.0ms/step | 15.0 samples/s
+Step 7880/50462 | Loss: 2.8345 | LR: 4.75e-04 | 3991.1ms/step | 15.0 samples/s
+Step 7890/50462 | Loss: 2.8348 | LR: 4.75e-04 | 3989.5ms/step | 15.0 samples/s
+Step 7900/50462 | Loss: 2.8098 | LR: 4.75e-04 | 3993.4ms/step | 15.0 samples/s
+Step 7910/50462 | Loss: 2.8336 | LR: 4.75e-04 | 3992.3ms/step | 15.0 samples/s
+Step 7920/50462 | Loss: 2.7925 | LR: 4.75e-04 | 3981.3ms/step | 15.1 samples/s
+Step 7930/50462 | Loss: 2.8408 | LR: 4.75e-04 | 3982.2ms/step | 15.1 samples/s
+Step 7940/50462 | Loss: 2.8175 | LR: 4.75e-04 | 3985.8ms/step | 15.1 samples/s
+Step 7950/50462 | Loss: 2.8373 | LR: 4.75e-04 | 3992.8ms/step | 15.0 samples/s
+Step 7960/50462 | Loss: 2.8222 | LR: 4.74e-04 | 3984.2ms/step | 15.1 samples/s
+Step 7970/50462 | Loss: 2.8284 | LR: 4.74e-04 | 3979.4ms/step | 15.1 samples/s
+Step 7980/50462 | Loss: 2.8198 | LR: 4.74e-04 | 3987.7ms/step | 15.0 samples/s
+Step 7990/50462 | Loss: 2.8009 | LR: 4.74e-04 | 3994.5ms/step | 15.0 samples/s
+Step 8000/50462 | Loss: 2.7861 | LR: 4.74e-04 | 3987.0ms/step | 15.0 samples/s
+Validation at step 8000: Loss = 2.8100, Perplexity = 16.61
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-8000
+Step 8010/50462 | Loss: 2.8335 | LR: 4.74e-04 | 6054.8ms/step | 9.9 samples/s
+Step 8020/50462 | Loss: 2.8219 | LR: 4.74e-04 | 3994.7ms/step | 15.0 samples/s
+Step 8030/50462 | Loss: 2.8228 | LR: 4.74e-04 | 3987.8ms/step | 15.0 samples/s
+Step 8040/50462 | Loss: 2.8083 | LR: 4.74e-04 | 3993.0ms/step | 15.0 samples/s
+Step 8050/50462 | Loss: 2.8118 | LR: 4.74e-04 | 3990.3ms/step | 15.0 samples/s
+Step 8060/50462 | Loss: 2.8212 | LR: 4.74e-04 | 3982.5ms/step | 15.1 samples/s
+Step 8070/50462 | Loss: 2.8422 | LR: 4.74e-04 | 3986.4ms/step | 15.1 samples/s
+Step 8080/50462 | Loss: 2.8430 | LR: 4.74e-04 | 3991.5ms/step | 15.0 samples/s
+Step 8090/50462 | Loss: 2.8367 | LR: 4.74e-04 | 3992.1ms/step | 15.0 samples/s
+Step 8100/50462 | Loss: 2.8240 | LR: 4.74e-04 | 3985.3ms/step | 15.1 samples/s
+Step 8110/50462 | Loss: 2.8125 | LR: 4.73e-04 | 3989.4ms/step | 15.0 samples/s
+Step 8120/50462 | Loss: 2.8082 | LR: 4.73e-04 | 3994.3ms/step | 15.0 samples/s
+Step 8130/50462 | Loss: 2.7942 | LR: 4.73e-04 | 3986.6ms/step | 15.1 samples/s
+Step 8140/50462 | Loss: 2.8086 | LR: 4.73e-04 | 3995.2ms/step | 15.0 samples/s
+Step 8150/50462 | Loss: 2.8039 | LR: 4.73e-04 | 3986.9ms/step | 15.0 samples/s
+Step 8160/50462 | Loss: 2.7899 | LR: 4.73e-04 | 3986.3ms/step | 15.1 samples/s
+Step 8170/50462 | Loss: 2.8159 | LR: 4.73e-04 | 3989.6ms/step | 15.0 samples/s
+Step 8180/50462 | Loss: 2.7747 | LR: 4.73e-04 | 3996.7ms/step | 15.0 samples/s
+Step 8190/50462 | Loss: 2.8307 | LR: 4.73e-04 | 3991.6ms/step | 15.0 samples/s
+Step 8200/50462 | Loss: 2.7888 | LR: 4.73e-04 | 3995.6ms/step | 15.0 samples/s
+Step 8210/50462 | Loss: 2.7951 | LR: 4.73e-04 | 3986.7ms/step | 15.0 samples/s
+Step 8220/50462 | Loss: 2.8003 | LR: 4.73e-04 | 3982.6ms/step | 15.1 samples/s
+Step 8230/50462 | Loss: 2.8143 | LR: 4.73e-04 | 3983.5ms/step | 15.1 samples/s
+Step 8240/50462 | Loss: 2.8044 | LR: 4.73e-04 | 3982.1ms/step | 15.1 samples/s
+Step 8250/50462 | Loss: 2.8275 | LR: 4.72e-04 | 3992.8ms/step | 15.0 samples/s
+Validation at step 8250: Loss = 2.8045, Perplexity = 16.52
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-8250
+Step 8260/50462 | Loss: 2.8104 | LR: 4.72e-04 | 6121.8ms/step | 9.8 samples/s
+Step 8270/50462 | Loss: 2.8274 | LR: 4.72e-04 | 3991.9ms/step | 15.0 samples/s
+Step 8280/50462 | Loss: 2.8333 | LR: 4.72e-04 | 3983.0ms/step | 15.1 samples/s
+Step 8290/50462 | Loss: 2.7986 | LR: 4.72e-04 | 3988.6ms/step | 15.0 samples/s
+Step 8300/50462 | Loss: 2.8244 | LR: 4.72e-04 | 3991.5ms/step | 15.0 samples/s
+Step 8310/50462 | Loss: 2.8502 | LR: 4.72e-04 | 3991.0ms/step | 15.0 samples/s
+Step 8320/50462 | Loss: 2.7884 | LR: 4.72e-04 | 3986.8ms/step | 15.0 samples/s
+Step 8330/50462 | Loss: 2.7988 | LR: 4.72e-04 | 3978.9ms/step | 15.1 samples/s
+Step 8340/50462 | Loss: 2.7984 | LR: 4.72e-04 | 3997.2ms/step | 15.0 samples/s
+Step 8350/50462 | Loss: 2.8084 | LR: 4.72e-04 | 3983.8ms/step | 15.1 samples/s
+Step 8360/50462 | Loss: 2.8067 | LR: 4.72e-04 | 3983.1ms/step | 15.1 samples/s
+Step 8370/50462 | Loss: 2.8403 | LR: 4.72e-04 | 3999.1ms/step | 15.0 samples/s
+Step 8380/50462 | Loss: 2.8079 | LR: 4.71e-04 | 3987.6ms/step | 15.0 samples/s
+Step 8390/50462 | Loss: 2.8098 | LR: 4.71e-04 | 3990.8ms/step | 15.0 samples/s
+Step 8400/50462 | Loss: 2.8074 | LR: 4.71e-04 | 4001.1ms/step | 15.0 samples/s
+Step 8410/50462 | Loss: 2.8192 | LR: 4.71e-04 | 3983.4ms/step | 15.1 samples/s
+Step 8420/50462 | Loss: 2.8376 | LR: 4.71e-04 | 3984.4ms/step | 15.1 samples/s
+Step 8430/50462 | Loss: 2.8525 | LR: 4.71e-04 | 3985.0ms/step | 15.1 samples/s
+Step 8440/50462 | Loss: 2.8333 | LR: 4.71e-04 | 3996.0ms/step | 15.0 samples/s
+Step 8450/50462 | Loss: 2.7824 | LR: 4.71e-04 | 3999.1ms/step | 15.0 samples/s
+Step 8460/50462 | Loss: 2.8119 | LR: 4.71e-04 | 4000.9ms/step | 15.0 samples/s
+Step 8470/50462 | Loss: 2.8299 | LR: 4.71e-04 | 3996.1ms/step | 15.0 samples/s
+Step 8480/50462 | Loss: 2.8240 | LR: 4.71e-04 | 3973.6ms/step | 15.1 samples/s
+Step 8490/50462 | Loss: 2.8089 | LR: 4.71e-04 | 3989.4ms/step | 15.0 samples/s
+Step 8500/50462 | Loss: 2.8089 | LR: 4.71e-04 | 3979.9ms/step | 15.1 samples/s
+Validation at step 8500: Loss = 2.7959, Perplexity = 16.38
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-8500
+Step 8510/50462 | Loss: 2.8111 | LR: 4.71e-04 | 6130.9ms/step | 9.8 samples/s
+Step 8520/50462 | Loss: 2.8297 | LR: 4.70e-04 | 3987.4ms/step | 15.0 samples/s
+Step 8530/50462 | Loss: 2.8274 | LR: 4.70e-04 | 3991.3ms/step | 15.0 samples/s
+Step 8540/50462 | Loss: 2.8277 | LR: 4.70e-04 | 3997.8ms/step | 15.0 samples/s
+Step 8550/50462 | Loss: 2.8181 | LR: 4.70e-04 | 3993.5ms/step | 15.0 samples/s
+Step 8560/50462 | Loss: 2.7797 | LR: 4.70e-04 | 3983.6ms/step | 15.1 samples/s
+Step 8570/50462 | Loss: 2.8074 | LR: 4.70e-04 | 4006.0ms/step | 15.0 samples/s
+Step 8580/50462 | Loss: 2.8043 | LR: 4.70e-04 | 3988.7ms/step | 15.0 samples/s
+Step 8590/50462 | Loss: 2.8084 | LR: 4.70e-04 | 3977.2ms/step | 15.1 samples/s
+Step 8600/50462 | Loss: 2.7942 | LR: 4.70e-04 | 3988.1ms/step | 15.0 samples/s
+Step 8610/50462 | Loss: 2.8083 | LR: 4.70e-04 | 3980.1ms/step | 15.1 samples/s
+Step 8620/50462 | Loss: 2.7954 | LR: 4.70e-04 | 3993.6ms/step | 15.0 samples/s
+Step 8630/50462 | Loss: 2.8053 | LR: 4.70e-04 | 3992.3ms/step | 15.0 samples/s
+Step 8640/50462 | Loss: 2.8089 | LR: 4.70e-04 | 3993.1ms/step | 15.0 samples/s
+Step 8650/50462 | Loss: 2.8089 | LR: 4.69e-04 | 3986.8ms/step | 15.0 samples/s
+Step 8660/50462 | Loss: 2.7867 | LR: 4.69e-04 | 3980.6ms/step | 15.1 samples/s
+Step 8670/50462 | Loss: 2.8130 | LR: 4.69e-04 | 3974.7ms/step | 15.1 samples/s
+Step 8680/50462 | Loss: 2.8018 | LR: 4.69e-04 | 3981.8ms/step | 15.1 samples/s
+Step 8690/50462 | Loss: 2.8352 | LR: 4.69e-04 | 3987.0ms/step | 15.0 samples/s
+Step 8700/50462 | Loss: 2.8085 | LR: 4.69e-04 | 3996.1ms/step | 15.0 samples/s
+Step 8710/50462 | Loss: 2.8171 | LR: 4.69e-04 | 4001.0ms/step | 15.0 samples/s
+Step 8720/50462 | Loss: 2.7673 | LR: 4.69e-04 | 3982.2ms/step | 15.1 samples/s
+Step 8730/50462 | Loss: 2.8184 | LR: 4.69e-04 | 3993.6ms/step | 15.0 samples/s
+Step 8740/50462 | Loss: 2.7951 | LR: 4.69e-04 | 3995.0ms/step | 15.0 samples/s
+Step 8750/50462 | Loss: 2.7770 | LR: 4.69e-04 | 3985.8ms/step | 15.1 samples/s
+Validation at step 8750: Loss = 2.7914, Perplexity = 16.30
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-8750
+Step 8760/50462 | Loss: 2.7900 | LR: 4.69e-04 | 6110.8ms/step | 9.8 samples/s
+Step 8770/50462 | Loss: 2.7880 | LR: 4.69e-04 | 3994.3ms/step | 15.0 samples/s
+Step 8780/50462 | Loss: 2.8073 | LR: 4.68e-04 | 3983.9ms/step | 15.1 samples/s
+Step 8790/50462 | Loss: 2.8209 | LR: 4.68e-04 | 3984.5ms/step | 15.1 samples/s
+Step 8800/50462 | Loss: 2.7785 | LR: 4.68e-04 | 3977.0ms/step | 15.1 samples/s
+Step 8810/50462 | Loss: 2.7963 | LR: 4.68e-04 | 3998.2ms/step | 15.0 samples/s
+Step 8820/50462 | Loss: 2.8362 | LR: 4.68e-04 | 3986.7ms/step | 15.1 samples/s
+Step 8830/50462 | Loss: 2.7823 | LR: 4.68e-04 | 3993.4ms/step | 15.0 samples/s
+Step 8840/50462 | Loss: 2.7884 | LR: 4.68e-04 | 3981.2ms/step | 15.1 samples/s
+Step 8850/50462 | Loss: 2.8276 | LR: 4.68e-04 | 3989.5ms/step | 15.0 samples/s
+Step 8860/50462 | Loss: 2.7981 | LR: 4.68e-04 | 3991.5ms/step | 15.0 samples/s
+Step 8870/50462 | Loss: 2.8038 | LR: 4.68e-04 | 3986.0ms/step | 15.1 samples/s
+Step 8880/50462 | Loss: 2.7669 | LR: 4.68e-04 | 3986.6ms/step | 15.1 samples/s
+Step 8890/50462 | Loss: 2.7701 | LR: 4.68e-04 | 3982.9ms/step | 15.1 samples/s
+Step 8900/50462 | Loss: 2.7895 | LR: 4.68e-04 | 4006.6ms/step | 15.0 samples/s
+Step 8910/50462 | Loss: 2.8061 | LR: 4.67e-04 | 3987.5ms/step | 15.0 samples/s
+Step 8920/50462 | Loss: 2.7801 | LR: 4.67e-04 | 3991.7ms/step | 15.0 samples/s
+Step 8930/50462 | Loss: 2.8126 | LR: 4.67e-04 | 3990.2ms/step | 15.0 samples/s
+Step 8940/50462 | Loss: 2.8277 | LR: 4.67e-04 | 3990.2ms/step | 15.0 samples/s
+Step 8950/50462 | Loss: 2.8140 | LR: 4.67e-04 | 3996.1ms/step | 15.0 samples/s
+Step 8960/50462 | Loss: 2.7785 | LR: 4.67e-04 | 3986.6ms/step | 15.1 samples/s
+Step 8970/50462 | Loss: 2.8070 | LR: 4.67e-04 | 3987.4ms/step | 15.0 samples/s
+Step 8980/50462 | Loss: 2.8130 | LR: 4.67e-04 | 3978.8ms/step | 15.1 samples/s
+Step 8990/50462 | Loss: 2.7913 | LR: 4.67e-04 | 3975.9ms/step | 15.1 samples/s
+Step 9000/50462 | Loss: 2.7816 | LR: 4.67e-04 | 3993.3ms/step | 15.0 samples/s
+Validation at step 9000: Loss = 2.7855, Perplexity = 16.21
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-9000
+Step 9010/50462 | Loss: 2.8090 | LR: 4.67e-04 | 6250.4ms/step | 9.6 samples/s
+Step 9020/50462 | Loss: 2.7929 | LR: 4.67e-04 | 3997.3ms/step | 15.0 samples/s
+Step 9030/50462 | Loss: 2.8274 | LR: 4.67e-04 | 3995.1ms/step | 15.0 samples/s
+Step 9040/50462 | Loss: 2.8089 | LR: 4.66e-04 | 3984.6ms/step | 15.1 samples/s
+Step 9050/50462 | Loss: 2.7855 | LR: 4.66e-04 | 3992.1ms/step | 15.0 samples/s

epfl_final/SparseAtt/wandb/run-20260826_175546-8ynn1we6/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260826_175546-8ynn1we6/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-08-26T17:55:46.977037Z",
+  "args":  [
+    "--config",
+    "configs/p2_1p2b_mixed_seed71_4gpu.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/SparseAtt.git",
+    "commit":  "8f862f749eb4d5c17f688eb1562235e3fe5b2550"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-train-4-gpu-0-14",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "5965646430208"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-e3358200-f776-7214-8269-c2c219f03c2e"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-a3a94461-36d7-5e8a-af9b-6209869fa7f4"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-0c2f0695-54b3-664a-b12d-640d1c9a0f2e"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-7a2d68b3-dda3-ec4d-f88e-205cb866ebb6"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "4xgro68c3harxx20bi9kn2ju95cli6x4"
+}

epfl_final/SparseAtt/wandb/run-20260826_175546-8ynn1we6/logs/debug.log

@@ -0,0 +1,19 @@
+2026-08-26 17:55:47,004 INFO    MainThread:212 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-08-26 17:55:47,005 INFO    MainThread:212 [wandb_setup.py:_flush():81] Configure stats pid to 212
+2026-08-26 17:55:47,005 INFO    MainThread:212 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-26 17:55:47,005 INFO    MainThread:212 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260826_175546-8ynn1we6/logs/debug.log
+2026-08-26 17:55:47,005 INFO    MainThread:212 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260826_175546-8ynn1we6/logs/debug-internal.log
+2026-08-26 17:55:47,005 INFO    MainThread:212 [wandb_init.py:init():850] calling init triggers
+2026-08-26 17:55:47,005 INFO    MainThread:212 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 15, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 71, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b mixed 8k seed71 4gpu', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'mixed', 'seed71'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed', 'sparse_mask_seed': 71, 'sparse_mixed_nrb_set': [63, 47, 31, 21, 11], 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-08-26 17:55:47,005 INFO    MainThread:212 [wandb_init.py:init():898] starting backend
+2026-08-26 17:55:47,933 INFO    MainThread:212 [wandb_init.py:init():913] sending inform_init request
+2026-08-26 17:55:48,473 INFO    MainThread:212 [wandb_init.py:init():918] backend started and connected
+2026-08-26 17:55:48,480 INFO    MainThread:212 [wandb_init.py:init():988] updated telemetry
+2026-08-26 17:55:48,532 INFO    MainThread:212 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-08-26 17:55:49,040 INFO    MainThread:212 [wandb_init.py:init():1056] starting run threads in backend
+2026-08-26 17:55:49,661 INFO    MainThread:212 [wandb_run.py:_console_start():2554] atexit reg
+2026-08-26 17:55:49,661 INFO    MainThread:212 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-08-26 17:55:49,661 INFO    MainThread:212 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-08-26 17:55:49,661 INFO    MainThread:212 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-08-26 17:55:49,669 INFO    MainThread:212 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/run-20260826_175546-8ynn1we6/run-8ynn1we6.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c3b9ad4e02fbbe46e4cd1aa686ff9533d934b1e599bc02a946fc95f6f32c730
+size 12484608

epfl_final/SparseAtt/wandb/run-20260827_022420-rgmkfpnl/files/config.yaml

@@ -0,0 +1,205 @@
+_wandb:
+    value:
+        cli_version: 0.26.1
+        e:
+            ge3aug5csshnod8dmxmlezb8wuau8hkz:
+                args:
+                    - --config
+                    - configs/p2_1p2b_dense_seed70_6gpu_resume41750.json
+                codePath: train.py
+                codePathLocal: train.py
+                cpu_count: 96
+                cpu_count_logical: 192
+                cudaVersion: "13.0"
+                disk:
+                    /:
+                        total: "9374931091456"
+                        used: "6433026093056"
+                email: ruben.navasardyan@mail.utoronto.ca
+                executable: /mloscratch/homes/navasard/conda/bin/python3.13
+                git:
+                    commit: 8f862f749eb4d5c17f688eb1562235e3fe5b2550
+                    remote: https://github.com/rubentium/SparseAtt.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 6
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bab1e315-c967-43cf-bdfb-cac1a8dfdfac
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-f65cda91-ff87-9bff-fb3d-3e1b3f9f8cf0
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-069bdfaf-ef1c-f7f2-f9e3-2249bb9dfc41
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-4c1d91e9-bf01-f8fd-1e8b-79aa8e0831df
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-086f9f0b-a6d8-6263-83e6-c2b66721f47f
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bdb5e23e-0700-20a0-0b21-1ad0901f08b7
+                host: h100-train-6-gpu-0-44
+                memory:
+                    total: "1623005732864"
+                os: Linux-6.8.0-124-generic-x86_64-with-glibc2.35
+                program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
+                python: CPython 3.13.13
+                root: /mloscratch/homes/navasard/sparse/SparseAtt
+                startedAt: "2026-08-27T02:24:20.312598Z"
+                writerId: ge3aug5csshnod8dmxmlezb8wuau8hkz
+        m: []
+        python_version: 3.13.13
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 5
+                - 13
+                - 14
+                - 15
+                - 16
+                - 61
+                - 62
+            "4": 3.13.13
+            "5": 0.26.1
+            "6": 5.7.0
+            "12": 0.26.1
+            "13": linux-x86_64
+attention_dropout:
+    value: null
+attention_type:
+    value: sparse_stochastic
+checkpoint_activations:
+    value: false
+dense_mode:
+    value: true
+dense_step_intervals:
+    value: null
+eval_batches:
+    value: 50
+eval_steps:
+    value: 250
+force_new_run:
+    value: false
+gradient_accumulation_steps:
+    value: 10
+head_dim:
+    value: null
+hidden_size:
+    value: 1536
+init_from_scratch:
+    value: true
+intermediate_size:
+    value: 6144
+learning_rate:
+    value: 0.0005
+logging_steps:
+    value: 10
+lr_scheduler:
+    value: cosine
+max_grad_norm:
+    value: 1
+max_seq_length:
+    value: 8192
+max_steps:
+    value: 50462
+mixed_precision:
+    value: bf16
+model_dtype:
+    value: bfloat16
+model_path:
+    value: /mloscratch/homes/navasard/qwen3_06b_base
+num_attention_heads:
+    value: 12
+num_epochs:
+    value: 1
+num_hidden_layers:
+    value: 28
+num_key_value_heads:
+    value: 4
+num_workers:
+    value: 8
+output_dir:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810
+per_device_batch_size:
+    value: 1
+resume_from_checkpoint:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-41750
+rope_train_position_scale_set:
+    value: null
+save_steps:
+    value: 250
+seed:
+    value: 70
+sparse_block_size:
+    value: 128
+sparse_layer_indices:
+    value: null
+sparse_mask_cache_dir:
+    value: null
+sparse_mask_pool_size:
+    value: 1
+sparse_mask_seed:
+    value: 0
+sparse_mixed_nrb_set:
+    value: null
+sparse_num_random_blocks:
+    value: 31
+sparse_val_dense_compare:
+    value: false
+strategy:
+    value: ddp
+train_path:
+    value: /mloscratch/homes/navasard/data/fineweb_large/tokenized
+val_path:
+    value: null
+verbose:
+    value: true
+wandb_enabled:
+    value: true
+wandb_name:
+    value: 1.2b dense 8k seed70 resume41750 6gpu
+wandb_project:
+    value: stochastic_sparsity
+wandb_tags:
+    value:
+        - qwen3
+        - 1.2b
+        - 8k
+        - p2
+        - dense
+        - seed71
+warmup_steps:
+    value: 750
+weight_decay:
+    value: 0.01

epfl_final/SparseAtt/wandb/run-20260827_022420-rgmkfpnl/files/output.log

@@ -0,0 +1,954 @@
+Starting training...
+Validation at step 41750: Loss = 2.4998, Perplexity = 12.18
+wandb: WARNING Tried to log to step 41750 that is less than the current step 41851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 41751: dense (dense step interval)
+Step 41760/50462 | Loss: 2.4765 | LR: 3.69e-05 | 27125.5ms/step | 2.2 samples/s
+wandb: WARNING Tried to log to step 41760 that is less than the current step 41851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 41770/50462 | Loss: 2.4496 | LR: 3.68e-05 | 1910.1ms/step | 31.4 samples/s
+wandb: WARNING Tried to log to step 41770 that is less than the current step 41851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 41780/50462 | Loss: 2.4236 | LR: 3.67e-05 | 1914.1ms/step | 31.3 samples/s
+wandb: WARNING Tried to log to step 41780 that is less than the current step 41851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 41790/50462 | Loss: 2.4385 | LR: 3.66e-05 | 1913.0ms/step | 31.4 samples/s
+wandb: WARNING Tried to log to step 41790 that is less than the current step 41851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 41800/50462 | Loss: 2.4967 | LR: 3.65e-05 | 1914.8ms/step | 31.3 samples/s
+wandb: WARNING Tried to log to step 41800 that is less than the current step 41851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 41810/50462 | Loss: 2.4740 | LR: 3.64e-05 | 1916.3ms/step | 31.3 samples/s
+wandb: WARNING Tried to log to step 41810 that is less than the current step 41851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 41820/50462 | Loss: 2.4658 | LR: 3.64e-05 | 1915.8ms/step | 31.3 samples/s
+wandb: WARNING Tried to log to step 41820 that is less than the current step 41851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 41830/50462 | Loss: 2.4844 | LR: 3.63e-05 | 1917.8ms/step | 31.3 samples/s
+wandb: WARNING Tried to log to step 41830 that is less than the current step 41851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 41840/50462 | Loss: 2.4547 | LR: 3.62e-05 | 1918.1ms/step | 31.3 samples/s
+wandb: WARNING Tried to log to step 41840 that is less than the current step 41851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 41850/50462 | Loss: 2.4263 | LR: 3.61e-05 | 1918.1ms/step | 31.3 samples/s
+wandb: WARNING Tried to log to step 41850 that is less than the current step 41851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 41860/50462 | Loss: 2.4356 | LR: 3.60e-05 | 1917.3ms/step | 31.3 samples/s
+Step 41870/50462 | Loss: 2.4848 | LR: 3.60e-05 | 1919.7ms/step | 31.3 samples/s
+Step 41880/50462 | Loss: 2.4195 | LR: 3.59e-05 | 1920.1ms/step | 31.2 samples/s
+Step 41890/50462 | Loss: 2.4288 | LR: 3.58e-05 | 1918.8ms/step | 31.3 samples/s
+Step 41900/50462 | Loss: 2.4447 | LR: 3.57e-05 | 1920.3ms/step | 31.2 samples/s
+Step 41910/50462 | Loss: 2.4391 | LR: 3.56e-05 | 1919.8ms/step | 31.3 samples/s
+Step 41920/50462 | Loss: 2.5013 | LR: 3.55e-05 | 1919.4ms/step | 31.3 samples/s
+Step 41930/50462 | Loss: 2.4701 | LR: 3.55e-05 | 1918.1ms/step | 31.3 samples/s
+Step 41940/50462 | Loss: 2.4601 | LR: 3.54e-05 | 1919.5ms/step | 31.3 samples/s
+Step 41950/50462 | Loss: 2.4752 | LR: 3.53e-05 | 1918.9ms/step | 31.3 samples/s
+Step 41960/50462 | Loss: 2.4323 | LR: 3.52e-05 | 1919.8ms/step | 31.3 samples/s
+Step 41970/50462 | Loss: 2.4556 | LR: 3.51e-05 | 1918.4ms/step | 31.3 samples/s
+Step 41980/50462 | Loss: 2.4609 | LR: 3.51e-05 | 1919.0ms/step | 31.3 samples/s
+Step 41990/50462 | Loss: 2.4705 | LR: 3.50e-05 | 1917.8ms/step | 31.3 samples/s
+Step 42000/50462 | Loss: 2.4515 | LR: 3.49e-05 | 1918.7ms/step | 31.3 samples/s
+Validation at step 42000: Loss = 2.4997, Perplexity = 12.18
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-42000
+Step 42010/50462 | Loss: 2.4335 | LR: 3.48e-05 | 3284.3ms/step | 18.3 samples/s
+Step 42020/50462 | Loss: 2.4252 | LR: 3.47e-05 | 1918.8ms/step | 31.3 samples/s
+Step 42030/50462 | Loss: 2.4807 | LR: 3.47e-05 | 1919.7ms/step | 31.3 samples/s
+Step 42040/50462 | Loss: 2.4371 | LR: 3.46e-05 | 1921.0ms/step | 31.2 samples/s
+Step 42050/50462 | Loss: 2.4613 | LR: 3.45e-05 | 1920.8ms/step | 31.2 samples/s
+Step 42060/50462 | Loss: 2.4550 | LR: 3.44e-05 | 1923.5ms/step | 31.2 samples/s
+Step 42070/50462 | Loss: 2.4377 | LR: 3.43e-05 | 1921.2ms/step | 31.2 samples/s
+Step 42080/50462 | Loss: 2.4887 | LR: 3.43e-05 | 1921.2ms/step | 31.2 samples/s
+Step 42090/50462 | Loss: 2.4232 | LR: 3.42e-05 | 1922.5ms/step | 31.2 samples/s
+Step 42100/50462 | Loss: 2.4605 | LR: 3.41e-05 | 1922.8ms/step | 31.2 samples/s
+Step 42110/50462 | Loss: 2.4762 | LR: 3.40e-05 | 1919.4ms/step | 31.3 samples/s
+Step 42120/50462 | Loss: 2.4436 | LR: 3.39e-05 | 1924.1ms/step | 31.2 samples/s
+Step 42130/50462 | Loss: 2.4249 | LR: 3.39e-05 | 1923.5ms/step | 31.2 samples/s
+Step 42140/50462 | Loss: 2.4420 | LR: 3.38e-05 | 1922.4ms/step | 31.2 samples/s
+Step 42150/50462 | Loss: 2.4723 | LR: 3.37e-05 | 1921.3ms/step | 31.2 samples/s
+Step 42160/50462 | Loss: 2.4360 | LR: 3.36e-05 | 1922.5ms/step | 31.2 samples/s
+Step 42170/50462 | Loss: 2.4758 | LR: 3.35e-05 | 1923.2ms/step | 31.2 samples/s
+Step 42180/50462 | Loss: 2.4251 | LR: 3.35e-05 | 1922.6ms/step | 31.2 samples/s
+Step 42190/50462 | Loss: 2.4669 | LR: 3.34e-05 | 1922.1ms/step | 31.2 samples/s
+Step 42200/50462 | Loss: 2.4345 | LR: 3.33e-05 | 1923.9ms/step | 31.2 samples/s
+Step 42210/50462 | Loss: 2.4191 | LR: 3.32e-05 | 1923.9ms/step | 31.2 samples/s
+Step 42220/50462 | Loss: 2.4376 | LR: 3.32e-05 | 1925.2ms/step | 31.2 samples/s
+Step 42230/50462 | Loss: 2.4225 | LR: 3.31e-05 | 1924.5ms/step | 31.2 samples/s
+Step 42240/50462 | Loss: 2.4419 | LR: 3.30e-05 | 1923.8ms/step | 31.2 samples/s
+Step 42250/50462 | Loss: 2.4459 | LR: 3.29e-05 | 1923.2ms/step | 31.2 samples/s
+Validation at step 42250: Loss = 2.4996, Perplexity = 12.18
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-42250
+Step 42260/50462 | Loss: 2.4690 | LR: 3.28e-05 | 3288.4ms/step | 18.2 samples/s
+Step 42270/50462 | Loss: 2.4109 | LR: 3.28e-05 | 1918.5ms/step | 31.3 samples/s
+Step 42280/50462 | Loss: 2.4810 | LR: 3.27e-05 | 1918.2ms/step | 31.3 samples/s
+Step 42290/50462 | Loss: 2.4652 | LR: 3.26e-05 | 1920.9ms/step | 31.2 samples/s
+Step 42300/50462 | Loss: 2.4609 | LR: 3.25e-05 | 1920.8ms/step | 31.2 samples/s
+Step 42310/50462 | Loss: 2.4420 | LR: 3.24e-05 | 1921.5ms/step | 31.2 samples/s
+Step 42320/50462 | Loss: 2.4595 | LR: 3.24e-05 | 1920.7ms/step | 31.2 samples/s
+Step 42330/50462 | Loss: 2.4655 | LR: 3.23e-05 | 1921.4ms/step | 31.2 samples/s
+Step 42340/50462 | Loss: 2.4250 | LR: 3.22e-05 | 1921.4ms/step | 31.2 samples/s
+Step 42350/50462 | Loss: 2.4435 | LR: 3.21e-05 | 1921.1ms/step | 31.2 samples/s
+Step 42360/50462 | Loss: 2.4586 | LR: 3.21e-05 | 1921.8ms/step | 31.2 samples/s
+Step 42370/50462 | Loss: 2.4205 | LR: 3.20e-05 | 1923.9ms/step | 31.2 samples/s
+Step 42380/50462 | Loss: 2.4001 | LR: 3.19e-05 | 1920.1ms/step | 31.2 samples/s
+Step 42390/50462 | Loss: 2.4502 | LR: 3.18e-05 | 1919.1ms/step | 31.3 samples/s
+Step 42400/50462 | Loss: 2.4204 | LR: 3.18e-05 | 1922.4ms/step | 31.2 samples/s
+Step 42410/50462 | Loss: 2.4973 | LR: 3.17e-05 | 1922.7ms/step | 31.2 samples/s
+Step 42420/50462 | Loss: 2.4565 | LR: 3.16e-05 | 1922.9ms/step | 31.2 samples/s
+Step 42430/50462 | Loss: 2.4520 | LR: 3.15e-05 | 1925.3ms/step | 31.2 samples/s
+Step 42440/50462 | Loss: 2.4613 | LR: 3.14e-05 | 1924.3ms/step | 31.2 samples/s
+Step 42450/50462 | Loss: 2.4724 | LR: 3.14e-05 | 1922.9ms/step | 31.2 samples/s
+Step 42460/50462 | Loss: 2.4924 | LR: 3.13e-05 | 1923.1ms/step | 31.2 samples/s
+Step 42470/50462 | Loss: 2.4884 | LR: 3.12e-05 | 1922.4ms/step | 31.2 samples/s
+Step 42480/50462 | Loss: 2.4620 | LR: 3.11e-05 | 1920.3ms/step | 31.2 samples/s
+Step 42490/50462 | Loss: 2.4762 | LR: 3.11e-05 | 1924.7ms/step | 31.2 samples/s
+Step 42500/50462 | Loss: 2.4652 | LR: 3.10e-05 | 1924.8ms/step | 31.2 samples/s
+Validation at step 42500: Loss = 2.4997, Perplexity = 12.18
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-42500
+Step 42510/50462 | Loss: 2.4799 | LR: 3.09e-05 | 3280.6ms/step | 18.3 samples/s
+Step 42520/50462 | Loss: 2.4559 | LR: 3.08e-05 | 1918.6ms/step | 31.3 samples/s
+Step 42530/50462 | Loss: 2.4718 | LR: 3.08e-05 | 1916.3ms/step | 31.3 samples/s
+Step 42540/50462 | Loss: 2.4174 | LR: 3.07e-05 | 1913.1ms/step | 31.4 samples/s
+Step 42550/50462 | Loss: 2.4554 | LR: 3.06e-05 | 1915.9ms/step | 31.3 samples/s
+Step 42560/50462 | Loss: 2.4561 | LR: 3.05e-05 | 1917.2ms/step | 31.3 samples/s
+Step 42570/50462 | Loss: 2.4430 | LR: 3.05e-05 | 1916.8ms/step | 31.3 samples/s
+Step 42580/50462 | Loss: 2.4743 | LR: 3.04e-05 | 1916.4ms/step | 31.3 samples/s
+Step 42590/50462 | Loss: 2.4770 | LR: 3.03e-05 | 1916.3ms/step | 31.3 samples/s
+Step 42600/50462 | Loss: 2.4816 | LR: 3.02e-05 | 1918.2ms/step | 31.3 samples/s
+Step 42610/50462 | Loss: 2.4614 | LR: 3.02e-05 | 1916.3ms/step | 31.3 samples/s
+Step 42620/50462 | Loss: 2.4648 | LR: 3.01e-05 | 1916.7ms/step | 31.3 samples/s
+Step 42630/50462 | Loss: 2.4493 | LR: 3.00e-05 | 1917.4ms/step | 31.3 samples/s
+Step 42640/50462 | Loss: 2.4549 | LR: 2.99e-05 | 1919.6ms/step | 31.3 samples/s
+Step 42650/50462 | Loss: 2.4691 | LR: 2.99e-05 | 1917.3ms/step | 31.3 samples/s
+Step 42660/50462 | Loss: 2.4372 | LR: 2.98e-05 | 1916.8ms/step | 31.3 samples/s
+Step 42670/50462 | Loss: 2.4466 | LR: 2.97e-05 | 1914.5ms/step | 31.3 samples/s
+Step 42680/50462 | Loss: 2.4520 | LR: 2.96e-05 | 1916.4ms/step | 31.3 samples/s
+Step 42690/50462 | Loss: 2.4741 | LR: 2.96e-05 | 1917.8ms/step | 31.3 samples/s
+Step 42700/50462 | Loss: 2.4279 | LR: 2.95e-05 | 1916.4ms/step | 31.3 samples/s
+Step 42710/50462 | Loss: 2.4447 | LR: 2.94e-05 | 1919.1ms/step | 31.3 samples/s
+Step 42720/50462 | Loss: 2.4124 | LR: 2.93e-05 | 1918.8ms/step | 31.3 samples/s
+Step 42730/50462 | Loss: 2.4278 | LR: 2.93e-05 | 1918.3ms/step | 31.3 samples/s
+Step 42740/50462 | Loss: 2.4914 | LR: 2.92e-05 | 1917.6ms/step | 31.3 samples/s
+Step 42750/50462 | Loss: 2.4426 | LR: 2.91e-05 | 1918.2ms/step | 31.3 samples/s
+Validation at step 42750: Loss = 2.4997, Perplexity = 12.18
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-42750
+Step 42760/50462 | Loss: 2.4383 | LR: 2.90e-05 | 3276.0ms/step | 18.3 samples/s
+Step 42770/50462 | Loss: 2.4460 | LR: 2.90e-05 | 1913.9ms/step | 31.3 samples/s
+Step 42780/50462 | Loss: 2.4482 | LR: 2.89e-05 | 1918.4ms/step | 31.3 samples/s
+Step 42790/50462 | Loss: 2.4689 | LR: 2.88e-05 | 1922.2ms/step | 31.2 samples/s
+Step 42800/50462 | Loss: 2.4269 | LR: 2.87e-05 | 1914.0ms/step | 31.3 samples/s
+Step 42810/50462 | Loss: 2.4772 | LR: 2.87e-05 | 1917.5ms/step | 31.3 samples/s
+Step 42820/50462 | Loss: 2.4598 | LR: 2.86e-05 | 1916.4ms/step | 31.3 samples/s
+Step 42830/50462 | Loss: 2.4822 | LR: 2.85e-05 | 1917.5ms/step | 31.3 samples/s
+Step 42840/50462 | Loss: 2.4376 | LR: 2.84e-05 | 1917.5ms/step | 31.3 samples/s
+Step 42850/50462 | Loss: 2.4690 | LR: 2.84e-05 | 1915.6ms/step | 31.3 samples/s
+Step 42860/50462 | Loss: 2.4457 | LR: 2.83e-05 | 1915.3ms/step | 31.3 samples/s
+Step 42870/50462 | Loss: 2.4663 | LR: 2.82e-05 | 1918.1ms/step | 31.3 samples/s
+Step 42880/50462 | Loss: 2.4254 | LR: 2.82e-05 | 1921.7ms/step | 31.2 samples/s
+Step 42890/50462 | Loss: 2.4646 | LR: 2.81e-05 | 1917.2ms/step | 31.3 samples/s
+Step 42900/50462 | Loss: 2.4320 | LR: 2.80e-05 | 1920.1ms/step | 31.2 samples/s
+Step 42910/50462 | Loss: 2.4421 | LR: 2.79e-05 | 1920.4ms/step | 31.2 samples/s
+Step 42920/50462 | Loss: 2.4584 | LR: 2.79e-05 | 1921.0ms/step | 31.2 samples/s
+Step 42930/50462 | Loss: 2.4675 | LR: 2.78e-05 | 1917.1ms/step | 31.3 samples/s
+Step 42940/50462 | Loss: 2.4397 | LR: 2.77e-05 | 1918.4ms/step | 31.3 samples/s
+Step 42950/50462 | Loss: 2.4409 | LR: 2.76e-05 | 1917.7ms/step | 31.3 samples/s
+Step 42960/50462 | Loss: 2.4326 | LR: 2.76e-05 | 1920.4ms/step | 31.2 samples/s
+Step 42970/50462 | Loss: 2.4749 | LR: 2.75e-05 | 1919.4ms/step | 31.3 samples/s
+Step 42980/50462 | Loss: 2.4205 | LR: 2.74e-05 | 1915.8ms/step | 31.3 samples/s
+Step 42990/50462 | Loss: 2.4393 | LR: 2.74e-05 | 1917.7ms/step | 31.3 samples/s
+Step 43000/50462 | Loss: 2.4520 | LR: 2.73e-05 | 1917.1ms/step | 31.3 samples/s
+Validation at step 43000: Loss = 2.4995, Perplexity = 12.18
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-43000
+Step 43010/50462 | Loss: 2.4904 | LR: 2.72e-05 | 3275.3ms/step | 18.3 samples/s
+Step 43020/50462 | Loss: 2.4611 | LR: 2.71e-05 | 1917.3ms/step | 31.3 samples/s
+Step 43030/50462 | Loss: 2.4958 | LR: 2.71e-05 | 1917.3ms/step | 31.3 samples/s
+Step 43040/50462 | Loss: 2.4301 | LR: 2.70e-05 | 1919.0ms/step | 31.3 samples/s
+Step 43050/50462 | Loss: 2.4283 | LR: 2.69e-05 | 1918.2ms/step | 31.3 samples/s
+Step 43060/50462 | Loss: 2.4670 | LR: 2.69e-05 | 1919.3ms/step | 31.3 samples/s
+Step 43070/50462 | Loss: 2.4626 | LR: 2.68e-05 | 1918.4ms/step | 31.3 samples/s
+Step 43080/50462 | Loss: 2.4339 | LR: 2.67e-05 | 1916.9ms/step | 31.3 samples/s
+Step 43090/50462 | Loss: 2.4369 | LR: 2.66e-05 | 1917.3ms/step | 31.3 samples/s
+Step 43100/50462 | Loss: 2.4060 | LR: 2.66e-05 | 1916.0ms/step | 31.3 samples/s
+Step 43110/50462 | Loss: 2.4322 | LR: 2.65e-05 | 1916.7ms/step | 31.3 samples/s
+Step 43120/50462 | Loss: 2.4421 | LR: 2.64e-05 | 1920.8ms/step | 31.2 samples/s
+Step 43130/50462 | Loss: 2.4280 | LR: 2.64e-05 | 1918.3ms/step | 31.3 samples/s
+Step 43140/50462 | Loss: 2.4364 | LR: 2.63e-05 | 1915.2ms/step | 31.3 samples/s
+Step 43150/50462 | Loss: 2.4483 | LR: 2.62e-05 | 1921.7ms/step | 31.2 samples/s
+Step 43160/50462 | Loss: 2.4592 | LR: 2.61e-05 | 1924.6ms/step | 31.2 samples/s
+Step 43170/50462 | Loss: 2.4678 | LR: 2.61e-05 | 1921.7ms/step | 31.2 samples/s
+Step 43180/50462 | Loss: 2.4234 | LR: 2.60e-05 | 1917.6ms/step | 31.3 samples/s
+Step 43190/50462 | Loss: 2.4756 | LR: 2.59e-05 | 1919.0ms/step | 31.3 samples/s
+Step 43200/50462 | Loss: 2.4294 | LR: 2.59e-05 | 1918.6ms/step | 31.3 samples/s
+Step 43210/50462 | Loss: 2.4174 | LR: 2.58e-05 | 1927.6ms/step | 31.1 samples/s
+Step 43220/50462 | Loss: 2.4747 | LR: 2.57e-05 | 1917.4ms/step | 31.3 samples/s
+Step 43230/50462 | Loss: 2.4487 | LR: 2.57e-05 | 1920.2ms/step | 31.2 samples/s
+Step 43240/50462 | Loss: 2.4113 | LR: 2.56e-05 | 1918.2ms/step | 31.3 samples/s
+Step 43250/50462 | Loss: 2.4730 | LR: 2.55e-05 | 1916.9ms/step | 31.3 samples/s
+Validation at step 43250: Loss = 2.4995, Perplexity = 12.18
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-43250
+Step 43260/50462 | Loss: 2.4482 | LR: 2.54e-05 | 3346.3ms/step | 17.9 samples/s
+Step 43270/50462 | Loss: 2.4502 | LR: 2.54e-05 | 1917.0ms/step | 31.3 samples/s
+Step 43280/50462 | Loss: 2.4405 | LR: 2.53e-05 | 1917.3ms/step | 31.3 samples/s
+Step 43290/50462 | Loss: 2.4461 | LR: 2.52e-05 | 1916.4ms/step | 31.3 samples/s
+Step 43300/50462 | Loss: 2.4342 | LR: 2.52e-05 | 1916.8ms/step | 31.3 samples/s
+Step 43310/50462 | Loss: 2.4266 | LR: 2.51e-05 | 1917.0ms/step | 31.3 samples/s
+Step 43320/50462 | Loss: 2.4717 | LR: 2.50e-05 | 1923.6ms/step | 31.2 samples/s
+Step 43330/50462 | Loss: 2.4409 | LR: 2.50e-05 | 1924.6ms/step | 31.2 samples/s
+Step 43340/50462 | Loss: 2.4457 | LR: 2.49e-05 | 1922.3ms/step | 31.2 samples/s
+Step 43350/50462 | Loss: 2.4038 | LR: 2.48e-05 | 1923.2ms/step | 31.2 samples/s
+Step 43360/50462 | Loss: 2.4263 | LR: 2.48e-05 | 1922.0ms/step | 31.2 samples/s
+Step 43370/50462 | Loss: 2.4600 | LR: 2.47e-05 | 1926.4ms/step | 31.1 samples/s
+Step 43380/50462 | Loss: 2.4333 | LR: 2.46e-05 | 1922.0ms/step | 31.2 samples/s
+Step 43390/50462 | Loss: 2.4387 | LR: 2.46e-05 | 1922.3ms/step | 31.2 samples/s
+Step 43400/50462 | Loss: 2.4163 | LR: 2.45e-05 | 1927.5ms/step | 31.1 samples/s
+Step 43410/50462 | Loss: 2.4875 | LR: 2.44e-05 | 1923.8ms/step | 31.2 samples/s
+Step 43420/50462 | Loss: 2.4688 | LR: 2.44e-05 | 1922.6ms/step | 31.2 samples/s
+Step 43430/50462 | Loss: 2.4565 | LR: 2.43e-05 | 1922.6ms/step | 31.2 samples/s
+Step 43440/50462 | Loss: 2.4940 | LR: 2.42e-05 | 1925.5ms/step | 31.2 samples/s
+Step 43450/50462 | Loss: 2.4718 | LR: 2.41e-05 | 1925.2ms/step | 31.2 samples/s
+Step 43460/50462 | Loss: 2.4385 | LR: 2.41e-05 | 1924.7ms/step | 31.2 samples/s
+Step 43470/50462 | Loss: 2.4937 | LR: 2.40e-05 | 1924.2ms/step | 31.2 samples/s
+Step 43480/50462 | Loss: 2.4716 | LR: 2.39e-05 | 1922.1ms/step | 31.2 samples/s
+Step 43490/50462 | Loss: 2.4598 | LR: 2.39e-05 | 1926.3ms/step | 31.1 samples/s
+Step 43500/50462 | Loss: 2.4390 | LR: 2.38e-05 | 1926.8ms/step | 31.1 samples/s
+Validation at step 43500: Loss = 2.4995, Perplexity = 12.18
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-43500
+Step 43510/50462 | Loss: 2.4165 | LR: 2.37e-05 | 3278.5ms/step | 18.3 samples/s
+Step 43520/50462 | Loss: 2.4469 | LR: 2.37e-05 | 1916.2ms/step | 31.3 samples/s
+Step 43530/50462 | Loss: 2.4435 | LR: 2.36e-05 | 1920.8ms/step | 31.2 samples/s
+Step 43540/50462 | Loss: 2.4546 | LR: 2.35e-05 | 1914.7ms/step | 31.3 samples/s
+Step 43550/50462 | Loss: 2.4757 | LR: 2.35e-05 | 1919.3ms/step | 31.3 samples/s
+Step 43560/50462 | Loss: 2.4365 | LR: 2.34e-05 | 1919.7ms/step | 31.3 samples/s
+Step 43570/50462 | Loss: 2.4800 | LR: 2.33e-05 | 1923.8ms/step | 31.2 samples/s
+Step 43580/50462 | Loss: 2.4194 | LR: 2.33e-05 | 1923.8ms/step | 31.2 samples/s
+Step 43590/50462 | Loss: 2.4146 | LR: 2.32e-05 | 1924.6ms/step | 31.2 samples/s
+Step 43600/50462 | Loss: 2.4287 | LR: 2.31e-05 | 1923.0ms/step | 31.2 samples/s
+Step 43610/50462 | Loss: 2.4385 | LR: 2.31e-05 | 1919.6ms/step | 31.3 samples/s
+Step 43620/50462 | Loss: 2.4355 | LR: 2.30e-05 | 1917.5ms/step | 31.3 samples/s
+Step 43630/50462 | Loss: 2.4303 | LR: 2.29e-05 | 1917.8ms/step | 31.3 samples/s
+Step 43640/50462 | Loss: 2.4537 | LR: 2.29e-05 | 1918.3ms/step | 31.3 samples/s
+Step 43650/50462 | Loss: 2.4379 | LR: 2.28e-05 | 1923.9ms/step | 31.2 samples/s
+Step 43660/50462 | Loss: 2.4450 | LR: 2.27e-05 | 1922.6ms/step | 31.2 samples/s
+Step 43670/50462 | Loss: 2.4526 | LR: 2.27e-05 | 1920.9ms/step | 31.2 samples/s
+Step 43680/50462 | Loss: 2.3862 | LR: 2.26e-05 | 1923.7ms/step | 31.2 samples/s
+Step 43690/50462 | Loss: 2.4481 | LR: 2.25e-05 | 1920.9ms/step | 31.2 samples/s
+Step 43700/50462 | Loss: 2.4789 | LR: 2.25e-05 | 1923.0ms/step | 31.2 samples/s
+Step 43710/50462 | Loss: 2.4246 | LR: 2.24e-05 | 1921.9ms/step | 31.2 samples/s
+Step 43720/50462 | Loss: 2.4366 | LR: 2.24e-05 | 1921.3ms/step | 31.2 samples/s
+Step 43730/50462 | Loss: 2.4544 | LR: 2.23e-05 | 1919.6ms/step | 31.3 samples/s
+Step 43740/50462 | Loss: 2.4309 | LR: 2.22e-05 | 1917.4ms/step | 31.3 samples/s
+Step 43750/50462 | Loss: 2.4466 | LR: 2.22e-05 | 1919.3ms/step | 31.3 samples/s
+Validation at step 43750: Loss = 2.4995, Perplexity = 12.18
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-43750
+Step 43760/50462 | Loss: 2.4365 | LR: 2.21e-05 | 3258.8ms/step | 18.4 samples/s
+Step 43770/50462 | Loss: 2.4684 | LR: 2.20e-05 | 1921.9ms/step | 31.2 samples/s
+Step 43780/50462 | Loss: 2.4653 | LR: 2.20e-05 | 1922.1ms/step | 31.2 samples/s
+Step 43790/50462 | Loss: 2.4065 | LR: 2.19e-05 | 1918.8ms/step | 31.3 samples/s
+Step 43800/50462 | Loss: 2.4566 | LR: 2.18e-05 | 1918.2ms/step | 31.3 samples/s
+Step 43810/50462 | Loss: 2.4250 | LR: 2.18e-05 | 1917.9ms/step | 31.3 samples/s
+Step 43820/50462 | Loss: 2.5246 | LR: 2.17e-05 | 1917.3ms/step | 31.3 samples/s
+Step 43830/50462 | Loss: 2.4840 | LR: 2.16e-05 | 1916.8ms/step | 31.3 samples/s
+Step 43840/50462 | Loss: 2.4121 | LR: 2.16e-05 | 1916.7ms/step | 31.3 samples/s
+Step 43850/50462 | Loss: 2.3902 | LR: 2.15e-05 | 1917.1ms/step | 31.3 samples/s
+Step 43860/50462 | Loss: 2.4531 | LR: 2.14e-05 | 1917.6ms/step | 31.3 samples/s
+Step 43870/50462 | Loss: 2.4381 | LR: 2.14e-05 | 1919.1ms/step | 31.3 samples/s
+Step 43880/50462 | Loss: 2.4783 | LR: 2.13e-05 | 1920.1ms/step | 31.2 samples/s
+Step 43890/50462 | Loss: 2.4974 | LR: 2.13e-05 | 1919.0ms/step | 31.3 samples/s
+Step 43900/50462 | Loss: 2.4290 | LR: 2.12e-05 | 1922.2ms/step | 31.2 samples/s
+Step 43910/50462 | Loss: 2.4281 | LR: 2.11e-05 | 1924.4ms/step | 31.2 samples/s
+Step 43920/50462 | Loss: 2.4434 | LR: 2.11e-05 | 1923.0ms/step | 31.2 samples/s
+Step 43930/50462 | Loss: 2.4642 | LR: 2.10e-05 | 1922.2ms/step | 31.2 samples/s
+Step 43940/50462 | Loss: 2.4623 | LR: 2.09e-05 | 1923.2ms/step | 31.2 samples/s
+Step 43950/50462 | Loss: 2.4378 | LR: 2.09e-05 | 1924.2ms/step | 31.2 samples/s
+Step 43960/50462 | Loss: 2.4489 | LR: 2.08e-05 | 1925.6ms/step | 31.2 samples/s
+Step 43970/50462 | Loss: 2.4478 | LR: 2.07e-05 | 1925.0ms/step | 31.2 samples/s
+Step 43980/50462 | Loss: 2.4438 | LR: 2.07e-05 | 1922.3ms/step | 31.2 samples/s
+Step 43990/50462 | Loss: 2.4061 | LR: 2.06e-05 | 1924.1ms/step | 31.2 samples/s
+Step 44000/50462 | Loss: 2.4359 | LR: 2.06e-05 | 1922.2ms/step | 31.2 samples/s
+Validation at step 44000: Loss = 2.4995, Perplexity = 12.18
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-44000
+Step 44010/50462 | Loss: 2.4252 | LR: 2.05e-05 | 3301.3ms/step | 18.2 samples/s
+Step 44020/50462 | Loss: 2.4414 | LR: 2.04e-05 | 1915.6ms/step | 31.3 samples/s
+Step 44030/50462 | Loss: 2.4204 | LR: 2.04e-05 | 1917.8ms/step | 31.3 samples/s
+Step 44040/50462 | Loss: 2.4698 | LR: 2.03e-05 | 1916.9ms/step | 31.3 samples/s
+Step 44050/50462 | Loss: 2.4514 | LR: 2.02e-05 | 1917.0ms/step | 31.3 samples/s
+Step 44060/50462 | Loss: 2.4378 | LR: 2.02e-05 | 1919.8ms/step | 31.3 samples/s
+Step 44070/50462 | Loss: 2.4463 | LR: 2.01e-05 | 1921.9ms/step | 31.2 samples/s
+Step 44080/50462 | Loss: 2.4620 | LR: 2.01e-05 | 1918.5ms/step | 31.3 samples/s
+Step 44090/50462 | Loss: 2.4916 | LR: 2.00e-05 | 1922.8ms/step | 31.2 samples/s
+Step 44100/50462 | Loss: 2.4125 | LR: 1.99e-05 | 1919.2ms/step | 31.3 samples/s
+Step 44110/50462 | Loss: 2.4522 | LR: 1.99e-05 | 1920.2ms/step | 31.2 samples/s
+Step 44120/50462 | Loss: 2.4452 | LR: 1.98e-05 | 1917.5ms/step | 31.3 samples/s
+Step 44130/50462 | Loss: 2.4243 | LR: 1.97e-05 | 1914.4ms/step | 31.3 samples/s
+Step 44140/50462 | Loss: 2.4771 | LR: 1.97e-05 | 1918.5ms/step | 31.3 samples/s
+Step 44150/50462 | Loss: 2.4300 | LR: 1.96e-05 | 1919.2ms/step | 31.3 samples/s
+Step 44160/50462 | Loss: 2.4304 | LR: 1.96e-05 | 1918.8ms/step | 31.3 samples/s
+Step 44170/50462 | Loss: 2.4386 | LR: 1.95e-05 | 1918.7ms/step | 31.3 samples/s
+Step 44180/50462 | Loss: 2.4345 | LR: 1.94e-05 | 1919.7ms/step | 31.3 samples/s
+Step 44190/50462 | Loss: 2.4743 | LR: 1.94e-05 | 1917.6ms/step | 31.3 samples/s
+Step 44200/50462 | Loss: 2.4926 | LR: 1.93e-05 | 1917.8ms/step | 31.3 samples/s
+Step 44210/50462 | Loss: 2.4853 | LR: 1.93e-05 | 1919.5ms/step | 31.3 samples/s
+Step 44220/50462 | Loss: 2.4625 | LR: 1.92e-05 | 1920.7ms/step | 31.2 samples/s
+Step 44230/50462 | Loss: 2.4772 | LR: 1.91e-05 | 1918.9ms/step | 31.3 samples/s
+Step 44240/50462 | Loss: 2.5026 | LR: 1.91e-05 | 1919.2ms/step | 31.3 samples/s
+Step 44250/50462 | Loss: 2.4596 | LR: 1.90e-05 | 1919.5ms/step | 31.3 samples/s
+Validation at step 44250: Loss = 2.4990, Perplexity = 12.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-44250
+Step 44260/50462 | Loss: 2.4803 | LR: 1.90e-05 | 3251.3ms/step | 18.5 samples/s
+Step 44270/50462 | Loss: 2.4711 | LR: 1.89e-05 | 1916.0ms/step | 31.3 samples/s
+Step 44280/50462 | Loss: 2.4804 | LR: 1.88e-05 | 1918.4ms/step | 31.3 samples/s
+Step 44290/50462 | Loss: 2.4909 | LR: 1.88e-05 | 1919.7ms/step | 31.3 samples/s
+Step 44300/50462 | Loss: 2.4937 | LR: 1.87e-05 | 1919.4ms/step | 31.3 samples/s
+Step 44310/50462 | Loss: 2.4347 | LR: 1.87e-05 | 1923.9ms/step | 31.2 samples/s
+Step 44320/50462 | Loss: 2.4798 | LR: 1.86e-05 | 1924.2ms/step | 31.2 samples/s
+Step 44330/50462 | Loss: 2.4634 | LR: 1.85e-05 | 1920.9ms/step | 31.2 samples/s
+Step 44340/50462 | Loss: 2.4542 | LR: 1.85e-05 | 1923.5ms/step | 31.2 samples/s
+Step 44350/50462 | Loss: 2.4359 | LR: 1.84e-05 | 1921.9ms/step | 31.2 samples/s
+Step 44360/50462 | Loss: 2.4608 | LR: 1.84e-05 | 1923.0ms/step | 31.2 samples/s
+Step 44370/50462 | Loss: 2.4880 | LR: 1.83e-05 | 1923.1ms/step | 31.2 samples/s
+Step 44380/50462 | Loss: 2.4324 | LR: 1.82e-05 | 1925.8ms/step | 31.2 samples/s
+Step 44390/50462 | Loss: 2.4410 | LR: 1.82e-05 | 1925.0ms/step | 31.2 samples/s
+Step 44400/50462 | Loss: 2.4964 | LR: 1.81e-05 | 1925.9ms/step | 31.2 samples/s
+Step 44410/50462 | Loss: 2.4777 | LR: 1.81e-05 | 1924.4ms/step | 31.2 samples/s
+Step 44420/50462 | Loss: 2.4611 | LR: 1.80e-05 | 1922.7ms/step | 31.2 samples/s
+Step 44430/50462 | Loss: 2.4499 | LR: 1.79e-05 | 1927.5ms/step | 31.1 samples/s
+Step 44440/50462 | Loss: 2.4295 | LR: 1.79e-05 | 1926.6ms/step | 31.1 samples/s
+Step 44450/50462 | Loss: 2.4659 | LR: 1.78e-05 | 1922.5ms/step | 31.2 samples/s
+Step 44460/50462 | Loss: 2.5139 | LR: 1.78e-05 | 1920.8ms/step | 31.2 samples/s
+Step 44470/50462 | Loss: 2.4357 | LR: 1.77e-05 | 1925.5ms/step | 31.2 samples/s
+Step 44480/50462 | Loss: 2.4974 | LR: 1.77e-05 | 1922.4ms/step | 31.2 samples/s
+Step 44490/50462 | Loss: 2.4779 | LR: 1.76e-05 | 1922.2ms/step | 31.2 samples/s
+Step 44500/50462 | Loss: 2.4207 | LR: 1.75e-05 | 1923.1ms/step | 31.2 samples/s
+Validation at step 44500: Loss = 2.4989, Perplexity = 12.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-44500
+Step 44510/50462 | Loss: 2.4830 | LR: 1.75e-05 | 3274.3ms/step | 18.3 samples/s
+Step 44520/50462 | Loss: 2.4782 | LR: 1.74e-05 | 1918.0ms/step | 31.3 samples/s
+Step 44530/50462 | Loss: 2.4588 | LR: 1.74e-05 | 1921.2ms/step | 31.2 samples/s
+Step 44540/50462 | Loss: 2.4542 | LR: 1.73e-05 | 1918.9ms/step | 31.3 samples/s
+Step 44550/50462 | Loss: 2.5151 | LR: 1.72e-05 | 1915.7ms/step | 31.3 samples/s
+Step 44560/50462 | Loss: 2.4647 | LR: 1.72e-05 | 1916.2ms/step | 31.3 samples/s
+Step 44570/50462 | Loss: 2.4655 | LR: 1.71e-05 | 1917.4ms/step | 31.3 samples/s
+Step 44580/50462 | Loss: 2.4762 | LR: 1.71e-05 | 1917.9ms/step | 31.3 samples/s
+Step 44590/50462 | Loss: 2.4583 | LR: 1.70e-05 | 1918.5ms/step | 31.3 samples/s
+Step 44600/50462 | Loss: 2.4609 | LR: 1.70e-05 | 1915.5ms/step | 31.3 samples/s
+Step 44610/50462 | Loss: 2.4555 | LR: 1.69e-05 | 1917.9ms/step | 31.3 samples/s
+Step 44620/50462 | Loss: 2.4735 | LR: 1.68e-05 | 1916.6ms/step | 31.3 samples/s
+Step 44630/50462 | Loss: 2.4707 | LR: 1.68e-05 | 1918.0ms/step | 31.3 samples/s
+Step 44640/50462 | Loss: 2.4972 | LR: 1.67e-05 | 1918.0ms/step | 31.3 samples/s
+Step 44650/50462 | Loss: 2.4945 | LR: 1.67e-05 | 1917.8ms/step | 31.3 samples/s
+Step 44660/50462 | Loss: 2.5098 | LR: 1.66e-05 | 1918.4ms/step | 31.3 samples/s
+Step 44670/50462 | Loss: 2.4512 | LR: 1.66e-05 | 1918.7ms/step | 31.3 samples/s
+Step 44680/50462 | Loss: 2.4673 | LR: 1.65e-05 | 1919.0ms/step | 31.3 samples/s
+Step 44690/50462 | Loss: 2.4847 | LR: 1.64e-05 | 1917.0ms/step | 31.3 samples/s
+Step 44700/50462 | Loss: 2.4670 | LR: 1.64e-05 | 1917.0ms/step | 31.3 samples/s
+Step 44710/50462 | Loss: 2.4709 | LR: 1.63e-05 | 1915.9ms/step | 31.3 samples/s
+Step 44720/50462 | Loss: 2.4967 | LR: 1.63e-05 | 1919.8ms/step | 31.3 samples/s
+Step 44730/50462 | Loss: 2.4918 | LR: 1.62e-05 | 1916.4ms/step | 31.3 samples/s
+Step 44740/50462 | Loss: 2.4737 | LR: 1.62e-05 | 1917.5ms/step | 31.3 samples/s
+Step 44750/50462 | Loss: 2.4531 | LR: 1.61e-05 | 1917.3ms/step | 31.3 samples/s
+Validation at step 44750: Loss = 2.4989, Perplexity = 12.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-44750
+Step 44760/50462 | Loss: 2.4520 | LR: 1.61e-05 | 3258.3ms/step | 18.4 samples/s
+Step 44770/50462 | Loss: 2.4371 | LR: 1.60e-05 | 1918.1ms/step | 31.3 samples/s
+Step 44780/50462 | Loss: 2.4705 | LR: 1.59e-05 | 1919.3ms/step | 31.3 samples/s
+Step 44790/50462 | Loss: 2.5051 | LR: 1.59e-05 | 1920.8ms/step | 31.2 samples/s
+Step 44800/50462 | Loss: 2.4233 | LR: 1.58e-05 | 1923.4ms/step | 31.2 samples/s
+Step 44810/50462 | Loss: 2.4711 | LR: 1.58e-05 | 1921.8ms/step | 31.2 samples/s
+Step 44820/50462 | Loss: 2.4165 | LR: 1.57e-05 | 1920.4ms/step | 31.2 samples/s
+Step 44830/50462 | Loss: 2.4607 | LR: 1.57e-05 | 1926.1ms/step | 31.2 samples/s
+Step 44840/50462 | Loss: 2.4703 | LR: 1.56e-05 | 1924.7ms/step | 31.2 samples/s
+Step 44850/50462 | Loss: 2.4942 | LR: 1.56e-05 | 1924.5ms/step | 31.2 samples/s
+Step 44860/50462 | Loss: 2.4232 | LR: 1.55e-05 | 1921.1ms/step | 31.2 samples/s
+Step 44870/50462 | Loss: 2.4886 | LR: 1.54e-05 | 1922.1ms/step | 31.2 samples/s
+Step 44880/50462 | Loss: 2.4622 | LR: 1.54e-05 | 1921.8ms/step | 31.2 samples/s
+Step 44890/50462 | Loss: 2.4408 | LR: 1.53e-05 | 1921.2ms/step | 31.2 samples/s
+Step 44900/50462 | Loss: 2.4750 | LR: 1.53e-05 | 1925.0ms/step | 31.2 samples/s
+Step 44910/50462 | Loss: 2.4820 | LR: 1.52e-05 | 1921.9ms/step | 31.2 samples/s
+Step 44920/50462 | Loss: 2.4474 | LR: 1.52e-05 | 1918.8ms/step | 31.3 samples/s
+Step 44930/50462 | Loss: 2.4809 | LR: 1.51e-05 | 1923.2ms/step | 31.2 samples/s
+Step 44940/50462 | Loss: 2.4949 | LR: 1.51e-05 | 1927.2ms/step | 31.1 samples/s
+Step 44950/50462 | Loss: 2.4415 | LR: 1.50e-05 | 1923.8ms/step | 31.2 samples/s
+Step 44960/50462 | Loss: 2.4358 | LR: 1.50e-05 | 1923.1ms/step | 31.2 samples/s
+Step 44970/50462 | Loss: 2.4449 | LR: 1.49e-05 | 1921.4ms/step | 31.2 samples/s
+Step 44980/50462 | Loss: 2.4818 | LR: 1.49e-05 | 1922.1ms/step | 31.2 samples/s
+Step 44990/50462 | Loss: 2.4448 | LR: 1.48e-05 | 1925.0ms/step | 31.2 samples/s
+Step 45000/50462 | Loss: 2.4615 | LR: 1.47e-05 | 1922.8ms/step | 31.2 samples/s
+Validation at step 45000: Loss = 2.4988, Perplexity = 12.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-45000
+Step 45010/50462 | Loss: 2.4564 | LR: 1.47e-05 | 3297.9ms/step | 18.2 samples/s
+Step 45020/50462 | Loss: 2.4558 | LR: 1.46e-05 | 1913.5ms/step | 31.4 samples/s
+Step 45030/50462 | Loss: 2.4511 | LR: 1.46e-05 | 1921.1ms/step | 31.2 samples/s
+Step 45040/50462 | Loss: 2.4349 | LR: 1.45e-05 | 1919.2ms/step | 31.3 samples/s
+Step 45050/50462 | Loss: 2.4490 | LR: 1.45e-05 | 1921.6ms/step | 31.2 samples/s
+Step 45060/50462 | Loss: 2.4522 | LR: 1.44e-05 | 1921.0ms/step | 31.2 samples/s
+Step 45070/50462 | Loss: 2.4234 | LR: 1.44e-05 | 1922.6ms/step | 31.2 samples/s
+Step 45080/50462 | Loss: 2.4709 | LR: 1.43e-05 | 1925.0ms/step | 31.2 samples/s
+Step 45090/50462 | Loss: 2.4800 | LR: 1.43e-05 | 1922.0ms/step | 31.2 samples/s
+Step 45100/50462 | Loss: 2.4691 | LR: 1.42e-05 | 1924.3ms/step | 31.2 samples/s
+Step 45110/50462 | Loss: 2.4342 | LR: 1.42e-05 | 1921.2ms/step | 31.2 samples/s
+Step 45120/50462 | Loss: 2.4736 | LR: 1.41e-05 | 1925.0ms/step | 31.2 samples/s
+Step 45130/50462 | Loss: 2.4437 | LR: 1.41e-05 | 1926.2ms/step | 31.1 samples/s
+Step 45140/50462 | Loss: 2.5367 | LR: 1.40e-05 | 1925.4ms/step | 31.2 samples/s
+Step 45150/50462 | Loss: 2.4483 | LR: 1.40e-05 | 1925.4ms/step | 31.2 samples/s
+Step 45160/50462 | Loss: 2.4879 | LR: 1.39e-05 | 1923.7ms/step | 31.2 samples/s
+Step 45170/50462 | Loss: 2.4924 | LR: 1.39e-05 | 1925.0ms/step | 31.2 samples/s
+Step 45180/50462 | Loss: 2.4560 | LR: 1.38e-05 | 1925.9ms/step | 31.2 samples/s
+Step 45190/50462 | Loss: 2.4832 | LR: 1.37e-05 | 1918.1ms/step | 31.3 samples/s
+Step 45200/50462 | Loss: 2.4594 | LR: 1.37e-05 | 1919.9ms/step | 31.3 samples/s
+Step 45210/50462 | Loss: 2.4270 | LR: 1.36e-05 | 1918.0ms/step | 31.3 samples/s
+Step 45220/50462 | Loss: 2.4418 | LR: 1.36e-05 | 1921.2ms/step | 31.2 samples/s
+Step 45230/50462 | Loss: 2.5007 | LR: 1.35e-05 | 1916.5ms/step | 31.3 samples/s
+Step 45240/50462 | Loss: 2.5106 | LR: 1.35e-05 | 1919.5ms/step | 31.3 samples/s
+Step 45250/50462 | Loss: 2.4533 | LR: 1.34e-05 | 1916.5ms/step | 31.3 samples/s
+Validation at step 45250: Loss = 2.4988, Perplexity = 12.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-45250
+Step 45260/50462 | Loss: 2.4836 | LR: 1.34e-05 | 3259.1ms/step | 18.4 samples/s
+Step 45270/50462 | Loss: 2.5005 | LR: 1.33e-05 | 1914.5ms/step | 31.3 samples/s
+Step 45280/50462 | Loss: 2.4417 | LR: 1.33e-05 | 1919.6ms/step | 31.3 samples/s
+Step 45290/50462 | Loss: 2.4753 | LR: 1.32e-05 | 1920.2ms/step | 31.2 samples/s
+Step 45300/50462 | Loss: 2.5373 | LR: 1.32e-05 | 1921.4ms/step | 31.2 samples/s
+Step 45310/50462 | Loss: 2.4478 | LR: 1.31e-05 | 1921.8ms/step | 31.2 samples/s
+Step 45320/50462 | Loss: 2.4725 | LR: 1.31e-05 | 1921.9ms/step | 31.2 samples/s
+Step 45330/50462 | Loss: 2.4741 | LR: 1.30e-05 | 1924.7ms/step | 31.2 samples/s
+Step 45340/50462 | Loss: 2.4498 | LR: 1.30e-05 | 1922.6ms/step | 31.2 samples/s
+Step 45350/50462 | Loss: 2.4930 | LR: 1.29e-05 | 1923.5ms/step | 31.2 samples/s
+Step 45360/50462 | Loss: 2.4648 | LR: 1.29e-05 | 1921.3ms/step | 31.2 samples/s
+Step 45370/50462 | Loss: 2.4564 | LR: 1.28e-05 | 1921.6ms/step | 31.2 samples/s
+Step 45380/50462 | Loss: 2.4427 | LR: 1.28e-05 | 1925.5ms/step | 31.2 samples/s
+Step 45390/50462 | Loss: 2.4447 | LR: 1.27e-05 | 1923.7ms/step | 31.2 samples/s
+Step 45400/50462 | Loss: 2.4423 | LR: 1.27e-05 | 1922.4ms/step | 31.2 samples/s
+Step 45410/50462 | Loss: 2.4823 | LR: 1.26e-05 | 1925.1ms/step | 31.2 samples/s
+Step 45420/50462 | Loss: 2.4698 | LR: 1.26e-05 | 1924.6ms/step | 31.2 samples/s
+Step 45430/50462 | Loss: 2.5209 | LR: 1.25e-05 | 1923.3ms/step | 31.2 samples/s
+Step 45440/50462 | Loss: 2.4659 | LR: 1.25e-05 | 1922.7ms/step | 31.2 samples/s
+Step 45450/50462 | Loss: 2.5022 | LR: 1.24e-05 | 1927.5ms/step | 31.1 samples/s
+Step 45460/50462 | Loss: 2.4470 | LR: 1.24e-05 | 1928.4ms/step | 31.1 samples/s
+Step 45470/50462 | Loss: 2.4433 | LR: 1.23e-05 | 1925.4ms/step | 31.2 samples/s
+Step 45480/50462 | Loss: 2.4502 | LR: 1.23e-05 | 1924.0ms/step | 31.2 samples/s
+Step 45490/50462 | Loss: 2.4775 | LR: 1.22e-05 | 1923.6ms/step | 31.2 samples/s
+Step 45500/50462 | Loss: 2.4669 | LR: 1.22e-05 | 1925.9ms/step | 31.2 samples/s
+Validation at step 45500: Loss = 2.4988, Perplexity = 12.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-45500
+Step 45510/50462 | Loss: 2.4472 | LR: 1.21e-05 | 3256.6ms/step | 18.4 samples/s
+Step 45520/50462 | Loss: 2.4466 | LR: 1.21e-05 | 1918.1ms/step | 31.3 samples/s
+Step 45530/50462 | Loss: 2.4601 | LR: 1.20e-05 | 1918.6ms/step | 31.3 samples/s
+Step 45540/50462 | Loss: 2.4692 | LR: 1.20e-05 | 1921.5ms/step | 31.2 samples/s
+Step 45550/50462 | Loss: 2.4633 | LR: 1.19e-05 | 1922.5ms/step | 31.2 samples/s
+Step 45560/50462 | Loss: 2.4711 | LR: 1.19e-05 | 1925.7ms/step | 31.2 samples/s
+Step 45570/50462 | Loss: 2.4809 | LR: 1.19e-05 | 1922.6ms/step | 31.2 samples/s
+Step 45580/50462 | Loss: 2.4433 | LR: 1.18e-05 | 1923.5ms/step | 31.2 samples/s
+Step 45590/50462 | Loss: 2.4561 | LR: 1.18e-05 | 1921.5ms/step | 31.2 samples/s
+Step 45600/50462 | Loss: 2.5045 | LR: 1.17e-05 | 1922.3ms/step | 31.2 samples/s
+Step 45610/50462 | Loss: 2.4789 | LR: 1.17e-05 | 1917.8ms/step | 31.3 samples/s
+Step 45620/50462 | Loss: 2.4775 | LR: 1.16e-05 | 1917.5ms/step | 31.3 samples/s
+Step 45630/50462 | Loss: 2.4762 | LR: 1.16e-05 | 1920.6ms/step | 31.2 samples/s
+Step 45640/50462 | Loss: 2.5028 | LR: 1.15e-05 | 1919.1ms/step | 31.3 samples/s
+Step 45650/50462 | Loss: 2.4313 | LR: 1.15e-05 | 1919.1ms/step | 31.3 samples/s
+Step 45660/50462 | Loss: 2.4793 | LR: 1.14e-05 | 1919.2ms/step | 31.3 samples/s
+Step 45670/50462 | Loss: 2.4838 | LR: 1.14e-05 | 1918.9ms/step | 31.3 samples/s
+Step 45680/50462 | Loss: 2.4650 | LR: 1.13e-05 | 1914.8ms/step | 31.3 samples/s
+Step 45690/50462 | Loss: 2.4721 | LR: 1.13e-05 | 1918.7ms/step | 31.3 samples/s
+Step 45700/50462 | Loss: 2.5494 | LR: 1.12e-05 | 1919.1ms/step | 31.3 samples/s
+Step 45710/50462 | Loss: 2.4502 | LR: 1.12e-05 | 1917.5ms/step | 31.3 samples/s
+Step 45720/50462 | Loss: 2.4356 | LR: 1.11e-05 | 1920.6ms/step | 31.2 samples/s
+Step 45730/50462 | Loss: 2.4605 | LR: 1.11e-05 | 1918.5ms/step | 31.3 samples/s
+Step 45740/50462 | Loss: 2.4800 | LR: 1.10e-05 | 1917.8ms/step | 31.3 samples/s
+Step 45750/50462 | Loss: 2.4708 | LR: 1.10e-05 | 1917.3ms/step | 31.3 samples/s
+Validation at step 45750: Loss = 2.4988, Perplexity = 12.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-45750
+Step 45760/50462 | Loss: 2.4833 | LR: 1.10e-05 | 3266.3ms/step | 18.4 samples/s
+Step 45770/50462 | Loss: 2.4648 | LR: 1.09e-05 | 1917.2ms/step | 31.3 samples/s
+Step 45780/50462 | Loss: 2.4417 | LR: 1.09e-05 | 1917.4ms/step | 31.3 samples/s
+Step 45790/50462 | Loss: 2.4261 | LR: 1.08e-05 | 1916.5ms/step | 31.3 samples/s
+Step 45800/50462 | Loss: 2.5306 | LR: 1.08e-05 | 1918.6ms/step | 31.3 samples/s
+Step 45810/50462 | Loss: 2.4634 | LR: 1.07e-05 | 1917.1ms/step | 31.3 samples/s
+Step 45820/50462 | Loss: 2.4529 | LR: 1.07e-05 | 1919.8ms/step | 31.3 samples/s
+Step 45830/50462 | Loss: 2.4374 | LR: 1.06e-05 | 1919.7ms/step | 31.3 samples/s
+Step 45840/50462 | Loss: 2.4661 | LR: 1.06e-05 | 1916.9ms/step | 31.3 samples/s
+Step 45850/50462 | Loss: 2.4758 | LR: 1.05e-05 | 1916.7ms/step | 31.3 samples/s
+Step 45860/50462 | Loss: 2.4522 | LR: 1.05e-05 | 1919.5ms/step | 31.3 samples/s
+Step 45870/50462 | Loss: 2.4374 | LR: 1.05e-05 | 1919.9ms/step | 31.3 samples/s
+Step 45880/50462 | Loss: 2.5036 | LR: 1.04e-05 | 1918.5ms/step | 31.3 samples/s
+Step 45890/50462 | Loss: 2.4574 | LR: 1.04e-05 | 1919.1ms/step | 31.3 samples/s
+Step 45900/50462 | Loss: 2.4612 | LR: 1.03e-05 | 1921.5ms/step | 31.2 samples/s
+Step 45910/50462 | Loss: 2.4739 | LR: 1.03e-05 | 1919.9ms/step | 31.3 samples/s
+Step 45920/50462 | Loss: 2.4628 | LR: 1.02e-05 | 1917.6ms/step | 31.3 samples/s
+Step 45930/50462 | Loss: 2.5061 | LR: 1.02e-05 | 1919.6ms/step | 31.3 samples/s
+Step 45940/50462 | Loss: 2.4795 | LR: 1.01e-05 | 1920.5ms/step | 31.2 samples/s
+Step 45950/50462 | Loss: 2.4315 | LR: 1.01e-05 | 1919.0ms/step | 31.3 samples/s
+Step 45960/50462 | Loss: 2.4746 | LR: 1.01e-05 | 1918.9ms/step | 31.3 samples/s
+Step 45970/50462 | Loss: 2.4716 | LR: 1.00e-05 | 1917.6ms/step | 31.3 samples/s
+Step 45980/50462 | Loss: 2.4656 | LR: 9.96e-06 | 1918.6ms/step | 31.3 samples/s
+Step 45990/50462 | Loss: 2.4984 | LR: 9.92e-06 | 1919.3ms/step | 31.3 samples/s
+Step 46000/50462 | Loss: 2.4757 | LR: 9.87e-06 | 1919.4ms/step | 31.3 samples/s
+Validation at step 46000: Loss = 2.4987, Perplexity = 12.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-46000
+Step 46010/50462 | Loss: 2.4585 | LR: 9.83e-06 | 3253.9ms/step | 18.4 samples/s
+Step 46020/50462 | Loss: 2.4217 | LR: 9.79e-06 | 1920.0ms/step | 31.3 samples/s
+Step 46030/50462 | Loss: 2.4556 | LR: 9.74e-06 | 1921.3ms/step | 31.2 samples/s
+Step 46040/50462 | Loss: 2.4790 | LR: 9.70e-06 | 1924.0ms/step | 31.2 samples/s
+Step 46050/50462 | Loss: 2.4203 | LR: 9.65e-06 | 1924.2ms/step | 31.2 samples/s
+Step 46060/50462 | Loss: 2.4171 | LR: 9.61e-06 | 1925.3ms/step | 31.2 samples/s
+Step 46070/50462 | Loss: 2.4707 | LR: 9.57e-06 | 1922.5ms/step | 31.2 samples/s
+Step 46080/50462 | Loss: 2.4632 | LR: 9.52e-06 | 1920.8ms/step | 31.2 samples/s
+Step 46090/50462 | Loss: 2.4981 | LR: 9.48e-06 | 1926.1ms/step | 31.2 samples/s
+Step 46100/50462 | Loss: 2.5062 | LR: 9.44e-06 | 1923.3ms/step | 31.2 samples/s
+Step 46110/50462 | Loss: 2.4697 | LR: 9.40e-06 | 1925.7ms/step | 31.2 samples/s
+Step 46120/50462 | Loss: 2.4609 | LR: 9.35e-06 | 1921.6ms/step | 31.2 samples/s
+Step 46130/50462 | Loss: 2.4842 | LR: 9.31e-06 | 1923.0ms/step | 31.2 samples/s
+Step 46140/50462 | Loss: 2.5019 | LR: 9.27e-06 | 1924.3ms/step | 31.2 samples/s
+Step 46150/50462 | Loss: 2.4675 | LR: 9.22e-06 | 1922.2ms/step | 31.2 samples/s
+Step 46160/50462 | Loss: 2.4752 | LR: 9.18e-06 | 1924.1ms/step | 31.2 samples/s
+Step 46170/50462 | Loss: 2.4904 | LR: 9.14e-06 | 1921.1ms/step | 31.2 samples/s
+Step 46180/50462 | Loss: 2.5061 | LR: 9.10e-06 | 1927.1ms/step | 31.1 samples/s
+Step 46190/50462 | Loss: 2.4397 | LR: 9.06e-06 | 1921.3ms/step | 31.2 samples/s
+Step 46200/50462 | Loss: 2.4606 | LR: 9.01e-06 | 1924.9ms/step | 31.2 samples/s
+Step 46210/50462 | Loss: 2.4374 | LR: 8.97e-06 | 1923.3ms/step | 31.2 samples/s
+Step 46220/50462 | Loss: 2.4603 | LR: 8.93e-06 | 1926.2ms/step | 31.1 samples/s
+Step 46230/50462 | Loss: 2.4428 | LR: 8.89e-06 | 1924.9ms/step | 31.2 samples/s
+Step 46240/50462 | Loss: 2.4603 | LR: 8.85e-06 | 1924.5ms/step | 31.2 samples/s
+Step 46250/50462 | Loss: 2.4302 | LR: 8.80e-06 | 1927.6ms/step | 31.1 samples/s
+Validation at step 46250: Loss = 2.4988, Perplexity = 12.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-46250
+Step 46260/50462 | Loss: 2.5131 | LR: 8.76e-06 | 3305.1ms/step | 18.2 samples/s
+Step 46270/50462 | Loss: 2.4862 | LR: 8.72e-06 | 1918.8ms/step | 31.3 samples/s
+Step 46280/50462 | Loss: 2.4912 | LR: 8.68e-06 | 1922.8ms/step | 31.2 samples/s
+Step 46290/50462 | Loss: 2.4960 | LR: 8.64e-06 | 1923.7ms/step | 31.2 samples/s
+Step 46300/50462 | Loss: 2.4733 | LR: 8.60e-06 | 1921.9ms/step | 31.2 samples/s
+Step 46310/50462 | Loss: 2.4538 | LR: 8.56e-06 | 1922.6ms/step | 31.2 samples/s
+Step 46320/50462 | Loss: 2.4543 | LR: 8.52e-06 | 1923.9ms/step | 31.2 samples/s
+Step 46330/50462 | Loss: 2.4846 | LR: 8.47e-06 | 1923.6ms/step | 31.2 samples/s
+Step 46340/50462 | Loss: 2.4799 | LR: 8.43e-06 | 1920.3ms/step | 31.2 samples/s
+Step 46350/50462 | Loss: 2.4437 | LR: 8.39e-06 | 1922.9ms/step | 31.2 samples/s
+Step 46360/50462 | Loss: 2.4783 | LR: 8.35e-06 | 1925.1ms/step | 31.2 samples/s
+Step 46370/50462 | Loss: 2.4630 | LR: 8.31e-06 | 1928.3ms/step | 31.1 samples/s
+Step 46380/50462 | Loss: 2.4529 | LR: 8.27e-06 | 1921.9ms/step | 31.2 samples/s
+Step 46390/50462 | Loss: 2.4523 | LR: 8.23e-06 | 1924.4ms/step | 31.2 samples/s
+Step 46400/50462 | Loss: 2.4968 | LR: 8.19e-06 | 1923.1ms/step | 31.2 samples/s
+Step 46410/50462 | Loss: 2.4479 | LR: 8.15e-06 | 1924.4ms/step | 31.2 samples/s
+Step 46420/50462 | Loss: 2.4419 | LR: 8.11e-06 | 1921.1ms/step | 31.2 samples/s
+Step 46430/50462 | Loss: 2.4370 | LR: 8.07e-06 | 1924.7ms/step | 31.2 samples/s
+Step 46440/50462 | Loss: 2.4812 | LR: 8.03e-06 | 1926.3ms/step | 31.1 samples/s
+Step 46450/50462 | Loss: 2.4597 | LR: 7.99e-06 | 1922.6ms/step | 31.2 samples/s
+Step 46460/50462 | Loss: 2.4850 | LR: 7.95e-06 | 1925.3ms/step | 31.2 samples/s
+Step 46470/50462 | Loss: 2.4079 | LR: 7.91e-06 | 1924.7ms/step | 31.2 samples/s
+Step 46480/50462 | Loss: 2.4684 | LR: 7.87e-06 | 1926.0ms/step | 31.2 samples/s
+Step 46490/50462 | Loss: 2.4641 | LR: 7.83e-06 | 1925.4ms/step | 31.2 samples/s
+Step 46500/50462 | Loss: 2.4763 | LR: 7.80e-06 | 1924.2ms/step | 31.2 samples/s
+Validation at step 46500: Loss = 2.4987, Perplexity = 12.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-46500
+Step 46510/50462 | Loss: 2.4691 | LR: 7.76e-06 | 3379.2ms/step | 17.8 samples/s
+Step 46520/50462 | Loss: 2.4238 | LR: 7.72e-06 | 1914.9ms/step | 31.3 samples/s
+Step 46530/50462 | Loss: 2.4831 | LR: 7.68e-06 | 1919.3ms/step | 31.3 samples/s
+Step 46540/50462 | Loss: 2.5015 | LR: 7.64e-06 | 1921.6ms/step | 31.2 samples/s
+Step 46550/50462 | Loss: 2.4583 | LR: 7.60e-06 | 1924.9ms/step | 31.2 samples/s
+Step 46560/50462 | Loss: 2.4948 | LR: 7.56e-06 | 1924.7ms/step | 31.2 samples/s
+Step 46570/50462 | Loss: 2.4723 | LR: 7.52e-06 | 1923.2ms/step | 31.2 samples/s
+Step 46580/50462 | Loss: 2.4477 | LR: 7.49e-06 | 1921.8ms/step | 31.2 samples/s
+Step 46590/50462 | Loss: 2.4772 | LR: 7.45e-06 | 1919.9ms/step | 31.3 samples/s
+Step 46600/50462 | Loss: 2.4728 | LR: 7.41e-06 | 1921.2ms/step | 31.2 samples/s
+Step 46610/50462 | Loss: 2.4965 | LR: 7.37e-06 | 1916.2ms/step | 31.3 samples/s
+Step 46620/50462 | Loss: 2.4331 | LR: 7.33e-06 | 1919.2ms/step | 31.3 samples/s
+Step 46630/50462 | Loss: 2.4514 | LR: 7.29e-06 | 1915.5ms/step | 31.3 samples/s
+Step 46640/50462 | Loss: 2.4242 | LR: 7.26e-06 | 1918.1ms/step | 31.3 samples/s
+Step 46650/50462 | Loss: 2.5266 | LR: 7.22e-06 | 1919.0ms/step | 31.3 samples/s
+Step 46660/50462 | Loss: 2.4518 | LR: 7.18e-06 | 1919.6ms/step | 31.3 samples/s
+Step 46670/50462 | Loss: 2.4514 | LR: 7.14e-06 | 1918.6ms/step | 31.3 samples/s
+Step 46680/50462 | Loss: 2.4667 | LR: 7.11e-06 | 1920.5ms/step | 31.2 samples/s
+Step 46690/50462 | Loss: 2.4927 | LR: 7.07e-06 | 1921.2ms/step | 31.2 samples/s
+Step 46700/50462 | Loss: 2.4737 | LR: 7.03e-06 | 1920.9ms/step | 31.2 samples/s
+Step 46710/50462 | Loss: 2.4743 | LR: 6.99e-06 | 1917.6ms/step | 31.3 samples/s
+Step 46720/50462 | Loss: 2.4817 | LR: 6.96e-06 | 1920.0ms/step | 31.3 samples/s
+Step 46730/50462 | Loss: 2.4341 | LR: 6.92e-06 | 1917.6ms/step | 31.3 samples/s
+Step 46740/50462 | Loss: 2.4589 | LR: 6.88e-06 | 1917.3ms/step | 31.3 samples/s
+Step 46750/50462 | Loss: 2.3727 | LR: 6.85e-06 | 1919.4ms/step | 31.3 samples/s
+Validation at step 46750: Loss = 2.4988, Perplexity = 12.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-46750
+Step 46760/50462 | Loss: 2.4908 | LR: 6.81e-06 | 3259.7ms/step | 18.4 samples/s
+Step 46770/50462 | Loss: 2.4642 | LR: 6.77e-06 | 1917.2ms/step | 31.3 samples/s
+Step 46780/50462 | Loss: 2.4925 | LR: 6.74e-06 | 1919.8ms/step | 31.3 samples/s
+Step 46790/50462 | Loss: 2.4438 | LR: 6.70e-06 | 1920.5ms/step | 31.2 samples/s
+Step 46800/50462 | Loss: 2.4609 | LR: 6.66e-06 | 1916.0ms/step | 31.3 samples/s
+Step 46810/50462 | Loss: 2.4237 | LR: 6.63e-06 | 1922.1ms/step | 31.2 samples/s
+Step 46820/50462 | Loss: 2.4838 | LR: 6.59e-06 | 1921.4ms/step | 31.2 samples/s
+Step 46830/50462 | Loss: 2.4396 | LR: 6.56e-06 | 1924.3ms/step | 31.2 samples/s
+Step 46840/50462 | Loss: 2.4829 | LR: 6.52e-06 | 1924.1ms/step | 31.2 samples/s
+Step 46850/50462 | Loss: 2.4540 | LR: 6.48e-06 | 1926.4ms/step | 31.1 samples/s
+Step 46860/50462 | Loss: 2.4880 | LR: 6.45e-06 | 1925.9ms/step | 31.2 samples/s
+Step 46870/50462 | Loss: 2.4926 | LR: 6.41e-06 | 1923.0ms/step | 31.2 samples/s
+Step 46880/50462 | Loss: 2.4483 | LR: 6.38e-06 | 1924.0ms/step | 31.2 samples/s
+Step 46890/50462 | Loss: 2.4671 | LR: 6.34e-06 | 1924.3ms/step | 31.2 samples/s
+Step 46900/50462 | Loss: 2.4385 | LR: 6.31e-06 | 1925.6ms/step | 31.2 samples/s
+Step 46910/50462 | Loss: 2.4853 | LR: 6.27e-06 | 1926.8ms/step | 31.1 samples/s
+Step 46920/50462 | Loss: 2.4426 | LR: 6.24e-06 | 1924.7ms/step | 31.2 samples/s
+Step 46930/50462 | Loss: 2.4577 | LR: 6.20e-06 | 1922.6ms/step | 31.2 samples/s
+Step 46940/50462 | Loss: 2.4792 | LR: 6.17e-06 | 1925.1ms/step | 31.2 samples/s
+Step 46950/50462 | Loss: 2.4722 | LR: 6.13e-06 | 1920.2ms/step | 31.2 samples/s
+Step 46960/50462 | Loss: 2.4641 | LR: 6.10e-06 | 1925.4ms/step | 31.2 samples/s
+Step 46970/50462 | Loss: 2.4542 | LR: 6.06e-06 | 1922.8ms/step | 31.2 samples/s
+Step 46980/50462 | Loss: 2.4552 | LR: 6.03e-06 | 1925.5ms/step | 31.2 samples/s
+Step 46990/50462 | Loss: 2.4703 | LR: 5.99e-06 | 1921.4ms/step | 31.2 samples/s
+Step 47000/50462 | Loss: 2.4782 | LR: 5.96e-06 | 1926.4ms/step | 31.1 samples/s
+Validation at step 47000: Loss = 2.4988, Perplexity = 12.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-47000
+Step 47010/50462 | Loss: 2.4535 | LR: 5.93e-06 | 3250.4ms/step | 18.5 samples/s
+Step 47020/50462 | Loss: 2.4650 | LR: 5.89e-06 | 1917.8ms/step | 31.3 samples/s
+Step 47030/50462 | Loss: 2.4534 | LR: 5.86e-06 | 1921.7ms/step | 31.2 samples/s
+Step 47040/50462 | Loss: 2.4926 | LR: 5.82e-06 | 1916.4ms/step | 31.3 samples/s
+Step 47050/50462 | Loss: 2.4695 | LR: 5.79e-06 | 1916.4ms/step | 31.3 samples/s
+Step 47060/50462 | Loss: 2.4439 | LR: 5.76e-06 | 1917.8ms/step | 31.3 samples/s
+Step 47070/50462 | Loss: 2.4558 | LR: 5.72e-06 | 1919.2ms/step | 31.3 samples/s
+Step 47080/50462 | Loss: 2.4468 | LR: 5.69e-06 | 1917.5ms/step | 31.3 samples/s
+Step 47090/50462 | Loss: 2.4588 | LR: 5.65e-06 | 1917.3ms/step | 31.3 samples/s
+Step 47100/50462 | Loss: 2.4943 | LR: 5.62e-06 | 1919.1ms/step | 31.3 samples/s
+Step 47110/50462 | Loss: 2.4844 | LR: 5.59e-06 | 1919.1ms/step | 31.3 samples/s
+Step 47120/50462 | Loss: 2.4444 | LR: 5.56e-06 | 1920.5ms/step | 31.2 samples/s
+Step 47130/50462 | Loss: 2.4195 | LR: 5.52e-06 | 1919.7ms/step | 31.3 samples/s
+Step 47140/50462 | Loss: 2.4494 | LR: 5.49e-06 | 1919.1ms/step | 31.3 samples/s
+Step 47150/50462 | Loss: 2.4933 | LR: 5.46e-06 | 1917.7ms/step | 31.3 samples/s
+Step 47160/50462 | Loss: 2.4474 | LR: 5.42e-06 | 1919.0ms/step | 31.3 samples/s
+Step 47170/50462 | Loss: 2.4742 | LR: 5.39e-06 | 1914.4ms/step | 31.3 samples/s
+Step 47180/50462 | Loss: 2.4621 | LR: 5.36e-06 | 1917.5ms/step | 31.3 samples/s
+Step 47190/50462 | Loss: 2.4557 | LR: 5.33e-06 | 1917.9ms/step | 31.3 samples/s
+Step 47200/50462 | Loss: 2.4809 | LR: 5.29e-06 | 1916.7ms/step | 31.3 samples/s
+Step 47210/50462 | Loss: 2.4132 | LR: 5.26e-06 | 1915.9ms/step | 31.3 samples/s
+Step 47220/50462 | Loss: 2.4667 | LR: 5.23e-06 | 1918.4ms/step | 31.3 samples/s
+Step 47230/50462 | Loss: 2.4535 | LR: 5.20e-06 | 1918.5ms/step | 31.3 samples/s
+Step 47240/50462 | Loss: 2.4722 | LR: 5.16e-06 | 1921.0ms/step | 31.2 samples/s
+Step 47250/50462 | Loss: 2.4480 | LR: 5.13e-06 | 1918.1ms/step | 31.3 samples/s
+Validation at step 47250: Loss = 2.4988, Perplexity = 12.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-47250
+Step 47260/50462 | Loss: 2.4855 | LR: 5.10e-06 | 3246.2ms/step | 18.5 samples/s
+Step 47270/50462 | Loss: 2.4627 | LR: 5.07e-06 | 1918.3ms/step | 31.3 samples/s
+Step 47280/50462 | Loss: 2.4669 | LR: 5.04e-06 | 1919.0ms/step | 31.3 samples/s
+Step 47290/50462 | Loss: 2.4736 | LR: 5.01e-06 | 1920.0ms/step | 31.3 samples/s
+Step 47300/50462 | Loss: 2.4853 | LR: 4.97e-06 | 1915.0ms/step | 31.3 samples/s
+Step 47310/50462 | Loss: 2.4795 | LR: 4.94e-06 | 1916.7ms/step | 31.3 samples/s
+Step 47320/50462 | Loss: 2.4753 | LR: 4.91e-06 | 1917.1ms/step | 31.3 samples/s
+Step 47330/50462 | Loss: 2.4399 | LR: 4.88e-06 | 1917.4ms/step | 31.3 samples/s
+Step 47340/50462 | Loss: 2.4534 | LR: 4.85e-06 | 1916.5ms/step | 31.3 samples/s
+Step 47350/50462 | Loss: 2.4654 | LR: 4.82e-06 | 1916.9ms/step | 31.3 samples/s
+Step 47360/50462 | Loss: 2.4646 | LR: 4.79e-06 | 1916.5ms/step | 31.3 samples/s
+Step 47370/50462 | Loss: 2.4799 | LR: 4.76e-06 | 1917.1ms/step | 31.3 samples/s
+Step 47380/50462 | Loss: 2.5009 | LR: 4.73e-06 | 1919.5ms/step | 31.3 samples/s
+Step 47390/50462 | Loss: 2.4429 | LR: 4.70e-06 | 1915.6ms/step | 31.3 samples/s
+Step 47400/50462 | Loss: 2.4791 | LR: 4.67e-06 | 1915.1ms/step | 31.3 samples/s
+Step 47410/50462 | Loss: 2.4885 | LR: 4.64e-06 | 1915.8ms/step | 31.3 samples/s
+Step 47420/50462 | Loss: 2.4061 | LR: 4.61e-06 | 1917.6ms/step | 31.3 samples/s
+Step 47430/50462 | Loss: 2.4439 | LR: 4.58e-06 | 1917.3ms/step | 31.3 samples/s
+Step 47440/50462 | Loss: 2.4648 | LR: 4.55e-06 | 1919.3ms/step | 31.3 samples/s
+Step 47450/50462 | Loss: 2.5031 | LR: 4.52e-06 | 1917.8ms/step | 31.3 samples/s
+Step 47460/50462 | Loss: 2.4547 | LR: 4.49e-06 | 1919.5ms/step | 31.3 samples/s
+Step 47470/50462 | Loss: 2.4539 | LR: 4.46e-06 | 1916.7ms/step | 31.3 samples/s
+Step 47480/50462 | Loss: 2.4742 | LR: 4.43e-06 | 1916.8ms/step | 31.3 samples/s
+Step 47490/50462 | Loss: 2.4467 | LR: 4.40e-06 | 1916.9ms/step | 31.3 samples/s
+Step 47500/50462 | Loss: 2.4457 | LR: 4.37e-06 | 1919.1ms/step | 31.3 samples/s
+Validation at step 47500: Loss = 2.4988, Perplexity = 12.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-47500
+Step 47510/50462 | Loss: 2.4867 | LR: 4.34e-06 | 3279.4ms/step | 18.3 samples/s
+Step 47520/50462 | Loss: 2.4672 | LR: 4.31e-06 | 1917.2ms/step | 31.3 samples/s
+Step 47530/50462 | Loss: 2.4825 | LR: 4.28e-06 | 1914.2ms/step | 31.3 samples/s
+Step 47540/50462 | Loss: 2.5235 | LR: 4.25e-06 | 1917.8ms/step | 31.3 samples/s
+Step 47550/50462 | Loss: 2.4669 | LR: 4.22e-06 | 1919.8ms/step | 31.3 samples/s
+Step 47560/50462 | Loss: 2.4963 | LR: 4.19e-06 | 1919.6ms/step | 31.3 samples/s
+Step 47570/50462 | Loss: 2.4652 | LR: 4.16e-06 | 1917.0ms/step | 31.3 samples/s
+Step 47580/50462 | Loss: 2.4827 | LR: 4.13e-06 | 1914.1ms/step | 31.3 samples/s
+Step 47590/50462 | Loss: 2.4916 | LR: 4.11e-06 | 1916.7ms/step | 31.3 samples/s
+Step 47600/50462 | Loss: 2.4791 | LR: 4.08e-06 | 1916.5ms/step | 31.3 samples/s
+Step 47610/50462 | Loss: 2.4443 | LR: 4.05e-06 | 1918.4ms/step | 31.3 samples/s
+Step 47620/50462 | Loss: 2.4982 | LR: 4.02e-06 | 1918.0ms/step | 31.3 samples/s
+Step 47630/50462 | Loss: 2.4646 | LR: 3.99e-06 | 1915.9ms/step | 31.3 samples/s
+Step 47640/50462 | Loss: 2.4753 | LR: 3.97e-06 | 1916.3ms/step | 31.3 samples/s
+Step 47650/50462 | Loss: 2.4668 | LR: 3.94e-06 | 1915.7ms/step | 31.3 samples/s
+Step 47660/50462 | Loss: 2.3972 | LR: 3.91e-06 | 1918.7ms/step | 31.3 samples/s
+Step 47670/50462 | Loss: 2.4377 | LR: 3.88e-06 | 1918.6ms/step | 31.3 samples/s
+Step 47680/50462 | Loss: 2.4626 | LR: 3.85e-06 | 1916.1ms/step | 31.3 samples/s
+Step 47690/50462 | Loss: 2.4618 | LR: 3.83e-06 | 1914.7ms/step | 31.3 samples/s
+Step 47700/50462 | Loss: 2.4584 | LR: 3.80e-06 | 1914.0ms/step | 31.3 samples/s
+Step 47710/50462 | Loss: 2.4769 | LR: 3.77e-06 | 1917.2ms/step | 31.3 samples/s
+Step 47720/50462 | Loss: 2.4632 | LR: 3.74e-06 | 1915.7ms/step | 31.3 samples/s
+Step 47730/50462 | Loss: 2.4365 | LR: 3.72e-06 | 1918.4ms/step | 31.3 samples/s
+Step 47740/50462 | Loss: 2.5154 | LR: 3.69e-06 | 1918.2ms/step | 31.3 samples/s
+Step 47750/50462 | Loss: 2.5042 | LR: 3.66e-06 | 1915.1ms/step | 31.3 samples/s
+Validation at step 47750: Loss = 2.4988, Perplexity = 12.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-47750
+Step 47760/50462 | Loss: 2.4718 | LR: 3.64e-06 | 3250.1ms/step | 18.5 samples/s
+Step 47770/50462 | Loss: 2.4704 | LR: 3.61e-06 | 1919.6ms/step | 31.3 samples/s
+Step 47780/50462 | Loss: 2.4749 | LR: 3.58e-06 | 1918.6ms/step | 31.3 samples/s
+Step 47790/50462 | Loss: 2.4595 | LR: 3.56e-06 | 1921.7ms/step | 31.2 samples/s
+Step 47800/50462 | Loss: 2.4364 | LR: 3.53e-06 | 1919.0ms/step | 31.3 samples/s
+Step 47810/50462 | Loss: 2.4587 | LR: 3.50e-06 | 1916.3ms/step | 31.3 samples/s
+Step 47820/50462 | Loss: 2.4801 | LR: 3.48e-06 | 1917.2ms/step | 31.3 samples/s
+Step 47830/50462 | Loss: 2.4532 | LR: 3.45e-06 | 1921.1ms/step | 31.2 samples/s
+Step 47840/50462 | Loss: 2.4763 | LR: 3.42e-06 | 1920.0ms/step | 31.2 samples/s
+Step 47850/50462 | Loss: 2.4456 | LR: 3.40e-06 | 1918.8ms/step | 31.3 samples/s
+Step 47860/50462 | Loss: 2.4704 | LR: 3.37e-06 | 1915.3ms/step | 31.3 samples/s
+Step 47870/50462 | Loss: 2.5078 | LR: 3.35e-06 | 1917.8ms/step | 31.3 samples/s
+Step 47880/50462 | Loss: 2.4665 | LR: 3.32e-06 | 1919.4ms/step | 31.3 samples/s
+Step 47890/50462 | Loss: 2.4418 | LR: 3.30e-06 | 1917.9ms/step | 31.3 samples/s
+Step 47900/50462 | Loss: 2.4607 | LR: 3.27e-06 | 1917.7ms/step | 31.3 samples/s
+Step 47910/50462 | Loss: 2.5139 | LR: 3.24e-06 | 1919.3ms/step | 31.3 samples/s
+Step 47920/50462 | Loss: 2.4614 | LR: 3.22e-06 | 1919.1ms/step | 31.3 samples/s
+Step 47930/50462 | Loss: 2.4829 | LR: 3.19e-06 | 1917.7ms/step | 31.3 samples/s
+Step 47940/50462 | Loss: 2.4583 | LR: 3.17e-06 | 1918.5ms/step | 31.3 samples/s
+Step 47950/50462 | Loss: 2.4685 | LR: 3.14e-06 | 1916.2ms/step | 31.3 samples/s
+Step 47960/50462 | Loss: 2.4592 | LR: 3.12e-06 | 1919.1ms/step | 31.3 samples/s
+Step 47970/50462 | Loss: 2.4420 | LR: 3.09e-06 | 1919.2ms/step | 31.3 samples/s
+Step 47980/50462 | Loss: 2.4619 | LR: 3.07e-06 | 1917.2ms/step | 31.3 samples/s
+Step 47990/50462 | Loss: 2.4489 | LR: 3.04e-06 | 1918.7ms/step | 31.3 samples/s
+Step 48000/50462 | Loss: 2.4927 | LR: 3.02e-06 | 1917.0ms/step | 31.3 samples/s
+Validation at step 48000: Loss = 2.4988, Perplexity = 12.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-48000
+Step 48010/50462 | Loss: 2.4353 | LR: 3.00e-06 | 3264.6ms/step | 18.4 samples/s
+Step 48020/50462 | Loss: 2.4998 | LR: 2.97e-06 | 1917.0ms/step | 31.3 samples/s
+Step 48030/50462 | Loss: 2.4524 | LR: 2.95e-06 | 1920.8ms/step | 31.2 samples/s
+Step 48040/50462 | Loss: 2.4857 | LR: 2.92e-06 | 1923.1ms/step | 31.2 samples/s
+Step 48050/50462 | Loss: 2.4942 | LR: 2.90e-06 | 1923.7ms/step | 31.2 samples/s
+Step 48060/50462 | Loss: 2.4287 | LR: 2.87e-06 | 1922.4ms/step | 31.2 samples/s
+Step 48070/50462 | Loss: 2.4427 | LR: 2.85e-06 | 1923.2ms/step | 31.2 samples/s
+Step 48080/50462 | Loss: 2.5093 | LR: 2.83e-06 | 1923.2ms/step | 31.2 samples/s
+Step 48090/50462 | Loss: 2.4518 | LR: 2.80e-06 | 1922.8ms/step | 31.2 samples/s
+Step 48100/50462 | Loss: 2.4384 | LR: 2.78e-06 | 1921.6ms/step | 31.2 samples/s
+Step 48110/50462 | Loss: 2.4345 | LR: 2.76e-06 | 1923.8ms/step | 31.2 samples/s
+Step 48120/50462 | Loss: 2.4388 | LR: 2.73e-06 | 1922.3ms/step | 31.2 samples/s
+Step 48130/50462 | Loss: 2.5105 | LR: 2.71e-06 | 1922.3ms/step | 31.2 samples/s
+Step 48140/50462 | Loss: 2.4855 | LR: 2.69e-06 | 1925.0ms/step | 31.2 samples/s
+Step 48150/50462 | Loss: 2.4560 | LR: 2.66e-06 | 1923.3ms/step | 31.2 samples/s
+Step 48160/50462 | Loss: 2.4398 | LR: 2.64e-06 | 1925.8ms/step | 31.2 samples/s
+Step 48170/50462 | Loss: 2.4493 | LR: 2.62e-06 | 1926.3ms/step | 31.1 samples/s
+Step 48180/50462 | Loss: 2.4542 | LR: 2.60e-06 | 1923.7ms/step | 31.2 samples/s
+Step 48190/50462 | Loss: 2.4509 | LR: 2.57e-06 | 1924.1ms/step | 31.2 samples/s
+Step 48200/50462 | Loss: 2.4344 | LR: 2.55e-06 | 1921.5ms/step | 31.2 samples/s
+Step 48210/50462 | Loss: 2.4642 | LR: 2.53e-06 | 1925.2ms/step | 31.2 samples/s
+Step 48220/50462 | Loss: 2.4660 | LR: 2.51e-06 | 1924.2ms/step | 31.2 samples/s
+Step 48230/50462 | Loss: 2.4688 | LR: 2.48e-06 | 1925.1ms/step | 31.2 samples/s
+Step 48240/50462 | Loss: 2.5138 | LR: 2.46e-06 | 1923.7ms/step | 31.2 samples/s
+Step 48250/50462 | Loss: 2.4803 | LR: 2.44e-06 | 1923.0ms/step | 31.2 samples/s
+Validation at step 48250: Loss = 2.4988, Perplexity = 12.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-48250
+Step 48260/50462 | Loss: 2.4407 | LR: 2.42e-06 | 3253.6ms/step | 18.4 samples/s
+Step 48270/50462 | Loss: 2.4776 | LR: 2.39e-06 | 1913.4ms/step | 31.4 samples/s
+Step 48280/50462 | Loss: 2.4794 | LR: 2.37e-06 | 1918.6ms/step | 31.3 samples/s
+Step 48290/50462 | Loss: 2.4778 | LR: 2.35e-06 | 1917.6ms/step | 31.3 samples/s
+Step 48300/50462 | Loss: 2.4407 | LR: 2.33e-06 | 1914.1ms/step | 31.3 samples/s
+Step 48310/50462 | Loss: 2.4481 | LR: 2.31e-06 | 1918.3ms/step | 31.3 samples/s
+Step 48320/50462 | Loss: 2.4377 | LR: 2.29e-06 | 1918.9ms/step | 31.3 samples/s
+Step 48330/50462 | Loss: 2.5010 | LR: 2.27e-06 | 1920.3ms/step | 31.2 samples/s
+Step 48340/50462 | Loss: 2.4903 | LR: 2.24e-06 | 1916.2ms/step | 31.3 samples/s
+Step 48350/50462 | Loss: 2.4931 | LR: 2.22e-06 | 1917.0ms/step | 31.3 samples/s
+Step 48360/50462 | Loss: 2.4656 | LR: 2.20e-06 | 1917.5ms/step | 31.3 samples/s
+Step 48370/50462 | Loss: 2.4389 | LR: 2.18e-06 | 1919.1ms/step | 31.3 samples/s
+Step 48380/50462 | Loss: 2.5167 | LR: 2.16e-06 | 1920.9ms/step | 31.2 samples/s
+Step 48390/50462 | Loss: 2.4923 | LR: 2.14e-06 | 1918.0ms/step | 31.3 samples/s
+Step 48400/50462 | Loss: 2.4492 | LR: 2.12e-06 | 1917.7ms/step | 31.3 samples/s
+Step 48410/50462 | Loss: 2.4649 | LR: 2.10e-06 | 1919.0ms/step | 31.3 samples/s
+Step 48420/50462 | Loss: 2.4215 | LR: 2.08e-06 | 1918.6ms/step | 31.3 samples/s
+Step 48430/50462 | Loss: 2.4341 | LR: 2.06e-06 | 1919.2ms/step | 31.3 samples/s
+Step 48440/50462 | Loss: 2.3954 | LR: 2.04e-06 | 1920.9ms/step | 31.2 samples/s
+Step 48450/50462 | Loss: 2.4823 | LR: 2.02e-06 | 1917.7ms/step | 31.3 samples/s
+Step 48460/50462 | Loss: 2.4719 | LR: 2.00e-06 | 1919.9ms/step | 31.3 samples/s
+Step 48470/50462 | Loss: 2.4613 | LR: 1.98e-06 | 1919.7ms/step | 31.3 samples/s
+Step 48480/50462 | Loss: 2.4654 | LR: 1.96e-06 | 1918.4ms/step | 31.3 samples/s
+Step 48490/50462 | Loss: 2.5074 | LR: 1.94e-06 | 1920.5ms/step | 31.2 samples/s
+Step 48500/50462 | Loss: 2.4540 | LR: 1.92e-06 | 1920.0ms/step | 31.2 samples/s
+Validation at step 48500: Loss = 2.4989, Perplexity = 12.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-48500
+Step 48510/50462 | Loss: 2.4430 | LR: 1.90e-06 | 3248.8ms/step | 18.5 samples/s
+Step 48520/50462 | Loss: 2.4406 | LR: 1.88e-06 | 1915.7ms/step | 31.3 samples/s
+Step 48530/50462 | Loss: 2.4425 | LR: 1.86e-06 | 1919.8ms/step | 31.3 samples/s
+Step 48540/50462 | Loss: 2.4552 | LR: 1.84e-06 | 1915.2ms/step | 31.3 samples/s
+Step 48550/50462 | Loss: 2.4557 | LR: 1.82e-06 | 1918.6ms/step | 31.3 samples/s
+Step 48560/50462 | Loss: 2.4222 | LR: 1.80e-06 | 1913.8ms/step | 31.4 samples/s
+Step 48570/50462 | Loss: 2.4386 | LR: 1.78e-06 | 1917.3ms/step | 31.3 samples/s
+Step 48580/50462 | Loss: 2.4402 | LR: 1.77e-06 | 1917.4ms/step | 31.3 samples/s
+Step 48590/50462 | Loss: 2.4742 | LR: 1.75e-06 | 1918.0ms/step | 31.3 samples/s
+Step 48600/50462 | Loss: 2.4750 | LR: 1.73e-06 | 1917.5ms/step | 31.3 samples/s
+Step 48610/50462 | Loss: 2.5085 | LR: 1.71e-06 | 1916.2ms/step | 31.3 samples/s
+Step 48620/50462 | Loss: 2.4287 | LR: 1.69e-06 | 1916.2ms/step | 31.3 samples/s
+Step 48630/50462 | Loss: 2.5159 | LR: 1.67e-06 | 1918.0ms/step | 31.3 samples/s
+Step 48640/50462 | Loss: 2.4347 | LR: 1.66e-06 | 1917.7ms/step | 31.3 samples/s
+Step 48650/50462 | Loss: 2.4472 | LR: 1.64e-06 | 1917.9ms/step | 31.3 samples/s
+Step 48660/50462 | Loss: 2.4389 | LR: 1.62e-06 | 1919.0ms/step | 31.3 samples/s
+Step 48670/50462 | Loss: 2.4259 | LR: 1.60e-06 | 1919.3ms/step | 31.3 samples/s
+Step 48680/50462 | Loss: 2.4301 | LR: 1.58e-06 | 1917.9ms/step | 31.3 samples/s
+Step 48690/50462 | Loss: 2.4910 | LR: 1.57e-06 | 1916.7ms/step | 31.3 samples/s
+Step 48700/50462 | Loss: 2.4669 | LR: 1.55e-06 | 1916.6ms/step | 31.3 samples/s
+Step 48710/50462 | Loss: 2.4263 | LR: 1.53e-06 | 1918.7ms/step | 31.3 samples/s
+Step 48720/50462 | Loss: 2.4289 | LR: 1.51e-06 | 1920.3ms/step | 31.2 samples/s
+Step 48730/50462 | Loss: 2.4532 | LR: 1.50e-06 | 1918.8ms/step | 31.3 samples/s
+Step 48740/50462 | Loss: 2.4840 | LR: 1.48e-06 | 1917.3ms/step | 31.3 samples/s
+Step 48750/50462 | Loss: 2.4612 | LR: 1.46e-06 | 1917.5ms/step | 31.3 samples/s
+Validation at step 48750: Loss = 2.4989, Perplexity = 12.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-48750
+Step 48760/50462 | Loss: 2.4310 | LR: 1.44e-06 | 3245.9ms/step | 18.5 samples/s
+Step 48770/50462 | Loss: 2.4619 | LR: 1.43e-06 | 1913.7ms/step | 31.4 samples/s
+Step 48780/50462 | Loss: 2.4302 | LR: 1.41e-06 | 1918.1ms/step | 31.3 samples/s
+Step 48790/50462 | Loss: 2.4586 | LR: 1.39e-06 | 1920.8ms/step | 31.2 samples/s
+Step 48800/50462 | Loss: 2.4736 | LR: 1.38e-06 | 1922.9ms/step | 31.2 samples/s
+Step 48810/50462 | Loss: 2.4688 | LR: 1.36e-06 | 1923.2ms/step | 31.2 samples/s
+Step 48820/50462 | Loss: 2.4595 | LR: 1.34e-06 | 1922.3ms/step | 31.2 samples/s
+Step 48830/50462 | Loss: 2.4388 | LR: 1.33e-06 | 1924.2ms/step | 31.2 samples/s
+Step 48840/50462 | Loss: 2.5066 | LR: 1.31e-06 | 1922.7ms/step | 31.2 samples/s
+Step 48850/50462 | Loss: 2.4724 | LR: 1.30e-06 | 1921.5ms/step | 31.2 samples/s
+Step 48860/50462 | Loss: 2.4585 | LR: 1.28e-06 | 1917.0ms/step | 31.3 samples/s
+Step 48870/50462 | Loss: 2.5023 | LR: 1.26e-06 | 1918.4ms/step | 31.3 samples/s
+Step 48880/50462 | Loss: 2.4467 | LR: 1.25e-06 | 1918.8ms/step | 31.3 samples/s
+Step 48890/50462 | Loss: 2.4677 | LR: 1.23e-06 | 1916.5ms/step | 31.3 samples/s
+Step 48900/50462 | Loss: 2.4269 | LR: 1.22e-06 | 1919.1ms/step | 31.3 samples/s
+Step 48910/50462 | Loss: 2.4301 | LR: 1.20e-06 | 1919.7ms/step | 31.3 samples/s
+Step 48920/50462 | Loss: 2.4694 | LR: 1.19e-06 | 1917.5ms/step | 31.3 samples/s
+Step 48930/50462 | Loss: 2.4361 | LR: 1.17e-06 | 1920.1ms/step | 31.2 samples/s
+Step 48940/50462 | Loss: 2.4731 | LR: 1.16e-06 | 1918.7ms/step | 31.3 samples/s
+Step 48950/50462 | Loss: 2.4650 | LR: 1.14e-06 | 1915.6ms/step | 31.3 samples/s
+Step 48960/50462 | Loss: 2.4940 | LR: 1.13e-06 | 1917.8ms/step | 31.3 samples/s
+Step 48970/50462 | Loss: 2.4244 | LR: 1.11e-06 | 1918.6ms/step | 31.3 samples/s
+Step 48980/50462 | Loss: 2.4672 | LR: 1.10e-06 | 1916.5ms/step | 31.3 samples/s
+Step 48990/50462 | Loss: 2.4576 | LR: 1.08e-06 | 1920.0ms/step | 31.2 samples/s
+Step 49000/50462 | Loss: 2.4554 | LR: 1.07e-06 | 1919.7ms/step | 31.3 samples/s
+Validation at step 49000: Loss = 2.4989, Perplexity = 12.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-49000
+Step 49010/50462 | Loss: 2.4918 | LR: 1.05e-06 | 3249.7ms/step | 18.5 samples/s
+Step 49020/50462 | Loss: 2.4728 | LR: 1.04e-06 | 1915.5ms/step | 31.3 samples/s
+Step 49030/50462 | Loss: 2.4415 | LR: 1.02e-06 | 1920.6ms/step | 31.2 samples/s
+Step 49040/50462 | Loss: 2.4416 | LR: 1.01e-06 | 1923.0ms/step | 31.2 samples/s
+Step 49050/50462 | Loss: 2.4504 | LR: 9.95e-07 | 1921.9ms/step | 31.2 samples/s
+Step 49060/50462 | Loss: 2.4626 | LR: 9.81e-07 | 1917.4ms/step | 31.3 samples/s
+Step 49070/50462 | Loss: 2.5310 | LR: 9.67e-07 | 1918.8ms/step | 31.3 samples/s
+Step 49080/50462 | Loss: 2.4718 | LR: 9.53e-07 | 1920.6ms/step | 31.2 samples/s
+Step 49090/50462 | Loss: 2.4717 | LR: 9.39e-07 | 1916.2ms/step | 31.3 samples/s
+Step 49100/50462 | Loss: 2.4692 | LR: 9.25e-07 | 1915.1ms/step | 31.3 samples/s
+Step 49110/50462 | Loss: 2.4824 | LR: 9.12e-07 | 1916.9ms/step | 31.3 samples/s
+Step 49120/50462 | Loss: 2.4652 | LR: 8.99e-07 | 1917.9ms/step | 31.3 samples/s
+Step 49130/50462 | Loss: 2.4763 | LR: 8.85e-07 | 1917.7ms/step | 31.3 samples/s
+Step 49140/50462 | Loss: 2.4745 | LR: 8.72e-07 | 1915.5ms/step | 31.3 samples/s
+Step 49150/50462 | Loss: 2.4613 | LR: 8.59e-07 | 1919.5ms/step | 31.3 samples/s
+Step 49160/50462 | Loss: 2.4523 | LR: 8.46e-07 | 1920.1ms/step | 31.2 samples/s
+Step 49170/50462 | Loss: 2.4351 | LR: 8.33e-07 | 1917.8ms/step | 31.3 samples/s
+Step 49180/50462 | Loss: 2.4550 | LR: 8.20e-07 | 1914.5ms/step | 31.3 samples/s
+Step 49190/50462 | Loss: 2.4864 | LR: 8.07e-07 | 1918.3ms/step | 31.3 samples/s
+Step 49200/50462 | Loss: 2.4399 | LR: 7.95e-07 | 1916.4ms/step | 31.3 samples/s
+Step 49210/50462 | Loss: 2.4556 | LR: 7.82e-07 | 1916.4ms/step | 31.3 samples/s
+Step 49220/50462 | Loss: 2.4866 | LR: 7.70e-07 | 1916.8ms/step | 31.3 samples/s
+Step 49230/50462 | Loss: 2.4462 | LR: 7.57e-07 | 1915.8ms/step | 31.3 samples/s
+Step 49240/50462 | Loss: 2.4963 | LR: 7.45e-07 | 1917.7ms/step | 31.3 samples/s
+Step 49250/50462 | Loss: 2.4353 | LR: 7.33e-07 | 1918.4ms/step | 31.3 samples/s
+Validation at step 49250: Loss = 2.4989, Perplexity = 12.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-49250
+Step 49260/50462 | Loss: 2.4547 | LR: 7.21e-07 | 3329.2ms/step | 18.0 samples/s
+Step 49270/50462 | Loss: 2.4553 | LR: 7.09e-07 | 1919.7ms/step | 31.3 samples/s
+Step 49280/50462 | Loss: 2.4529 | LR: 6.97e-07 | 1919.3ms/step | 31.3 samples/s
+Step 49290/50462 | Loss: 2.4322 | LR: 6.85e-07 | 1923.9ms/step | 31.2 samples/s
+Step 49300/50462 | Loss: 2.4783 | LR: 6.74e-07 | 1920.4ms/step | 31.2 samples/s
+Step 49310/50462 | Loss: 2.4220 | LR: 6.62e-07 | 1921.1ms/step | 31.2 samples/s
+Step 49320/50462 | Loss: 2.4773 | LR: 6.51e-07 | 1917.9ms/step | 31.3 samples/s
+Step 49330/50462 | Loss: 2.4386 | LR: 6.39e-07 | 1915.7ms/step | 31.3 samples/s
+Step 49340/50462 | Loss: 2.4521 | LR: 6.28e-07 | 1916.7ms/step | 31.3 samples/s
+Step 49350/50462 | Loss: 2.4953 | LR: 6.17e-07 | 1919.7ms/step | 31.3 samples/s
+Step 49360/50462 | Loss: 2.5013 | LR: 6.06e-07 | 1919.9ms/step | 31.3 samples/s
+Step 49370/50462 | Loss: 2.4876 | LR: 5.95e-07 | 1918.2ms/step | 31.3 samples/s
+Step 49380/50462 | Loss: 2.4638 | LR: 5.84e-07 | 1917.4ms/step | 31.3 samples/s
+Step 49390/50462 | Loss: 2.4245 | LR: 5.73e-07 | 1922.5ms/step | 31.2 samples/s
+Step 49400/50462 | Loss: 2.4566 | LR: 5.63e-07 | 1921.0ms/step | 31.2 samples/s
+Step 49410/50462 | Loss: 2.4551 | LR: 5.52e-07 | 1919.1ms/step | 31.3 samples/s
+Step 49420/50462 | Loss: 2.4716 | LR: 5.42e-07 | 1917.6ms/step | 31.3 samples/s
+Step 49430/50462 | Loss: 2.4680 | LR: 5.31e-07 | 1918.9ms/step | 31.3 samples/s
+Step 49440/50462 | Loss: 2.4413 | LR: 5.21e-07 | 1917.3ms/step | 31.3 samples/s
+Step 49450/50462 | Loss: 2.4589 | LR: 5.11e-07 | 1916.9ms/step | 31.3 samples/s
+Step 49460/50462 | Loss: 2.4852 | LR: 5.01e-07 | 1918.9ms/step | 31.3 samples/s
+Step 49470/50462 | Loss: 2.4431 | LR: 4.91e-07 | 1920.2ms/step | 31.2 samples/s
+Step 49480/50462 | Loss: 2.4691 | LR: 4.81e-07 | 1917.8ms/step | 31.3 samples/s
+Step 49490/50462 | Loss: 2.4631 | LR: 4.72e-07 | 1919.6ms/step | 31.3 samples/s
+Step 49500/50462 | Loss: 2.4153 | LR: 4.62e-07 | 1918.6ms/step | 31.3 samples/s
+Validation at step 49500: Loss = 2.4989, Perplexity = 12.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-49500
+Step 49510/50462 | Loss: 2.4739 | LR: 4.52e-07 | 3253.7ms/step | 18.4 samples/s
+Step 49520/50462 | Loss: 2.4585 | LR: 4.43e-07 | 1916.6ms/step | 31.3 samples/s
+Step 49530/50462 | Loss: 2.4405 | LR: 4.34e-07 | 1920.0ms/step | 31.3 samples/s
+Step 49540/50462 | Loss: 2.4827 | LR: 4.24e-07 | 1920.8ms/step | 31.2 samples/s
+Step 49550/50462 | Loss: 2.4285 | LR: 4.15e-07 | 1921.7ms/step | 31.2 samples/s
+Step 49560/50462 | Loss: 2.4084 | LR: 4.06e-07 | 1922.7ms/step | 31.2 samples/s
+Step 49570/50462 | Loss: 2.4576 | LR: 3.97e-07 | 1922.2ms/step | 31.2 samples/s
+Step 49580/50462 | Loss: 2.4087 | LR: 3.88e-07 | 1922.6ms/step | 31.2 samples/s
+Step 49590/50462 | Loss: 2.4472 | LR: 3.79e-07 | 1923.9ms/step | 31.2 samples/s
+Step 49600/50462 | Loss: 2.4458 | LR: 3.71e-07 | 1924.2ms/step | 31.2 samples/s
+Step 49610/50462 | Loss: 2.4779 | LR: 3.62e-07 | 1924.7ms/step | 31.2 samples/s
+Step 49620/50462 | Loss: 2.4345 | LR: 3.54e-07 | 1923.4ms/step | 31.2 samples/s
+Step 49630/50462 | Loss: 2.4247 | LR: 3.45e-07 | 1922.7ms/step | 31.2 samples/s
+Step 49640/50462 | Loss: 2.4311 | LR: 3.37e-07 | 1923.9ms/step | 31.2 samples/s
+Step 49650/50462 | Loss: 2.4190 | LR: 3.29e-07 | 1922.6ms/step | 31.2 samples/s
+Step 49660/50462 | Loss: 2.4283 | LR: 3.21e-07 | 1920.0ms/step | 31.2 samples/s
+Step 49670/50462 | Loss: 2.4728 | LR: 3.13e-07 | 1924.0ms/step | 31.2 samples/s
+Step 49680/50462 | Loss: 2.4499 | LR: 3.05e-07 | 1925.0ms/step | 31.2 samples/s
+Step 49690/50462 | Loss: 2.4583 | LR: 2.97e-07 | 1923.0ms/step | 31.2 samples/s
+Step 49700/50462 | Loss: 2.4532 | LR: 2.90e-07 | 1924.4ms/step | 31.2 samples/s
+Step 49710/50462 | Loss: 2.4434 | LR: 2.82e-07 | 1919.9ms/step | 31.3 samples/s
+Step 49720/50462 | Loss: 2.4492 | LR: 2.75e-07 | 1921.1ms/step | 31.2 samples/s
+Step 49730/50462 | Loss: 2.4780 | LR: 2.67e-07 | 1922.4ms/step | 31.2 samples/s
+Step 49740/50462 | Loss: 2.4489 | LR: 2.60e-07 | 1923.5ms/step | 31.2 samples/s
+Step 49750/50462 | Loss: 2.4648 | LR: 2.53e-07 | 1924.2ms/step | 31.2 samples/s
+Validation at step 49750: Loss = 2.4989, Perplexity = 12.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-49750
+Step 49760/50462 | Loss: 2.4704 | LR: 2.46e-07 | 3263.1ms/step | 18.4 samples/s
+Step 49770/50462 | Loss: 2.4019 | LR: 2.39e-07 | 1915.4ms/step | 31.3 samples/s
+Step 49780/50462 | Loss: 2.4313 | LR: 2.32e-07 | 1919.8ms/step | 31.3 samples/s
+Step 49790/50462 | Loss: 2.5284 | LR: 2.25e-07 | 1917.8ms/step | 31.3 samples/s
+Step 49800/50462 | Loss: 2.4334 | LR: 2.19e-07 | 1916.2ms/step | 31.3 samples/s
+Step 49810/50462 | Loss: 2.4540 | LR: 2.12e-07 | 1917.0ms/step | 31.3 samples/s
+Step 49820/50462 | Loss: 2.4802 | LR: 2.06e-07 | 1916.8ms/step | 31.3 samples/s
+Step 49830/50462 | Loss: 2.4738 | LR: 1.99e-07 | 1916.0ms/step | 31.3 samples/s
+Step 49840/50462 | Loss: 2.4790 | LR: 1.93e-07 | 1917.8ms/step | 31.3 samples/s
+Step 49850/50462 | Loss: 2.4824 | LR: 1.87e-07 | 1918.4ms/step | 31.3 samples/s
+Step 49860/50462 | Loss: 2.4529 | LR: 1.81e-07 | 1921.7ms/step | 31.2 samples/s
+Step 49870/50462 | Loss: 2.5313 | LR: 1.75e-07 | 1917.6ms/step | 31.3 samples/s
+Step 49880/50462 | Loss: 2.4425 | LR: 1.69e-07 | 1920.2ms/step | 31.2 samples/s
+Step 49890/50462 | Loss: 2.4782 | LR: 1.63e-07 | 1919.7ms/step | 31.3 samples/s
+Step 49900/50462 | Loss: 2.4688 | LR: 1.58e-07 | 1919.0ms/step | 31.3 samples/s
+Step 49910/50462 | Loss: 2.4431 | LR: 1.52e-07 | 1919.4ms/step | 31.3 samples/s
+Step 49920/50462 | Loss: 2.4735 | LR: 1.47e-07 | 1919.3ms/step | 31.3 samples/s
+Step 49930/50462 | Loss: 2.4415 | LR: 1.41e-07 | 1917.8ms/step | 31.3 samples/s
+Step 49940/50462 | Loss: 2.4553 | LR: 1.36e-07 | 1918.8ms/step | 31.3 samples/s
+Step 49950/50462 | Loss: 2.4586 | LR: 1.31e-07 | 1917.3ms/step | 31.3 samples/s
+Step 49960/50462 | Loss: 2.4505 | LR: 1.26e-07 | 1920.3ms/step | 31.2 samples/s
+Step 49970/50462 | Loss: 2.4707 | LR: 1.21e-07 | 1917.7ms/step | 31.3 samples/s
+Step 49980/50462 | Loss: 2.4379 | LR: 1.16e-07 | 1919.7ms/step | 31.3 samples/s
+Step 49990/50462 | Loss: 2.5049 | LR: 1.11e-07 | 1920.3ms/step | 31.2 samples/s
+Step 50000/50462 | Loss: 2.4293 | LR: 1.07e-07 | 1920.7ms/step | 31.2 samples/s
+Validation at step 50000: Loss = 2.4989, Perplexity = 12.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-50000
+Step 50010/50462 | Loss: 2.4104 | LR: 1.02e-07 | 3251.5ms/step | 18.5 samples/s
+Step 50020/50462 | Loss: 2.4556 | LR: 9.75e-08 | 1920.0ms/step | 31.2 samples/s
+Step 50030/50462 | Loss: 2.4608 | LR: 9.32e-08 | 1920.2ms/step | 31.2 samples/s
+Step 50040/50462 | Loss: 2.4836 | LR: 8.89e-08 | 1920.4ms/step | 31.2 samples/s
+Step 50050/50462 | Loss: 2.5032 | LR: 8.47e-08 | 1923.8ms/step | 31.2 samples/s
+Step 50060/50462 | Loss: 2.4885 | LR: 8.07e-08 | 1924.3ms/step | 31.2 samples/s
+Step 50070/50462 | Loss: 2.4883 | LR: 7.67e-08 | 1922.0ms/step | 31.2 samples/s
+Step 50080/50462 | Loss: 2.4687 | LR: 7.28e-08 | 1922.3ms/step | 31.2 samples/s
+Step 50090/50462 | Loss: 2.4525 | LR: 6.91e-08 | 1923.5ms/step | 31.2 samples/s
+Step 50100/50462 | Loss: 2.4403 | LR: 6.54e-08 | 1925.6ms/step | 31.2 samples/s
+Step 50110/50462 | Loss: 2.4738 | LR: 6.19e-08 | 1924.8ms/step | 31.2 samples/s
+Step 50120/50462 | Loss: 2.4606 | LR: 5.84e-08 | 1921.1ms/step | 31.2 samples/s
+Step 50130/50462 | Loss: 2.4388 | LR: 5.50e-08 | 1923.9ms/step | 31.2 samples/s
+Step 50140/50462 | Loss: 2.4341 | LR: 5.18e-08 | 1920.7ms/step | 31.2 samples/s
+Step 50150/50462 | Loss: 2.4438 | LR: 4.86e-08 | 1922.6ms/step | 31.2 samples/s
+Step 50160/50462 | Loss: 2.4943 | LR: 4.55e-08 | 1923.6ms/step | 31.2 samples/s
+Step 50170/50462 | Loss: 2.4167 | LR: 4.26e-08 | 1924.5ms/step | 31.2 samples/s
+Step 50180/50462 | Loss: 2.4217 | LR: 3.97e-08 | 1922.9ms/step | 31.2 samples/s
+Step 50190/50462 | Loss: 2.4289 | LR: 3.69e-08 | 1922.2ms/step | 31.2 samples/s
+Step 50200/50462 | Loss: 2.4397 | LR: 3.43e-08 | 1921.6ms/step | 31.2 samples/s
+Step 50210/50462 | Loss: 2.4460 | LR: 3.17e-08 | 1919.7ms/step | 31.3 samples/s
+Step 50220/50462 | Loss: 2.4347 | LR: 2.92e-08 | 1922.1ms/step | 31.2 samples/s
+Step 50230/50462 | Loss: 2.4324 | LR: 2.69e-08 | 1920.3ms/step | 31.2 samples/s
+Step 50240/50462 | Loss: 2.4404 | LR: 2.46e-08 | 1922.8ms/step | 31.2 samples/s
+Step 50250/50462 | Loss: 2.4152 | LR: 2.24e-08 | 1923.0ms/step | 31.2 samples/s
+Validation at step 50250: Loss = 2.4989, Perplexity = 12.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-50250
+Step 50260/50462 | Loss: 2.4820 | LR: 2.04e-08 | 3297.3ms/step | 18.2 samples/s
+Step 50270/50462 | Loss: 2.4292 | LR: 1.84e-08 | 1914.0ms/step | 31.3 samples/s
+Step 50280/50462 | Loss: 2.4780 | LR: 1.65e-08 | 1916.3ms/step | 31.3 samples/s
+Step 50290/50462 | Loss: 2.4465 | LR: 1.48e-08 | 1920.7ms/step | 31.2 samples/s
+Step 50300/50462 | Loss: 2.4276 | LR: 1.31e-08 | 1918.2ms/step | 31.3 samples/s
+Step 50310/50462 | Loss: 2.4382 | LR: 1.15e-08 | 1916.8ms/step | 31.3 samples/s
+Step 50320/50462 | Loss: 2.4418 | LR: 1.01e-08 | 1914.9ms/step | 31.3 samples/s
+Step 50330/50462 | Loss: 2.4617 | LR: 8.70e-09 | 1918.3ms/step | 31.3 samples/s
+Step 50340/50462 | Loss: 2.4766 | LR: 7.43e-09 | 1915.3ms/step | 31.3 samples/s
+Step 50350/50462 | Loss: 2.4444 | LR: 6.26e-09 | 1917.6ms/step | 31.3 samples/s
+Step 50360/50462 | Loss: 2.4418 | LR: 5.19e-09 | 1919.7ms/step | 31.3 samples/s
+Step 50370/50462 | Loss: 2.5199 | LR: 4.23e-09 | 1917.4ms/step | 31.3 samples/s
+Step 50380/50462 | Loss: 2.4488 | LR: 3.36e-09 | 1920.1ms/step | 31.2 samples/s
+Step 50390/50462 | Loss: 2.4309 | LR: 2.59e-09 | 1921.6ms/step | 31.2 samples/s
+Step 50400/50462 | Loss: 2.4418 | LR: 1.92e-09 | 1916.5ms/step | 31.3 samples/s
+Step 50410/50462 | Loss: 2.4476 | LR: 1.35e-09 | 1919.0ms/step | 31.3 samples/s
+Step 50420/50462 | Loss: 2.4333 | LR: 8.81e-10 | 1916.2ms/step | 31.3 samples/s
+Step 50430/50462 | Loss: 2.4399 | LR: 5.11e-10 | 1916.0ms/step | 31.3 samples/s
+Step 50440/50462 | Loss: 2.4397 | LR: 2.42e-10 | 1920.8ms/step | 31.2 samples/s
+Step 50450/50462 | Loss: 2.4503 | LR: 7.19e-11 | 1920.5ms/step | 31.2 samples/s
+Step 50460/50462 | Loss: 2.4393 | LR: 2.00e-12 | 1917.9ms/step | 31.3 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-50462

epfl_final/SparseAtt/wandb/run-20260827_022420-rgmkfpnl/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260827_022420-rgmkfpnl/files/wandb-metadata.json

@@ -0,0 +1,79 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-08-27T02:24:20.312598Z",
+  "args":  [
+    "--config",
+    "configs/p2_1p2b_dense_seed70_6gpu_resume41750.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/SparseAtt.git",
+    "commit":  "8f862f749eb4d5c17f688eb1562235e3fe5b2550"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-train-6-gpu-0-44",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  6,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "6433026093056"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bab1e315-c967-43cf-bdfb-cac1a8dfdfac"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-f65cda91-ff87-9bff-fb3d-3e1b3f9f8cf0"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-069bdfaf-ef1c-f7f2-f9e3-2249bb9dfc41"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-4c1d91e9-bf01-f8fd-1e8b-79aa8e0831df"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-086f9f0b-a6d8-6263-83e6-c2b66721f47f"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bdb5e23e-0700-20a0-0b21-1ad0901f08b7"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "ge3aug5csshnod8dmxmlezb8wuau8hkz"
+}

epfl_final/SparseAtt/wandb/run-20260827_022420-rgmkfpnl/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb.runtime":34816,"train/loss":2.439325326681137,"eval/uses_dense_attention":1,"speed/steps_per_sec":0.5214109132394723,"_wandb":{"runtime":246286},"eval/dense_interval_end":-1,"train/learning_rate":1.9968584286100823e-12,"train/global_step":50460,"eval/loss":2.498921811580658,"eval/uses_sparse_attention":0,"train/epoch":0.9999603662161627,"eval/dense_interval_active":0,"_runtime":246286.089952201,"eval/perplexity":12.169366015267467,"_step":50460,"speed/ms_per_step":1917.8731679916382,"speed/samples_per_sec":31.28465479436834,"_timestamp":1.787814898830027e+09,"eval/dense_interval_start":-1}

epfl_final/SparseAtt/wandb/run-20260827_022420-rgmkfpnl/logs/debug.log

@@ -0,0 +1,25 @@
+2026-08-27 02:24:20,335 INFO    MainThread:241 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-08-27 02:24:20,335 INFO    MainThread:241 [wandb_setup.py:_flush():81] Configure stats pid to 241
+2026-08-27 02:24:20,335 INFO    MainThread:241 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-27 02:24:20,335 INFO    MainThread:241 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260827_022420-rgmkfpnl/logs/debug.log
+2026-08-27 02:24:20,336 INFO    MainThread:241 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260827_022420-rgmkfpnl/logs/debug-internal.log
+2026-08-27 02:24:20,336 INFO    MainThread:241 [wandb_init.py:init():850] calling init triggers
+2026-08-27 02:24:20,336 INFO    MainThread:241 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-41750', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 70, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b dense 8k seed70 resume41750 6gpu', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'dense', 'seed71'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
+2026-08-27 02:24:20,336 INFO    MainThread:241 [wandb_init.py:init():898] starting backend
+2026-08-27 02:24:21,013 INFO    MainThread:241 [wandb_init.py:init():913] sending inform_init request
+2026-08-27 02:24:21,462 INFO    MainThread:241 [wandb_init.py:init():918] backend started and connected
+2026-08-27 02:24:21,466 INFO    MainThread:241 [wandb_init.py:init():988] updated telemetry
+2026-08-27 02:24:21,499 INFO    MainThread:241 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-08-27 02:24:21,977 INFO    MainThread:241 [wandb_init.py:init():1051] run resumed
+2026-08-27 02:24:21,980 INFO    MainThread:241 [wandb_init.py:init():1056] starting run threads in backend
+2026-08-27 02:24:22,320 INFO    MainThread:241 [wandb_run.py:_console_start():2554] atexit reg
+2026-08-27 02:24:22,320 INFO    MainThread:241 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-08-27 02:24:22,320 INFO    MainThread:241 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-08-27 02:24:22,320 INFO    MainThread:241 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-08-27 02:24:22,332 INFO    MainThread:241 [wandb_init.py:init():1094] run started, returning control to user process
+2026-08-27 07:15:12,068 INFO    MainThread:241 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/rgmkfpnl
+2026-08-27 07:15:12,069 INFO    MainThread:241 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
+2026-08-27 07:15:12,070 INFO    MainThread:241 [wandb_run.py:_restore():2502] restore
+2026-08-27 07:15:12,070 INFO    MainThread:241 [wandb_run.py:_restore():2508] restore done
+2026-08-27 07:15:13,837 INFO    MainThread:241 [wandb_run.py:_footer_sync_info():3889] logging synced files

epfl_final/SparseAtt/wandb/run-20260827_022420-rgmkfpnl/run-rgmkfpnl.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88ad3db70030593a4d8384dae08604c293beb757e20fad8ffd1e85ac66a8ef84
+size 8395912

epfl_final/SparseAtt/wandb/run-20260827_041913-8ynn1we6/files/config.yaml

@@ -0,0 +1,199 @@
+_wandb:
+    value:
+        cli_version: 0.26.1
+        e:
+            4y7vrshm704jr2z23pf52r9efbq38ko7:
+                args:
+                    - --config
+                    - configs/p2_1p2b_mixed_seed71_4gpu_resume9000.json
+                codePath: train.py
+                codePathLocal: train.py
+                cpu_count: 96
+                cpu_count_logical: 192
+                cudaVersion: "13.0"
+                disk:
+                    /:
+                        total: "9374931091456"
+                        used: "5970348429312"
+                email: ruben.navasardyan@mail.utoronto.ca
+                executable: /mloscratch/homes/navasard/conda/bin/python3.13
+                git:
+                    commit: 8f862f749eb4d5c17f688eb1562235e3fe5b2550
+                    remote: https://github.com/rubentium/SparseAtt.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 4
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-e3358200-f776-7214-8269-c2c219f03c2e
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-a3a94461-36d7-5e8a-af9b-6209869fa7f4
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-0c2f0695-54b3-664a-b12d-640d1c9a0f2e
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-7a2d68b3-dda3-ec4d-f88e-205cb866ebb6
+                host: h100-train-4-gpu-0-15
+                memory:
+                    total: "1623005732864"
+                os: Linux-6.8.0-124-generic-x86_64-with-glibc2.35
+                program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
+                python: CPython 3.13.13
+                root: /mloscratch/homes/navasard/sparse/SparseAtt
+                startedAt: "2026-08-27T04:19:13.272934Z"
+                writerId: 4y7vrshm704jr2z23pf52r9efbq38ko7
+        m: []
+        python_version: 3.13.13
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 5
+                - 13
+                - 14
+                - 15
+                - 16
+                - 61
+                - 62
+            "4": 3.13.13
+            "5": 0.26.1
+            "6": 5.7.0
+            "12": 0.26.1
+            "13": linux-x86_64
+attention_dropout:
+    value: null
+attention_type:
+    value: sparse_stochastic
+checkpoint_activations:
+    value: true
+dense_mode:
+    value: false
+dense_step_intervals:
+    value: null
+eval_batches:
+    value: 50
+eval_steps:
+    value: 250
+force_new_run:
+    value: false
+gradient_accumulation_steps:
+    value: 15
+head_dim:
+    value: null
+hidden_size:
+    value: 1536
+init_from_scratch:
+    value: true
+intermediate_size:
+    value: 6144
+learning_rate:
+    value: 0.0005
+logging_steps:
+    value: 10
+lr_scheduler:
+    value: cosine
+max_grad_norm:
+    value: 1
+max_seq_length:
+    value: 8192
+max_steps:
+    value: 50462
+mixed_precision:
+    value: bf16
+model_dtype:
+    value: bfloat16
+model_path:
+    value: /mloscratch/homes/navasard/qwen3_06b_base
+num_attention_heads:
+    value: 12
+num_epochs:
+    value: 1
+num_hidden_layers:
+    value: null
+num_key_value_heads:
+    value: 4
+num_workers:
+    value: 8
+output_dir:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205
+per_device_batch_size:
+    value: 1
+resume_from_checkpoint:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-9000
+rope_train_position_scale_set:
+    value: null
+save_steps:
+    value: 250
+seed:
+    value: 71
+sparse_block_size:
+    value: 128
+sparse_layer_indices:
+    value: null
+sparse_mask_cache_dir:
+    value: /mloscratch/homes/navasard/sparse/mask_cache_8k_mixed
+sparse_mask_pool_size:
+    value: 10000
+sparse_mask_seed:
+    value: 71
+sparse_mixed_nrb_set:
+    value:
+        - 63
+        - 47
+        - 31
+        - 21
+        - 11
+sparse_num_random_blocks:
+    value: 35
+sparse_val_dense_compare:
+    value: false
+strategy:
+    value: ddp
+train_path:
+    value: /mloscratch/homes/navasard/data/fineweb_large/tokenized
+val_path:
+    value: null
+verbose:
+    value: true
+wandb_enabled:
+    value: true
+wandb_name:
+    value: 1.2b mixed 8k seed71 4gpu
+wandb_project:
+    value: stochastic_sparsity
+wandb_tags:
+    value:
+        - qwen3
+        - 1.2b
+        - 8k
+        - p2
+        - mixed
+        - seed71
+warmup_steps:
+    value: 750
+weight_decay:
+    value: 0.01

epfl_final/SparseAtt/wandb/run-20260827_041913-8ynn1we6/files/output.log

@@ -0,0 +1,843 @@
+Starting training...
+Validation at step 9000: Loss = 2.7854, Perplexity = 16.21
+wandb: WARNING Tried to log to step 9000 that is less than the current step 9041. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 9001: sparse_stochastic (configured sparse attention)
+Step 9010/50462 | Loss: 2.8091 | LR: 4.67e-04 | 33212.1ms/step | 1.8 samples/s
+wandb: WARNING Tried to log to step 9010 that is less than the current step 9041. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 9020/50462 | Loss: 2.7925 | LR: 4.67e-04 | 4013.8ms/step | 14.9 samples/s
+wandb: WARNING Tried to log to step 9020 that is less than the current step 9041. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 9030/50462 | Loss: 2.8282 | LR: 4.67e-04 | 4001.6ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 9030 that is less than the current step 9041. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 9040/50462 | Loss: 2.8079 | LR: 4.66e-04 | 4004.4ms/step | 15.0 samples/s
+wandb: WARNING Tried to log to step 9040 that is less than the current step 9041. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 9050/50462 | Loss: 2.7873 | LR: 4.66e-04 | 4007.4ms/step | 15.0 samples/s
+Step 9060/50462 | Loss: 2.7973 | LR: 4.66e-04 | 4001.6ms/step | 15.0 samples/s
+Step 9070/50462 | Loss: 2.8019 | LR: 4.66e-04 | 4010.6ms/step | 15.0 samples/s
+Step 9080/50462 | Loss: 2.7690 | LR: 4.66e-04 | 4000.4ms/step | 15.0 samples/s
+Step 9090/50462 | Loss: 2.7811 | LR: 4.66e-04 | 4008.8ms/step | 15.0 samples/s
+Step 9100/50462 | Loss: 2.8276 | LR: 4.66e-04 | 4007.2ms/step | 15.0 samples/s
+Step 9110/50462 | Loss: 2.8081 | LR: 4.66e-04 | 3996.2ms/step | 15.0 samples/s
+Step 9120/50462 | Loss: 2.7911 | LR: 4.66e-04 | 4006.1ms/step | 15.0 samples/s
+Step 9130/50462 | Loss: 2.7803 | LR: 4.66e-04 | 4010.7ms/step | 15.0 samples/s
+Step 9140/50462 | Loss: 2.8171 | LR: 4.66e-04 | 4009.3ms/step | 15.0 samples/s
+Step 9150/50462 | Loss: 2.7921 | LR: 4.66e-04 | 4009.8ms/step | 15.0 samples/s
+Step 9160/50462 | Loss: 2.7957 | LR: 4.66e-04 | 4011.5ms/step | 15.0 samples/s
+Step 9170/50462 | Loss: 2.7601 | LR: 4.65e-04 | 4017.6ms/step | 14.9 samples/s
+Step 9180/50462 | Loss: 2.7732 | LR: 4.65e-04 | 4009.5ms/step | 15.0 samples/s
+Step 9190/50462 | Loss: 2.7932 | LR: 4.65e-04 | 3990.4ms/step | 15.0 samples/s
+Step 9200/50462 | Loss: 2.7715 | LR: 4.65e-04 | 4009.4ms/step | 15.0 samples/s
+Step 9210/50462 | Loss: 2.7882 | LR: 4.65e-04 | 4007.9ms/step | 15.0 samples/s
+Step 9220/50462 | Loss: 2.7896 | LR: 4.65e-04 | 4013.2ms/step | 15.0 samples/s
+Step 9230/50462 | Loss: 2.7986 | LR: 4.65e-04 | 4007.9ms/step | 15.0 samples/s
+Step 9240/50462 | Loss: 2.8204 | LR: 4.65e-04 | 4003.1ms/step | 15.0 samples/s
+Step 9250/50462 | Loss: 2.7785 | LR: 4.65e-04 | 4013.2ms/step | 15.0 samples/s
+Validation at step 9250: Loss = 2.7794, Perplexity = 16.11
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-9250
+Step 9260/50462 | Loss: 2.7804 | LR: 4.65e-04 | 5982.8ms/step | 10.0 samples/s
+Step 9270/50462 | Loss: 2.8018 | LR: 4.65e-04 | 4000.9ms/step | 15.0 samples/s
+Step 9280/50462 | Loss: 2.7998 | LR: 4.65e-04 | 3993.3ms/step | 15.0 samples/s
+Step 9290/50462 | Loss: 2.7832 | LR: 4.64e-04 | 4004.8ms/step | 15.0 samples/s
+Step 9300/50462 | Loss: 2.7845 | LR: 4.64e-04 | 4016.9ms/step | 14.9 samples/s
+Step 9310/50462 | Loss: 2.7566 | LR: 4.64e-04 | 4003.3ms/step | 15.0 samples/s
+Step 9320/50462 | Loss: 2.7646 | LR: 4.64e-04 | 4001.5ms/step | 15.0 samples/s
+Step 9330/50462 | Loss: 2.7832 | LR: 4.64e-04 | 4005.7ms/step | 15.0 samples/s
+Step 9340/50462 | Loss: 2.8124 | LR: 4.64e-04 | 4016.1ms/step | 14.9 samples/s
+Step 9350/50462 | Loss: 2.7685 | LR: 4.64e-04 | 3998.1ms/step | 15.0 samples/s
+Step 9360/50462 | Loss: 2.7854 | LR: 4.64e-04 | 4016.6ms/step | 14.9 samples/s
+Step 9370/50462 | Loss: 2.7918 | LR: 4.64e-04 | 4014.2ms/step | 14.9 samples/s
+Step 9380/50462 | Loss: 2.7746 | LR: 4.64e-04 | 4003.8ms/step | 15.0 samples/s
+Step 9390/50462 | Loss: 2.7925 | LR: 4.64e-04 | 4013.1ms/step | 15.0 samples/s
+Step 9400/50462 | Loss: 2.7793 | LR: 4.64e-04 | 4004.8ms/step | 15.0 samples/s
+Step 9410/50462 | Loss: 2.7684 | LR: 4.63e-04 | 3997.8ms/step | 15.0 samples/s
+Step 9420/50462 | Loss: 2.7700 | LR: 4.63e-04 | 4006.2ms/step | 15.0 samples/s
+Step 9430/50462 | Loss: 2.7788 | LR: 4.63e-04 | 3989.7ms/step | 15.0 samples/s
+Step 9440/50462 | Loss: 2.7604 | LR: 4.63e-04 | 4027.1ms/step | 14.9 samples/s
+Step 9450/50462 | Loss: 2.7730 | LR: 4.63e-04 | 4036.0ms/step | 14.9 samples/s
+Step 9460/50462 | Loss: 2.7926 | LR: 4.63e-04 | 4011.8ms/step | 15.0 samples/s
+Step 9470/50462 | Loss: 2.7794 | LR: 4.63e-04 | 3997.6ms/step | 15.0 samples/s
+Step 9480/50462 | Loss: 2.7810 | LR: 4.63e-04 | 4009.3ms/step | 15.0 samples/s
+Step 9490/50462 | Loss: 2.8041 | LR: 4.63e-04 | 4024.7ms/step | 14.9 samples/s
+Step 9500/50462 | Loss: 2.7358 | LR: 4.63e-04 | 4008.9ms/step | 15.0 samples/s
+Validation at step 9500: Loss = 2.7734, Perplexity = 16.01
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-9500
+Step 9510/50462 | Loss: 2.7890 | LR: 4.63e-04 | 6000.9ms/step | 10.0 samples/s
+Step 9520/50462 | Loss: 2.8333 | LR: 4.63e-04 | 4020.6ms/step | 14.9 samples/s
+Step 9530/50462 | Loss: 2.7958 | LR: 4.62e-04 | 4009.0ms/step | 15.0 samples/s
+Step 9540/50462 | Loss: 2.7498 | LR: 4.62e-04 | 4029.0ms/step | 14.9 samples/s
+Step 9550/50462 | Loss: 2.7954 | LR: 4.62e-04 | 4016.6ms/step | 14.9 samples/s
+Step 9560/50462 | Loss: 2.8040 | LR: 4.62e-04 | 4017.6ms/step | 14.9 samples/s
+Step 9570/50462 | Loss: 2.7674 | LR: 4.62e-04 | 4010.0ms/step | 15.0 samples/s
+Step 9580/50462 | Loss: 2.7965 | LR: 4.62e-04 | 4020.6ms/step | 14.9 samples/s
+Step 9590/50462 | Loss: 2.7485 | LR: 4.62e-04 | 4022.7ms/step | 14.9 samples/s
+Step 9600/50462 | Loss: 2.7978 | LR: 4.62e-04 | 4027.8ms/step | 14.9 samples/s
+Step 9610/50462 | Loss: 2.7807 | LR: 4.62e-04 | 4009.7ms/step | 15.0 samples/s
+Step 9620/50462 | Loss: 2.8190 | LR: 4.62e-04 | 4012.5ms/step | 15.0 samples/s
+Step 9630/50462 | Loss: 2.7913 | LR: 4.62e-04 | 3999.7ms/step | 15.0 samples/s
+Step 9640/50462 | Loss: 2.8000 | LR: 4.62e-04 | 4003.3ms/step | 15.0 samples/s
+Step 9650/50462 | Loss: 2.7853 | LR: 4.61e-04 | 4010.7ms/step | 15.0 samples/s
+Step 9660/50462 | Loss: 2.7714 | LR: 4.61e-04 | 4007.8ms/step | 15.0 samples/s
+Step 9670/50462 | Loss: 2.7968 | LR: 4.61e-04 | 4008.4ms/step | 15.0 samples/s
+Step 9680/50462 | Loss: 2.8063 | LR: 4.61e-04 | 4030.9ms/step | 14.9 samples/s
+Step 9690/50462 | Loss: 2.7536 | LR: 4.61e-04 | 4022.5ms/step | 14.9 samples/s
+Step 9700/50462 | Loss: 2.8054 | LR: 4.61e-04 | 4013.4ms/step | 14.9 samples/s
+Step 9710/50462 | Loss: 2.8032 | LR: 4.61e-04 | 4002.8ms/step | 15.0 samples/s
+Step 9720/50462 | Loss: 2.7689 | LR: 4.61e-04 | 4017.1ms/step | 14.9 samples/s
+Step 9730/50462 | Loss: 2.7855 | LR: 4.61e-04 | 3996.0ms/step | 15.0 samples/s
+Step 9740/50462 | Loss: 2.7976 | LR: 4.61e-04 | 3999.4ms/step | 15.0 samples/s
+Step 9750/50462 | Loss: 2.7677 | LR: 4.61e-04 | 4000.9ms/step | 15.0 samples/s
+Validation at step 9750: Loss = 2.7690, Perplexity = 15.94
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-9750
+Step 9760/50462 | Loss: 2.7987 | LR: 4.61e-04 | 5930.9ms/step | 10.1 samples/s
+Step 9770/50462 | Loss: 2.7818 | LR: 4.60e-04 | 4003.0ms/step | 15.0 samples/s
+Step 9780/50462 | Loss: 2.7751 | LR: 4.60e-04 | 4018.1ms/step | 14.9 samples/s
+Step 9790/50462 | Loss: 2.7655 | LR: 4.60e-04 | 4025.3ms/step | 14.9 samples/s
+Step 9800/50462 | Loss: 2.7655 | LR: 4.60e-04 | 4007.3ms/step | 15.0 samples/s
+Step 9810/50462 | Loss: 2.7983 | LR: 4.60e-04 | 3995.7ms/step | 15.0 samples/s
+Step 9820/50462 | Loss: 2.7760 | LR: 4.60e-04 | 4003.4ms/step | 15.0 samples/s
+Step 9830/50462 | Loss: 2.7868 | LR: 4.60e-04 | 4016.1ms/step | 14.9 samples/s
+Step 9840/50462 | Loss: 2.8156 | LR: 4.60e-04 | 4027.3ms/step | 14.9 samples/s
+Step 9850/50462 | Loss: 2.7879 | LR: 4.60e-04 | 3996.6ms/step | 15.0 samples/s
+Step 9860/50462 | Loss: 2.7679 | LR: 4.60e-04 | 3995.0ms/step | 15.0 samples/s
+Step 9870/50462 | Loss: 2.8054 | LR: 4.60e-04 | 4004.0ms/step | 15.0 samples/s
+Step 9880/50462 | Loss: 2.8005 | LR: 4.60e-04 | 3996.4ms/step | 15.0 samples/s
+Step 9890/50462 | Loss: 2.7654 | LR: 4.59e-04 | 4005.1ms/step | 15.0 samples/s
+Step 9900/50462 | Loss: 2.7543 | LR: 4.59e-04 | 4002.5ms/step | 15.0 samples/s
+Step 9910/50462 | Loss: 2.7570 | LR: 4.59e-04 | 4006.4ms/step | 15.0 samples/s
+Step 9920/50462 | Loss: 2.7457 | LR: 4.59e-04 | 4023.0ms/step | 14.9 samples/s
+Step 9930/50462 | Loss: 2.7815 | LR: 4.59e-04 | 4020.8ms/step | 14.9 samples/s
+Step 9940/50462 | Loss: 2.7767 | LR: 4.59e-04 | 4019.6ms/step | 14.9 samples/s
+Step 9950/50462 | Loss: 2.7577 | LR: 4.59e-04 | 4012.7ms/step | 15.0 samples/s
+Step 9960/50462 | Loss: 2.7550 | LR: 4.59e-04 | 4011.5ms/step | 15.0 samples/s
+Step 9970/50462 | Loss: 2.7528 | LR: 4.59e-04 | 4035.0ms/step | 14.9 samples/s
+Step 9980/50462 | Loss: 2.7666 | LR: 4.59e-04 | 4013.4ms/step | 14.9 samples/s
+Step 9990/50462 | Loss: 2.7721 | LR: 4.59e-04 | 4011.5ms/step | 15.0 samples/s
+Step 10000/50462 | Loss: 2.7700 | LR: 4.58e-04 | 4022.0ms/step | 14.9 samples/s
+Validation at step 10000: Loss = 2.7622, Perplexity = 15.83
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-10000
+Step 10010/50462 | Loss: 2.7522 | LR: 4.58e-04 | 5998.1ms/step | 10.0 samples/s
+Step 10020/50462 | Loss: 2.7823 | LR: 4.58e-04 | 3998.2ms/step | 15.0 samples/s
+Step 10030/50462 | Loss: 2.8078 | LR: 4.58e-04 | 4002.9ms/step | 15.0 samples/s
+Step 10040/50462 | Loss: 2.7914 | LR: 4.58e-04 | 4019.7ms/step | 14.9 samples/s
+Step 10050/50462 | Loss: 2.7574 | LR: 4.58e-04 | 4030.3ms/step | 14.9 samples/s
+Step 10060/50462 | Loss: 2.7273 | LR: 4.58e-04 | 4006.2ms/step | 15.0 samples/s
+Step 10070/50462 | Loss: 2.8417 | LR: 4.58e-04 | 3995.3ms/step | 15.0 samples/s
+Step 10080/50462 | Loss: 2.7588 | LR: 4.58e-04 | 4008.2ms/step | 15.0 samples/s
+Step 10090/50462 | Loss: 2.7834 | LR: 4.58e-04 | 4016.1ms/step | 14.9 samples/s
+Step 10100/50462 | Loss: 2.7816 | LR: 4.58e-04 | 4002.9ms/step | 15.0 samples/s
+Step 10110/50462 | Loss: 2.7854 | LR: 4.58e-04 | 4007.9ms/step | 15.0 samples/s
+Step 10120/50462 | Loss: 2.7835 | LR: 4.57e-04 | 3993.0ms/step | 15.0 samples/s
+Step 10130/50462 | Loss: 2.7858 | LR: 4.57e-04 | 4006.4ms/step | 15.0 samples/s
+Step 10140/50462 | Loss: 2.7612 | LR: 4.57e-04 | 3994.8ms/step | 15.0 samples/s
+Step 10150/50462 | Loss: 2.7578 | LR: 4.57e-04 | 4015.7ms/step | 14.9 samples/s
+Step 10160/50462 | Loss: 2.7749 | LR: 4.57e-04 | 4010.0ms/step | 15.0 samples/s
+Step 10170/50462 | Loss: 2.7626 | LR: 4.57e-04 | 4020.8ms/step | 14.9 samples/s
+Step 10180/50462 | Loss: 2.7465 | LR: 4.57e-04 | 3996.4ms/step | 15.0 samples/s
+Step 10190/50462 | Loss: 2.7885 | LR: 4.57e-04 | 4015.7ms/step | 14.9 samples/s
+Step 10200/50462 | Loss: 2.7600 | LR: 4.57e-04 | 4009.1ms/step | 15.0 samples/s
+Step 10210/50462 | Loss: 2.7452 | LR: 4.57e-04 | 4014.9ms/step | 14.9 samples/s
+Step 10220/50462 | Loss: 2.7782 | LR: 4.57e-04 | 4002.3ms/step | 15.0 samples/s
+Step 10230/50462 | Loss: 2.8057 | LR: 4.56e-04 | 3999.0ms/step | 15.0 samples/s
+Step 10240/50462 | Loss: 2.7887 | LR: 4.56e-04 | 4014.9ms/step | 14.9 samples/s
+Step 10250/50462 | Loss: 2.7859 | LR: 4.56e-04 | 4017.9ms/step | 14.9 samples/s
+Validation at step 10250: Loss = 2.7583, Perplexity = 15.77
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-10250
+Step 10260/50462 | Loss: 2.7715 | LR: 4.56e-04 | 5956.9ms/step | 10.1 samples/s
+Step 10270/50462 | Loss: 2.7671 | LR: 4.56e-04 | 4009.4ms/step | 15.0 samples/s
+Step 10280/50462 | Loss: 2.7783 | LR: 4.56e-04 | 4015.1ms/step | 14.9 samples/s
+Step 10290/50462 | Loss: 2.7477 | LR: 4.56e-04 | 4013.7ms/step | 14.9 samples/s
+Step 10300/50462 | Loss: 2.7672 | LR: 4.56e-04 | 4003.5ms/step | 15.0 samples/s
+Step 10310/50462 | Loss: 2.7693 | LR: 4.56e-04 | 4013.9ms/step | 14.9 samples/s
+Step 10320/50462 | Loss: 2.7800 | LR: 4.56e-04 | 4004.7ms/step | 15.0 samples/s
+Step 10330/50462 | Loss: 2.7363 | LR: 4.56e-04 | 4003.6ms/step | 15.0 samples/s
+Step 10340/50462 | Loss: 2.7906 | LR: 4.55e-04 | 4002.0ms/step | 15.0 samples/s
+Step 10350/50462 | Loss: 2.7677 | LR: 4.55e-04 | 4008.3ms/step | 15.0 samples/s
+Step 10360/50462 | Loss: 2.7531 | LR: 4.55e-04 | 4006.6ms/step | 15.0 samples/s
+Step 10370/50462 | Loss: 2.7567 | LR: 4.55e-04 | 4011.5ms/step | 15.0 samples/s
+Step 10380/50462 | Loss: 2.7613 | LR: 4.55e-04 | 3998.0ms/step | 15.0 samples/s
+Step 10390/50462 | Loss: 2.7889 | LR: 4.55e-04 | 4000.4ms/step | 15.0 samples/s
+Step 10400/50462 | Loss: 2.7567 | LR: 4.55e-04 | 4000.8ms/step | 15.0 samples/s
+Step 10410/50462 | Loss: 2.7774 | LR: 4.55e-04 | 4003.7ms/step | 15.0 samples/s
+Step 10420/50462 | Loss: 2.7993 | LR: 4.55e-04 | 3984.6ms/step | 15.1 samples/s
+Step 10430/50462 | Loss: 2.7464 | LR: 4.55e-04 | 4008.8ms/step | 15.0 samples/s
+Step 10440/50462 | Loss: 2.7540 | LR: 4.55e-04 | 4009.1ms/step | 15.0 samples/s
+Step 10450/50462 | Loss: 2.7874 | LR: 4.54e-04 | 3996.2ms/step | 15.0 samples/s
+Step 10460/50462 | Loss: 2.7556 | LR: 4.54e-04 | 3989.9ms/step | 15.0 samples/s
+Step 10470/50462 | Loss: 2.7406 | LR: 4.54e-04 | 3997.5ms/step | 15.0 samples/s
+Step 10480/50462 | Loss: 2.7754 | LR: 4.54e-04 | 4001.1ms/step | 15.0 samples/s
+Step 10490/50462 | Loss: 2.7781 | LR: 4.54e-04 | 4000.0ms/step | 15.0 samples/s
+Step 10500/50462 | Loss: 2.7503 | LR: 4.54e-04 | 4000.0ms/step | 15.0 samples/s
+Validation at step 10500: Loss = 2.7546, Perplexity = 15.71
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-10500
+Step 10510/50462 | Loss: 2.7822 | LR: 4.54e-04 | 6017.2ms/step | 10.0 samples/s
+Step 10520/50462 | Loss: 2.7762 | LR: 4.54e-04 | 4005.6ms/step | 15.0 samples/s
+Step 10530/50462 | Loss: 2.7848 | LR: 4.54e-04 | 4006.2ms/step | 15.0 samples/s
+Step 10540/50462 | Loss: 2.7547 | LR: 4.54e-04 | 4004.7ms/step | 15.0 samples/s
+Step 10550/50462 | Loss: 2.7609 | LR: 4.54e-04 | 3995.8ms/step | 15.0 samples/s
+Step 10560/50462 | Loss: 2.7357 | LR: 4.53e-04 | 4005.0ms/step | 15.0 samples/s
+Step 10570/50462 | Loss: 2.7904 | LR: 4.53e-04 | 4003.3ms/step | 15.0 samples/s
+Step 10580/50462 | Loss: 2.7617 | LR: 4.53e-04 | 4006.8ms/step | 15.0 samples/s
+Step 10590/50462 | Loss: 2.7675 | LR: 4.53e-04 | 3996.7ms/step | 15.0 samples/s
+Step 10600/50462 | Loss: 2.7778 | LR: 4.53e-04 | 4005.9ms/step | 15.0 samples/s
+Step 10610/50462 | Loss: 2.7245 | LR: 4.53e-04 | 4007.5ms/step | 15.0 samples/s
+Step 10620/50462 | Loss: 2.7218 | LR: 4.53e-04 | 4013.1ms/step | 15.0 samples/s
+Step 10630/50462 | Loss: 2.8031 | LR: 4.53e-04 | 4012.0ms/step | 15.0 samples/s
+Step 10640/50462 | Loss: 2.7737 | LR: 4.53e-04 | 4006.3ms/step | 15.0 samples/s
+Step 10650/50462 | Loss: 2.7773 | LR: 4.53e-04 | 3999.8ms/step | 15.0 samples/s
+Step 10660/50462 | Loss: 2.7596 | LR: 4.53e-04 | 4010.0ms/step | 15.0 samples/s
+Step 10670/50462 | Loss: 2.7974 | LR: 4.52e-04 | 4003.7ms/step | 15.0 samples/s
+Step 10680/50462 | Loss: 2.7702 | LR: 4.52e-04 | 4012.3ms/step | 15.0 samples/s
+Step 10690/50462 | Loss: 2.7551 | LR: 4.52e-04 | 3992.7ms/step | 15.0 samples/s
+Step 10700/50462 | Loss: 2.7783 | LR: 4.52e-04 | 4000.7ms/step | 15.0 samples/s
+Step 10710/50462 | Loss: 2.7403 | LR: 4.52e-04 | 4006.1ms/step | 15.0 samples/s
+Step 10720/50462 | Loss: 2.7526 | LR: 4.52e-04 | 3997.1ms/step | 15.0 samples/s
+Step 10730/50462 | Loss: 2.7844 | LR: 4.52e-04 | 3999.1ms/step | 15.0 samples/s
+Step 10740/50462 | Loss: 2.7458 | LR: 4.52e-04 | 4003.3ms/step | 15.0 samples/s
+Step 10750/50462 | Loss: 2.7594 | LR: 4.52e-04 | 4000.4ms/step | 15.0 samples/s
+Validation at step 10750: Loss = 2.7473, Perplexity = 15.60
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-10750
+Step 10760/50462 | Loss: 2.7814 | LR: 4.52e-04 | 5948.5ms/step | 10.1 samples/s
+Step 10770/50462 | Loss: 2.7489 | LR: 4.52e-04 | 4023.6ms/step | 14.9 samples/s
+Step 10780/50462 | Loss: 2.7713 | LR: 4.51e-04 | 4001.9ms/step | 15.0 samples/s
+Step 10790/50462 | Loss: 2.7356 | LR: 4.51e-04 | 4006.7ms/step | 15.0 samples/s
+Step 10800/50462 | Loss: 2.7471 | LR: 4.51e-04 | 3987.2ms/step | 15.0 samples/s
+Step 10810/50462 | Loss: 2.7234 | LR: 4.51e-04 | 3990.3ms/step | 15.0 samples/s
+Step 10820/50462 | Loss: 2.7613 | LR: 4.51e-04 | 4010.7ms/step | 15.0 samples/s
+Step 10830/50462 | Loss: 2.7532 | LR: 4.51e-04 | 4006.3ms/step | 15.0 samples/s
+Step 10840/50462 | Loss: 2.7685 | LR: 4.51e-04 | 3994.9ms/step | 15.0 samples/s
+Step 10850/50462 | Loss: 2.7235 | LR: 4.51e-04 | 4007.6ms/step | 15.0 samples/s
+Step 10860/50462 | Loss: 2.7693 | LR: 4.51e-04 | 3985.0ms/step | 15.1 samples/s
+Step 10870/50462 | Loss: 2.7393 | LR: 4.51e-04 | 4013.7ms/step | 14.9 samples/s
+Step 10880/50462 | Loss: 2.7286 | LR: 4.50e-04 | 3998.4ms/step | 15.0 samples/s
+Step 10890/50462 | Loss: 2.7566 | LR: 4.50e-04 | 4014.0ms/step | 14.9 samples/s
+Step 10900/50462 | Loss: 2.7191 | LR: 4.50e-04 | 4006.4ms/step | 15.0 samples/s
+Step 10910/50462 | Loss: 2.7261 | LR: 4.50e-04 | 4000.9ms/step | 15.0 samples/s
+Step 10920/50462 | Loss: 2.7555 | LR: 4.50e-04 | 4005.0ms/step | 15.0 samples/s
+Step 10930/50462 | Loss: 2.7509 | LR: 4.50e-04 | 4003.1ms/step | 15.0 samples/s
+Step 10940/50462 | Loss: 2.7291 | LR: 4.50e-04 | 3996.3ms/step | 15.0 samples/s
+Step 10950/50462 | Loss: 2.7506 | LR: 4.50e-04 | 4002.0ms/step | 15.0 samples/s
+Step 10960/50462 | Loss: 2.7430 | LR: 4.50e-04 | 4015.9ms/step | 14.9 samples/s
+Step 10970/50462 | Loss: 2.7365 | LR: 4.50e-04 | 4015.1ms/step | 14.9 samples/s
+Step 10980/50462 | Loss: 2.7219 | LR: 4.50e-04 | 4018.6ms/step | 14.9 samples/s
+Step 10990/50462 | Loss: 2.7261 | LR: 4.49e-04 | 4007.2ms/step | 15.0 samples/s
+Step 11000/50462 | Loss: 2.7243 | LR: 4.49e-04 | 4007.5ms/step | 15.0 samples/s
+Validation at step 11000: Loss = 2.7432, Perplexity = 15.54
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-11000
+Step 11010/50462 | Loss: 2.7214 | LR: 4.49e-04 | 5955.1ms/step | 10.1 samples/s
+Step 11020/50462 | Loss: 2.7647 | LR: 4.49e-04 | 3997.5ms/step | 15.0 samples/s
+Step 11030/50462 | Loss: 2.7368 | LR: 4.49e-04 | 4002.7ms/step | 15.0 samples/s
+Step 11040/50462 | Loss: 2.7476 | LR: 4.49e-04 | 3997.3ms/step | 15.0 samples/s
+Step 11050/50462 | Loss: 2.7255 | LR: 4.49e-04 | 4006.8ms/step | 15.0 samples/s
+Step 11060/50462 | Loss: 2.7727 | LR: 4.49e-04 | 3989.8ms/step | 15.0 samples/s
+Step 11070/50462 | Loss: 2.8017 | LR: 4.49e-04 | 3999.0ms/step | 15.0 samples/s
+Step 11080/50462 | Loss: 2.7473 | LR: 4.49e-04 | 4006.6ms/step | 15.0 samples/s
+Step 11090/50462 | Loss: 2.7514 | LR: 4.48e-04 | 3991.7ms/step | 15.0 samples/s
+Step 11100/50462 | Loss: 2.7485 | LR: 4.48e-04 | 3997.2ms/step | 15.0 samples/s
+Step 11110/50462 | Loss: 2.7595 | LR: 4.48e-04 | 4019.6ms/step | 14.9 samples/s
+Step 11120/50462 | Loss: 2.7541 | LR: 4.48e-04 | 3994.7ms/step | 15.0 samples/s
+Step 11130/50462 | Loss: 2.7548 | LR: 4.48e-04 | 3994.1ms/step | 15.0 samples/s
+Step 11140/50462 | Loss: 2.7464 | LR: 4.48e-04 | 4014.9ms/step | 14.9 samples/s
+Step 11150/50462 | Loss: 2.7513 | LR: 4.48e-04 | 3999.5ms/step | 15.0 samples/s
+Step 11160/50462 | Loss: 2.7720 | LR: 4.48e-04 | 3981.6ms/step | 15.1 samples/s
+Step 11170/50462 | Loss: 2.7546 | LR: 4.48e-04 | 4004.3ms/step | 15.0 samples/s
+Step 11180/50462 | Loss: 2.7569 | LR: 4.48e-04 | 3999.6ms/step | 15.0 samples/s
+Step 11190/50462 | Loss: 2.6974 | LR: 4.48e-04 | 3994.6ms/step | 15.0 samples/s
+Step 11200/50462 | Loss: 2.7683 | LR: 4.47e-04 | 3999.2ms/step | 15.0 samples/s
+Step 11210/50462 | Loss: 2.7808 | LR: 4.47e-04 | 4005.5ms/step | 15.0 samples/s
+Step 11220/50462 | Loss: 2.7438 | LR: 4.47e-04 | 4007.8ms/step | 15.0 samples/s
+Step 11230/50462 | Loss: 2.7442 | LR: 4.47e-04 | 4000.8ms/step | 15.0 samples/s
+Step 11240/50462 | Loss: 2.7258 | LR: 4.47e-04 | 4002.9ms/step | 15.0 samples/s
+Step 11250/50462 | Loss: 2.7774 | LR: 4.47e-04 | 3999.9ms/step | 15.0 samples/s
+Validation at step 11250: Loss = 2.7383, Perplexity = 15.46
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-11250
+Step 11260/50462 | Loss: 2.7482 | LR: 4.47e-04 | 5911.0ms/step | 10.2 samples/s
+Step 11270/50462 | Loss: 2.7537 | LR: 4.47e-04 | 4009.1ms/step | 15.0 samples/s
+Step 11280/50462 | Loss: 2.7845 | LR: 4.47e-04 | 4004.9ms/step | 15.0 samples/s
+Step 11290/50462 | Loss: 2.7146 | LR: 4.47e-04 | 4010.0ms/step | 15.0 samples/s
+Step 11300/50462 | Loss: 2.7343 | LR: 4.46e-04 | 4006.4ms/step | 15.0 samples/s
+Step 11310/50462 | Loss: 2.7579 | LR: 4.46e-04 | 4003.5ms/step | 15.0 samples/s
+Step 11320/50462 | Loss: 2.7377 | LR: 4.46e-04 | 4003.5ms/step | 15.0 samples/s
+Step 11330/50462 | Loss: 2.7307 | LR: 4.46e-04 | 3993.6ms/step | 15.0 samples/s
+Step 11340/50462 | Loss: 2.7399 | LR: 4.46e-04 | 4011.4ms/step | 15.0 samples/s
+Step 11350/50462 | Loss: 2.7761 | LR: 4.46e-04 | 4005.1ms/step | 15.0 samples/s
+Step 11360/50462 | Loss: 2.7609 | LR: 4.46e-04 | 4009.7ms/step | 15.0 samples/s
+Step 11370/50462 | Loss: 2.7604 | LR: 4.46e-04 | 4010.1ms/step | 15.0 samples/s
+Step 11380/50462 | Loss: 2.7495 | LR: 4.46e-04 | 3995.7ms/step | 15.0 samples/s
+Step 11390/50462 | Loss: 2.7562 | LR: 4.46e-04 | 4003.3ms/step | 15.0 samples/s
+Step 11400/50462 | Loss: 2.7793 | LR: 4.45e-04 | 4009.8ms/step | 15.0 samples/s
+Step 11410/50462 | Loss: 2.7256 | LR: 4.45e-04 | 4020.8ms/step | 14.9 samples/s
+Step 11420/50462 | Loss: 2.7611 | LR: 4.45e-04 | 3996.0ms/step | 15.0 samples/s
+Step 11430/50462 | Loss: 2.7607 | LR: 4.45e-04 | 3996.6ms/step | 15.0 samples/s
+Step 11440/50462 | Loss: 2.8004 | LR: 4.45e-04 | 4004.7ms/step | 15.0 samples/s
+Step 11450/50462 | Loss: 2.7822 | LR: 4.45e-04 | 3999.3ms/step | 15.0 samples/s
+Step 11460/50462 | Loss: 2.7254 | LR: 4.45e-04 | 4003.3ms/step | 15.0 samples/s
+Step 11470/50462 | Loss: 2.7348 | LR: 4.45e-04 | 4007.5ms/step | 15.0 samples/s
+Step 11480/50462 | Loss: 2.7093 | LR: 4.45e-04 | 4001.4ms/step | 15.0 samples/s
+Step 11490/50462 | Loss: 2.7410 | LR: 4.45e-04 | 3995.8ms/step | 15.0 samples/s
+Step 11500/50462 | Loss: 2.7605 | LR: 4.44e-04 | 4002.5ms/step | 15.0 samples/s
+Validation at step 11500: Loss = 2.7345, Perplexity = 15.40
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-11500
+Step 11510/50462 | Loss: 2.7282 | LR: 4.44e-04 | 5921.4ms/step | 10.1 samples/s
+Step 11520/50462 | Loss: 2.7744 | LR: 4.44e-04 | 3996.1ms/step | 15.0 samples/s
+Step 11530/50462 | Loss: 2.7482 | LR: 4.44e-04 | 3996.7ms/step | 15.0 samples/s
+Step 11540/50462 | Loss: 2.7426 | LR: 4.44e-04 | 3992.3ms/step | 15.0 samples/s
+Step 11550/50462 | Loss: 2.7404 | LR: 4.44e-04 | 3993.7ms/step | 15.0 samples/s
+Step 11560/50462 | Loss: 2.7534 | LR: 4.44e-04 | 3995.8ms/step | 15.0 samples/s
+Step 11570/50462 | Loss: 2.7215 | LR: 4.44e-04 | 4008.4ms/step | 15.0 samples/s
+Step 11580/50462 | Loss: 2.7125 | LR: 4.44e-04 | 4008.9ms/step | 15.0 samples/s
+Step 11590/50462 | Loss: 2.7161 | LR: 4.44e-04 | 4001.0ms/step | 15.0 samples/s
+Step 11600/50462 | Loss: 2.7975 | LR: 4.43e-04 | 4003.2ms/step | 15.0 samples/s
+Step 11610/50462 | Loss: 2.7513 | LR: 4.43e-04 | 4006.5ms/step | 15.0 samples/s
+Step 11620/50462 | Loss: 2.7424 | LR: 4.43e-04 | 3995.1ms/step | 15.0 samples/s
+Step 11630/50462 | Loss: 2.7417 | LR: 4.43e-04 | 3995.0ms/step | 15.0 samples/s
+Step 11640/50462 | Loss: 2.7611 | LR: 4.43e-04 | 3998.3ms/step | 15.0 samples/s
+Step 11650/50462 | Loss: 2.7156 | LR: 4.43e-04 | 4003.5ms/step | 15.0 samples/s
+Step 11660/50462 | Loss: 2.7413 | LR: 4.43e-04 | 3996.3ms/step | 15.0 samples/s
+Step 11670/50462 | Loss: 2.7323 | LR: 4.43e-04 | 3983.8ms/step | 15.1 samples/s
+Step 11680/50462 | Loss: 2.7608 | LR: 4.43e-04 | 3991.5ms/step | 15.0 samples/s
+Step 11690/50462 | Loss: 2.7328 | LR: 4.43e-04 | 4001.5ms/step | 15.0 samples/s
+Step 11700/50462 | Loss: 2.7550 | LR: 4.42e-04 | 3996.0ms/step | 15.0 samples/s
+Step 11710/50462 | Loss: 2.7333 | LR: 4.42e-04 | 3999.4ms/step | 15.0 samples/s
+Step 11720/50462 | Loss: 2.7306 | LR: 4.42e-04 | 3999.9ms/step | 15.0 samples/s
+Step 11730/50462 | Loss: 2.7378 | LR: 4.42e-04 | 4008.8ms/step | 15.0 samples/s
+Step 11740/50462 | Loss: 2.7761 | LR: 4.42e-04 | 3996.6ms/step | 15.0 samples/s
+Step 11750/50462 | Loss: 2.7373 | LR: 4.42e-04 | 3996.7ms/step | 15.0 samples/s
+Validation at step 11750: Loss = 2.7298, Perplexity = 15.33
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-11750
+Step 11760/50462 | Loss: 2.7383 | LR: 4.42e-04 | 5936.2ms/step | 10.1 samples/s
+Step 11770/50462 | Loss: 2.7394 | LR: 4.42e-04 | 3996.5ms/step | 15.0 samples/s
+Step 11780/50462 | Loss: 2.7333 | LR: 4.42e-04 | 3997.3ms/step | 15.0 samples/s
+Step 11790/50462 | Loss: 2.6860 | LR: 4.42e-04 | 4007.7ms/step | 15.0 samples/s
+Step 11800/50462 | Loss: 2.7590 | LR: 4.41e-04 | 4010.6ms/step | 15.0 samples/s
+Step 11810/50462 | Loss: 2.7127 | LR: 4.41e-04 | 4002.8ms/step | 15.0 samples/s
+Step 11820/50462 | Loss: 2.7438 | LR: 4.41e-04 | 3993.4ms/step | 15.0 samples/s
+Step 11830/50462 | Loss: 2.7128 | LR: 4.41e-04 | 4000.0ms/step | 15.0 samples/s
+Step 11840/50462 | Loss: 2.7282 | LR: 4.41e-04 | 3998.0ms/step | 15.0 samples/s
+Step 11850/50462 | Loss: 2.7445 | LR: 4.41e-04 | 3995.4ms/step | 15.0 samples/s
+Step 11860/50462 | Loss: 2.7620 | LR: 4.41e-04 | 4006.8ms/step | 15.0 samples/s
+Step 11870/50462 | Loss: 2.7295 | LR: 4.41e-04 | 4013.8ms/step | 14.9 samples/s
+Step 11880/50462 | Loss: 2.7397 | LR: 4.41e-04 | 4010.0ms/step | 15.0 samples/s
+Step 11890/50462 | Loss: 2.7120 | LR: 4.41e-04 | 4031.1ms/step | 14.9 samples/s
+Step 11900/50462 | Loss: 2.7625 | LR: 4.40e-04 | 4012.9ms/step | 15.0 samples/s
+Step 11910/50462 | Loss: 2.7419 | LR: 4.40e-04 | 4004.7ms/step | 15.0 samples/s
+Step 11920/50462 | Loss: 2.7244 | LR: 4.40e-04 | 4017.0ms/step | 14.9 samples/s
+Step 11930/50462 | Loss: 2.7522 | LR: 4.40e-04 | 4001.4ms/step | 15.0 samples/s
+Step 11940/50462 | Loss: 2.7157 | LR: 4.40e-04 | 3994.5ms/step | 15.0 samples/s
+Step 11950/50462 | Loss: 2.7273 | LR: 4.40e-04 | 3991.1ms/step | 15.0 samples/s
+Step 11960/50462 | Loss: 2.7368 | LR: 4.40e-04 | 3994.6ms/step | 15.0 samples/s
+Step 11970/50462 | Loss: 2.7674 | LR: 4.40e-04 | 3992.1ms/step | 15.0 samples/s
+Step 11980/50462 | Loss: 2.7751 | LR: 4.40e-04 | 4001.6ms/step | 15.0 samples/s
+Step 11990/50462 | Loss: 2.7128 | LR: 4.40e-04 | 4002.4ms/step | 15.0 samples/s
+Step 12000/50462 | Loss: 2.7442 | LR: 4.39e-04 | 4002.3ms/step | 15.0 samples/s
+Validation at step 12000: Loss = 2.7273, Perplexity = 15.29
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-12000
+Step 12010/50462 | Loss: 2.7284 | LR: 4.39e-04 | 5944.5ms/step | 10.1 samples/s
+Step 12020/50462 | Loss: 2.7647 | LR: 4.39e-04 | 4002.7ms/step | 15.0 samples/s
+Step 12030/50462 | Loss: 2.7263 | LR: 4.39e-04 | 4010.0ms/step | 15.0 samples/s
+Step 12040/50462 | Loss: 2.7228 | LR: 4.39e-04 | 4008.7ms/step | 15.0 samples/s
+Step 12050/50462 | Loss: 2.6870 | LR: 4.39e-04 | 4006.3ms/step | 15.0 samples/s
+Step 12060/50462 | Loss: 2.7512 | LR: 4.39e-04 | 3994.9ms/step | 15.0 samples/s
+Step 12070/50462 | Loss: 2.7394 | LR: 4.39e-04 | 4003.9ms/step | 15.0 samples/s
+Step 12080/50462 | Loss: 2.7607 | LR: 4.39e-04 | 3989.0ms/step | 15.0 samples/s
+Step 12090/50462 | Loss: 2.7265 | LR: 4.39e-04 | 4002.3ms/step | 15.0 samples/s
+Step 12100/50462 | Loss: 2.7744 | LR: 4.38e-04 | 3987.7ms/step | 15.0 samples/s
+Step 12110/50462 | Loss: 2.7556 | LR: 4.38e-04 | 3998.8ms/step | 15.0 samples/s
+Step 12120/50462 | Loss: 2.7295 | LR: 4.38e-04 | 4016.8ms/step | 14.9 samples/s
+Step 12130/50462 | Loss: 2.7295 | LR: 4.38e-04 | 3998.8ms/step | 15.0 samples/s
+Step 12140/50462 | Loss: 2.7051 | LR: 4.38e-04 | 4001.1ms/step | 15.0 samples/s
+Step 12150/50462 | Loss: 2.7180 | LR: 4.38e-04 | 4011.5ms/step | 15.0 samples/s
+Step 12160/50462 | Loss: 2.7362 | LR: 4.38e-04 | 3993.8ms/step | 15.0 samples/s
+Step 12170/50462 | Loss: 2.7286 | LR: 4.38e-04 | 3998.7ms/step | 15.0 samples/s
+Step 12180/50462 | Loss: 2.7109 | LR: 4.38e-04 | 3991.5ms/step | 15.0 samples/s
+Step 12190/50462 | Loss: 2.7851 | LR: 4.37e-04 | 4007.3ms/step | 15.0 samples/s
+Step 12200/50462 | Loss: 2.7442 | LR: 4.37e-04 | 4000.0ms/step | 15.0 samples/s
+Step 12210/50462 | Loss: 2.7657 | LR: 4.37e-04 | 4014.9ms/step | 14.9 samples/s
+Step 12220/50462 | Loss: 2.7276 | LR: 4.37e-04 | 4013.2ms/step | 15.0 samples/s
+Step 12230/50462 | Loss: 2.7675 | LR: 4.37e-04 | 4006.6ms/step | 15.0 samples/s
+Step 12240/50462 | Loss: 2.7509 | LR: 4.37e-04 | 4003.6ms/step | 15.0 samples/s
+Step 12250/50462 | Loss: 2.7018 | LR: 4.37e-04 | 4003.0ms/step | 15.0 samples/s
+Validation at step 12250: Loss = 2.7189, Perplexity = 15.16
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-12250
+Step 12260/50462 | Loss: 2.7517 | LR: 4.37e-04 | 5940.5ms/step | 10.1 samples/s
+Step 12270/50462 | Loss: 2.7690 | LR: 4.37e-04 | 3996.1ms/step | 15.0 samples/s
+Step 12280/50462 | Loss: 2.7482 | LR: 4.37e-04 | 4000.7ms/step | 15.0 samples/s
+Step 12290/50462 | Loss: 2.7436 | LR: 4.36e-04 | 3997.1ms/step | 15.0 samples/s
+Step 12300/50462 | Loss: 2.7249 | LR: 4.36e-04 | 3995.8ms/step | 15.0 samples/s
+Step 12310/50462 | Loss: 2.7160 | LR: 4.36e-04 | 4000.3ms/step | 15.0 samples/s
+Step 12320/50462 | Loss: 2.7505 | LR: 4.36e-04 | 3998.2ms/step | 15.0 samples/s
+Step 12330/50462 | Loss: 2.7264 | LR: 4.36e-04 | 3998.0ms/step | 15.0 samples/s
+Step 12340/50462 | Loss: 2.7554 | LR: 4.36e-04 | 3997.3ms/step | 15.0 samples/s
+Step 12350/50462 | Loss: 2.7178 | LR: 4.36e-04 | 3997.3ms/step | 15.0 samples/s
+Step 12360/50462 | Loss: 2.7270 | LR: 4.36e-04 | 4000.1ms/step | 15.0 samples/s
+Step 12370/50462 | Loss: 2.6729 | LR: 4.36e-04 | 4010.1ms/step | 15.0 samples/s
+Step 12380/50462 | Loss: 2.7103 | LR: 4.35e-04 | 4007.4ms/step | 15.0 samples/s
+Step 12390/50462 | Loss: 2.7260 | LR: 4.35e-04 | 4005.7ms/step | 15.0 samples/s
+Step 12400/50462 | Loss: 2.7350 | LR: 4.35e-04 | 3997.6ms/step | 15.0 samples/s
+Step 12410/50462 | Loss: 2.7680 | LR: 4.35e-04 | 3994.3ms/step | 15.0 samples/s
+Step 12420/50462 | Loss: 2.7107 | LR: 4.35e-04 | 3994.0ms/step | 15.0 samples/s
+Step 12430/50462 | Loss: 2.7738 | LR: 4.35e-04 | 3991.3ms/step | 15.0 samples/s
+Step 12440/50462 | Loss: 2.7578 | LR: 4.35e-04 | 3998.7ms/step | 15.0 samples/s
+Step 12450/50462 | Loss: 2.7450 | LR: 4.35e-04 | 4014.2ms/step | 14.9 samples/s
+Step 12460/50462 | Loss: 2.7268 | LR: 4.35e-04 | 3992.9ms/step | 15.0 samples/s
+Step 12470/50462 | Loss: 2.7653 | LR: 4.35e-04 | 3984.2ms/step | 15.1 samples/s
+Step 12480/50462 | Loss: 2.7024 | LR: 4.34e-04 | 4010.0ms/step | 15.0 samples/s
+Step 12490/50462 | Loss: 2.7345 | LR: 4.34e-04 | 4008.5ms/step | 15.0 samples/s
+Step 12500/50462 | Loss: 2.7430 | LR: 4.34e-04 | 4002.6ms/step | 15.0 samples/s
+Validation at step 12500: Loss = 2.7179, Perplexity = 15.15
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-12500
+Step 12510/50462 | Loss: 2.6974 | LR: 4.34e-04 | 5950.7ms/step | 10.1 samples/s
+Step 12520/50462 | Loss: 2.7356 | LR: 4.34e-04 | 3997.7ms/step | 15.0 samples/s
+Step 12530/50462 | Loss: 2.7346 | LR: 4.34e-04 | 4006.4ms/step | 15.0 samples/s
+Step 12540/50462 | Loss: 2.7033 | LR: 4.34e-04 | 3995.1ms/step | 15.0 samples/s
+Step 12550/50462 | Loss: 2.7267 | LR: 4.34e-04 | 4006.2ms/step | 15.0 samples/s
+Step 12560/50462 | Loss: 2.7581 | LR: 4.34e-04 | 4012.5ms/step | 15.0 samples/s
+Step 12570/50462 | Loss: 2.7357 | LR: 4.33e-04 | 4003.2ms/step | 15.0 samples/s
+Step 12580/50462 | Loss: 2.7400 | LR: 4.33e-04 | 3992.5ms/step | 15.0 samples/s
+Step 12590/50462 | Loss: 2.6940 | LR: 4.33e-04 | 3996.7ms/step | 15.0 samples/s
+Step 12600/50462 | Loss: 2.7445 | LR: 4.33e-04 | 3996.8ms/step | 15.0 samples/s
+Step 12610/50462 | Loss: 2.7227 | LR: 4.33e-04 | 3994.0ms/step | 15.0 samples/s
+Step 12620/50462 | Loss: 2.7160 | LR: 4.33e-04 | 3996.2ms/step | 15.0 samples/s
+Step 12630/50462 | Loss: 2.7080 | LR: 4.33e-04 | 4009.6ms/step | 15.0 samples/s
+Step 12640/50462 | Loss: 2.7078 | LR: 4.33e-04 | 4011.0ms/step | 15.0 samples/s
+Step 12650/50462 | Loss: 2.7409 | LR: 4.33e-04 | 3997.4ms/step | 15.0 samples/s
+Step 12660/50462 | Loss: 2.7543 | LR: 4.32e-04 | 3996.8ms/step | 15.0 samples/s
+Step 12670/50462 | Loss: 2.7070 | LR: 4.32e-04 | 3996.1ms/step | 15.0 samples/s
+Step 12680/50462 | Loss: 2.7174 | LR: 4.32e-04 | 3997.2ms/step | 15.0 samples/s
+Step 12690/50462 | Loss: 2.7116 | LR: 4.32e-04 | 4008.7ms/step | 15.0 samples/s
+Step 12700/50462 | Loss: 2.7589 | LR: 4.32e-04 | 4002.6ms/step | 15.0 samples/s
+Step 12710/50462 | Loss: 2.6953 | LR: 4.32e-04 | 3994.0ms/step | 15.0 samples/s
+Step 12720/50462 | Loss: 2.7325 | LR: 4.32e-04 | 4004.8ms/step | 15.0 samples/s
+Step 12730/50462 | Loss: 2.7333 | LR: 4.32e-04 | 3992.0ms/step | 15.0 samples/s
+Step 12740/50462 | Loss: 2.6959 | LR: 4.32e-04 | 3990.7ms/step | 15.0 samples/s
+Step 12750/50462 | Loss: 2.7115 | LR: 4.31e-04 | 4014.2ms/step | 14.9 samples/s
+Validation at step 12750: Loss = 2.7101, Perplexity = 15.03
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-12750
+Step 12760/50462 | Loss: 2.7077 | LR: 4.31e-04 | 5951.1ms/step | 10.1 samples/s
+Step 12770/50462 | Loss: 2.7293 | LR: 4.31e-04 | 4002.7ms/step | 15.0 samples/s
+Step 12780/50462 | Loss: 2.7153 | LR: 4.31e-04 | 3994.4ms/step | 15.0 samples/s
+Step 12790/50462 | Loss: 2.6865 | LR: 4.31e-04 | 4004.8ms/step | 15.0 samples/s
+Step 12800/50462 | Loss: 2.7184 | LR: 4.31e-04 | 4010.0ms/step | 15.0 samples/s
+Step 12810/50462 | Loss: 2.7341 | LR: 4.31e-04 | 4016.2ms/step | 14.9 samples/s
+Step 12820/50462 | Loss: 2.7343 | LR: 4.31e-04 | 4017.4ms/step | 14.9 samples/s
+Step 12830/50462 | Loss: 2.7257 | LR: 4.31e-04 | 4014.8ms/step | 14.9 samples/s
+Step 12840/50462 | Loss: 2.7431 | LR: 4.31e-04 | 4019.7ms/step | 14.9 samples/s
+Step 12850/50462 | Loss: 2.7587 | LR: 4.30e-04 | 4027.6ms/step | 14.9 samples/s
+Step 12860/50462 | Loss: 2.7432 | LR: 4.30e-04 | 4007.3ms/step | 15.0 samples/s
+Step 12870/50462 | Loss: 2.7215 | LR: 4.30e-04 | 4004.2ms/step | 15.0 samples/s
+Step 12880/50462 | Loss: 2.7103 | LR: 4.30e-04 | 4004.5ms/step | 15.0 samples/s
+Step 12890/50462 | Loss: 2.7055 | LR: 4.30e-04 | 4006.4ms/step | 15.0 samples/s
+Step 12900/50462 | Loss: 2.7241 | LR: 4.30e-04 | 4005.2ms/step | 15.0 samples/s
+Step 12910/50462 | Loss: 2.7304 | LR: 4.30e-04 | 3999.1ms/step | 15.0 samples/s
+Step 12920/50462 | Loss: 2.7388 | LR: 4.30e-04 | 3993.9ms/step | 15.0 samples/s
+Step 12930/50462 | Loss: 2.7314 | LR: 4.30e-04 | 3996.1ms/step | 15.0 samples/s
+Step 12940/50462 | Loss: 2.7155 | LR: 4.29e-04 | 3998.2ms/step | 15.0 samples/s
+Step 12950/50462 | Loss: 2.7387 | LR: 4.29e-04 | 4002.9ms/step | 15.0 samples/s
+Step 12960/50462 | Loss: 2.7002 | LR: 4.29e-04 | 4011.9ms/step | 15.0 samples/s
+Step 12970/50462 | Loss: 2.7117 | LR: 4.29e-04 | 4008.2ms/step | 15.0 samples/s
+Step 12980/50462 | Loss: 2.7404 | LR: 4.29e-04 | 3992.6ms/step | 15.0 samples/s
+Step 12990/50462 | Loss: 2.7001 | LR: 4.29e-04 | 3999.7ms/step | 15.0 samples/s
+Step 13000/50462 | Loss: 2.7291 | LR: 4.29e-04 | 3998.1ms/step | 15.0 samples/s
+Validation at step 13000: Loss = 2.7081, Perplexity = 15.00
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-13000
+Step 13010/50462 | Loss: 2.7394 | LR: 4.29e-04 | 5939.1ms/step | 10.1 samples/s
+Step 13020/50462 | Loss: 2.7118 | LR: 4.29e-04 | 4004.1ms/step | 15.0 samples/s
+Step 13030/50462 | Loss: 2.7123 | LR: 4.28e-04 | 4016.8ms/step | 14.9 samples/s
+Step 13040/50462 | Loss: 2.7056 | LR: 4.28e-04 | 3999.1ms/step | 15.0 samples/s
+Step 13050/50462 | Loss: 2.7411 | LR: 4.28e-04 | 4008.8ms/step | 15.0 samples/s
+Step 13060/50462 | Loss: 2.7523 | LR: 4.28e-04 | 4007.3ms/step | 15.0 samples/s
+Step 13070/50462 | Loss: 2.7464 | LR: 4.28e-04 | 3993.3ms/step | 15.0 samples/s
+Step 13080/50462 | Loss: 2.7417 | LR: 4.28e-04 | 3988.4ms/step | 15.0 samples/s
+Step 13090/50462 | Loss: 2.6946 | LR: 4.28e-04 | 4013.8ms/step | 14.9 samples/s
+Step 13100/50462 | Loss: 2.7249 | LR: 4.28e-04 | 4002.0ms/step | 15.0 samples/s
+Step 13110/50462 | Loss: 2.7409 | LR: 4.28e-04 | 4005.3ms/step | 15.0 samples/s
+Step 13120/50462 | Loss: 2.7333 | LR: 4.27e-04 | 4004.1ms/step | 15.0 samples/s
+Step 13130/50462 | Loss: 2.6898 | LR: 4.27e-04 | 4014.5ms/step | 14.9 samples/s
+Step 13140/50462 | Loss: 2.7452 | LR: 4.27e-04 | 3998.0ms/step | 15.0 samples/s
+Step 13150/50462 | Loss: 2.7367 | LR: 4.27e-04 | 4006.6ms/step | 15.0 samples/s
+Step 13160/50462 | Loss: 2.7367 | LR: 4.27e-04 | 3998.7ms/step | 15.0 samples/s
+Step 13170/50462 | Loss: 2.6949 | LR: 4.27e-04 | 4001.5ms/step | 15.0 samples/s
+Step 13180/50462 | Loss: 2.7264 | LR: 4.27e-04 | 4001.3ms/step | 15.0 samples/s
+Step 13190/50462 | Loss: 2.7416 | LR: 4.27e-04 | 3993.4ms/step | 15.0 samples/s
+Step 13200/50462 | Loss: 2.7085 | LR: 4.27e-04 | 3998.5ms/step | 15.0 samples/s
+Step 13210/50462 | Loss: 2.7054 | LR: 4.26e-04 | 3993.4ms/step | 15.0 samples/s
+Step 13220/50462 | Loss: 2.6968 | LR: 4.26e-04 | 4007.6ms/step | 15.0 samples/s
+Step 13230/50462 | Loss: 2.7258 | LR: 4.26e-04 | 3988.7ms/step | 15.0 samples/s
+Step 13240/50462 | Loss: 2.7488 | LR: 4.26e-04 | 3986.1ms/step | 15.1 samples/s
+Step 13250/50462 | Loss: 2.7041 | LR: 4.26e-04 | 4010.2ms/step | 15.0 samples/s
+Validation at step 13250: Loss = 2.7036, Perplexity = 14.93
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-13250
+Step 13260/50462 | Loss: 2.7161 | LR: 4.26e-04 | 5960.3ms/step | 10.1 samples/s
+Step 13270/50462 | Loss: 2.7502 | LR: 4.26e-04 | 3997.0ms/step | 15.0 samples/s
+Step 13280/50462 | Loss: 2.6771 | LR: 4.26e-04 | 3992.9ms/step | 15.0 samples/s
+Step 13290/50462 | Loss: 2.7242 | LR: 4.26e-04 | 4008.8ms/step | 15.0 samples/s
+Step 13300/50462 | Loss: 2.7166 | LR: 4.25e-04 | 4012.4ms/step | 15.0 samples/s
+Step 13310/50462 | Loss: 2.6944 | LR: 4.25e-04 | 3999.5ms/step | 15.0 samples/s
+Step 13320/50462 | Loss: 2.7126 | LR: 4.25e-04 | 4005.6ms/step | 15.0 samples/s
+Step 13330/50462 | Loss: 2.7379 | LR: 4.25e-04 | 3997.8ms/step | 15.0 samples/s
+Step 13340/50462 | Loss: 2.7420 | LR: 4.25e-04 | 4000.7ms/step | 15.0 samples/s
+Step 13350/50462 | Loss: 2.7141 | LR: 4.25e-04 | 3997.8ms/step | 15.0 samples/s
+Step 13360/50462 | Loss: 2.7335 | LR: 4.25e-04 | 4003.2ms/step | 15.0 samples/s
+Step 13370/50462 | Loss: 2.7248 | LR: 4.25e-04 | 3995.9ms/step | 15.0 samples/s
+Step 13380/50462 | Loss: 2.7106 | LR: 4.25e-04 | 4005.6ms/step | 15.0 samples/s
+Step 13390/50462 | Loss: 2.7043 | LR: 4.24e-04 | 3998.4ms/step | 15.0 samples/s
+Step 13400/50462 | Loss: 2.6888 | LR: 4.24e-04 | 4004.1ms/step | 15.0 samples/s
+Step 13410/50462 | Loss: 2.7079 | LR: 4.24e-04 | 3998.3ms/step | 15.0 samples/s
+Step 13420/50462 | Loss: 2.7151 | LR: 4.24e-04 | 3997.1ms/step | 15.0 samples/s
+Step 13430/50462 | Loss: 2.7141 | LR: 4.24e-04 | 4004.2ms/step | 15.0 samples/s
+Step 13440/50462 | Loss: 2.7024 | LR: 4.24e-04 | 4013.6ms/step | 14.9 samples/s
+Step 13450/50462 | Loss: 2.7386 | LR: 4.24e-04 | 3993.0ms/step | 15.0 samples/s
+Step 13460/50462 | Loss: 2.7413 | LR: 4.24e-04 | 3999.5ms/step | 15.0 samples/s
+Step 13470/50462 | Loss: 2.7432 | LR: 4.23e-04 | 3987.7ms/step | 15.0 samples/s
+Step 13480/50462 | Loss: 2.7240 | LR: 4.23e-04 | 3996.5ms/step | 15.0 samples/s
+Step 13490/50462 | Loss: 2.7032 | LR: 4.23e-04 | 4008.6ms/step | 15.0 samples/s
+Step 13500/50462 | Loss: 2.7047 | LR: 4.23e-04 | 3985.5ms/step | 15.1 samples/s
+Validation at step 13500: Loss = 2.6996, Perplexity = 14.87
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-13500
+Step 13510/50462 | Loss: 2.7484 | LR: 4.23e-04 | 5922.3ms/step | 10.1 samples/s
+Step 13520/50462 | Loss: 2.7141 | LR: 4.23e-04 | 4010.6ms/step | 15.0 samples/s
+Step 13530/50462 | Loss: 2.7066 | LR: 4.23e-04 | 3990.6ms/step | 15.0 samples/s
+Step 13540/50462 | Loss: 2.7008 | LR: 4.23e-04 | 4018.7ms/step | 14.9 samples/s
+Step 13550/50462 | Loss: 2.6896 | LR: 4.23e-04 | 3997.8ms/step | 15.0 samples/s
+Step 13560/50462 | Loss: 2.6937 | LR: 4.22e-04 | 4006.4ms/step | 15.0 samples/s
+Step 13570/50462 | Loss: 2.7231 | LR: 4.22e-04 | 4005.3ms/step | 15.0 samples/s
+Step 13580/50462 | Loss: 2.7208 | LR: 4.22e-04 | 3992.6ms/step | 15.0 samples/s
+Step 13590/50462 | Loss: 2.7068 | LR: 4.22e-04 | 4010.8ms/step | 15.0 samples/s
+Step 13600/50462 | Loss: 2.7421 | LR: 4.22e-04 | 4003.7ms/step | 15.0 samples/s
+Step 13610/50462 | Loss: 2.7066 | LR: 4.22e-04 | 3992.3ms/step | 15.0 samples/s
+Step 13620/50462 | Loss: 2.7203 | LR: 4.22e-04 | 3994.4ms/step | 15.0 samples/s
+Step 13630/50462 | Loss: 2.6964 | LR: 4.22e-04 | 4004.9ms/step | 15.0 samples/s
+Step 13640/50462 | Loss: 2.7278 | LR: 4.22e-04 | 4000.6ms/step | 15.0 samples/s
+Step 13650/50462 | Loss: 2.7370 | LR: 4.21e-04 | 4000.7ms/step | 15.0 samples/s
+Step 13660/50462 | Loss: 2.6911 | LR: 4.21e-04 | 4000.8ms/step | 15.0 samples/s
+Step 13670/50462 | Loss: 2.6779 | LR: 4.21e-04 | 3998.4ms/step | 15.0 samples/s
+Step 13680/50462 | Loss: 2.7161 | LR: 4.21e-04 | 3993.1ms/step | 15.0 samples/s
+Step 13690/50462 | Loss: 2.7239 | LR: 4.21e-04 | 3984.9ms/step | 15.1 samples/s
+Step 13700/50462 | Loss: 2.6948 | LR: 4.21e-04 | 4003.9ms/step | 15.0 samples/s
+Step 13710/50462 | Loss: 2.7390 | LR: 4.21e-04 | 4003.1ms/step | 15.0 samples/s
+Step 13720/50462 | Loss: 2.7250 | LR: 4.21e-04 | 3996.8ms/step | 15.0 samples/s
+Step 13730/50462 | Loss: 2.7013 | LR: 4.21e-04 | 4014.8ms/step | 14.9 samples/s
+Step 13740/50462 | Loss: 2.7149 | LR: 4.20e-04 | 4014.9ms/step | 14.9 samples/s
+Step 13750/50462 | Loss: 2.7726 | LR: 4.20e-04 | 4004.3ms/step | 15.0 samples/s
+Validation at step 13750: Loss = 2.6963, Perplexity = 14.82
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-13750
+Step 13760/50462 | Loss: 2.7386 | LR: 4.20e-04 | 5946.5ms/step | 10.1 samples/s
+Step 13770/50462 | Loss: 2.7221 | LR: 4.20e-04 | 3992.8ms/step | 15.0 samples/s
+Step 13780/50462 | Loss: 2.6998 | LR: 4.20e-04 | 4010.3ms/step | 15.0 samples/s
+Step 13790/50462 | Loss: 2.6668 | LR: 4.20e-04 | 4003.3ms/step | 15.0 samples/s
+Step 13800/50462 | Loss: 2.7022 | LR: 4.20e-04 | 3996.9ms/step | 15.0 samples/s
+Step 13810/50462 | Loss: 2.7199 | LR: 4.20e-04 | 4000.0ms/step | 15.0 samples/s
+Step 13820/50462 | Loss: 2.6989 | LR: 4.19e-04 | 4002.9ms/step | 15.0 samples/s
+Step 13830/50462 | Loss: 2.6907 | LR: 4.19e-04 | 4002.6ms/step | 15.0 samples/s
+Step 13840/50462 | Loss: 2.6894 | LR: 4.19e-04 | 4001.3ms/step | 15.0 samples/s
+Step 13850/50462 | Loss: 2.7162 | LR: 4.19e-04 | 3991.7ms/step | 15.0 samples/s
+Step 13860/50462 | Loss: 2.6874 | LR: 4.19e-04 | 3995.8ms/step | 15.0 samples/s
+Step 13870/50462 | Loss: 2.7168 | LR: 4.19e-04 | 4009.9ms/step | 15.0 samples/s
+Step 13880/50462 | Loss: 2.7135 | LR: 4.19e-04 | 4005.0ms/step | 15.0 samples/s
+Step 13890/50462 | Loss: 2.7234 | LR: 4.19e-04 | 3999.8ms/step | 15.0 samples/s
+Step 13900/50462 | Loss: 2.6944 | LR: 4.19e-04 | 4000.0ms/step | 15.0 samples/s
+Step 13910/50462 | Loss: 2.7246 | LR: 4.18e-04 | 4009.1ms/step | 15.0 samples/s
+Step 13920/50462 | Loss: 2.7159 | LR: 4.18e-04 | 4004.8ms/step | 15.0 samples/s
+Step 13930/50462 | Loss: 2.6742 | LR: 4.18e-04 | 4004.1ms/step | 15.0 samples/s
+Step 13940/50462 | Loss: 2.7449 | LR: 4.18e-04 | 4005.0ms/step | 15.0 samples/s
+Step 13950/50462 | Loss: 2.7130 | LR: 4.18e-04 | 4000.8ms/step | 15.0 samples/s
+Step 13960/50462 | Loss: 2.7247 | LR: 4.18e-04 | 4004.4ms/step | 15.0 samples/s
+Step 13970/50462 | Loss: 2.7051 | LR: 4.18e-04 | 4004.5ms/step | 15.0 samples/s
+Step 13980/50462 | Loss: 2.6832 | LR: 4.18e-04 | 4005.3ms/step | 15.0 samples/s
+Step 13990/50462 | Loss: 2.6649 | LR: 4.17e-04 | 3994.4ms/step | 15.0 samples/s
+Step 14000/50462 | Loss: 2.7159 | LR: 4.17e-04 | 3993.1ms/step | 15.0 samples/s
+Validation at step 14000: Loss = 2.6923, Perplexity = 14.77
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-14000
+Step 14010/50462 | Loss: 2.6930 | LR: 4.17e-04 | 5921.1ms/step | 10.1 samples/s
+Step 14020/50462 | Loss: 2.6788 | LR: 4.17e-04 | 3994.7ms/step | 15.0 samples/s
+Step 14030/50462 | Loss: 2.7316 | LR: 4.17e-04 | 3995.9ms/step | 15.0 samples/s
+Step 14040/50462 | Loss: 2.7074 | LR: 4.17e-04 | 3997.9ms/step | 15.0 samples/s
+Step 14050/50462 | Loss: 2.7336 | LR: 4.17e-04 | 3999.5ms/step | 15.0 samples/s
+Step 14060/50462 | Loss: 2.7254 | LR: 4.17e-04 | 3999.1ms/step | 15.0 samples/s
+Step 14070/50462 | Loss: 2.7105 | LR: 4.17e-04 | 4006.5ms/step | 15.0 samples/s
+Step 14080/50462 | Loss: 2.6787 | LR: 4.16e-04 | 3994.9ms/step | 15.0 samples/s
+Step 14090/50462 | Loss: 2.7325 | LR: 4.16e-04 | 3987.8ms/step | 15.0 samples/s
+Step 14100/50462 | Loss: 2.7090 | LR: 4.16e-04 | 3986.9ms/step | 15.0 samples/s
+Step 14110/50462 | Loss: 2.6993 | LR: 4.16e-04 | 4007.7ms/step | 15.0 samples/s
+Step 14120/50462 | Loss: 2.6969 | LR: 4.16e-04 | 3984.7ms/step | 15.1 samples/s
+Step 14130/50462 | Loss: 2.7156 | LR: 4.16e-04 | 3992.4ms/step | 15.0 samples/s
+Step 14140/50462 | Loss: 2.6935 | LR: 4.16e-04 | 4001.9ms/step | 15.0 samples/s
+Step 14150/50462 | Loss: 2.7330 | LR: 4.16e-04 | 4005.9ms/step | 15.0 samples/s
+Step 14160/50462 | Loss: 2.7095 | LR: 4.15e-04 | 3990.1ms/step | 15.0 samples/s
+Step 14170/50462 | Loss: 2.6860 | LR: 4.15e-04 | 3996.4ms/step | 15.0 samples/s
+Step 14180/50462 | Loss: 2.7231 | LR: 4.15e-04 | 3999.3ms/step | 15.0 samples/s
+Step 14190/50462 | Loss: 2.7028 | LR: 4.15e-04 | 3986.8ms/step | 15.0 samples/s
+Step 14200/50462 | Loss: 2.7165 | LR: 4.15e-04 | 4000.6ms/step | 15.0 samples/s
+Step 14210/50462 | Loss: 2.6719 | LR: 4.15e-04 | 4000.9ms/step | 15.0 samples/s
+Step 14220/50462 | Loss: 2.7232 | LR: 4.15e-04 | 3997.9ms/step | 15.0 samples/s
+Step 14230/50462 | Loss: 2.6985 | LR: 4.15e-04 | 4010.2ms/step | 15.0 samples/s
+Step 14240/50462 | Loss: 2.7162 | LR: 4.15e-04 | 3996.9ms/step | 15.0 samples/s
+Step 14250/50462 | Loss: 2.7115 | LR: 4.14e-04 | 4015.4ms/step | 14.9 samples/s
+Validation at step 14250: Loss = 2.6906, Perplexity = 14.74
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-14250
+Step 14260/50462 | Loss: 2.6816 | LR: 4.14e-04 | 6084.4ms/step | 9.9 samples/s
+Step 14270/50462 | Loss: 2.6791 | LR: 4.14e-04 | 4004.4ms/step | 15.0 samples/s
+Step 14280/50462 | Loss: 2.7185 | LR: 4.14e-04 | 4006.3ms/step | 15.0 samples/s
+Step 14290/50462 | Loss: 2.6968 | LR: 4.14e-04 | 4010.4ms/step | 15.0 samples/s
+Step 14300/50462 | Loss: 2.6737 | LR: 4.14e-04 | 4006.6ms/step | 15.0 samples/s
+Step 14310/50462 | Loss: 2.6986 | LR: 4.14e-04 | 3997.8ms/step | 15.0 samples/s
+Step 14320/50462 | Loss: 2.6786 | LR: 4.14e-04 | 4012.1ms/step | 15.0 samples/s
+Step 14330/50462 | Loss: 2.6911 | LR: 4.13e-04 | 3972.5ms/step | 15.1 samples/s
+Step 14340/50462 | Loss: 2.7122 | LR: 4.13e-04 | 3991.1ms/step | 15.0 samples/s
+Step 14350/50462 | Loss: 2.7203 | LR: 4.13e-04 | 4004.1ms/step | 15.0 samples/s
+Step 14360/50462 | Loss: 2.7216 | LR: 4.13e-04 | 4006.0ms/step | 15.0 samples/s
+Step 14370/50462 | Loss: 2.7211 | LR: 4.13e-04 | 3996.5ms/step | 15.0 samples/s
+Step 14380/50462 | Loss: 2.7079 | LR: 4.13e-04 | 3995.6ms/step | 15.0 samples/s
+Step 14390/50462 | Loss: 2.7338 | LR: 4.13e-04 | 3997.9ms/step | 15.0 samples/s
+Step 14400/50462 | Loss: 2.7082 | LR: 4.13e-04 | 3997.9ms/step | 15.0 samples/s
+Step 14410/50462 | Loss: 2.6905 | LR: 4.12e-04 | 4001.0ms/step | 15.0 samples/s
+Step 14420/50462 | Loss: 2.7088 | LR: 4.12e-04 | 3999.6ms/step | 15.0 samples/s
+Step 14430/50462 | Loss: 2.7221 | LR: 4.12e-04 | 4004.4ms/step | 15.0 samples/s
+Step 14440/50462 | Loss: 2.7090 | LR: 4.12e-04 | 3999.5ms/step | 15.0 samples/s
+Step 14450/50462 | Loss: 2.7099 | LR: 4.12e-04 | 3997.5ms/step | 15.0 samples/s
+Step 14460/50462 | Loss: 2.7187 | LR: 4.12e-04 | 3994.1ms/step | 15.0 samples/s
+Step 14470/50462 | Loss: 2.7017 | LR: 4.12e-04 | 4002.5ms/step | 15.0 samples/s
+Step 14480/50462 | Loss: 2.6946 | LR: 4.12e-04 | 4011.5ms/step | 15.0 samples/s
+Step 14490/50462 | Loss: 2.7131 | LR: 4.12e-04 | 4002.3ms/step | 15.0 samples/s
+Step 14500/50462 | Loss: 2.6929 | LR: 4.11e-04 | 4010.8ms/step | 15.0 samples/s
+Validation at step 14500: Loss = 2.6873, Perplexity = 14.69
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-14500
+Step 14510/50462 | Loss: 2.6744 | LR: 4.11e-04 | 5954.0ms/step | 10.1 samples/s
+Step 14520/50462 | Loss: 2.6705 | LR: 4.11e-04 | 3989.7ms/step | 15.0 samples/s
+Step 14530/50462 | Loss: 2.7070 | LR: 4.11e-04 | 4004.0ms/step | 15.0 samples/s
+Step 14540/50462 | Loss: 2.7087 | LR: 4.11e-04 | 3998.0ms/step | 15.0 samples/s
+Step 14550/50462 | Loss: 2.6832 | LR: 4.11e-04 | 4011.3ms/step | 15.0 samples/s
+Step 14560/50462 | Loss: 2.6990 | LR: 4.11e-04 | 4007.0ms/step | 15.0 samples/s
+Step 14570/50462 | Loss: 2.6601 | LR: 4.11e-04 | 3993.2ms/step | 15.0 samples/s
+Step 14580/50462 | Loss: 2.6865 | LR: 4.10e-04 | 4007.8ms/step | 15.0 samples/s
+Step 14590/50462 | Loss: 2.6622 | LR: 4.10e-04 | 4011.9ms/step | 15.0 samples/s
+Step 14600/50462 | Loss: 2.6926 | LR: 4.10e-04 | 3997.3ms/step | 15.0 samples/s
+Step 14610/50462 | Loss: 2.6685 | LR: 4.10e-04 | 4005.1ms/step | 15.0 samples/s
+Step 14620/50462 | Loss: 2.6869 | LR: 4.10e-04 | 3987.9ms/step | 15.0 samples/s
+Step 14630/50462 | Loss: 2.7435 | LR: 4.10e-04 | 3998.7ms/step | 15.0 samples/s
+Step 14640/50462 | Loss: 2.7124 | LR: 4.10e-04 | 4010.7ms/step | 15.0 samples/s
+Step 14650/50462 | Loss: 2.6984 | LR: 4.10e-04 | 4008.8ms/step | 15.0 samples/s
+Step 14660/50462 | Loss: 2.6877 | LR: 4.09e-04 | 3986.9ms/step | 15.0 samples/s
+Step 14670/50462 | Loss: 2.6749 | LR: 4.09e-04 | 4008.9ms/step | 15.0 samples/s
+Step 14680/50462 | Loss: 2.7013 | LR: 4.09e-04 | 3992.6ms/step | 15.0 samples/s
+Step 14690/50462 | Loss: 2.6927 | LR: 4.09e-04 | 3995.1ms/step | 15.0 samples/s
+Step 14700/50462 | Loss: 2.7089 | LR: 4.09e-04 | 3994.7ms/step | 15.0 samples/s
+Step 14710/50462 | Loss: 2.7132 | LR: 4.09e-04 | 3993.0ms/step | 15.0 samples/s
+Step 14720/50462 | Loss: 2.7136 | LR: 4.09e-04 | 4011.5ms/step | 15.0 samples/s
+Step 14730/50462 | Loss: 2.7004 | LR: 4.09e-04 | 3989.8ms/step | 15.0 samples/s
+Step 14740/50462 | Loss: 2.6837 | LR: 4.08e-04 | 3995.5ms/step | 15.0 samples/s
+Step 14750/50462 | Loss: 2.7129 | LR: 4.08e-04 | 4008.1ms/step | 15.0 samples/s
+Validation at step 14750: Loss = 2.6805, Perplexity = 14.59
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-14750
+Step 14760/50462 | Loss: 2.7035 | LR: 4.08e-04 | 5958.8ms/step | 10.1 samples/s
+Step 14770/50462 | Loss: 2.6762 | LR: 4.08e-04 | 4007.9ms/step | 15.0 samples/s
+Step 14780/50462 | Loss: 2.6985 | LR: 4.08e-04 | 4003.3ms/step | 15.0 samples/s
+Step 14790/50462 | Loss: 2.6526 | LR: 4.08e-04 | 4002.2ms/step | 15.0 samples/s
+Step 14800/50462 | Loss: 2.7067 | LR: 4.08e-04 | 4001.9ms/step | 15.0 samples/s
+Step 14810/50462 | Loss: 2.7120 | LR: 4.08e-04 | 4004.1ms/step | 15.0 samples/s
+Step 14820/50462 | Loss: 2.7313 | LR: 4.08e-04 | 4003.4ms/step | 15.0 samples/s
+Step 14830/50462 | Loss: 2.7019 | LR: 4.07e-04 | 4002.8ms/step | 15.0 samples/s
+Step 14840/50462 | Loss: 2.7373 | LR: 4.07e-04 | 4000.3ms/step | 15.0 samples/s
+Step 14850/50462 | Loss: 2.6544 | LR: 4.07e-04 | 3994.0ms/step | 15.0 samples/s
+Step 14860/50462 | Loss: 2.6611 | LR: 4.07e-04 | 4001.1ms/step | 15.0 samples/s
+Step 14870/50462 | Loss: 2.7268 | LR: 4.07e-04 | 3985.1ms/step | 15.1 samples/s
+Step 14880/50462 | Loss: 2.7210 | LR: 4.07e-04 | 3985.4ms/step | 15.1 samples/s
+Step 14890/50462 | Loss: 2.6907 | LR: 4.07e-04 | 3997.1ms/step | 15.0 samples/s
+Step 14900/50462 | Loss: 2.6941 | LR: 4.07e-04 | 4000.7ms/step | 15.0 samples/s
+Step 14910/50462 | Loss: 2.7115 | LR: 4.06e-04 | 3994.4ms/step | 15.0 samples/s
+Step 14920/50462 | Loss: 2.6868 | LR: 4.06e-04 | 3991.8ms/step | 15.0 samples/s
+Step 14930/50462 | Loss: 2.7060 | LR: 4.06e-04 | 3981.2ms/step | 15.1 samples/s
+Step 14940/50462 | Loss: 2.6670 | LR: 4.06e-04 | 3994.3ms/step | 15.0 samples/s
+Step 14950/50462 | Loss: 2.6729 | LR: 4.06e-04 | 4003.1ms/step | 15.0 samples/s
+Step 14960/50462 | Loss: 2.6691 | LR: 4.06e-04 | 3991.8ms/step | 15.0 samples/s
+Step 14970/50462 | Loss: 2.7069 | LR: 4.06e-04 | 3997.9ms/step | 15.0 samples/s
+Step 14980/50462 | Loss: 2.6845 | LR: 4.06e-04 | 3989.0ms/step | 15.0 samples/s
+Step 14990/50462 | Loss: 2.7173 | LR: 4.05e-04 | 4003.0ms/step | 15.0 samples/s
+Step 15000/50462 | Loss: 2.6713 | LR: 4.05e-04 | 4006.0ms/step | 15.0 samples/s
+Validation at step 15000: Loss = 2.6792, Perplexity = 14.57
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-15000
+Step 15010/50462 | Loss: 2.6937 | LR: 4.05e-04 | 6013.1ms/step | 10.0 samples/s
+Step 15020/50462 | Loss: 2.7303 | LR: 4.05e-04 | 4002.7ms/step | 15.0 samples/s
+Step 15030/50462 | Loss: 2.7113 | LR: 4.05e-04 | 4019.6ms/step | 14.9 samples/s
+Step 15040/50462 | Loss: 2.6920 | LR: 4.05e-04 | 3996.3ms/step | 15.0 samples/s
+Step 15050/50462 | Loss: 2.6758 | LR: 4.05e-04 | 3996.5ms/step | 15.0 samples/s
+Step 15060/50462 | Loss: 2.6598 | LR: 4.05e-04 | 4014.1ms/step | 14.9 samples/s
+Step 15070/50462 | Loss: 2.6430 | LR: 4.04e-04 | 4003.5ms/step | 15.0 samples/s
+Step 15080/50462 | Loss: 2.6923 | LR: 4.04e-04 | 3993.3ms/step | 15.0 samples/s
+Step 15090/50462 | Loss: 2.7004 | LR: 4.04e-04 | 4008.4ms/step | 15.0 samples/s
+Step 15100/50462 | Loss: 2.6658 | LR: 4.04e-04 | 4000.0ms/step | 15.0 samples/s
+Step 15110/50462 | Loss: 2.7128 | LR: 4.04e-04 | 3991.5ms/step | 15.0 samples/s
+Step 15120/50462 | Loss: 2.6990 | LR: 4.04e-04 | 3998.3ms/step | 15.0 samples/s
+Step 15130/50462 | Loss: 2.7253 | LR: 4.04e-04 | 4003.8ms/step | 15.0 samples/s
+Step 15140/50462 | Loss: 2.7077 | LR: 4.04e-04 | 3997.6ms/step | 15.0 samples/s
+Step 15150/50462 | Loss: 2.7189 | LR: 4.03e-04 | 3998.8ms/step | 15.0 samples/s
+Step 15160/50462 | Loss: 2.6988 | LR: 4.03e-04 | 4010.2ms/step | 15.0 samples/s
+Step 15170/50462 | Loss: 2.7020 | LR: 4.03e-04 | 4007.6ms/step | 15.0 samples/s
+Step 15180/50462 | Loss: 2.6635 | LR: 4.03e-04 | 4001.0ms/step | 15.0 samples/s
+Step 15190/50462 | Loss: 2.7306 | LR: 4.03e-04 | 4004.3ms/step | 15.0 samples/s
+Step 15200/50462 | Loss: 2.6955 | LR: 4.03e-04 | 4000.1ms/step | 15.0 samples/s
+Step 15210/50462 | Loss: 2.6890 | LR: 4.03e-04 | 4004.3ms/step | 15.0 samples/s
+Step 15220/50462 | Loss: 2.7510 | LR: 4.03e-04 | 3996.7ms/step | 15.0 samples/s
+Step 15230/50462 | Loss: 2.6728 | LR: 4.02e-04 | 3980.4ms/step | 15.1 samples/s
+Step 15240/50462 | Loss: 2.6988 | LR: 4.02e-04 | 3993.5ms/step | 15.0 samples/s
+Step 15250/50462 | Loss: 2.6923 | LR: 4.02e-04 | 3991.7ms/step | 15.0 samples/s
+Validation at step 15250: Loss = 2.6763, Perplexity = 14.53
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-15250
+Step 15260/50462 | Loss: 2.7068 | LR: 4.02e-04 | 6066.1ms/step | 9.9 samples/s
+Step 15270/50462 | Loss: 2.6928 | LR: 4.02e-04 | 4000.8ms/step | 15.0 samples/s
+Step 15280/50462 | Loss: 2.6962 | LR: 4.02e-04 | 3995.9ms/step | 15.0 samples/s
+Step 15290/50462 | Loss: 2.6840 | LR: 4.02e-04 | 4001.4ms/step | 15.0 samples/s
+Step 15300/50462 | Loss: 2.6961 | LR: 4.02e-04 | 4000.4ms/step | 15.0 samples/s
+Step 15310/50462 | Loss: 2.6974 | LR: 4.01e-04 | 4001.1ms/step | 15.0 samples/s
+Step 15320/50462 | Loss: 2.7122 | LR: 4.01e-04 | 3999.2ms/step | 15.0 samples/s
+Step 15330/50462 | Loss: 2.6945 | LR: 4.01e-04 | 3999.1ms/step | 15.0 samples/s
+Step 15340/50462 | Loss: 2.6732 | LR: 4.01e-04 | 3998.7ms/step | 15.0 samples/s
+Step 15350/50462 | Loss: 2.7148 | LR: 4.01e-04 | 4003.0ms/step | 15.0 samples/s
+Step 15360/50462 | Loss: 2.6785 | LR: 4.01e-04 | 4007.3ms/step | 15.0 samples/s
+Step 15370/50462 | Loss: 2.7030 | LR: 4.01e-04 | 4001.6ms/step | 15.0 samples/s
+Step 15380/50462 | Loss: 2.6818 | LR: 4.01e-04 | 4000.0ms/step | 15.0 samples/s
+Step 15390/50462 | Loss: 2.6808 | LR: 4.00e-04 | 4004.5ms/step | 15.0 samples/s
+Step 15400/50462 | Loss: 2.6713 | LR: 4.00e-04 | 4005.5ms/step | 15.0 samples/s
+Step 15410/50462 | Loss: 2.6813 | LR: 4.00e-04 | 4006.9ms/step | 15.0 samples/s
+Step 15420/50462 | Loss: 2.6806 | LR: 4.00e-04 | 4007.3ms/step | 15.0 samples/s
+Step 15430/50462 | Loss: 2.6926 | LR: 4.00e-04 | 3995.1ms/step | 15.0 samples/s
+Step 15440/50462 | Loss: 2.7027 | LR: 4.00e-04 | 4004.7ms/step | 15.0 samples/s
+Step 15450/50462 | Loss: 2.6810 | LR: 4.00e-04 | 4002.6ms/step | 15.0 samples/s
+Step 15460/50462 | Loss: 2.6878 | LR: 4.00e-04 | 3995.6ms/step | 15.0 samples/s
+Step 15470/50462 | Loss: 2.6637 | LR: 3.99e-04 | 3999.1ms/step | 15.0 samples/s
+Step 15480/50462 | Loss: 2.6901 | LR: 3.99e-04 | 4009.1ms/step | 15.0 samples/s
+Step 15490/50462 | Loss: 2.6843 | LR: 3.99e-04 | 4005.3ms/step | 15.0 samples/s
+Step 15500/50462 | Loss: 2.6736 | LR: 3.99e-04 | 4001.8ms/step | 15.0 samples/s
+Validation at step 15500: Loss = 2.6721, Perplexity = 14.47
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-15500
+Step 15510/50462 | Loss: 2.6940 | LR: 3.99e-04 | 6012.7ms/step | 10.0 samples/s
+Step 15520/50462 | Loss: 2.6892 | LR: 3.99e-04 | 4008.7ms/step | 15.0 samples/s
+Step 15530/50462 | Loss: 2.7013 | LR: 3.99e-04 | 3996.3ms/step | 15.0 samples/s
+Step 15540/50462 | Loss: 2.6735 | LR: 3.99e-04 | 3994.5ms/step | 15.0 samples/s
+Step 15550/50462 | Loss: 2.6949 | LR: 3.98e-04 | 3997.5ms/step | 15.0 samples/s
+Step 15560/50462 | Loss: 2.6824 | LR: 3.98e-04 | 3997.1ms/step | 15.0 samples/s
+Step 15570/50462 | Loss: 2.6875 | LR: 3.98e-04 | 4000.9ms/step | 15.0 samples/s
+Step 15580/50462 | Loss: 2.6621 | LR: 3.98e-04 | 3990.1ms/step | 15.0 samples/s
+Step 15590/50462 | Loss: 2.6983 | LR: 3.98e-04 | 4001.3ms/step | 15.0 samples/s
+Step 15600/50462 | Loss: 2.6784 | LR: 3.98e-04 | 4001.7ms/step | 15.0 samples/s
+Step 15610/50462 | Loss: 2.6939 | LR: 3.98e-04 | 3993.5ms/step | 15.0 samples/s
+Step 15620/50462 | Loss: 2.6711 | LR: 3.98e-04 | 3998.0ms/step | 15.0 samples/s
+Step 15630/50462 | Loss: 2.6886 | LR: 3.97e-04 | 4008.3ms/step | 15.0 samples/s
+Step 15640/50462 | Loss: 2.6951 | LR: 3.97e-04 | 3991.8ms/step | 15.0 samples/s
+Step 15650/50462 | Loss: 2.6743 | LR: 3.97e-04 | 3992.5ms/step | 15.0 samples/s
+Step 15660/50462 | Loss: 2.6770 | LR: 3.97e-04 | 3997.8ms/step | 15.0 samples/s
+Step 15670/50462 | Loss: 2.6966 | LR: 3.97e-04 | 3998.0ms/step | 15.0 samples/s
+Step 15680/50462 | Loss: 2.6900 | LR: 3.97e-04 | 4008.5ms/step | 15.0 samples/s
+Step 15690/50462 | Loss: 2.6783 | LR: 3.97e-04 | 4016.2ms/step | 14.9 samples/s
+Step 15700/50462 | Loss: 2.6934 | LR: 3.96e-04 | 3995.5ms/step | 15.0 samples/s
+Step 15710/50462 | Loss: 2.6731 | LR: 3.96e-04 | 3998.4ms/step | 15.0 samples/s
+Step 15720/50462 | Loss: 2.6985 | LR: 3.96e-04 | 4005.6ms/step | 15.0 samples/s
+Step 15730/50462 | Loss: 2.6346 | LR: 3.96e-04 | 4006.9ms/step | 15.0 samples/s
+Step 15740/50462 | Loss: 2.6626 | LR: 3.96e-04 | 4003.6ms/step | 15.0 samples/s
+Step 15750/50462 | Loss: 2.7045 | LR: 3.96e-04 | 4010.4ms/step | 15.0 samples/s
+Validation at step 15750: Loss = 2.6683, Perplexity = 14.42
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-15750
+Step 15760/50462 | Loss: 2.6591 | LR: 3.96e-04 | 6046.5ms/step | 9.9 samples/s
+Step 15770/50462 | Loss: 2.6972 | LR: 3.96e-04 | 3991.7ms/step | 15.0 samples/s
+Step 15780/50462 | Loss: 2.6706 | LR: 3.95e-04 | 3989.2ms/step | 15.0 samples/s
+Step 15790/50462 | Loss: 2.7131 | LR: 3.95e-04 | 4001.7ms/step | 15.0 samples/s
+Step 15800/50462 | Loss: 2.6989 | LR: 3.95e-04 | 4006.8ms/step | 15.0 samples/s
+Step 15810/50462 | Loss: 2.6869 | LR: 3.95e-04 | 4004.6ms/step | 15.0 samples/s
+Step 15820/50462 | Loss: 2.6753 | LR: 3.95e-04 | 3997.0ms/step | 15.0 samples/s
+Step 15830/50462 | Loss: 2.6832 | LR: 3.95e-04 | 3998.0ms/step | 15.0 samples/s
+Step 15840/50462 | Loss: 2.7027 | LR: 3.95e-04 | 3999.3ms/step | 15.0 samples/s
+Step 15850/50462 | Loss: 2.6943 | LR: 3.95e-04 | 3996.1ms/step | 15.0 samples/s
+Step 15860/50462 | Loss: 2.6525 | LR: 3.94e-04 | 3994.6ms/step | 15.0 samples/s
+Step 15870/50462 | Loss: 2.6637 | LR: 3.94e-04 | 4004.2ms/step | 15.0 samples/s
+Step 15880/50462 | Loss: 2.6612 | LR: 3.94e-04 | 3998.3ms/step | 15.0 samples/s
+Step 15890/50462 | Loss: 2.6943 | LR: 3.94e-04 | 3998.6ms/step | 15.0 samples/s
+Step 15900/50462 | Loss: 2.6912 | LR: 3.94e-04 | 4001.7ms/step | 15.0 samples/s
+Step 15910/50462 | Loss: 2.6520 | LR: 3.94e-04 | 3992.2ms/step | 15.0 samples/s
+Step 15920/50462 | Loss: 2.6672 | LR: 3.94e-04 | 3998.4ms/step | 15.0 samples/s
+Step 15930/50462 | Loss: 2.6634 | LR: 3.94e-04 | 3997.0ms/step | 15.0 samples/s
+Step 15940/50462 | Loss: 2.6743 | LR: 3.93e-04 | 4010.1ms/step | 15.0 samples/s
+Step 15950/50462 | Loss: 2.6842 | LR: 3.93e-04 | 3994.8ms/step | 15.0 samples/s
+Step 15960/50462 | Loss: 2.6739 | LR: 3.93e-04 | 4002.9ms/step | 15.0 samples/s
+Step 15970/50462 | Loss: 2.6622 | LR: 3.93e-04 | 3993.9ms/step | 15.0 samples/s
+Step 15980/50462 | Loss: 2.6941 | LR: 3.93e-04 | 3997.2ms/step | 15.0 samples/s
+Step 15990/50462 | Loss: 2.6825 | LR: 3.93e-04 | 4006.0ms/step | 15.0 samples/s
+Step 16000/50462 | Loss: 2.7037 | LR: 3.93e-04 | 4015.7ms/step | 14.9 samples/s
+Validation at step 16000: Loss = 2.6671, Perplexity = 14.40
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-16000
+Step 16010/50462 | Loss: 2.6731 | LR: 3.92e-04 | 6168.2ms/step | 9.7 samples/s
+Step 16020/50462 | Loss: 2.6575 | LR: 3.92e-04 | 4024.1ms/step | 14.9 samples/s
+Step 16030/50462 | Loss: 2.6512 | LR: 3.92e-04 | 4007.0ms/step | 15.0 samples/s
+Step 16040/50462 | Loss: 2.7290 | LR: 3.92e-04 | 4015.6ms/step | 14.9 samples/s
+Step 16050/50462 | Loss: 2.6769 | LR: 3.92e-04 | 4018.4ms/step | 14.9 samples/s
+Step 16060/50462 | Loss: 2.6550 | LR: 3.92e-04 | 4006.3ms/step | 15.0 samples/s
+Step 16070/50462 | Loss: 2.6760 | LR: 3.92e-04 | 3999.2ms/step | 15.0 samples/s
+Step 16080/50462 | Loss: 2.6521 | LR: 3.92e-04 | 4011.6ms/step | 15.0 samples/s
+Step 16090/50462 | Loss: 2.6779 | LR: 3.91e-04 | 4003.2ms/step | 15.0 samples/s
+Traceback (most recent call last):
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1506, in <module>
+    main()
+    ~~~~^^
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1503, in main
+    train(config)
+    ~~~~~^^^^^^^^
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1385, in train
+    loss.backward()
+    ~~~~~~~~~~~~~^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_tensor.py", line 631, in backward
+    torch.autograd.backward(
+    ~~~~~~~~~~~~~~~~~~~~~~~^
+        self, gradient, retain_graph, create_graph, inputs=inputs
+        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+    )
+    ^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/autograd/__init__.py", line 381, in backward
+    _engine_run_backward(
+    ~~~~~~~~~~~~~~~~~~~~^
+        tensors,
+        ^^^^^^^^
+    ...<5 lines>...
+        accumulate_grad=True,
+        ^^^^^^^^^^^^^^^^^^^^^
+    )
+    ^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/autograd/graph.py", line 869, in _engine_run_backward
+    return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+        t_outputs, *args, **kwargs
+        ^^^^^^^^^^^^^^^^^^^^^^^^^^
+    )  # Calls into the C++ engine to run the backward pass
+    ^
+KeyboardInterrupt
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1506, in <module>
+[rank0]:     main()
+[rank0]:     ~~~~^^
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1503, in main
+[rank0]:     train(config)
+[rank0]:     ~~~~~^^^^^^^^
+[rank0]:   File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1385, in train
+[rank0]:     loss.backward()
+[rank0]:     ~~~~~~~~~~~~~^^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_tensor.py", line 631, in backward
+[rank0]:     torch.autograd.backward(
+[rank0]:     ~~~~~~~~~~~~~~~~~~~~~~~^
+[rank0]:         self, gradient, retain_graph, create_graph, inputs=inputs
+[rank0]:         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:     )
+[rank0]:     ^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/autograd/__init__.py", line 381, in backward
+[rank0]:     _engine_run_backward(
+[rank0]:     ~~~~~~~~~~~~~~~~~~~~^
+[rank0]:         tensors,
+[rank0]:         ^^^^^^^^
+[rank0]:     ...<5 lines>...
+[rank0]:         accumulate_grad=True,
+[rank0]:         ^^^^^^^^^^^^^^^^^^^^^
+[rank0]:     )
+[rank0]:     ^
+[rank0]:   File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/autograd/graph.py", line 869, in _engine_run_backward
+[rank0]:     return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+[rank0]:            ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:         t_outputs, *args, **kwargs
+[rank0]:         ^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:     )  # Calls into the C++ engine to run the backward pass
+[rank0]:     ^
+[rank0]: KeyboardInterrupt

epfl_final/SparseAtt/wandb/run-20260827_041913-8ynn1we6/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260827_041913-8ynn1we6/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-08-27T04:19:13.272934Z",
+  "args":  [
+    "--config",
+    "configs/p2_1p2b_mixed_seed71_4gpu_resume9000.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/SparseAtt.git",
+    "commit":  "8f862f749eb4d5c17f688eb1562235e3fe5b2550"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-train-4-gpu-0-15",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "5970348429312"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-e3358200-f776-7214-8269-c2c219f03c2e"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-a3a94461-36d7-5e8a-af9b-6209869fa7f4"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-0c2f0695-54b3-664a-b12d-640d1c9a0f2e"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-7a2d68b3-dda3-ec4d-f88e-205cb866ebb6"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "4y7vrshm704jr2z23pf52r9efbq38ko7"
+}

epfl_final/SparseAtt/wandb/run-20260827_041913-8ynn1we6/files/wandb-summary.json

@@ -0,0 +1 @@
+{"eval/dense_interval_start":-1,"_wandb":{"runtime":66280},"eval/loss":2.6671423233569937,"_runtime":66280.326600509,"eval/dense_interval_active":0,"_step":16090,"train/epoch":0.318853790971424,"eval/dense_interval_end":-1,"_timestamp":1.7878335806846104e+09,"speed/ms_per_step":4003.239321708679,"speed/steps_per_sec":0.24979770621686836,"eval/uses_sparse_attention":1,"speed/samples_per_sec":14.987862373012101,"train/learning_rate":0.00039144353748140915,"_wandb.runtime":37052,"eval/perplexity":14.398763334665723,"train/global_step":16090,"eval/uses_dense_attention":0,"train/loss":2.6779270060857137}

epfl_final/SparseAtt/wandb/run-20260827_041913-8ynn1we6/logs/debug.log

@@ -0,0 +1,22 @@
+2026-08-27 04:19:13,293 INFO    MainThread:221 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-08-27 04:19:13,293 INFO    MainThread:221 [wandb_setup.py:_flush():81] Configure stats pid to 221
+2026-08-27 04:19:13,293 INFO    MainThread:221 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-27 04:19:13,293 INFO    MainThread:221 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260827_041913-8ynn1we6/logs/debug.log
+2026-08-27 04:19:13,293 INFO    MainThread:221 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260827_041913-8ynn1we6/logs/debug-internal.log
+2026-08-27 04:19:13,293 INFO    MainThread:221 [wandb_init.py:init():850] calling init triggers
+2026-08-27 04:19:13,293 INFO    MainThread:221 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-9000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 15, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 71, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b mixed 8k seed71 4gpu', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'mixed', 'seed71'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed', 'sparse_mask_seed': 71, 'sparse_mixed_nrb_set': [63, 47, 31, 21, 11], 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-08-27 04:19:13,293 INFO    MainThread:221 [wandb_init.py:init():898] starting backend
+2026-08-27 04:19:13,850 INFO    MainThread:221 [wandb_init.py:init():913] sending inform_init request
+2026-08-27 04:19:14,301 INFO    MainThread:221 [wandb_init.py:init():918] backend started and connected
+2026-08-27 04:19:14,307 INFO    MainThread:221 [wandb_init.py:init():988] updated telemetry
+2026-08-27 04:19:14,350 INFO    MainThread:221 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-08-27 04:19:14,798 INFO    MainThread:221 [wandb_init.py:init():1051] run resumed
+2026-08-27 04:19:14,804 INFO    MainThread:221 [wandb_init.py:init():1056] starting run threads in backend
+2026-08-27 04:19:15,106 INFO    MainThread:221 [wandb_run.py:_console_start():2554] atexit reg
+2026-08-27 04:19:15,107 INFO    MainThread:221 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-08-27 04:19:15,107 INFO    MainThread:221 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-08-27 04:19:15,107 INFO    MainThread:221 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-08-27 04:19:15,113 INFO    MainThread:221 [wandb_init.py:init():1094] run started, returning control to user process
+2026-08-27 12:26:23,132 INFO    wandb-AsyncioManager-main:221 [service_client.py:_forward_responses():134] Reached EOF.
+2026-08-27 12:26:23,132 INFO    wandb-AsyncioManager-main:221 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.

epfl_final/SparseAtt/wandb/run-20260827_041913-8ynn1we6/run-8ynn1we6.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4e50fa97d41e130314f9312f1161d99eb8814790cad19c0826bbc4390876dea
+size 9781895

epfl_final/SparseAtt/wandb/run-20260827_071828-z1i60278/files/wandb-metadata.json

@@ -0,0 +1,79 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-08-27T07:18:28.779930Z",
+  "args":  [
+    "--config",
+    "configs/p2_1p2b_mixed_seed72_6gpu_after_dense70.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/SparseAtt.git",
+    "commit":  "8f862f749eb4d5c17f688eb1562235e3fe5b2550"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-train-6-gpu-0-44",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  6,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "6435044397056"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bab1e315-c967-43cf-bdfb-cac1a8dfdfac"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-f65cda91-ff87-9bff-fb3d-3e1b3f9f8cf0"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-069bdfaf-ef1c-f7f2-f9e3-2249bb9dfc41"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-4c1d91e9-bf01-f8fd-1e8b-79aa8e0831df"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-086f9f0b-a6d8-6263-83e6-c2b66721f47f"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bdb5e23e-0700-20a0-0b21-1ad0901f08b7"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "ylybg6vzzrd3xxpj4m926034iom31ku7"
+}