EPFL final checkpoints and W&B history 301-350 of 1463
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +4 -0
- epfl_final/SparseAtt/wandb/run-20260707_035639-2qqtcms1/files/output.log +2 -0
- epfl_final/SparseAtt/wandb/run-20260707_035639-2qqtcms1/files/requirements.txt +141 -0
- epfl_final/SparseAtt/wandb/run-20260707_035639-2qqtcms1/files/wandb-metadata.json +65 -0
- epfl_final/SparseAtt/wandb/run-20260707_035639-2qqtcms1/logs/debug-internal.log +12 -0
- epfl_final/SparseAtt/wandb/run-20260707_035639-2qqtcms1/logs/debug.log +19 -0
- epfl_final/SparseAtt/wandb/run-20260707_035639-2qqtcms1/run-2qqtcms1.wandb +0 -0
- epfl_final/SparseAtt/wandb/run-20260707_035639-l6anse2f/files/output.log +2 -0
- epfl_final/SparseAtt/wandb/run-20260707_035639-l6anse2f/files/requirements.txt +141 -0
- epfl_final/SparseAtt/wandb/run-20260707_035639-l6anse2f/files/wandb-metadata.json +65 -0
- epfl_final/SparseAtt/wandb/run-20260707_035639-l6anse2f/logs/debug-internal.log +12 -0
- epfl_final/SparseAtt/wandb/run-20260707_035639-l6anse2f/logs/debug.log +19 -0
- epfl_final/SparseAtt/wandb/run-20260707_035639-l6anse2f/run-l6anse2f.wandb +0 -0
- epfl_final/SparseAtt/wandb/run-20260707_035711-9iiocss5/logs/debug-internal.log +7 -0
- epfl_final/SparseAtt/wandb/run-20260707_035711-9iiocss5/logs/debug.log +10 -0
- epfl_final/SparseAtt/wandb/run-20260707_035711-9iiocss5/run-9iiocss5.wandb +0 -0
- epfl_final/SparseAtt/wandb/run-20260707_035814-kxni9nyw/files/config.yaml +186 -0
- epfl_final/SparseAtt/wandb/run-20260707_035814-kxni9nyw/files/output.log +1230 -0
- epfl_final/SparseAtt/wandb/run-20260707_035814-kxni9nyw/files/requirements.txt +141 -0
- epfl_final/SparseAtt/wandb/run-20260707_035814-kxni9nyw/files/wandb-metadata.json +65 -0
- epfl_final/SparseAtt/wandb/run-20260707_035814-kxni9nyw/files/wandb-summary.json +1 -0
- epfl_final/SparseAtt/wandb/run-20260707_035814-kxni9nyw/logs/debug-internal.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260707_035814-kxni9nyw/logs/debug.log +24 -0
- epfl_final/SparseAtt/wandb/run-20260707_035814-kxni9nyw/run-kxni9nyw.wandb +3 -0
- epfl_final/SparseAtt/wandb/run-20260707_035818-m10lf13v/files/config.yaml +186 -0
- epfl_final/SparseAtt/wandb/run-20260707_035818-m10lf13v/files/output.log +1230 -0
- epfl_final/SparseAtt/wandb/run-20260707_035818-m10lf13v/files/requirements.txt +141 -0
- epfl_final/SparseAtt/wandb/run-20260707_035818-m10lf13v/files/wandb-metadata.json +65 -0
- epfl_final/SparseAtt/wandb/run-20260707_035818-m10lf13v/files/wandb-summary.json +1 -0
- epfl_final/SparseAtt/wandb/run-20260707_035818-m10lf13v/logs/debug-internal.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260707_035818-m10lf13v/logs/debug.log +24 -0
- epfl_final/SparseAtt/wandb/run-20260707_035818-m10lf13v/run-m10lf13v.wandb +3 -0
- epfl_final/SparseAtt/wandb/run-20260707_035828-jmi9928p/files/config.yaml +185 -0
- epfl_final/SparseAtt/wandb/run-20260707_035828-jmi9928p/files/output.log +1230 -0
- epfl_final/SparseAtt/wandb/run-20260707_035828-jmi9928p/files/requirements.txt +141 -0
- epfl_final/SparseAtt/wandb/run-20260707_035828-jmi9928p/files/wandb-metadata.json +65 -0
- epfl_final/SparseAtt/wandb/run-20260707_035828-jmi9928p/files/wandb-summary.json +1 -0
- epfl_final/SparseAtt/wandb/run-20260707_035828-jmi9928p/logs/debug-internal.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260707_035828-jmi9928p/logs/debug.log +24 -0
- epfl_final/SparseAtt/wandb/run-20260707_035828-jmi9928p/run-jmi9928p.wandb +3 -0
- epfl_final/SparseAtt/wandb/run-20260707_035835-sh1m3lmh/files/config.yaml +185 -0
- epfl_final/SparseAtt/wandb/run-20260707_035835-sh1m3lmh/files/output.log +1230 -0
- epfl_final/SparseAtt/wandb/run-20260707_035835-sh1m3lmh/files/requirements.txt +141 -0
- epfl_final/SparseAtt/wandb/run-20260707_035835-sh1m3lmh/files/wandb-metadata.json +65 -0
- epfl_final/SparseAtt/wandb/run-20260707_035835-sh1m3lmh/files/wandb-summary.json +1 -0
- epfl_final/SparseAtt/wandb/run-20260707_035835-sh1m3lmh/logs/debug-internal.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260707_035835-sh1m3lmh/logs/debug.log +24 -0
- epfl_final/SparseAtt/wandb/run-20260707_035835-sh1m3lmh/run-sh1m3lmh.wandb +3 -0
- epfl_final/SparseAtt/wandb/run-20260707_170941-qlqhc3gy/files/wandb-metadata.json +65 -0
- epfl_final/SparseAtt/wandb/run-20260707_170941-qlqhc3gy/logs/debug-internal.log +0 -0
.gitattributes
CHANGED
|
@@ -39,3 +39,7 @@ epfl_final/SparseAtt/wandb/run-20260707_034612-dbtl6d2o/run-dbtl6d2o.wandb filte
|
|
| 39 |
epfl_final/SparseAtt/wandb/run-20260707_034612-qlqhc3gy/run-qlqhc3gy.wandb filter=lfs diff=lfs merge=lfs -text
|
| 40 |
epfl_final/SparseAtt/wandb/run-20260707_034618-lgjoqt2r/run-lgjoqt2r.wandb filter=lfs diff=lfs merge=lfs -text
|
| 41 |
epfl_final/SparseAtt/wandb/run-20260707_034748-304r0ilz/run-304r0ilz.wandb filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
epfl_final/SparseAtt/wandb/run-20260707_034612-qlqhc3gy/run-qlqhc3gy.wandb filter=lfs diff=lfs merge=lfs -text
|
| 40 |
epfl_final/SparseAtt/wandb/run-20260707_034618-lgjoqt2r/run-lgjoqt2r.wandb filter=lfs diff=lfs merge=lfs -text
|
| 41 |
epfl_final/SparseAtt/wandb/run-20260707_034748-304r0ilz/run-304r0ilz.wandb filter=lfs diff=lfs merge=lfs -text
|
| 42 |
+
epfl_final/SparseAtt/wandb/run-20260707_035814-kxni9nyw/run-kxni9nyw.wandb filter=lfs diff=lfs merge=lfs -text
|
| 43 |
+
epfl_final/SparseAtt/wandb/run-20260707_035818-m10lf13v/run-m10lf13v.wandb filter=lfs diff=lfs merge=lfs -text
|
| 44 |
+
epfl_final/SparseAtt/wandb/run-20260707_035828-jmi9928p/run-jmi9928p.wandb filter=lfs diff=lfs merge=lfs -text
|
| 45 |
+
epfl_final/SparseAtt/wandb/run-20260707_035835-sh1m3lmh/run-sh1m3lmh.wandb filter=lfs diff=lfs merge=lfs -text
|
epfl_final/SparseAtt/wandb/run-20260707_035639-2qqtcms1/files/output.log
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Starting training...
|
| 2 |
+
Switching attention mode for step 1: sparse_stochastic (configured sparse attention)
|
epfl_final/SparseAtt/wandb/run-20260707_035639-2qqtcms1/files/requirements.txt
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
epfl_final/SparseAtt/wandb/run-20260707_035639-2qqtcms1/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-07-07T03:56:39.542977Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"configs/ladder30k/fixed_s1.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/TokenPrune.git",
|
| 14 |
+
"commit": "88298c7a418622530ca5426b22b1e5ebd10ca0cb"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "h100-train-4-gpu-0-0",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 23 |
+
"gpu_count": 4,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "9374931091456",
|
| 27 |
+
"used": "4873917313024"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1623005732864"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-85844f64-d144-1c84-306d-f88b5f232a77"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 43 |
+
"memoryTotal": "85520809984",
|
| 44 |
+
"cudaCores": 16896,
|
| 45 |
+
"architecture": "Hopper",
|
| 46 |
+
"uuid": "GPU-4e4c2343-4f5b-a2ff-312e-03d402a27803"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 50 |
+
"memoryTotal": "85520809984",
|
| 51 |
+
"cudaCores": 16896,
|
| 52 |
+
"architecture": "Hopper",
|
| 53 |
+
"uuid": "GPU-fbf0ccd1-8bd4-5f57-6e74-6c0cf7eb70c9"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 57 |
+
"memoryTotal": "85520809984",
|
| 58 |
+
"cudaCores": 16896,
|
| 59 |
+
"architecture": "Hopper",
|
| 60 |
+
"uuid": "GPU-410cba62-f098-3454-c960-ebc55bd081af"
|
| 61 |
+
}
|
| 62 |
+
],
|
| 63 |
+
"cudaVersion": "13.0",
|
| 64 |
+
"writerId": "xjwh06dqjc62a68cflurgqcztrlhzf0i"
|
| 65 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260707_035639-2qqtcms1/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-07-07T03:56:39.827790868Z","level":"INFO","msg":"wandb-core"}
|
| 2 |
+
{"time":"2026-07-07T03:56:39.828307381Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
|
| 3 |
+
{"time":"2026-07-07T03:56:40.143924204Z","level":"INFO","msg":"stream: created new stream","id":"2qqtcms1"}
|
| 4 |
+
{"time":"2026-07-07T03:56:40.144011986Z","level":"INFO","msg":"handler: started"}
|
| 5 |
+
{"time":"2026-07-07T03:56:40.145908802Z","level":"INFO","msg":"stream: started"}
|
| 6 |
+
{"time":"2026-07-07T03:56:40.145946068Z","level":"INFO","msg":"writer: started","stream_id":"2qqtcms1"}
|
| 7 |
+
{"time":"2026-07-07T03:56:40.14597465Z","level":"INFO","msg":"sender: started"}
|
| 8 |
+
{"time":"2026-07-07T03:56:41.219234175Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
|
| 9 |
+
{"time":"2026-07-07T03:56:41.707934006Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 10 |
+
{"time":"2026-07-07T03:56:56.219527714Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"console_offset":1,"console_lines":1,"uploaded_len":2}
|
| 11 |
+
{"time":"2026-07-07T03:56:56.533450855Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 12 |
+
{"time":"2026-07-07T03:57:11.220049325Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
|
epfl_final/SparseAtt/wandb/run-20260707_035639-2qqtcms1/logs/debug.log
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-07-07 03:56:39,562 INFO MainThread:235 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-07-07 03:56:39,563 INFO MainThread:235 [wandb_setup.py:_flush():81] Configure stats pid to 235
|
| 3 |
+
2026-07-07 03:56:39,563 INFO MainThread:235 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-07-07 03:56:39,563 INFO MainThread:235 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260707_035639-2qqtcms1/logs/debug.log
|
| 5 |
+
2026-07-07 03:56:39,563 INFO MainThread:235 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260707_035639-2qqtcms1/logs/debug-internal.log
|
| 6 |
+
2026-07-07 03:56:39,563 INFO MainThread:235 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-07-07 03:56:39,563 INFO MainThread:235 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/longctx30k/tokenized', 'val_path': None, 'max_seq_length': 30720, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035549', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 11700, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 20, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 20, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m fixed s1 30k', 'wandb_tags': ['qwen3', '365m', '30k', 'scaleladder', 'fixed', 'seed1'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 120, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_30k_fixed_s1', 'sparse_mask_seed': 1, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-07-07 03:56:39,563 INFO MainThread:235 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-07-07 03:56:39,815 INFO MainThread:235 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-07-07 03:56:40,146 INFO MainThread:235 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-07-07 03:56:40,148 INFO MainThread:235 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-07-07 03:56:40,161 INFO MainThread:235 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-07-07 03:56:41,008 INFO MainThread:235 [wandb_init.py:init():1056] starting run threads in backend
|
| 15 |
+
2026-07-07 03:56:41,211 INFO MainThread:235 [wandb_run.py:_console_start():2554] atexit reg
|
| 16 |
+
2026-07-07 03:56:41,212 INFO MainThread:235 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 17 |
+
2026-07-07 03:56:41,212 INFO MainThread:235 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 18 |
+
2026-07-07 03:56:41,212 INFO MainThread:235 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 19 |
+
2026-07-07 03:56:41,215 INFO MainThread:235 [wandb_init.py:init():1094] run started, returning control to user process
|
epfl_final/SparseAtt/wandb/run-20260707_035639-2qqtcms1/run-2qqtcms1.wandb
ADDED
|
Binary file (7 Bytes). View file
|
|
|
epfl_final/SparseAtt/wandb/run-20260707_035639-l6anse2f/files/output.log
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Starting training...
|
| 2 |
+
Switching attention mode for step 1: sparse_stochastic (configured sparse attention)
|
epfl_final/SparseAtt/wandb/run-20260707_035639-l6anse2f/files/requirements.txt
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
epfl_final/SparseAtt/wandb/run-20260707_035639-l6anse2f/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-07-07T03:56:39.556970Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"configs/ladder30k/fixed_s2.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/TokenPrune.git",
|
| 14 |
+
"commit": "88298c7a418622530ca5426b22b1e5ebd10ca0cb"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "h100-train-4-gpu-0-0",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 23 |
+
"gpu_count": 4,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "9374931091456",
|
| 27 |
+
"used": "4873917313024"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1623005732864"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-85844f64-d144-1c84-306d-f88b5f232a77"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 43 |
+
"memoryTotal": "85520809984",
|
| 44 |
+
"cudaCores": 16896,
|
| 45 |
+
"architecture": "Hopper",
|
| 46 |
+
"uuid": "GPU-4e4c2343-4f5b-a2ff-312e-03d402a27803"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 50 |
+
"memoryTotal": "85520809984",
|
| 51 |
+
"cudaCores": 16896,
|
| 52 |
+
"architecture": "Hopper",
|
| 53 |
+
"uuid": "GPU-fbf0ccd1-8bd4-5f57-6e74-6c0cf7eb70c9"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 57 |
+
"memoryTotal": "85520809984",
|
| 58 |
+
"cudaCores": 16896,
|
| 59 |
+
"architecture": "Hopper",
|
| 60 |
+
"uuid": "GPU-410cba62-f098-3454-c960-ebc55bd081af"
|
| 61 |
+
}
|
| 62 |
+
],
|
| 63 |
+
"cudaVersion": "13.0",
|
| 64 |
+
"writerId": "5ef3i21vedlfoanoivaaq2sz10084h4a"
|
| 65 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260707_035639-l6anse2f/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-07-07T03:56:39.832688862Z","level":"INFO","msg":"wandb-core"}
|
| 2 |
+
{"time":"2026-07-07T03:56:39.833224243Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
|
| 3 |
+
{"time":"2026-07-07T03:56:40.151767012Z","level":"INFO","msg":"stream: created new stream","id":"l6anse2f"}
|
| 4 |
+
{"time":"2026-07-07T03:56:40.151900802Z","level":"INFO","msg":"handler: started"}
|
| 5 |
+
{"time":"2026-07-07T03:56:40.15386587Z","level":"INFO","msg":"stream: started"}
|
| 6 |
+
{"time":"2026-07-07T03:56:40.153950828Z","level":"INFO","msg":"sender: started"}
|
| 7 |
+
{"time":"2026-07-07T03:56:40.153948384Z","level":"INFO","msg":"writer: started","stream_id":"l6anse2f"}
|
| 8 |
+
{"time":"2026-07-07T03:56:41.355281151Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
|
| 9 |
+
{"time":"2026-07-07T03:56:41.955465289Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 10 |
+
{"time":"2026-07-07T03:56:56.355627199Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"console_offset":1,"console_lines":1,"uploaded_len":2}
|
| 11 |
+
{"time":"2026-07-07T03:56:56.643186938Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 12 |
+
{"time":"2026-07-07T03:57:11.356167067Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
|
epfl_final/SparseAtt/wandb/run-20260707_035639-l6anse2f/logs/debug.log
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-07-07 03:56:39,577 INFO MainThread:245 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-07-07 03:56:39,577 INFO MainThread:245 [wandb_setup.py:_flush():81] Configure stats pid to 245
|
| 3 |
+
2026-07-07 03:56:39,577 INFO MainThread:245 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-07-07 03:56:39,577 INFO MainThread:245 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260707_035639-l6anse2f/logs/debug.log
|
| 5 |
+
2026-07-07 03:56:39,577 INFO MainThread:245 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260707_035639-l6anse2f/logs/debug-internal.log
|
| 6 |
+
2026-07-07 03:56:39,577 INFO MainThread:245 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-07-07 03:56:39,577 INFO MainThread:245 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/longctx30k/tokenized', 'val_path': None, 'max_seq_length': 30720, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035551', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 11700, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 20, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 20, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m fixed s2 30k', 'wandb_tags': ['qwen3', '365m', '30k', 'scaleladder', 'fixed', 'seed2'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 120, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_30k_fixed_s2', 'sparse_mask_seed': 2, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-07-07 03:56:39,577 INFO MainThread:245 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-07-07 03:56:39,820 INFO MainThread:245 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-07-07 03:56:40,154 INFO MainThread:245 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-07-07 03:56:40,157 INFO MainThread:245 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-07-07 03:56:40,170 INFO MainThread:245 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-07-07 03:56:41,134 INFO MainThread:245 [wandb_init.py:init():1056] starting run threads in backend
|
| 15 |
+
2026-07-07 03:56:41,347 INFO MainThread:245 [wandb_run.py:_console_start():2554] atexit reg
|
| 16 |
+
2026-07-07 03:56:41,347 INFO MainThread:245 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 17 |
+
2026-07-07 03:56:41,347 INFO MainThread:245 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 18 |
+
2026-07-07 03:56:41,347 INFO MainThread:245 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 19 |
+
2026-07-07 03:56:41,350 INFO MainThread:245 [wandb_init.py:init():1094] run started, returning control to user process
|
epfl_final/SparseAtt/wandb/run-20260707_035639-l6anse2f/run-l6anse2f.wandb
ADDED
|
Binary file (7 Bytes). View file
|
|
|
epfl_final/SparseAtt/wandb/run-20260707_035711-9iiocss5/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-07-07T03:57:11.39971814Z","level":"INFO","msg":"wandb-core"}
|
| 2 |
+
{"time":"2026-07-07T03:57:11.400292399Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
|
| 3 |
+
{"time":"2026-07-07T03:57:11.70161626Z","level":"INFO","msg":"stream: created new stream","id":"9iiocss5"}
|
| 4 |
+
{"time":"2026-07-07T03:57:11.701707376Z","level":"INFO","msg":"handler: started"}
|
| 5 |
+
{"time":"2026-07-07T03:57:11.703650481Z","level":"INFO","msg":"stream: started"}
|
| 6 |
+
{"time":"2026-07-07T03:57:11.703720787Z","level":"INFO","msg":"writer: started","stream_id":"9iiocss5"}
|
| 7 |
+
{"time":"2026-07-07T03:57:11.703744352Z","level":"INFO","msg":"sender: started"}
|
epfl_final/SparseAtt/wandb/run-20260707_035711-9iiocss5/logs/debug.log
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-07-07 03:57:11,125 INFO MainThread:224 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-07-07 03:57:11,125 INFO MainThread:224 [wandb_setup.py:_flush():81] Configure stats pid to 224
|
| 3 |
+
2026-07-07 03:57:11,125 INFO MainThread:224 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-07-07 03:57:11,125 INFO MainThread:224 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260707_035711-9iiocss5/logs/debug.log
|
| 5 |
+
2026-07-07 03:57:11,125 INFO MainThread:224 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260707_035711-9iiocss5/logs/debug-internal.log
|
| 6 |
+
2026-07-07 03:57:11,125 INFO MainThread:224 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-07-07 03:57:11,125 INFO MainThread:224 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/longctx30k/tokenized', 'val_path': None, 'max_seq_length': 30720, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035549', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 11700, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 20, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 20, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m resampled 30k nrb120', 'wandb_tags': ['qwen3', '365m', '30k', 'scaleladder', 'resampled'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 120, 'sparse_mask_pool_size': 1000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_30k_b128_nrb120', 'sparse_mask_seed': 0, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-07-07 03:57:11,126 INFO MainThread:224 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-07-07 03:57:11,386 INFO MainThread:224 [wandb_init.py:init():913] sending inform_init request
|
epfl_final/SparseAtt/wandb/run-20260707_035711-9iiocss5/run-9iiocss5.wandb
ADDED
|
Binary file (7 Bytes). View file
|
|
|
epfl_final/SparseAtt/wandb/run-20260707_035814-kxni9nyw/files/config.yaml
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.26.1
|
| 4 |
+
e:
|
| 5 |
+
m8mrvef218tbffx2zbh913ea7l1zv5ur:
|
| 6 |
+
args:
|
| 7 |
+
- --config
|
| 8 |
+
- configs/ladder30k/fixed_s1.json
|
| 9 |
+
codePath: train.py
|
| 10 |
+
codePathLocal: train.py
|
| 11 |
+
cpu_count: 96
|
| 12 |
+
cpu_count_logical: 192
|
| 13 |
+
cudaVersion: "13.0"
|
| 14 |
+
disk:
|
| 15 |
+
/:
|
| 16 |
+
total: "9374931091456"
|
| 17 |
+
used: "4873970139136"
|
| 18 |
+
email: ruben.navasardyan@mail.utoronto.ca
|
| 19 |
+
executable: /mloscratch/homes/navasard/conda/bin/python
|
| 20 |
+
git:
|
| 21 |
+
commit: 88298c7a418622530ca5426b22b1e5ebd10ca0cb
|
| 22 |
+
remote: https://github.com/rubentium/TokenPrune.git
|
| 23 |
+
gpu: NVIDIA H100 80GB HBM3
|
| 24 |
+
gpu_count: 4
|
| 25 |
+
gpu_nvidia:
|
| 26 |
+
- architecture: Hopper
|
| 27 |
+
cudaCores: 16896
|
| 28 |
+
memoryTotal: "85520809984"
|
| 29 |
+
name: NVIDIA H100 80GB HBM3
|
| 30 |
+
uuid: GPU-85844f64-d144-1c84-306d-f88b5f232a77
|
| 31 |
+
- architecture: Hopper
|
| 32 |
+
cudaCores: 16896
|
| 33 |
+
memoryTotal: "85520809984"
|
| 34 |
+
name: NVIDIA H100 80GB HBM3
|
| 35 |
+
uuid: GPU-4e4c2343-4f5b-a2ff-312e-03d402a27803
|
| 36 |
+
- architecture: Hopper
|
| 37 |
+
cudaCores: 16896
|
| 38 |
+
memoryTotal: "85520809984"
|
| 39 |
+
name: NVIDIA H100 80GB HBM3
|
| 40 |
+
uuid: GPU-fbf0ccd1-8bd4-5f57-6e74-6c0cf7eb70c9
|
| 41 |
+
- architecture: Hopper
|
| 42 |
+
cudaCores: 16896
|
| 43 |
+
memoryTotal: "85520809984"
|
| 44 |
+
name: NVIDIA H100 80GB HBM3
|
| 45 |
+
uuid: GPU-410cba62-f098-3454-c960-ebc55bd081af
|
| 46 |
+
host: h100-train-4-gpu-0-0
|
| 47 |
+
memory:
|
| 48 |
+
total: "1623005732864"
|
| 49 |
+
os: Linux-6.8.0-124-generic-x86_64-with-glibc2.35
|
| 50 |
+
program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
|
| 51 |
+
python: CPython 3.13.13
|
| 52 |
+
root: /mloscratch/homes/navasard/sparse/SparseAtt
|
| 53 |
+
startedAt: "2026-07-07T03:58:14.345278Z"
|
| 54 |
+
writerId: m8mrvef218tbffx2zbh913ea7l1zv5ur
|
| 55 |
+
m: []
|
| 56 |
+
python_version: 3.13.13
|
| 57 |
+
t:
|
| 58 |
+
"1":
|
| 59 |
+
- 1
|
| 60 |
+
- 5
|
| 61 |
+
- 11
|
| 62 |
+
- 49
|
| 63 |
+
- 51
|
| 64 |
+
- 53
|
| 65 |
+
- 71
|
| 66 |
+
"2":
|
| 67 |
+
- 1
|
| 68 |
+
- 5
|
| 69 |
+
- 11
|
| 70 |
+
- 49
|
| 71 |
+
- 51
|
| 72 |
+
- 53
|
| 73 |
+
- 71
|
| 74 |
+
"3":
|
| 75 |
+
- 2
|
| 76 |
+
- 13
|
| 77 |
+
- 15
|
| 78 |
+
- 16
|
| 79 |
+
- 61
|
| 80 |
+
"4": 3.13.13
|
| 81 |
+
"5": 0.26.1
|
| 82 |
+
"6": 5.7.0
|
| 83 |
+
"12": 0.26.1
|
| 84 |
+
"13": linux-x86_64
|
| 85 |
+
attention_type:
|
| 86 |
+
value: sparse_stochastic
|
| 87 |
+
checkpoint_activations:
|
| 88 |
+
value: false
|
| 89 |
+
dense_mode:
|
| 90 |
+
value: false
|
| 91 |
+
dense_step_intervals:
|
| 92 |
+
value: null
|
| 93 |
+
eval_batches:
|
| 94 |
+
value: 50
|
| 95 |
+
eval_steps:
|
| 96 |
+
value: 250
|
| 97 |
+
force_new_run:
|
| 98 |
+
value: true
|
| 99 |
+
gradient_accumulation_steps:
|
| 100 |
+
value: 20
|
| 101 |
+
head_dim:
|
| 102 |
+
value: null
|
| 103 |
+
hidden_size:
|
| 104 |
+
value: 768
|
| 105 |
+
init_from_scratch:
|
| 106 |
+
value: true
|
| 107 |
+
intermediate_size:
|
| 108 |
+
value: 3072
|
| 109 |
+
learning_rate:
|
| 110 |
+
value: 0.0005
|
| 111 |
+
logging_steps:
|
| 112 |
+
value: 10
|
| 113 |
+
lr_scheduler:
|
| 114 |
+
value: cosine
|
| 115 |
+
max_grad_norm:
|
| 116 |
+
value: 1
|
| 117 |
+
max_seq_length:
|
| 118 |
+
value: 30720
|
| 119 |
+
max_steps:
|
| 120 |
+
value: 11700
|
| 121 |
+
mixed_precision:
|
| 122 |
+
value: bf16
|
| 123 |
+
model_dtype:
|
| 124 |
+
value: bfloat16
|
| 125 |
+
model_path:
|
| 126 |
+
value: /mloscratch/homes/navasard/qwen3_06b_base
|
| 127 |
+
num_attention_heads:
|
| 128 |
+
value: 6
|
| 129 |
+
num_epochs:
|
| 130 |
+
value: 1
|
| 131 |
+
num_hidden_layers:
|
| 132 |
+
value: 28
|
| 133 |
+
num_key_value_heads:
|
| 134 |
+
value: 2
|
| 135 |
+
num_workers:
|
| 136 |
+
value: 8
|
| 137 |
+
output_dir:
|
| 138 |
+
value: /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756
|
| 139 |
+
per_device_batch_size:
|
| 140 |
+
value: 1
|
| 141 |
+
resume_from_checkpoint:
|
| 142 |
+
value: null
|
| 143 |
+
save_steps:
|
| 144 |
+
value: 1000
|
| 145 |
+
seed:
|
| 146 |
+
value: 42
|
| 147 |
+
sparse_block_size:
|
| 148 |
+
value: 128
|
| 149 |
+
sparse_layer_indices:
|
| 150 |
+
value: null
|
| 151 |
+
sparse_mask_cache_dir:
|
| 152 |
+
value: /mloscratch/homes/navasard/sparse/mask_cache_30k_fixed_s1
|
| 153 |
+
sparse_mask_pool_size:
|
| 154 |
+
value: 1
|
| 155 |
+
sparse_mask_seed:
|
| 156 |
+
value: 1
|
| 157 |
+
sparse_num_random_blocks:
|
| 158 |
+
value: 120
|
| 159 |
+
sparse_val_dense_compare:
|
| 160 |
+
value: false
|
| 161 |
+
strategy:
|
| 162 |
+
value: ddp
|
| 163 |
+
train_path:
|
| 164 |
+
value: /mloscratch/homes/navasard/data/longctx30k/tokenized
|
| 165 |
+
val_path:
|
| 166 |
+
value: null
|
| 167 |
+
verbose:
|
| 168 |
+
value: true
|
| 169 |
+
wandb_enabled:
|
| 170 |
+
value: true
|
| 171 |
+
wandb_name:
|
| 172 |
+
value: 365m fixed s1 30k
|
| 173 |
+
wandb_project:
|
| 174 |
+
value: stochastic_sparsity
|
| 175 |
+
wandb_tags:
|
| 176 |
+
value:
|
| 177 |
+
- qwen3
|
| 178 |
+
- 365m
|
| 179 |
+
- 30k
|
| 180 |
+
- scaleladder
|
| 181 |
+
- fixed
|
| 182 |
+
- seed1
|
| 183 |
+
warmup_steps:
|
| 184 |
+
value: 750
|
| 185 |
+
weight_decay:
|
| 186 |
+
value: 0.01
|
epfl_final/SparseAtt/wandb/run-20260707_035814-kxni9nyw/files/output.log
ADDED
|
@@ -0,0 +1,1230 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Starting training...
|
| 2 |
+
Switching attention mode for step 1: sparse_stochastic (configured sparse attention)
|
| 3 |
+
Step 10/11700 | Loss: 12.0481 | LR: 6.67e-06 | 12242.1ms/step | 1.6 samples/s
|
| 4 |
+
Step 20/11700 | Loss: 11.9639 | LR: 1.33e-05 | 7668.7ms/step | 2.6 samples/s
|
| 5 |
+
Step 30/11700 | Loss: 11.6999 | LR: 2.00e-05 | 7668.2ms/step | 2.6 samples/s
|
| 6 |
+
Step 40/11700 | Loss: 11.1855 | LR: 2.67e-05 | 7651.9ms/step | 2.6 samples/s
|
| 7 |
+
Step 50/11700 | Loss: 10.8574 | LR: 3.33e-05 | 7638.5ms/step | 2.6 samples/s
|
| 8 |
+
Step 60/11700 | Loss: 10.5730 | LR: 4.00e-05 | 7630.7ms/step | 2.6 samples/s
|
| 9 |
+
Step 70/11700 | Loss: 10.2935 | LR: 4.67e-05 | 7624.0ms/step | 2.6 samples/s
|
| 10 |
+
Step 80/11700 | Loss: 10.0250 | LR: 5.33e-05 | 7617.0ms/step | 2.6 samples/s
|
| 11 |
+
Step 90/11700 | Loss: 9.7335 | LR: 6.00e-05 | 7618.1ms/step | 2.6 samples/s
|
| 12 |
+
Step 100/11700 | Loss: 9.3537 | LR: 6.67e-05 | 7627.8ms/step | 2.6 samples/s
|
| 13 |
+
Step 110/11700 | Loss: 8.8777 | LR: 7.33e-05 | 7624.6ms/step | 2.6 samples/s
|
| 14 |
+
Step 120/11700 | Loss: 8.3232 | LR: 8.00e-05 | 7627.7ms/step | 2.6 samples/s
|
| 15 |
+
Step 130/11700 | Loss: 7.8608 | LR: 8.67e-05 | 7631.8ms/step | 2.6 samples/s
|
| 16 |
+
Step 140/11700 | Loss: 7.5848 | LR: 9.33e-05 | 7638.4ms/step | 2.6 samples/s
|
| 17 |
+
Step 150/11700 | Loss: 7.3759 | LR: 1.00e-04 | 7643.7ms/step | 2.6 samples/s
|
| 18 |
+
Step 160/11700 | Loss: 7.1625 | LR: 1.07e-04 | 7649.8ms/step | 2.6 samples/s
|
| 19 |
+
Step 170/11700 | Loss: 7.0453 | LR: 1.13e-04 | 7657.7ms/step | 2.6 samples/s
|
| 20 |
+
Step 180/11700 | Loss: 6.9226 | LR: 1.20e-04 | 7665.7ms/step | 2.6 samples/s
|
| 21 |
+
Step 190/11700 | Loss: 6.8475 | LR: 1.27e-04 | 7669.9ms/step | 2.6 samples/s
|
| 22 |
+
Step 200/11700 | Loss: 6.6835 | LR: 1.33e-04 | 7672.2ms/step | 2.6 samples/s
|
| 23 |
+
Step 210/11700 | Loss: 6.5853 | LR: 1.40e-04 | 7676.6ms/step | 2.6 samples/s
|
| 24 |
+
Step 220/11700 | Loss: 6.4342 | LR: 1.47e-04 | 7680.1ms/step | 2.6 samples/s
|
| 25 |
+
Step 230/11700 | Loss: 6.3811 | LR: 1.53e-04 | 7684.1ms/step | 2.6 samples/s
|
| 26 |
+
Step 240/11700 | Loss: 6.2366 | LR: 1.60e-04 | 7677.1ms/step | 2.6 samples/s
|
| 27 |
+
Step 250/11700 | Loss: 6.2542 | LR: 1.67e-04 | 7681.6ms/step | 2.6 samples/s
|
| 28 |
+
Validation at step 250: Loss = 6.1219, Perplexity = 455.74
|
| 29 |
+
Step 260/11700 | Loss: 6.1660 | LR: 1.73e-04 | 21093.3ms/step | 0.9 samples/s
|
| 30 |
+
Step 270/11700 | Loss: 6.0283 | LR: 1.80e-04 | 7705.5ms/step | 2.6 samples/s
|
| 31 |
+
Step 280/11700 | Loss: 5.9767 | LR: 1.87e-04 | 7713.7ms/step | 2.6 samples/s
|
| 32 |
+
Step 290/11700 | Loss: 5.8484 | LR: 1.93e-04 | 7708.6ms/step | 2.6 samples/s
|
| 33 |
+
Step 300/11700 | Loss: 5.8329 | LR: 2.00e-04 | 7726.5ms/step | 2.6 samples/s
|
| 34 |
+
Step 310/11700 | Loss: 5.8043 | LR: 2.07e-04 | 7704.7ms/step | 2.6 samples/s
|
| 35 |
+
Step 320/11700 | Loss: 5.7098 | LR: 2.13e-04 | 7706.5ms/step | 2.6 samples/s
|
| 36 |
+
Step 330/11700 | Loss: 5.5965 | LR: 2.20e-04 | 7716.5ms/step | 2.6 samples/s
|
| 37 |
+
Step 340/11700 | Loss: 5.5504 | LR: 2.27e-04 | 7719.7ms/step | 2.6 samples/s
|
| 38 |
+
Step 350/11700 | Loss: 5.3912 | LR: 2.33e-04 | 7701.3ms/step | 2.6 samples/s
|
| 39 |
+
Step 360/11700 | Loss: 5.4623 | LR: 2.40e-04 | 7716.7ms/step | 2.6 samples/s
|
| 40 |
+
Step 370/11700 | Loss: 5.4942 | LR: 2.47e-04 | 7719.4ms/step | 2.6 samples/s
|
| 41 |
+
Step 380/11700 | Loss: 5.4840 | LR: 2.53e-04 | 7714.6ms/step | 2.6 samples/s
|
| 42 |
+
Step 390/11700 | Loss: 5.4318 | LR: 2.60e-04 | 7718.0ms/step | 2.6 samples/s
|
| 43 |
+
Step 400/11700 | Loss: 5.2920 | LR: 2.67e-04 | 7725.3ms/step | 2.6 samples/s
|
| 44 |
+
Step 410/11700 | Loss: 5.3159 | LR: 2.73e-04 | 7729.6ms/step | 2.6 samples/s
|
| 45 |
+
Step 420/11700 | Loss: 5.2723 | LR: 2.80e-04 | 7736.8ms/step | 2.6 samples/s
|
| 46 |
+
Step 430/11700 | Loss: 5.2204 | LR: 2.87e-04 | 7708.9ms/step | 2.6 samples/s
|
| 47 |
+
Step 440/11700 | Loss: 5.2186 | LR: 2.93e-04 | 7718.0ms/step | 2.6 samples/s
|
| 48 |
+
Step 450/11700 | Loss: 5.1471 | LR: 3.00e-04 | 7730.3ms/step | 2.6 samples/s
|
| 49 |
+
Step 460/11700 | Loss: 5.1613 | LR: 3.07e-04 | 7720.2ms/step | 2.6 samples/s
|
| 50 |
+
Step 470/11700 | Loss: 5.1233 | LR: 3.13e-04 | 7744.1ms/step | 2.6 samples/s
|
| 51 |
+
Step 480/11700 | Loss: 5.1200 | LR: 3.20e-04 | 7735.6ms/step | 2.6 samples/s
|
| 52 |
+
Step 490/11700 | Loss: 5.0295 | LR: 3.27e-04 | 7726.2ms/step | 2.6 samples/s
|
| 53 |
+
Step 500/11700 | Loss: 4.9870 | LR: 3.33e-04 | 7715.6ms/step | 2.6 samples/s
|
| 54 |
+
Validation at step 500: Loss = 4.9378, Perplexity = 139.46
|
| 55 |
+
Step 510/11700 | Loss: 4.9054 | LR: 3.40e-04 | 19236.1ms/step | 1.0 samples/s
|
| 56 |
+
Step 520/11700 | Loss: 4.9075 | LR: 3.47e-04 | 7717.4ms/step | 2.6 samples/s
|
| 57 |
+
Step 530/11700 | Loss: 4.8455 | LR: 3.53e-04 | 7716.2ms/step | 2.6 samples/s
|
| 58 |
+
Step 540/11700 | Loss: 4.7478 | LR: 3.60e-04 | 7743.2ms/step | 2.6 samples/s
|
| 59 |
+
Step 550/11700 | Loss: 4.7653 | LR: 3.67e-04 | 7741.2ms/step | 2.6 samples/s
|
| 60 |
+
Step 560/11700 | Loss: 4.7563 | LR: 3.73e-04 | 7749.7ms/step | 2.6 samples/s
|
| 61 |
+
Step 570/11700 | Loss: 4.5215 | LR: 3.80e-04 | 7742.4ms/step | 2.6 samples/s
|
| 62 |
+
Step 580/11700 | Loss: 4.5547 | LR: 3.87e-04 | 7725.1ms/step | 2.6 samples/s
|
| 63 |
+
Step 590/11700 | Loss: 4.5127 | LR: 3.93e-04 | 7738.4ms/step | 2.6 samples/s
|
| 64 |
+
Step 600/11700 | Loss: 4.5684 | LR: 4.00e-04 | 7736.3ms/step | 2.6 samples/s
|
| 65 |
+
Step 610/11700 | Loss: 4.5378 | LR: 4.07e-04 | 7729.6ms/step | 2.6 samples/s
|
| 66 |
+
Step 620/11700 | Loss: 4.5248 | LR: 4.13e-04 | 7740.9ms/step | 2.6 samples/s
|
| 67 |
+
Step 630/11700 | Loss: 4.3725 | LR: 4.20e-04 | 7747.3ms/step | 2.6 samples/s
|
| 68 |
+
Step 640/11700 | Loss: 4.4640 | LR: 4.27e-04 | 7743.0ms/step | 2.6 samples/s
|
| 69 |
+
Step 650/11700 | Loss: 4.3467 | LR: 4.33e-04 | 7715.0ms/step | 2.6 samples/s
|
| 70 |
+
Step 660/11700 | Loss: 4.4468 | LR: 4.40e-04 | 7735.0ms/step | 2.6 samples/s
|
| 71 |
+
Step 670/11700 | Loss: 4.3328 | LR: 4.47e-04 | 7720.5ms/step | 2.6 samples/s
|
| 72 |
+
Step 680/11700 | Loss: 4.2429 | LR: 4.53e-04 | 7732.2ms/step | 2.6 samples/s
|
| 73 |
+
Step 690/11700 | Loss: 4.0877 | LR: 4.60e-04 | 7736.0ms/step | 2.6 samples/s
|
| 74 |
+
Step 700/11700 | Loss: 4.2427 | LR: 4.67e-04 | 7745.8ms/step | 2.6 samples/s
|
| 75 |
+
Step 710/11700 | Loss: 4.2394 | LR: 4.73e-04 | 7742.0ms/step | 2.6 samples/s
|
| 76 |
+
Step 720/11700 | Loss: 4.1194 | LR: 4.80e-04 | 7745.0ms/step | 2.6 samples/s
|
| 77 |
+
Step 730/11700 | Loss: 3.9965 | LR: 4.87e-04 | 7748.7ms/step | 2.6 samples/s
|
| 78 |
+
Step 740/11700 | Loss: 4.1403 | LR: 4.93e-04 | 7740.3ms/step | 2.6 samples/s
|
| 79 |
+
Step 750/11700 | Loss: 4.0257 | LR: 5.00e-04 | 7754.3ms/step | 2.6 samples/s
|
| 80 |
+
Validation at step 750: Loss = 3.9729, Perplexity = 53.14
|
| 81 |
+
Step 760/11700 | Loss: 4.0513 | LR: 5.00e-04 | 19286.1ms/step | 1.0 samples/s
|
| 82 |
+
Step 770/11700 | Loss: 3.8243 | LR: 5.00e-04 | 7745.6ms/step | 2.6 samples/s
|
| 83 |
+
Step 780/11700 | Loss: 4.0187 | LR: 5.00e-04 | 7741.7ms/step | 2.6 samples/s
|
| 84 |
+
Step 790/11700 | Loss: 3.7950 | LR: 5.00e-04 | 7726.2ms/step | 2.6 samples/s
|
| 85 |
+
Step 800/11700 | Loss: 3.7916 | LR: 5.00e-04 | 7734.4ms/step | 2.6 samples/s
|
| 86 |
+
Step 810/11700 | Loss: 3.8328 | LR: 5.00e-04 | 7745.5ms/step | 2.6 samples/s
|
| 87 |
+
Step 820/11700 | Loss: 3.8211 | LR: 5.00e-04 | 7748.2ms/step | 2.6 samples/s
|
| 88 |
+
Step 830/11700 | Loss: 3.8070 | LR: 5.00e-04 | 7740.3ms/step | 2.6 samples/s
|
| 89 |
+
Step 840/11700 | Loss: 3.8021 | LR: 5.00e-04 | 7757.2ms/step | 2.6 samples/s
|
| 90 |
+
Step 850/11700 | Loss: 3.8368 | LR: 5.00e-04 | 7760.7ms/step | 2.6 samples/s
|
| 91 |
+
Step 860/11700 | Loss: 3.6708 | LR: 5.00e-04 | 7753.3ms/step | 2.6 samples/s
|
| 92 |
+
Step 870/11700 | Loss: 3.7118 | LR: 5.00e-04 | 7737.4ms/step | 2.6 samples/s
|
| 93 |
+
Step 880/11700 | Loss: 3.7644 | LR: 5.00e-04 | 7749.7ms/step | 2.6 samples/s
|
| 94 |
+
Step 890/11700 | Loss: 3.6054 | LR: 5.00e-04 | 7765.9ms/step | 2.6 samples/s
|
| 95 |
+
Step 900/11700 | Loss: 3.5428 | LR: 5.00e-04 | 7751.2ms/step | 2.6 samples/s
|
| 96 |
+
Step 910/11700 | Loss: 3.7092 | LR: 5.00e-04 | 7761.1ms/step | 2.6 samples/s
|
| 97 |
+
Step 920/11700 | Loss: 3.5774 | LR: 5.00e-04 | 7749.7ms/step | 2.6 samples/s
|
| 98 |
+
Step 930/11700 | Loss: 3.5651 | LR: 5.00e-04 | 7759.4ms/step | 2.6 samples/s
|
| 99 |
+
Step 940/11700 | Loss: 3.6396 | LR: 5.00e-04 | 7762.3ms/step | 2.6 samples/s
|
| 100 |
+
Step 950/11700 | Loss: 3.4800 | LR: 5.00e-04 | 7750.7ms/step | 2.6 samples/s
|
| 101 |
+
Step 960/11700 | Loss: 3.5553 | LR: 5.00e-04 | 7738.4ms/step | 2.6 samples/s
|
| 102 |
+
Step 970/11700 | Loss: 3.6108 | LR: 5.00e-04 | 7747.6ms/step | 2.6 samples/s
|
| 103 |
+
Step 980/11700 | Loss: 3.3437 | LR: 4.99e-04 | 7744.6ms/step | 2.6 samples/s
|
| 104 |
+
Step 990/11700 | Loss: 3.5572 | LR: 4.99e-04 | 7755.6ms/step | 2.6 samples/s
|
| 105 |
+
Step 1000/11700 | Loss: 3.4443 | LR: 4.99e-04 | 7740.1ms/step | 2.6 samples/s
|
| 106 |
+
Validation at step 1000: Loss = 3.4954, Perplexity = 32.96
|
| 107 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-1000
|
| 108 |
+
Step 1010/11700 | Loss: 3.5579 | LR: 4.99e-04 | 19617.8ms/step | 1.0 samples/s
|
| 109 |
+
Step 1020/11700 | Loss: 3.4608 | LR: 4.99e-04 | 7758.4ms/step | 2.6 samples/s
|
| 110 |
+
Step 1030/11700 | Loss: 3.4865 | LR: 4.99e-04 | 7740.5ms/step | 2.6 samples/s
|
| 111 |
+
Step 1040/11700 | Loss: 3.5065 | LR: 4.99e-04 | 7758.5ms/step | 2.6 samples/s
|
| 112 |
+
Step 1050/11700 | Loss: 3.4233 | LR: 4.99e-04 | 7757.3ms/step | 2.6 samples/s
|
| 113 |
+
Step 1060/11700 | Loss: 3.4798 | LR: 4.99e-04 | 7735.3ms/step | 2.6 samples/s
|
| 114 |
+
Step 1070/11700 | Loss: 3.4378 | LR: 4.99e-04 | 7746.1ms/step | 2.6 samples/s
|
| 115 |
+
Step 1080/11700 | Loss: 3.2996 | LR: 4.99e-04 | 7759.9ms/step | 2.6 samples/s
|
| 116 |
+
Step 1090/11700 | Loss: 3.3530 | LR: 4.99e-04 | 7751.9ms/step | 2.6 samples/s
|
| 117 |
+
Step 1100/11700 | Loss: 3.5537 | LR: 4.99e-04 | 7762.2ms/step | 2.6 samples/s
|
| 118 |
+
Step 1110/11700 | Loss: 3.5042 | LR: 4.99e-04 | 7749.6ms/step | 2.6 samples/s
|
| 119 |
+
Step 1120/11700 | Loss: 3.3953 | LR: 4.99e-04 | 7741.2ms/step | 2.6 samples/s
|
| 120 |
+
Step 1130/11700 | Loss: 3.3990 | LR: 4.99e-04 | 7749.6ms/step | 2.6 samples/s
|
| 121 |
+
Step 1140/11700 | Loss: 3.1649 | LR: 4.98e-04 | 7749.7ms/step | 2.6 samples/s
|
| 122 |
+
Step 1150/11700 | Loss: 3.3282 | LR: 4.98e-04 | 7750.3ms/step | 2.6 samples/s
|
| 123 |
+
Step 1160/11700 | Loss: 3.3397 | LR: 4.98e-04 | 7749.2ms/step | 2.6 samples/s
|
| 124 |
+
Step 1170/11700 | Loss: 3.3018 | LR: 4.98e-04 | 7732.0ms/step | 2.6 samples/s
|
| 125 |
+
Step 1180/11700 | Loss: 3.2404 | LR: 4.98e-04 | 7740.3ms/step | 2.6 samples/s
|
| 126 |
+
Step 1190/11700 | Loss: 3.2017 | LR: 4.98e-04 | 7745.0ms/step | 2.6 samples/s
|
| 127 |
+
Step 1200/11700 | Loss: 3.1345 | LR: 4.98e-04 | 7747.7ms/step | 2.6 samples/s
|
| 128 |
+
Step 1210/11700 | Loss: 3.4671 | LR: 4.98e-04 | 7761.1ms/step | 2.6 samples/s
|
| 129 |
+
Step 1220/11700 | Loss: 3.2681 | LR: 4.98e-04 | 7749.0ms/step | 2.6 samples/s
|
| 130 |
+
Step 1230/11700 | Loss: 3.3052 | LR: 4.98e-04 | 7753.3ms/step | 2.6 samples/s
|
| 131 |
+
Step 1240/11700 | Loss: 3.2441 | LR: 4.98e-04 | 7734.4ms/step | 2.6 samples/s
|
| 132 |
+
Step 1250/11700 | Loss: 3.2245 | LR: 4.97e-04 | 7748.7ms/step | 2.6 samples/s
|
| 133 |
+
Validation at step 1250: Loss = 3.2738, Perplexity = 26.41
|
| 134 |
+
Step 1260/11700 | Loss: 3.3045 | LR: 4.97e-04 | 19296.8ms/step | 1.0 samples/s
|
| 135 |
+
Step 1270/11700 | Loss: 3.1380 | LR: 4.97e-04 | 7749.9ms/step | 2.6 samples/s
|
| 136 |
+
Step 1280/11700 | Loss: 3.2447 | LR: 4.97e-04 | 7743.7ms/step | 2.6 samples/s
|
| 137 |
+
Step 1290/11700 | Loss: 3.2474 | LR: 4.97e-04 | 7757.3ms/step | 2.6 samples/s
|
| 138 |
+
Step 1300/11700 | Loss: 3.2476 | LR: 4.97e-04 | 7755.0ms/step | 2.6 samples/s
|
| 139 |
+
Step 1310/11700 | Loss: 3.2854 | LR: 4.97e-04 | 7757.5ms/step | 2.6 samples/s
|
| 140 |
+
Step 1320/11700 | Loss: 3.2521 | LR: 4.97e-04 | 7743.1ms/step | 2.6 samples/s
|
| 141 |
+
Step 1330/11700 | Loss: 3.1758 | LR: 4.97e-04 | 7739.0ms/step | 2.6 samples/s
|
| 142 |
+
Step 1340/11700 | Loss: 3.2265 | LR: 4.96e-04 | 7731.2ms/step | 2.6 samples/s
|
| 143 |
+
Step 1350/11700 | Loss: 3.2369 | LR: 4.96e-04 | 7758.0ms/step | 2.6 samples/s
|
| 144 |
+
Step 1360/11700 | Loss: 3.1989 | LR: 4.96e-04 | 7745.4ms/step | 2.6 samples/s
|
| 145 |
+
Step 1370/11700 | Loss: 3.1353 | LR: 4.96e-04 | 7730.8ms/step | 2.6 samples/s
|
| 146 |
+
Step 1380/11700 | Loss: 3.0074 | LR: 4.96e-04 | 7735.7ms/step | 2.6 samples/s
|
| 147 |
+
Step 1390/11700 | Loss: 3.1887 | LR: 4.96e-04 | 7744.3ms/step | 2.6 samples/s
|
| 148 |
+
Step 1400/11700 | Loss: 3.3018 | LR: 4.96e-04 | 7753.0ms/step | 2.6 samples/s
|
| 149 |
+
Step 1410/11700 | Loss: 3.2127 | LR: 4.96e-04 | 7738.8ms/step | 2.6 samples/s
|
| 150 |
+
Step 1420/11700 | Loss: 3.1101 | LR: 4.95e-04 | 7740.1ms/step | 2.6 samples/s
|
| 151 |
+
Step 1430/11700 | Loss: 3.1519 | LR: 4.95e-04 | 7733.4ms/step | 2.6 samples/s
|
| 152 |
+
Step 1440/11700 | Loss: 3.1423 | LR: 4.95e-04 | 7722.7ms/step | 2.6 samples/s
|
| 153 |
+
Step 1450/11700 | Loss: 3.0653 | LR: 4.95e-04 | 7721.9ms/step | 2.6 samples/s
|
| 154 |
+
Step 1460/11700 | Loss: 3.1189 | LR: 4.95e-04 | 7718.2ms/step | 2.6 samples/s
|
| 155 |
+
Step 1470/11700 | Loss: 3.1297 | LR: 4.95e-04 | 7717.5ms/step | 2.6 samples/s
|
| 156 |
+
Step 1480/11700 | Loss: 3.0623 | LR: 4.95e-04 | 7716.4ms/step | 2.6 samples/s
|
| 157 |
+
Step 1490/11700 | Loss: 3.2100 | LR: 4.94e-04 | 7718.4ms/step | 2.6 samples/s
|
| 158 |
+
Step 1500/11700 | Loss: 3.0466 | LR: 4.94e-04 | 7718.7ms/step | 2.6 samples/s
|
| 159 |
+
Validation at step 1500: Loss = 3.1303, Perplexity = 22.88
|
| 160 |
+
Step 1510/11700 | Loss: 3.1255 | LR: 4.94e-04 | 19214.2ms/step | 1.0 samples/s
|
| 161 |
+
Step 1520/11700 | Loss: 3.1178 | LR: 4.94e-04 | 7717.9ms/step | 2.6 samples/s
|
| 162 |
+
Step 1530/11700 | Loss: 3.1247 | LR: 4.94e-04 | 7714.6ms/step | 2.6 samples/s
|
| 163 |
+
Step 1540/11700 | Loss: 3.2733 | LR: 4.94e-04 | 7715.6ms/step | 2.6 samples/s
|
| 164 |
+
Step 1550/11700 | Loss: 3.1211 | LR: 4.93e-04 | 7713.9ms/step | 2.6 samples/s
|
| 165 |
+
Step 1560/11700 | Loss: 3.1404 | LR: 4.93e-04 | 7717.1ms/step | 2.6 samples/s
|
| 166 |
+
Step 1570/11700 | Loss: 3.1272 | LR: 4.93e-04 | 7715.1ms/step | 2.6 samples/s
|
| 167 |
+
Step 1580/11700 | Loss: 3.0854 | LR: 4.93e-04 | 7718.1ms/step | 2.6 samples/s
|
| 168 |
+
Step 1590/11700 | Loss: 3.2201 | LR: 4.93e-04 | 7722.6ms/step | 2.6 samples/s
|
| 169 |
+
Step 1600/11700 | Loss: 3.1082 | LR: 4.93e-04 | 7726.9ms/step | 2.6 samples/s
|
| 170 |
+
Step 1610/11700 | Loss: 3.1493 | LR: 4.92e-04 | 7727.9ms/step | 2.6 samples/s
|
| 171 |
+
Step 1620/11700 | Loss: 3.0263 | LR: 4.92e-04 | 7726.7ms/step | 2.6 samples/s
|
| 172 |
+
Step 1630/11700 | Loss: 2.9366 | LR: 4.92e-04 | 7729.3ms/step | 2.6 samples/s
|
| 173 |
+
Step 1640/11700 | Loss: 3.1180 | LR: 4.92e-04 | 7727.9ms/step | 2.6 samples/s
|
| 174 |
+
Step 1650/11700 | Loss: 3.0492 | LR: 4.92e-04 | 7727.2ms/step | 2.6 samples/s
|
| 175 |
+
Step 1660/11700 | Loss: 3.0831 | LR: 4.92e-04 | 7724.6ms/step | 2.6 samples/s
|
| 176 |
+
Step 1670/11700 | Loss: 3.2078 | LR: 4.91e-04 | 7726.1ms/step | 2.6 samples/s
|
| 177 |
+
Step 1680/11700 | Loss: 3.0567 | LR: 4.91e-04 | 7725.0ms/step | 2.6 samples/s
|
| 178 |
+
Step 1690/11700 | Loss: 2.9181 | LR: 4.91e-04 | 7727.8ms/step | 2.6 samples/s
|
| 179 |
+
Step 1700/11700 | Loss: 2.9730 | LR: 4.91e-04 | 7729.6ms/step | 2.6 samples/s
|
| 180 |
+
Step 1710/11700 | Loss: 3.1066 | LR: 4.91e-04 | 7725.5ms/step | 2.6 samples/s
|
| 181 |
+
Step 1720/11700 | Loss: 3.0302 | LR: 4.90e-04 | 7724.3ms/step | 2.6 samples/s
|
| 182 |
+
Step 1730/11700 | Loss: 3.0550 | LR: 4.90e-04 | 7717.7ms/step | 2.6 samples/s
|
| 183 |
+
Step 1740/11700 | Loss: 2.9553 | LR: 4.90e-04 | 7724.0ms/step | 2.6 samples/s
|
| 184 |
+
Step 1750/11700 | Loss: 2.9241 | LR: 4.90e-04 | 7726.0ms/step | 2.6 samples/s
|
| 185 |
+
Validation at step 1750: Loss = 3.0454, Perplexity = 21.02
|
| 186 |
+
Step 1760/11700 | Loss: 3.0235 | LR: 4.90e-04 | 19232.2ms/step | 1.0 samples/s
|
| 187 |
+
Step 1770/11700 | Loss: 3.0231 | LR: 4.89e-04 | 7716.8ms/step | 2.6 samples/s
|
| 188 |
+
Step 1780/11700 | Loss: 3.0465 | LR: 4.89e-04 | 7716.9ms/step | 2.6 samples/s
|
| 189 |
+
Step 1790/11700 | Loss: 3.0168 | LR: 4.89e-04 | 7717.2ms/step | 2.6 samples/s
|
| 190 |
+
Step 1800/11700 | Loss: 3.0394 | LR: 4.89e-04 | 7716.7ms/step | 2.6 samples/s
|
| 191 |
+
Step 1810/11700 | Loss: 3.1019 | LR: 4.89e-04 | 7714.4ms/step | 2.6 samples/s
|
| 192 |
+
Step 1820/11700 | Loss: 3.0397 | LR: 4.88e-04 | 7711.3ms/step | 2.6 samples/s
|
| 193 |
+
Step 1830/11700 | Loss: 3.0036 | LR: 4.88e-04 | 7714.1ms/step | 2.6 samples/s
|
| 194 |
+
Step 1840/11700 | Loss: 2.9925 | LR: 4.88e-04 | 7717.9ms/step | 2.6 samples/s
|
| 195 |
+
Step 1850/11700 | Loss: 3.0180 | LR: 4.88e-04 | 7717.8ms/step | 2.6 samples/s
|
| 196 |
+
Step 1860/11700 | Loss: 2.9950 | LR: 4.87e-04 | 7724.4ms/step | 2.6 samples/s
|
| 197 |
+
Step 1870/11700 | Loss: 2.9243 | LR: 4.87e-04 | 7720.2ms/step | 2.6 samples/s
|
| 198 |
+
Step 1880/11700 | Loss: 2.9448 | LR: 4.87e-04 | 7716.2ms/step | 2.6 samples/s
|
| 199 |
+
Step 1890/11700 | Loss: 3.0652 | LR: 4.87e-04 | 7712.1ms/step | 2.6 samples/s
|
| 200 |
+
Step 1900/11700 | Loss: 3.0094 | LR: 4.87e-04 | 7711.9ms/step | 2.6 samples/s
|
| 201 |
+
Step 1910/11700 | Loss: 3.0336 | LR: 4.86e-04 | 7705.0ms/step | 2.6 samples/s
|
| 202 |
+
Step 1920/11700 | Loss: 2.8256 | LR: 4.86e-04 | 7707.2ms/step | 2.6 samples/s
|
| 203 |
+
Step 1930/11700 | Loss: 3.0109 | LR: 4.86e-04 | 7705.3ms/step | 2.6 samples/s
|
| 204 |
+
Step 1940/11700 | Loss: 2.9488 | LR: 4.86e-04 | 7702.9ms/step | 2.6 samples/s
|
| 205 |
+
Step 1950/11700 | Loss: 2.8663 | LR: 4.85e-04 | 7701.3ms/step | 2.6 samples/s
|
| 206 |
+
Step 1960/11700 | Loss: 2.9363 | LR: 4.85e-04 | 7701.2ms/step | 2.6 samples/s
|
| 207 |
+
Step 1970/11700 | Loss: 2.9601 | LR: 4.85e-04 | 7704.9ms/step | 2.6 samples/s
|
| 208 |
+
Step 1980/11700 | Loss: 2.9415 | LR: 4.85e-04 | 7711.0ms/step | 2.6 samples/s
|
| 209 |
+
Step 1990/11700 | Loss: 2.8845 | LR: 4.84e-04 | 7718.8ms/step | 2.6 samples/s
|
| 210 |
+
Step 2000/11700 | Loss: 3.0976 | LR: 4.84e-04 | 7738.4ms/step | 2.6 samples/s
|
| 211 |
+
Validation at step 2000: Loss = 2.9675, Perplexity = 19.44
|
| 212 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-2000
|
| 213 |
+
Step 2010/11700 | Loss: 2.9858 | LR: 4.84e-04 | 19609.6ms/step | 1.0 samples/s
|
| 214 |
+
Step 2020/11700 | Loss: 3.0210 | LR: 4.84e-04 | 7712.6ms/step | 2.6 samples/s
|
| 215 |
+
Step 2030/11700 | Loss: 2.9242 | LR: 4.83e-04 | 7711.4ms/step | 2.6 samples/s
|
| 216 |
+
Step 2040/11700 | Loss: 2.9689 | LR: 4.83e-04 | 7717.7ms/step | 2.6 samples/s
|
| 217 |
+
Step 2050/11700 | Loss: 2.8396 | LR: 4.83e-04 | 7719.3ms/step | 2.6 samples/s
|
| 218 |
+
Step 2060/11700 | Loss: 2.9108 | LR: 4.83e-04 | 7716.6ms/step | 2.6 samples/s
|
| 219 |
+
Step 2070/11700 | Loss: 2.9181 | LR: 4.82e-04 | 7714.4ms/step | 2.6 samples/s
|
| 220 |
+
Step 2080/11700 | Loss: 2.8970 | LR: 4.82e-04 | 7714.0ms/step | 2.6 samples/s
|
| 221 |
+
Step 2090/11700 | Loss: 2.9291 | LR: 4.82e-04 | 7717.8ms/step | 2.6 samples/s
|
| 222 |
+
Step 2100/11700 | Loss: 3.0159 | LR: 4.81e-04 | 7719.0ms/step | 2.6 samples/s
|
| 223 |
+
Step 2110/11700 | Loss: 3.0049 | LR: 4.81e-04 | 7718.3ms/step | 2.6 samples/s
|
| 224 |
+
Step 2120/11700 | Loss: 2.9440 | LR: 4.81e-04 | 7718.5ms/step | 2.6 samples/s
|
| 225 |
+
Step 2130/11700 | Loss: 2.9360 | LR: 4.81e-04 | 7717.0ms/step | 2.6 samples/s
|
| 226 |
+
Step 2140/11700 | Loss: 3.0475 | LR: 4.80e-04 | 7714.6ms/step | 2.6 samples/s
|
| 227 |
+
Step 2150/11700 | Loss: 2.9680 | LR: 4.80e-04 | 7715.5ms/step | 2.6 samples/s
|
| 228 |
+
Step 2160/11700 | Loss: 2.9293 | LR: 4.80e-04 | 7714.0ms/step | 2.6 samples/s
|
| 229 |
+
Step 2170/11700 | Loss: 2.8703 | LR: 4.80e-04 | 7714.4ms/step | 2.6 samples/s
|
| 230 |
+
Step 2180/11700 | Loss: 2.8978 | LR: 4.79e-04 | 7718.9ms/step | 2.6 samples/s
|
| 231 |
+
Step 2190/11700 | Loss: 2.8808 | LR: 4.79e-04 | 7722.0ms/step | 2.6 samples/s
|
| 232 |
+
Step 2200/11700 | Loss: 2.9465 | LR: 4.79e-04 | 7717.3ms/step | 2.6 samples/s
|
| 233 |
+
Step 2210/11700 | Loss: 2.9855 | LR: 4.78e-04 | 7717.1ms/step | 2.6 samples/s
|
| 234 |
+
Step 2220/11700 | Loss: 2.7951 | LR: 4.78e-04 | 7714.0ms/step | 2.6 samples/s
|
| 235 |
+
Step 2230/11700 | Loss: 2.8323 | LR: 4.78e-04 | 7714.4ms/step | 2.6 samples/s
|
| 236 |
+
Step 2240/11700 | Loss: 2.8158 | LR: 4.78e-04 | 7713.5ms/step | 2.6 samples/s
|
| 237 |
+
Step 2250/11700 | Loss: 3.0016 | LR: 4.77e-04 | 7714.4ms/step | 2.6 samples/s
|
| 238 |
+
Validation at step 2250: Loss = 2.9025, Perplexity = 18.22
|
| 239 |
+
Step 2260/11700 | Loss: 2.7785 | LR: 4.77e-04 | 19205.7ms/step | 1.0 samples/s
|
| 240 |
+
Step 2270/11700 | Loss: 2.9910 | LR: 4.77e-04 | 7716.3ms/step | 2.6 samples/s
|
| 241 |
+
Step 2280/11700 | Loss: 2.9136 | LR: 4.76e-04 | 7716.6ms/step | 2.6 samples/s
|
| 242 |
+
Step 2290/11700 | Loss: 2.9289 | LR: 4.76e-04 | 7717.3ms/step | 2.6 samples/s
|
| 243 |
+
Step 2300/11700 | Loss: 2.8717 | LR: 4.76e-04 | 7718.3ms/step | 2.6 samples/s
|
| 244 |
+
Step 2310/11700 | Loss: 2.8606 | LR: 4.75e-04 | 7718.4ms/step | 2.6 samples/s
|
| 245 |
+
Step 2320/11700 | Loss: 3.0052 | LR: 4.75e-04 | 7714.6ms/step | 2.6 samples/s
|
| 246 |
+
Step 2330/11700 | Loss: 2.8291 | LR: 4.75e-04 | 7713.8ms/step | 2.6 samples/s
|
| 247 |
+
Step 2340/11700 | Loss: 2.8628 | LR: 4.74e-04 | 7716.4ms/step | 2.6 samples/s
|
| 248 |
+
Step 2350/11700 | Loss: 2.7823 | LR: 4.74e-04 | 7715.5ms/step | 2.6 samples/s
|
| 249 |
+
Step 2360/11700 | Loss: 2.8004 | LR: 4.74e-04 | 7717.4ms/step | 2.6 samples/s
|
| 250 |
+
Step 2370/11700 | Loss: 2.7834 | LR: 4.73e-04 | 7715.2ms/step | 2.6 samples/s
|
| 251 |
+
Step 2380/11700 | Loss: 2.7784 | LR: 4.73e-04 | 7717.2ms/step | 2.6 samples/s
|
| 252 |
+
Step 2390/11700 | Loss: 2.9225 | LR: 4.73e-04 | 7714.4ms/step | 2.6 samples/s
|
| 253 |
+
Step 2400/11700 | Loss: 2.8597 | LR: 4.73e-04 | 7714.1ms/step | 2.6 samples/s
|
| 254 |
+
Step 2410/11700 | Loss: 2.8766 | LR: 4.72e-04 | 7714.1ms/step | 2.6 samples/s
|
| 255 |
+
Step 2420/11700 | Loss: 2.7377 | LR: 4.72e-04 | 7714.1ms/step | 2.6 samples/s
|
| 256 |
+
Step 2430/11700 | Loss: 2.9091 | LR: 4.72e-04 | 7716.4ms/step | 2.6 samples/s
|
| 257 |
+
Step 2440/11700 | Loss: 2.9300 | LR: 4.71e-04 | 7714.6ms/step | 2.6 samples/s
|
| 258 |
+
Step 2450/11700 | Loss: 2.7659 | LR: 4.71e-04 | 7713.3ms/step | 2.6 samples/s
|
| 259 |
+
Step 2460/11700 | Loss: 2.8842 | LR: 4.71e-04 | 7714.5ms/step | 2.6 samples/s
|
| 260 |
+
Step 2470/11700 | Loss: 2.8932 | LR: 4.70e-04 | 7712.5ms/step | 2.6 samples/s
|
| 261 |
+
Step 2480/11700 | Loss: 2.8211 | LR: 4.70e-04 | 7715.6ms/step | 2.6 samples/s
|
| 262 |
+
Step 2490/11700 | Loss: 2.9127 | LR: 4.69e-04 | 7716.2ms/step | 2.6 samples/s
|
| 263 |
+
Step 2500/11700 | Loss: 2.7902 | LR: 4.69e-04 | 7709.2ms/step | 2.6 samples/s
|
| 264 |
+
Validation at step 2500: Loss = 2.8526, Perplexity = 17.33
|
| 265 |
+
Step 2510/11700 | Loss: 2.7599 | LR: 4.69e-04 | 19174.4ms/step | 1.0 samples/s
|
| 266 |
+
Step 2520/11700 | Loss: 2.9161 | LR: 4.68e-04 | 7709.5ms/step | 2.6 samples/s
|
| 267 |
+
Step 2530/11700 | Loss: 2.8117 | LR: 4.68e-04 | 7711.0ms/step | 2.6 samples/s
|
| 268 |
+
Step 2540/11700 | Loss: 2.7472 | LR: 4.68e-04 | 7710.6ms/step | 2.6 samples/s
|
| 269 |
+
Step 2550/11700 | Loss: 2.9180 | LR: 4.67e-04 | 7707.6ms/step | 2.6 samples/s
|
| 270 |
+
Step 2560/11700 | Loss: 2.8611 | LR: 4.67e-04 | 7707.9ms/step | 2.6 samples/s
|
| 271 |
+
Step 2570/11700 | Loss: 2.8233 | LR: 4.67e-04 | 7709.3ms/step | 2.6 samples/s
|
| 272 |
+
Step 2580/11700 | Loss: 2.8447 | LR: 4.66e-04 | 7711.0ms/step | 2.6 samples/s
|
| 273 |
+
Step 2590/11700 | Loss: 2.7847 | LR: 4.66e-04 | 7709.6ms/step | 2.6 samples/s
|
| 274 |
+
Step 2600/11700 | Loss: 2.8124 | LR: 4.66e-04 | 7710.3ms/step | 2.6 samples/s
|
| 275 |
+
Step 2610/11700 | Loss: 2.7369 | LR: 4.65e-04 | 7707.8ms/step | 2.6 samples/s
|
| 276 |
+
Step 2620/11700 | Loss: 2.7982 | LR: 4.65e-04 | 7709.8ms/step | 2.6 samples/s
|
| 277 |
+
Step 2630/11700 | Loss: 2.7409 | LR: 4.65e-04 | 7707.6ms/step | 2.6 samples/s
|
| 278 |
+
Step 2640/11700 | Loss: 2.8477 | LR: 4.64e-04 | 7706.6ms/step | 2.6 samples/s
|
| 279 |
+
Step 2650/11700 | Loss: 2.9116 | LR: 4.64e-04 | 7709.8ms/step | 2.6 samples/s
|
| 280 |
+
Step 2660/11700 | Loss: 2.8430 | LR: 4.63e-04 | 7710.7ms/step | 2.6 samples/s
|
| 281 |
+
Step 2670/11700 | Loss: 2.7835 | LR: 4.63e-04 | 7711.6ms/step | 2.6 samples/s
|
| 282 |
+
Step 2680/11700 | Loss: 2.8270 | LR: 4.63e-04 | 7708.9ms/step | 2.6 samples/s
|
| 283 |
+
Step 2690/11700 | Loss: 2.9312 | LR: 4.62e-04 | 7709.1ms/step | 2.6 samples/s
|
| 284 |
+
Step 2700/11700 | Loss: 2.8220 | LR: 4.62e-04 | 7713.5ms/step | 2.6 samples/s
|
| 285 |
+
Step 2710/11700 | Loss: 2.8186 | LR: 4.62e-04 | 7710.5ms/step | 2.6 samples/s
|
| 286 |
+
Step 2720/11700 | Loss: 2.8269 | LR: 4.61e-04 | 7713.2ms/step | 2.6 samples/s
|
| 287 |
+
Step 2730/11700 | Loss: 2.7283 | LR: 4.61e-04 | 7710.7ms/step | 2.6 samples/s
|
| 288 |
+
Step 2740/11700 | Loss: 2.8155 | LR: 4.60e-04 | 7712.2ms/step | 2.6 samples/s
|
| 289 |
+
Step 2750/11700 | Loss: 2.8666 | LR: 4.60e-04 | 7710.5ms/step | 2.6 samples/s
|
| 290 |
+
Validation at step 2750: Loss = 2.8081, Perplexity = 16.58
|
| 291 |
+
Step 2760/11700 | Loss: 2.6179 | LR: 4.60e-04 | 19185.1ms/step | 1.0 samples/s
|
| 292 |
+
Step 2770/11700 | Loss: 2.7682 | LR: 4.59e-04 | 7711.7ms/step | 2.6 samples/s
|
| 293 |
+
Step 2780/11700 | Loss: 2.7739 | LR: 4.59e-04 | 7708.4ms/step | 2.6 samples/s
|
| 294 |
+
Step 2790/11700 | Loss: 2.8229 | LR: 4.58e-04 | 7711.5ms/step | 2.6 samples/s
|
| 295 |
+
Step 2800/11700 | Loss: 2.7318 | LR: 4.58e-04 | 7709.8ms/step | 2.6 samples/s
|
| 296 |
+
Step 2810/11700 | Loss: 2.8317 | LR: 4.58e-04 | 7709.0ms/step | 2.6 samples/s
|
| 297 |
+
Step 2820/11700 | Loss: 2.8151 | LR: 4.57e-04 | 7708.1ms/step | 2.6 samples/s
|
| 298 |
+
Step 2830/11700 | Loss: 2.7838 | LR: 4.57e-04 | 7710.8ms/step | 2.6 samples/s
|
| 299 |
+
Step 2840/11700 | Loss: 2.8505 | LR: 4.56e-04 | 7709.1ms/step | 2.6 samples/s
|
| 300 |
+
Step 2850/11700 | Loss: 2.8951 | LR: 4.56e-04 | 7711.4ms/step | 2.6 samples/s
|
| 301 |
+
Step 2860/11700 | Loss: 2.7795 | LR: 4.56e-04 | 7709.9ms/step | 2.6 samples/s
|
| 302 |
+
Step 2870/11700 | Loss: 2.8504 | LR: 4.55e-04 | 7708.9ms/step | 2.6 samples/s
|
| 303 |
+
Step 2880/11700 | Loss: 2.5793 | LR: 4.55e-04 | 7708.8ms/step | 2.6 samples/s
|
| 304 |
+
Step 2890/11700 | Loss: 2.8273 | LR: 4.54e-04 | 7709.7ms/step | 2.6 samples/s
|
| 305 |
+
Step 2900/11700 | Loss: 2.7724 | LR: 4.54e-04 | 7708.7ms/step | 2.6 samples/s
|
| 306 |
+
Step 2910/11700 | Loss: 2.8468 | LR: 4.54e-04 | 7711.0ms/step | 2.6 samples/s
|
| 307 |
+
Step 2920/11700 | Loss: 2.7741 | LR: 4.53e-04 | 7712.1ms/step | 2.6 samples/s
|
| 308 |
+
Step 2930/11700 | Loss: 2.8561 | LR: 4.53e-04 | 7709.4ms/step | 2.6 samples/s
|
| 309 |
+
Step 2940/11700 | Loss: 2.7848 | LR: 4.52e-04 | 7707.6ms/step | 2.6 samples/s
|
| 310 |
+
Step 2950/11700 | Loss: 2.8354 | LR: 4.52e-04 | 7709.2ms/step | 2.6 samples/s
|
| 311 |
+
Step 2960/11700 | Loss: 2.7283 | LR: 4.51e-04 | 7705.9ms/step | 2.6 samples/s
|
| 312 |
+
Step 2970/11700 | Loss: 2.6797 | LR: 4.51e-04 | 7707.5ms/step | 2.6 samples/s
|
| 313 |
+
Step 2980/11700 | Loss: 2.7616 | LR: 4.51e-04 | 7701.6ms/step | 2.6 samples/s
|
| 314 |
+
Step 2990/11700 | Loss: 2.6946 | LR: 4.50e-04 | 7706.9ms/step | 2.6 samples/s
|
| 315 |
+
Step 3000/11700 | Loss: 2.7728 | LR: 4.50e-04 | 7709.9ms/step | 2.6 samples/s
|
| 316 |
+
Validation at step 3000: Loss = 2.7761, Perplexity = 16.06
|
| 317 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-3000
|
| 318 |
+
Step 3010/11700 | Loss: 2.7421 | LR: 4.49e-04 | 19559.3ms/step | 1.0 samples/s
|
| 319 |
+
Step 3020/11700 | Loss: 2.8878 | LR: 4.49e-04 | 7715.6ms/step | 2.6 samples/s
|
| 320 |
+
Step 3030/11700 | Loss: 2.7227 | LR: 4.48e-04 | 7719.5ms/step | 2.6 samples/s
|
| 321 |
+
Step 3040/11700 | Loss: 2.8291 | LR: 4.48e-04 | 7718.4ms/step | 2.6 samples/s
|
| 322 |
+
Step 3050/11700 | Loss: 2.7641 | LR: 4.48e-04 | 7715.0ms/step | 2.6 samples/s
|
| 323 |
+
Step 3060/11700 | Loss: 2.9472 | LR: 4.47e-04 | 7716.0ms/step | 2.6 samples/s
|
| 324 |
+
Step 3070/11700 | Loss: 2.7255 | LR: 4.47e-04 | 7714.0ms/step | 2.6 samples/s
|
| 325 |
+
Step 3080/11700 | Loss: 2.7223 | LR: 4.46e-04 | 7712.4ms/step | 2.6 samples/s
|
| 326 |
+
Step 3090/11700 | Loss: 2.7288 | LR: 4.46e-04 | 7714.3ms/step | 2.6 samples/s
|
| 327 |
+
Step 3100/11700 | Loss: 2.8627 | LR: 4.45e-04 | 7713.6ms/step | 2.6 samples/s
|
| 328 |
+
Step 3110/11700 | Loss: 2.6390 | LR: 4.45e-04 | 7716.0ms/step | 2.6 samples/s
|
| 329 |
+
Step 3120/11700 | Loss: 2.7126 | LR: 4.44e-04 | 7715.4ms/step | 2.6 samples/s
|
| 330 |
+
Step 3130/11700 | Loss: 2.7487 | LR: 4.44e-04 | 7714.8ms/step | 2.6 samples/s
|
| 331 |
+
Step 3140/11700 | Loss: 2.7026 | LR: 4.43e-04 | 7715.0ms/step | 2.6 samples/s
|
| 332 |
+
Step 3150/11700 | Loss: 2.7208 | LR: 4.43e-04 | 7713.7ms/step | 2.6 samples/s
|
| 333 |
+
Step 3160/11700 | Loss: 2.7216 | LR: 4.43e-04 | 7710.9ms/step | 2.6 samples/s
|
| 334 |
+
Step 3170/11700 | Loss: 2.8129 | LR: 4.42e-04 | 7709.7ms/step | 2.6 samples/s
|
| 335 |
+
Step 3180/11700 | Loss: 2.7776 | LR: 4.42e-04 | 7712.9ms/step | 2.6 samples/s
|
| 336 |
+
Step 3190/11700 | Loss: 2.6493 | LR: 4.41e-04 | 7716.1ms/step | 2.6 samples/s
|
| 337 |
+
Step 3200/11700 | Loss: 2.6807 | LR: 4.41e-04 | 7716.2ms/step | 2.6 samples/s
|
| 338 |
+
Step 3210/11700 | Loss: 2.7042 | LR: 4.40e-04 | 7718.7ms/step | 2.6 samples/s
|
| 339 |
+
Step 3220/11700 | Loss: 2.7263 | LR: 4.40e-04 | 7717.4ms/step | 2.6 samples/s
|
| 340 |
+
Step 3230/11700 | Loss: 2.6777 | LR: 4.39e-04 | 7717.5ms/step | 2.6 samples/s
|
| 341 |
+
Step 3240/11700 | Loss: 2.7455 | LR: 4.39e-04 | 7714.9ms/step | 2.6 samples/s
|
| 342 |
+
Step 3250/11700 | Loss: 2.8320 | LR: 4.38e-04 | 7714.9ms/step | 2.6 samples/s
|
| 343 |
+
Validation at step 3250: Loss = 2.7376, Perplexity = 15.45
|
| 344 |
+
Step 3260/11700 | Loss: 2.7464 | LR: 4.38e-04 | 19196.9ms/step | 1.0 samples/s
|
| 345 |
+
Step 3270/11700 | Loss: 2.6893 | LR: 4.37e-04 | 7715.5ms/step | 2.6 samples/s
|
| 346 |
+
Step 3280/11700 | Loss: 2.7472 | LR: 4.37e-04 | 7717.2ms/step | 2.6 samples/s
|
| 347 |
+
Step 3290/11700 | Loss: 2.6920 | LR: 4.37e-04 | 7716.4ms/step | 2.6 samples/s
|
| 348 |
+
Step 3300/11700 | Loss: 2.7550 | LR: 4.36e-04 | 7712.9ms/step | 2.6 samples/s
|
| 349 |
+
Step 3310/11700 | Loss: 2.7569 | LR: 4.36e-04 | 7714.5ms/step | 2.6 samples/s
|
| 350 |
+
Step 3320/11700 | Loss: 2.6759 | LR: 4.35e-04 | 7713.9ms/step | 2.6 samples/s
|
| 351 |
+
Step 3330/11700 | Loss: 2.6630 | LR: 4.35e-04 | 7715.6ms/step | 2.6 samples/s
|
| 352 |
+
Step 3340/11700 | Loss: 2.7988 | LR: 4.34e-04 | 7717.3ms/step | 2.6 samples/s
|
| 353 |
+
Step 3350/11700 | Loss: 2.6992 | LR: 4.34e-04 | 7713.1ms/step | 2.6 samples/s
|
| 354 |
+
Step 3360/11700 | Loss: 2.7821 | LR: 4.33e-04 | 7716.3ms/step | 2.6 samples/s
|
| 355 |
+
Step 3370/11700 | Loss: 2.6599 | LR: 4.33e-04 | 7715.0ms/step | 2.6 samples/s
|
| 356 |
+
Step 3380/11700 | Loss: 2.7246 | LR: 4.32e-04 | 7718.4ms/step | 2.6 samples/s
|
| 357 |
+
Step 3390/11700 | Loss: 2.6867 | LR: 4.32e-04 | 7720.8ms/step | 2.6 samples/s
|
| 358 |
+
Step 3400/11700 | Loss: 2.8002 | LR: 4.31e-04 | 7720.5ms/step | 2.6 samples/s
|
| 359 |
+
Step 3410/11700 | Loss: 2.7407 | LR: 4.31e-04 | 7722.9ms/step | 2.6 samples/s
|
| 360 |
+
Step 3420/11700 | Loss: 2.5686 | LR: 4.30e-04 | 7716.7ms/step | 2.6 samples/s
|
| 361 |
+
Step 3430/11700 | Loss: 2.7545 | LR: 4.30e-04 | 7718.9ms/step | 2.6 samples/s
|
| 362 |
+
Step 3440/11700 | Loss: 2.6100 | LR: 4.29e-04 | 7716.3ms/step | 2.6 samples/s
|
| 363 |
+
Step 3450/11700 | Loss: 2.7223 | LR: 4.29e-04 | 7718.4ms/step | 2.6 samples/s
|
| 364 |
+
Step 3460/11700 | Loss: 2.7068 | LR: 4.28e-04 | 7718.4ms/step | 2.6 samples/s
|
| 365 |
+
Step 3470/11700 | Loss: 2.5955 | LR: 4.28e-04 | 7717.0ms/step | 2.6 samples/s
|
| 366 |
+
Step 3480/11700 | Loss: 2.5179 | LR: 4.27e-04 | 7718.1ms/step | 2.6 samples/s
|
| 367 |
+
Step 3490/11700 | Loss: 2.8472 | LR: 4.27e-04 | 7715.2ms/step | 2.6 samples/s
|
| 368 |
+
Step 3500/11700 | Loss: 2.7590 | LR: 4.26e-04 | 7713.3ms/step | 2.6 samples/s
|
| 369 |
+
Validation at step 3500: Loss = 2.7072, Perplexity = 14.99
|
| 370 |
+
Step 3510/11700 | Loss: 2.7364 | LR: 4.26e-04 | 19196.4ms/step | 1.0 samples/s
|
| 371 |
+
Step 3520/11700 | Loss: 2.7248 | LR: 4.25e-04 | 7715.7ms/step | 2.6 samples/s
|
| 372 |
+
Step 3530/11700 | Loss: 2.7368 | LR: 4.25e-04 | 7713.1ms/step | 2.6 samples/s
|
| 373 |
+
Step 3540/11700 | Loss: 2.7452 | LR: 4.24e-04 | 7709.6ms/step | 2.6 samples/s
|
| 374 |
+
Step 3550/11700 | Loss: 2.6640 | LR: 4.24e-04 | 7715.0ms/step | 2.6 samples/s
|
| 375 |
+
Step 3560/11700 | Loss: 2.8521 | LR: 4.23e-04 | 7716.5ms/step | 2.6 samples/s
|
| 376 |
+
Step 3570/11700 | Loss: 2.7210 | LR: 4.23e-04 | 7719.2ms/step | 2.6 samples/s
|
| 377 |
+
Step 3580/11700 | Loss: 2.7349 | LR: 4.22e-04 | 7714.2ms/step | 2.6 samples/s
|
| 378 |
+
Step 3590/11700 | Loss: 2.6364 | LR: 4.22e-04 | 7716.7ms/step | 2.6 samples/s
|
| 379 |
+
Step 3600/11700 | Loss: 2.6346 | LR: 4.21e-04 | 7719.1ms/step | 2.6 samples/s
|
| 380 |
+
Step 3610/11700 | Loss: 2.7654 | LR: 4.20e-04 | 7719.9ms/step | 2.6 samples/s
|
| 381 |
+
Step 3620/11700 | Loss: 2.6394 | LR: 4.20e-04 | 7718.9ms/step | 2.6 samples/s
|
| 382 |
+
Step 3630/11700 | Loss: 2.6439 | LR: 4.19e-04 | 7717.8ms/step | 2.6 samples/s
|
| 383 |
+
Step 3640/11700 | Loss: 2.5920 | LR: 4.19e-04 | 7718.4ms/step | 2.6 samples/s
|
| 384 |
+
Step 3650/11700 | Loss: 2.7211 | LR: 4.18e-04 | 7719.4ms/step | 2.6 samples/s
|
| 385 |
+
Step 3660/11700 | Loss: 2.6780 | LR: 4.18e-04 | 7716.8ms/step | 2.6 samples/s
|
| 386 |
+
Step 3670/11700 | Loss: 2.6334 | LR: 4.17e-04 | 7715.9ms/step | 2.6 samples/s
|
| 387 |
+
Step 3680/11700 | Loss: 2.7337 | LR: 4.17e-04 | 7717.3ms/step | 2.6 samples/s
|
| 388 |
+
Step 3690/11700 | Loss: 2.6729 | LR: 4.16e-04 | 7721.8ms/step | 2.6 samples/s
|
| 389 |
+
Step 3700/11700 | Loss: 2.7145 | LR: 4.16e-04 | 7720.0ms/step | 2.6 samples/s
|
| 390 |
+
Step 3710/11700 | Loss: 2.6142 | LR: 4.15e-04 | 7718.6ms/step | 2.6 samples/s
|
| 391 |
+
Step 3720/11700 | Loss: 2.6243 | LR: 4.15e-04 | 7718.3ms/step | 2.6 samples/s
|
| 392 |
+
Step 3730/11700 | Loss: 2.6864 | LR: 4.14e-04 | 7716.6ms/step | 2.6 samples/s
|
| 393 |
+
Step 3740/11700 | Loss: 2.6381 | LR: 4.14e-04 | 7715.8ms/step | 2.6 samples/s
|
| 394 |
+
Step 3750/11700 | Loss: 2.7138 | LR: 4.13e-04 | 7714.8ms/step | 2.6 samples/s
|
| 395 |
+
Validation at step 3750: Loss = 2.6806, Perplexity = 14.59
|
| 396 |
+
Step 3760/11700 | Loss: 2.7229 | LR: 4.12e-04 | 19210.5ms/step | 1.0 samples/s
|
| 397 |
+
Step 3770/11700 | Loss: 2.7157 | LR: 4.12e-04 | 7721.9ms/step | 2.6 samples/s
|
| 398 |
+
Step 3780/11700 | Loss: 2.6655 | LR: 4.11e-04 | 7717.9ms/step | 2.6 samples/s
|
| 399 |
+
Step 3790/11700 | Loss: 2.8069 | LR: 4.11e-04 | 7715.5ms/step | 2.6 samples/s
|
| 400 |
+
Step 3800/11700 | Loss: 2.6959 | LR: 4.10e-04 | 7717.5ms/step | 2.6 samples/s
|
| 401 |
+
Step 3810/11700 | Loss: 2.6540 | LR: 4.10e-04 | 7714.1ms/step | 2.6 samples/s
|
| 402 |
+
Step 3820/11700 | Loss: 2.6414 | LR: 4.09e-04 | 7714.1ms/step | 2.6 samples/s
|
| 403 |
+
Step 3830/11700 | Loss: 2.6643 | LR: 4.09e-04 | 7716.2ms/step | 2.6 samples/s
|
| 404 |
+
Step 3840/11700 | Loss: 2.5925 | LR: 4.08e-04 | 7714.2ms/step | 2.6 samples/s
|
| 405 |
+
Step 3850/11700 | Loss: 2.7099 | LR: 4.07e-04 | 7714.6ms/step | 2.6 samples/s
|
| 406 |
+
Step 3860/11700 | Loss: 2.6287 | LR: 4.07e-04 | 7716.2ms/step | 2.6 samples/s
|
| 407 |
+
Step 3870/11700 | Loss: 2.6607 | LR: 4.06e-04 | 7714.3ms/step | 2.6 samples/s
|
| 408 |
+
Step 3880/11700 | Loss: 2.5449 | LR: 4.06e-04 | 7716.6ms/step | 2.6 samples/s
|
| 409 |
+
Step 3890/11700 | Loss: 2.7174 | LR: 4.05e-04 | 7715.7ms/step | 2.6 samples/s
|
| 410 |
+
Step 3900/11700 | Loss: 2.5914 | LR: 4.05e-04 | 7714.8ms/step | 2.6 samples/s
|
| 411 |
+
Step 3910/11700 | Loss: 2.6895 | LR: 4.04e-04 | 7714.2ms/step | 2.6 samples/s
|
| 412 |
+
Step 3920/11700 | Loss: 2.5903 | LR: 4.04e-04 | 7717.0ms/step | 2.6 samples/s
|
| 413 |
+
Step 3930/11700 | Loss: 2.6477 | LR: 4.03e-04 | 7719.8ms/step | 2.6 samples/s
|
| 414 |
+
Step 3940/11700 | Loss: 2.7823 | LR: 4.02e-04 | 7719.7ms/step | 2.6 samples/s
|
| 415 |
+
Step 3950/11700 | Loss: 2.6126 | LR: 4.02e-04 | 7715.8ms/step | 2.6 samples/s
|
| 416 |
+
Step 3960/11700 | Loss: 2.6623 | LR: 4.01e-04 | 7713.7ms/step | 2.6 samples/s
|
| 417 |
+
Step 3970/11700 | Loss: 2.6440 | LR: 4.01e-04 | 7706.2ms/step | 2.6 samples/s
|
| 418 |
+
Step 3980/11700 | Loss: 2.7042 | LR: 4.00e-04 | 7715.6ms/step | 2.6 samples/s
|
| 419 |
+
Step 3990/11700 | Loss: 2.6852 | LR: 4.00e-04 | 7715.8ms/step | 2.6 samples/s
|
| 420 |
+
Step 4000/11700 | Loss: 2.7364 | LR: 3.99e-04 | 7714.0ms/step | 2.6 samples/s
|
| 421 |
+
Validation at step 4000: Loss = 2.6616, Perplexity = 14.32
|
| 422 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-4000
|
| 423 |
+
Step 4010/11700 | Loss: 2.5560 | LR: 3.98e-04 | 19524.0ms/step | 1.0 samples/s
|
| 424 |
+
Step 4020/11700 | Loss: 2.6539 | LR: 3.98e-04 | 7716.0ms/step | 2.6 samples/s
|
| 425 |
+
Step 4030/11700 | Loss: 2.5463 | LR: 3.97e-04 | 7718.2ms/step | 2.6 samples/s
|
| 426 |
+
Step 4040/11700 | Loss: 2.6863 | LR: 3.97e-04 | 7717.0ms/step | 2.6 samples/s
|
| 427 |
+
Step 4050/11700 | Loss: 2.6830 | LR: 3.96e-04 | 7712.9ms/step | 2.6 samples/s
|
| 428 |
+
Step 4060/11700 | Loss: 2.5578 | LR: 3.95e-04 | 7714.1ms/step | 2.6 samples/s
|
| 429 |
+
Step 4070/11700 | Loss: 2.7375 | LR: 3.95e-04 | 7715.8ms/step | 2.6 samples/s
|
| 430 |
+
Step 4080/11700 | Loss: 2.6267 | LR: 3.94e-04 | 7716.2ms/step | 2.6 samples/s
|
| 431 |
+
Step 4090/11700 | Loss: 2.6041 | LR: 3.94e-04 | 7717.3ms/step | 2.6 samples/s
|
| 432 |
+
Step 4100/11700 | Loss: 2.6523 | LR: 3.93e-04 | 7720.0ms/step | 2.6 samples/s
|
| 433 |
+
Step 4110/11700 | Loss: 2.6349 | LR: 3.93e-04 | 7719.6ms/step | 2.6 samples/s
|
| 434 |
+
Step 4120/11700 | Loss: 2.6988 | LR: 3.92e-04 | 7718.9ms/step | 2.6 samples/s
|
| 435 |
+
Step 4130/11700 | Loss: 2.5694 | LR: 3.91e-04 | 7711.8ms/step | 2.6 samples/s
|
| 436 |
+
Step 4140/11700 | Loss: 2.6197 | LR: 3.91e-04 | 7716.2ms/step | 2.6 samples/s
|
| 437 |
+
Step 4150/11700 | Loss: 2.5599 | LR: 3.90e-04 | 7718.1ms/step | 2.6 samples/s
|
| 438 |
+
Step 4160/11700 | Loss: 2.6303 | LR: 3.90e-04 | 7720.3ms/step | 2.6 samples/s
|
| 439 |
+
Step 4170/11700 | Loss: 2.6883 | LR: 3.89e-04 | 7719.4ms/step | 2.6 samples/s
|
| 440 |
+
Step 4180/11700 | Loss: 2.6264 | LR: 3.88e-04 | 7719.6ms/step | 2.6 samples/s
|
| 441 |
+
Step 4190/11700 | Loss: 2.6592 | LR: 3.88e-04 | 7720.7ms/step | 2.6 samples/s
|
| 442 |
+
Step 4200/11700 | Loss: 2.5726 | LR: 3.87e-04 | 7721.4ms/step | 2.6 samples/s
|
| 443 |
+
Step 4210/11700 | Loss: 2.6774 | LR: 3.87e-04 | 7719.0ms/step | 2.6 samples/s
|
| 444 |
+
Step 4220/11700 | Loss: 2.6415 | LR: 3.86e-04 | 7719.0ms/step | 2.6 samples/s
|
| 445 |
+
Step 4230/11700 | Loss: 2.5621 | LR: 3.85e-04 | 7720.1ms/step | 2.6 samples/s
|
| 446 |
+
Step 4240/11700 | Loss: 2.6710 | LR: 3.85e-04 | 7717.9ms/step | 2.6 samples/s
|
| 447 |
+
Step 4250/11700 | Loss: 2.7735 | LR: 3.84e-04 | 7717.6ms/step | 2.6 samples/s
|
| 448 |
+
Validation at step 4250: Loss = 2.6440, Perplexity = 14.07
|
| 449 |
+
Step 4260/11700 | Loss: 2.7399 | LR: 3.84e-04 | 19205.9ms/step | 1.0 samples/s
|
| 450 |
+
Step 4270/11700 | Loss: 2.5886 | LR: 3.83e-04 | 7716.9ms/step | 2.6 samples/s
|
| 451 |
+
Step 4280/11700 | Loss: 2.6208 | LR: 3.82e-04 | 7717.8ms/step | 2.6 samples/s
|
| 452 |
+
Step 4290/11700 | Loss: 2.6278 | LR: 3.82e-04 | 7715.2ms/step | 2.6 samples/s
|
| 453 |
+
Step 4300/11700 | Loss: 2.6819 | LR: 3.81e-04 | 7709.7ms/step | 2.6 samples/s
|
| 454 |
+
Step 4310/11700 | Loss: 2.6059 | LR: 3.81e-04 | 7711.3ms/step | 2.6 samples/s
|
| 455 |
+
Step 4320/11700 | Loss: 2.5810 | LR: 3.80e-04 | 7714.6ms/step | 2.6 samples/s
|
| 456 |
+
Step 4330/11700 | Loss: 2.5922 | LR: 3.79e-04 | 7716.8ms/step | 2.6 samples/s
|
| 457 |
+
Step 4340/11700 | Loss: 2.6615 | LR: 3.79e-04 | 7715.8ms/step | 2.6 samples/s
|
| 458 |
+
Step 4350/11700 | Loss: 2.7378 | LR: 3.78e-04 | 7716.1ms/step | 2.6 samples/s
|
| 459 |
+
Step 4360/11700 | Loss: 2.5772 | LR: 3.77e-04 | 7715.6ms/step | 2.6 samples/s
|
| 460 |
+
Step 4370/11700 | Loss: 2.6086 | LR: 3.77e-04 | 7718.3ms/step | 2.6 samples/s
|
| 461 |
+
Step 4380/11700 | Loss: 2.5684 | LR: 3.76e-04 | 7716.2ms/step | 2.6 samples/s
|
| 462 |
+
Step 4390/11700 | Loss: 2.6421 | LR: 3.76e-04 | 7715.9ms/step | 2.6 samples/s
|
| 463 |
+
Step 4400/11700 | Loss: 2.6288 | LR: 3.75e-04 | 7716.1ms/step | 2.6 samples/s
|
| 464 |
+
Step 4410/11700 | Loss: 2.6481 | LR: 3.74e-04 | 7713.2ms/step | 2.6 samples/s
|
| 465 |
+
Step 4420/11700 | Loss: 2.6357 | LR: 3.74e-04 | 7713.4ms/step | 2.6 samples/s
|
| 466 |
+
Step 4430/11700 | Loss: 2.6620 | LR: 3.73e-04 | 7715.0ms/step | 2.6 samples/s
|
| 467 |
+
Step 4440/11700 | Loss: 2.6645 | LR: 3.73e-04 | 7714.9ms/step | 2.6 samples/s
|
| 468 |
+
Step 4450/11700 | Loss: 2.5937 | LR: 3.72e-04 | 7711.8ms/step | 2.6 samples/s
|
| 469 |
+
Step 4460/11700 | Loss: 2.7165 | LR: 3.71e-04 | 7716.8ms/step | 2.6 samples/s
|
| 470 |
+
Step 4470/11700 | Loss: 2.5258 | LR: 3.71e-04 | 7715.7ms/step | 2.6 samples/s
|
| 471 |
+
Step 4480/11700 | Loss: 2.5993 | LR: 3.70e-04 | 7713.2ms/step | 2.6 samples/s
|
| 472 |
+
Step 4490/11700 | Loss: 2.6097 | LR: 3.69e-04 | 7714.6ms/step | 2.6 samples/s
|
| 473 |
+
Step 4500/11700 | Loss: 2.6327 | LR: 3.69e-04 | 7711.7ms/step | 2.6 samples/s
|
| 474 |
+
Validation at step 4500: Loss = 2.6283, Perplexity = 13.85
|
| 475 |
+
Step 4510/11700 | Loss: 2.5235 | LR: 3.68e-04 | 19201.2ms/step | 1.0 samples/s
|
| 476 |
+
Step 4520/11700 | Loss: 2.6448 | LR: 3.67e-04 | 7720.6ms/step | 2.6 samples/s
|
| 477 |
+
Step 4530/11700 | Loss: 2.7404 | LR: 3.67e-04 | 7717.3ms/step | 2.6 samples/s
|
| 478 |
+
Step 4540/11700 | Loss: 2.6650 | LR: 3.66e-04 | 7715.7ms/step | 2.6 samples/s
|
| 479 |
+
Step 4550/11700 | Loss: 2.5984 | LR: 3.66e-04 | 7718.0ms/step | 2.6 samples/s
|
| 480 |
+
Step 4560/11700 | Loss: 2.5697 | LR: 3.65e-04 | 7720.0ms/step | 2.6 samples/s
|
| 481 |
+
Step 4570/11700 | Loss: 2.5925 | LR: 3.64e-04 | 7720.2ms/step | 2.6 samples/s
|
| 482 |
+
Step 4580/11700 | Loss: 2.5609 | LR: 3.64e-04 | 7718.0ms/step | 2.6 samples/s
|
| 483 |
+
Step 4590/11700 | Loss: 2.7103 | LR: 3.63e-04 | 7719.2ms/step | 2.6 samples/s
|
| 484 |
+
Step 4600/11700 | Loss: 2.4534 | LR: 3.62e-04 | 7721.0ms/step | 2.6 samples/s
|
| 485 |
+
Step 4610/11700 | Loss: 2.5077 | LR: 3.62e-04 | 7724.6ms/step | 2.6 samples/s
|
| 486 |
+
Step 4620/11700 | Loss: 2.6173 | LR: 3.61e-04 | 7722.3ms/step | 2.6 samples/s
|
| 487 |
+
Step 4630/11700 | Loss: 2.6259 | LR: 3.60e-04 | 7719.0ms/step | 2.6 samples/s
|
| 488 |
+
Step 4640/11700 | Loss: 2.5724 | LR: 3.60e-04 | 7736.0ms/step | 2.6 samples/s
|
| 489 |
+
Step 4650/11700 | Loss: 2.5720 | LR: 3.59e-04 | 7721.5ms/step | 2.6 samples/s
|
| 490 |
+
Step 4660/11700 | Loss: 2.6425 | LR: 3.59e-04 | 7716.2ms/step | 2.6 samples/s
|
| 491 |
+
Step 4670/11700 | Loss: 2.5621 | LR: 3.58e-04 | 7720.7ms/step | 2.6 samples/s
|
| 492 |
+
Step 4680/11700 | Loss: 2.6481 | LR: 3.57e-04 | 7716.0ms/step | 2.6 samples/s
|
| 493 |
+
Step 4690/11700 | Loss: 2.5433 | LR: 3.57e-04 | 7718.8ms/step | 2.6 samples/s
|
| 494 |
+
Step 4700/11700 | Loss: 2.5439 | LR: 3.56e-04 | 7719.8ms/step | 2.6 samples/s
|
| 495 |
+
Step 4710/11700 | Loss: 2.7022 | LR: 3.55e-04 | 7721.1ms/step | 2.6 samples/s
|
| 496 |
+
Step 4720/11700 | Loss: 2.5688 | LR: 3.55e-04 | 7720.1ms/step | 2.6 samples/s
|
| 497 |
+
Step 4730/11700 | Loss: 2.6315 | LR: 3.54e-04 | 7718.1ms/step | 2.6 samples/s
|
| 498 |
+
Step 4740/11700 | Loss: 2.6422 | LR: 3.53e-04 | 7719.1ms/step | 2.6 samples/s
|
| 499 |
+
Step 4750/11700 | Loss: 2.5868 | LR: 3.53e-04 | 7720.2ms/step | 2.6 samples/s
|
| 500 |
+
Validation at step 4750: Loss = 2.6047, Perplexity = 13.53
|
| 501 |
+
Step 4760/11700 | Loss: 2.6808 | LR: 3.52e-04 | 19215.9ms/step | 1.0 samples/s
|
| 502 |
+
Step 4770/11700 | Loss: 2.6380 | LR: 3.51e-04 | 7722.9ms/step | 2.6 samples/s
|
| 503 |
+
Step 4780/11700 | Loss: 2.6041 | LR: 3.51e-04 | 7718.5ms/step | 2.6 samples/s
|
| 504 |
+
Step 4790/11700 | Loss: 2.5810 | LR: 3.50e-04 | 7721.4ms/step | 2.6 samples/s
|
| 505 |
+
Step 4800/11700 | Loss: 2.5217 | LR: 3.49e-04 | 7724.4ms/step | 2.6 samples/s
|
| 506 |
+
Step 4810/11700 | Loss: 2.4948 | LR: 3.49e-04 | 7720.1ms/step | 2.6 samples/s
|
| 507 |
+
Step 4820/11700 | Loss: 2.5961 | LR: 3.48e-04 | 7724.8ms/step | 2.6 samples/s
|
| 508 |
+
Step 4830/11700 | Loss: 2.6159 | LR: 3.47e-04 | 7721.7ms/step | 2.6 samples/s
|
| 509 |
+
Step 4840/11700 | Loss: 2.6119 | LR: 3.47e-04 | 7722.0ms/step | 2.6 samples/s
|
| 510 |
+
Step 4850/11700 | Loss: 2.6202 | LR: 3.46e-04 | 7723.8ms/step | 2.6 samples/s
|
| 511 |
+
Step 4860/11700 | Loss: 2.5640 | LR: 3.45e-04 | 7726.6ms/step | 2.6 samples/s
|
| 512 |
+
Step 4870/11700 | Loss: 2.6902 | LR: 3.45e-04 | 7718.4ms/step | 2.6 samples/s
|
| 513 |
+
Step 4880/11700 | Loss: 2.6203 | LR: 3.44e-04 | 7720.0ms/step | 2.6 samples/s
|
| 514 |
+
Step 4890/11700 | Loss: 2.6811 | LR: 3.43e-04 | 7721.3ms/step | 2.6 samples/s
|
| 515 |
+
Step 4900/11700 | Loss: 2.6195 | LR: 3.43e-04 | 7720.2ms/step | 2.6 samples/s
|
| 516 |
+
Step 4910/11700 | Loss: 2.5678 | LR: 3.42e-04 | 7721.4ms/step | 2.6 samples/s
|
| 517 |
+
Step 4920/11700 | Loss: 2.5402 | LR: 3.41e-04 | 7722.3ms/step | 2.6 samples/s
|
| 518 |
+
Step 4930/11700 | Loss: 2.5083 | LR: 3.41e-04 | 7721.9ms/step | 2.6 samples/s
|
| 519 |
+
Step 4940/11700 | Loss: 2.6342 | LR: 3.40e-04 | 7721.2ms/step | 2.6 samples/s
|
| 520 |
+
Step 4950/11700 | Loss: 2.4520 | LR: 3.39e-04 | 7710.2ms/step | 2.6 samples/s
|
| 521 |
+
Step 4960/11700 | Loss: 2.6156 | LR: 3.39e-04 | 7714.1ms/step | 2.6 samples/s
|
| 522 |
+
Step 4970/11700 | Loss: 2.6009 | LR: 3.38e-04 | 7720.1ms/step | 2.6 samples/s
|
| 523 |
+
Step 4980/11700 | Loss: 2.7116 | LR: 3.37e-04 | 7722.6ms/step | 2.6 samples/s
|
| 524 |
+
Step 4990/11700 | Loss: 2.5289 | LR: 3.37e-04 | 7722.9ms/step | 2.6 samples/s
|
| 525 |
+
Step 5000/11700 | Loss: 2.6904 | LR: 3.36e-04 | 7722.5ms/step | 2.6 samples/s
|
| 526 |
+
Validation at step 5000: Loss = 2.5882, Perplexity = 13.31
|
| 527 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-5000
|
| 528 |
+
Step 5010/11700 | Loss: 2.6925 | LR: 3.35e-04 | 19560.1ms/step | 1.0 samples/s
|
| 529 |
+
Step 5020/11700 | Loss: 2.6726 | LR: 3.35e-04 | 7719.7ms/step | 2.6 samples/s
|
| 530 |
+
Step 5030/11700 | Loss: 2.6659 | LR: 3.34e-04 | 7722.0ms/step | 2.6 samples/s
|
| 531 |
+
Step 5040/11700 | Loss: 2.4974 | LR: 3.33e-04 | 7720.9ms/step | 2.6 samples/s
|
| 532 |
+
Step 5050/11700 | Loss: 2.4383 | LR: 3.33e-04 | 7715.2ms/step | 2.6 samples/s
|
| 533 |
+
Step 5060/11700 | Loss: 2.6081 | LR: 3.32e-04 | 7712.2ms/step | 2.6 samples/s
|
| 534 |
+
Step 5070/11700 | Loss: 2.5672 | LR: 3.31e-04 | 7720.7ms/step | 2.6 samples/s
|
| 535 |
+
Step 5080/11700 | Loss: 2.7004 | LR: 3.31e-04 | 7720.3ms/step | 2.6 samples/s
|
| 536 |
+
Step 5090/11700 | Loss: 2.6020 | LR: 3.30e-04 | 7723.4ms/step | 2.6 samples/s
|
| 537 |
+
Step 5100/11700 | Loss: 2.6994 | LR: 3.29e-04 | 7720.8ms/step | 2.6 samples/s
|
| 538 |
+
Step 5110/11700 | Loss: 2.6000 | LR: 3.29e-04 | 7720.7ms/step | 2.6 samples/s
|
| 539 |
+
Step 5120/11700 | Loss: 2.5516 | LR: 3.28e-04 | 7720.7ms/step | 2.6 samples/s
|
| 540 |
+
Step 5130/11700 | Loss: 2.4659 | LR: 3.27e-04 | 7721.8ms/step | 2.6 samples/s
|
| 541 |
+
Step 5140/11700 | Loss: 2.5177 | LR: 3.27e-04 | 7721.9ms/step | 2.6 samples/s
|
| 542 |
+
Step 5150/11700 | Loss: 2.5863 | LR: 3.26e-04 | 7724.0ms/step | 2.6 samples/s
|
| 543 |
+
Step 5160/11700 | Loss: 2.5313 | LR: 3.25e-04 | 7722.3ms/step | 2.6 samples/s
|
| 544 |
+
Step 5170/11700 | Loss: 2.5376 | LR: 3.25e-04 | 7719.3ms/step | 2.6 samples/s
|
| 545 |
+
Step 5180/11700 | Loss: 2.5189 | LR: 3.24e-04 | 7718.1ms/step | 2.6 samples/s
|
| 546 |
+
Step 5190/11700 | Loss: 2.5913 | LR: 3.23e-04 | 7724.2ms/step | 2.6 samples/s
|
| 547 |
+
Step 5200/11700 | Loss: 2.4286 | LR: 3.22e-04 | 7722.4ms/step | 2.6 samples/s
|
| 548 |
+
Step 5210/11700 | Loss: 2.5437 | LR: 3.22e-04 | 7726.2ms/step | 2.6 samples/s
|
| 549 |
+
Step 5220/11700 | Loss: 2.5985 | LR: 3.21e-04 | 7722.8ms/step | 2.6 samples/s
|
| 550 |
+
Step 5230/11700 | Loss: 2.6746 | LR: 3.20e-04 | 7724.9ms/step | 2.6 samples/s
|
| 551 |
+
Step 5240/11700 | Loss: 2.5506 | LR: 3.20e-04 | 7723.8ms/step | 2.6 samples/s
|
| 552 |
+
Step 5250/11700 | Loss: 2.5896 | LR: 3.19e-04 | 7724.0ms/step | 2.6 samples/s
|
| 553 |
+
Validation at step 5250: Loss = 2.5714, Perplexity = 13.08
|
| 554 |
+
Step 5260/11700 | Loss: 2.5822 | LR: 3.18e-04 | 19214.3ms/step | 1.0 samples/s
|
| 555 |
+
Step 5270/11700 | Loss: 2.5612 | LR: 3.18e-04 | 7722.1ms/step | 2.6 samples/s
|
| 556 |
+
Step 5280/11700 | Loss: 2.5177 | LR: 3.17e-04 | 7725.0ms/step | 2.6 samples/s
|
| 557 |
+
Step 5290/11700 | Loss: 2.5240 | LR: 3.16e-04 | 7719.2ms/step | 2.6 samples/s
|
| 558 |
+
Step 5300/11700 | Loss: 2.5436 | LR: 3.16e-04 | 7718.1ms/step | 2.6 samples/s
|
| 559 |
+
Step 5310/11700 | Loss: 2.5354 | LR: 3.15e-04 | 7720.3ms/step | 2.6 samples/s
|
| 560 |
+
Step 5320/11700 | Loss: 2.5096 | LR: 3.14e-04 | 7719.6ms/step | 2.6 samples/s
|
| 561 |
+
Step 5330/11700 | Loss: 2.4967 | LR: 3.13e-04 | 7719.3ms/step | 2.6 samples/s
|
| 562 |
+
Step 5340/11700 | Loss: 2.5308 | LR: 3.13e-04 | 7717.0ms/step | 2.6 samples/s
|
| 563 |
+
Step 5350/11700 | Loss: 2.5280 | LR: 3.12e-04 | 7722.9ms/step | 2.6 samples/s
|
| 564 |
+
Step 5360/11700 | Loss: 2.4557 | LR: 3.11e-04 | 7716.3ms/step | 2.6 samples/s
|
| 565 |
+
Step 5370/11700 | Loss: 2.5629 | LR: 3.11e-04 | 7720.4ms/step | 2.6 samples/s
|
| 566 |
+
Step 5380/11700 | Loss: 2.4602 | LR: 3.10e-04 | 7720.4ms/step | 2.6 samples/s
|
| 567 |
+
Step 5390/11700 | Loss: 2.5996 | LR: 3.09e-04 | 7720.2ms/step | 2.6 samples/s
|
| 568 |
+
Step 5400/11700 | Loss: 2.5090 | LR: 3.09e-04 | 7721.1ms/step | 2.6 samples/s
|
| 569 |
+
Step 5410/11700 | Loss: 2.4930 | LR: 3.08e-04 | 7719.7ms/step | 2.6 samples/s
|
| 570 |
+
Step 5420/11700 | Loss: 2.5601 | LR: 3.07e-04 | 7717.7ms/step | 2.6 samples/s
|
| 571 |
+
Step 5430/11700 | Loss: 2.6332 | LR: 3.07e-04 | 7715.9ms/step | 2.6 samples/s
|
| 572 |
+
Step 5440/11700 | Loss: 2.6383 | LR: 3.06e-04 | 7716.0ms/step | 2.6 samples/s
|
| 573 |
+
Step 5450/11700 | Loss: 2.5228 | LR: 3.05e-04 | 7718.5ms/step | 2.6 samples/s
|
| 574 |
+
Step 5460/11700 | Loss: 2.5616 | LR: 3.04e-04 | 7720.4ms/step | 2.6 samples/s
|
| 575 |
+
Step 5470/11700 | Loss: 2.4258 | LR: 3.04e-04 | 7719.5ms/step | 2.6 samples/s
|
| 576 |
+
Step 5480/11700 | Loss: 2.5287 | LR: 3.03e-04 | 7722.0ms/step | 2.6 samples/s
|
| 577 |
+
Step 5490/11700 | Loss: 2.6934 | LR: 3.02e-04 | 7717.7ms/step | 2.6 samples/s
|
| 578 |
+
Step 5500/11700 | Loss: 2.5979 | LR: 3.02e-04 | 7720.3ms/step | 2.6 samples/s
|
| 579 |
+
Validation at step 5500: Loss = 2.5559, Perplexity = 12.88
|
| 580 |
+
Step 5510/11700 | Loss: 2.4350 | LR: 3.01e-04 | 19206.0ms/step | 1.0 samples/s
|
| 581 |
+
Step 5520/11700 | Loss: 2.4161 | LR: 3.00e-04 | 7721.8ms/step | 2.6 samples/s
|
| 582 |
+
Step 5530/11700 | Loss: 2.5737 | LR: 3.00e-04 | 7720.8ms/step | 2.6 samples/s
|
| 583 |
+
Step 5540/11700 | Loss: 2.5536 | LR: 2.99e-04 | 7719.4ms/step | 2.6 samples/s
|
| 584 |
+
Step 5550/11700 | Loss: 2.5611 | LR: 2.98e-04 | 7718.5ms/step | 2.6 samples/s
|
| 585 |
+
Step 5560/11700 | Loss: 2.6005 | LR: 2.97e-04 | 7717.0ms/step | 2.6 samples/s
|
| 586 |
+
Step 5570/11700 | Loss: 2.4746 | LR: 2.97e-04 | 7721.7ms/step | 2.6 samples/s
|
| 587 |
+
Step 5580/11700 | Loss: 2.5653 | LR: 2.96e-04 | 7720.9ms/step | 2.6 samples/s
|
| 588 |
+
Step 5590/11700 | Loss: 2.5913 | LR: 2.95e-04 | 7721.9ms/step | 2.6 samples/s
|
| 589 |
+
Step 5600/11700 | Loss: 2.5367 | LR: 2.95e-04 | 7721.5ms/step | 2.6 samples/s
|
| 590 |
+
Step 5610/11700 | Loss: 2.3941 | LR: 2.94e-04 | 7722.8ms/step | 2.6 samples/s
|
| 591 |
+
Step 5620/11700 | Loss: 2.5537 | LR: 2.93e-04 | 7726.3ms/step | 2.6 samples/s
|
| 592 |
+
Step 5630/11700 | Loss: 2.4441 | LR: 2.92e-04 | 7722.4ms/step | 2.6 samples/s
|
| 593 |
+
Step 5640/11700 | Loss: 2.5754 | LR: 2.92e-04 | 7721.3ms/step | 2.6 samples/s
|
| 594 |
+
Step 5650/11700 | Loss: 2.6290 | LR: 2.91e-04 | 7722.2ms/step | 2.6 samples/s
|
| 595 |
+
Step 5660/11700 | Loss: 2.5369 | LR: 2.90e-04 | 7722.2ms/step | 2.6 samples/s
|
| 596 |
+
Step 5670/11700 | Loss: 2.5196 | LR: 2.90e-04 | 7721.1ms/step | 2.6 samples/s
|
| 597 |
+
Step 5680/11700 | Loss: 2.5938 | LR: 2.89e-04 | 7721.6ms/step | 2.6 samples/s
|
| 598 |
+
Step 5690/11700 | Loss: 2.5071 | LR: 2.88e-04 | 7723.6ms/step | 2.6 samples/s
|
| 599 |
+
Step 5700/11700 | Loss: 2.5291 | LR: 2.88e-04 | 7724.4ms/step | 2.6 samples/s
|
| 600 |
+
Step 5710/11700 | Loss: 2.5035 | LR: 2.87e-04 | 7721.2ms/step | 2.6 samples/s
|
| 601 |
+
Step 5720/11700 | Loss: 2.4795 | LR: 2.86e-04 | 7718.9ms/step | 2.6 samples/s
|
| 602 |
+
Step 5730/11700 | Loss: 2.5940 | LR: 2.85e-04 | 7721.3ms/step | 2.6 samples/s
|
| 603 |
+
Step 5740/11700 | Loss: 2.4616 | LR: 2.85e-04 | 7720.2ms/step | 2.6 samples/s
|
| 604 |
+
Step 5750/11700 | Loss: 2.4970 | LR: 2.84e-04 | 7721.9ms/step | 2.6 samples/s
|
| 605 |
+
Validation at step 5750: Loss = 2.5448, Perplexity = 12.74
|
| 606 |
+
Step 5760/11700 | Loss: 2.5357 | LR: 2.83e-04 | 19216.9ms/step | 1.0 samples/s
|
| 607 |
+
Step 5770/11700 | Loss: 2.6631 | LR: 2.83e-04 | 7717.8ms/step | 2.6 samples/s
|
| 608 |
+
Step 5780/11700 | Loss: 2.5033 | LR: 2.82e-04 | 7722.1ms/step | 2.6 samples/s
|
| 609 |
+
Step 5790/11700 | Loss: 2.6306 | LR: 2.81e-04 | 7719.7ms/step | 2.6 samples/s
|
| 610 |
+
Step 5800/11700 | Loss: 2.6255 | LR: 2.80e-04 | 7719.3ms/step | 2.6 samples/s
|
| 611 |
+
Step 5810/11700 | Loss: 2.5706 | LR: 2.80e-04 | 7721.5ms/step | 2.6 samples/s
|
| 612 |
+
Step 5820/11700 | Loss: 2.4624 | LR: 2.79e-04 | 7722.5ms/step | 2.6 samples/s
|
| 613 |
+
Step 5830/11700 | Loss: 2.5124 | LR: 2.78e-04 | 7716.7ms/step | 2.6 samples/s
|
| 614 |
+
Step 5840/11700 | Loss: 2.5031 | LR: 2.78e-04 | 7718.2ms/step | 2.6 samples/s
|
| 615 |
+
Step 5850/11700 | Loss: 2.5103 | LR: 2.77e-04 | 7715.6ms/step | 2.6 samples/s
|
| 616 |
+
Step 5860/11700 | Loss: 2.5819 | LR: 2.76e-04 | 7714.2ms/step | 2.6 samples/s
|
| 617 |
+
Step 5870/11700 | Loss: 2.5923 | LR: 2.75e-04 | 7717.4ms/step | 2.6 samples/s
|
| 618 |
+
Step 5880/11700 | Loss: 2.5760 | LR: 2.75e-04 | 7712.9ms/step | 2.6 samples/s
|
| 619 |
+
Step 5890/11700 | Loss: 2.5122 | LR: 2.74e-04 | 7716.2ms/step | 2.6 samples/s
|
| 620 |
+
Step 5900/11700 | Loss: 2.5238 | LR: 2.73e-04 | 7718.2ms/step | 2.6 samples/s
|
| 621 |
+
Step 5910/11700 | Loss: 2.5700 | LR: 2.73e-04 | 7722.8ms/step | 2.6 samples/s
|
| 622 |
+
Step 5920/11700 | Loss: 2.4785 | LR: 2.72e-04 | 7715.4ms/step | 2.6 samples/s
|
| 623 |
+
Step 5930/11700 | Loss: 2.5084 | LR: 2.71e-04 | 7710.8ms/step | 2.6 samples/s
|
| 624 |
+
Step 5940/11700 | Loss: 2.4368 | LR: 2.70e-04 | 7711.7ms/step | 2.6 samples/s
|
| 625 |
+
Step 5950/11700 | Loss: 2.6047 | LR: 2.70e-04 | 7711.6ms/step | 2.6 samples/s
|
| 626 |
+
Step 5960/11700 | Loss: 2.6292 | LR: 2.69e-04 | 7716.5ms/step | 2.6 samples/s
|
| 627 |
+
Step 5970/11700 | Loss: 2.5203 | LR: 2.68e-04 | 7719.5ms/step | 2.6 samples/s
|
| 628 |
+
Step 5980/11700 | Loss: 2.5397 | LR: 2.68e-04 | 7719.8ms/step | 2.6 samples/s
|
| 629 |
+
Step 5990/11700 | Loss: 2.5306 | LR: 2.67e-04 | 7715.8ms/step | 2.6 samples/s
|
| 630 |
+
Step 6000/11700 | Loss: 2.5254 | LR: 2.66e-04 | 7713.0ms/step | 2.6 samples/s
|
| 631 |
+
Validation at step 6000: Loss = 2.5337, Perplexity = 12.60
|
| 632 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-6000
|
| 633 |
+
Step 6010/11700 | Loss: 2.5803 | LR: 2.65e-04 | 19537.5ms/step | 1.0 samples/s
|
| 634 |
+
Step 6020/11700 | Loss: 2.4420 | LR: 2.65e-04 | 7707.9ms/step | 2.6 samples/s
|
| 635 |
+
Step 6030/11700 | Loss: 2.5021 | LR: 2.64e-04 | 7710.7ms/step | 2.6 samples/s
|
| 636 |
+
Step 6040/11700 | Loss: 2.4662 | LR: 2.63e-04 | 7712.1ms/step | 2.6 samples/s
|
| 637 |
+
Step 6050/11700 | Loss: 2.4687 | LR: 2.63e-04 | 7716.6ms/step | 2.6 samples/s
|
| 638 |
+
Step 6060/11700 | Loss: 2.5715 | LR: 2.62e-04 | 7715.5ms/step | 2.6 samples/s
|
| 639 |
+
Step 6070/11700 | Loss: 2.5612 | LR: 2.61e-04 | 7713.9ms/step | 2.6 samples/s
|
| 640 |
+
Step 6080/11700 | Loss: 2.5905 | LR: 2.60e-04 | 7716.0ms/step | 2.6 samples/s
|
| 641 |
+
Step 6090/11700 | Loss: 2.4596 | LR: 2.60e-04 | 7715.8ms/step | 2.6 samples/s
|
| 642 |
+
Step 6100/11700 | Loss: 2.5330 | LR: 2.59e-04 | 7716.2ms/step | 2.6 samples/s
|
| 643 |
+
Step 6110/11700 | Loss: 2.4657 | LR: 2.58e-04 | 7714.0ms/step | 2.6 samples/s
|
| 644 |
+
Step 6120/11700 | Loss: 2.5917 | LR: 2.58e-04 | 7715.6ms/step | 2.6 samples/s
|
| 645 |
+
Step 6130/11700 | Loss: 2.5221 | LR: 2.57e-04 | 7713.8ms/step | 2.6 samples/s
|
| 646 |
+
Step 6140/11700 | Loss: 2.5328 | LR: 2.56e-04 | 7713.1ms/step | 2.6 samples/s
|
| 647 |
+
Step 6150/11700 | Loss: 2.5058 | LR: 2.55e-04 | 7716.8ms/step | 2.6 samples/s
|
| 648 |
+
Step 6160/11700 | Loss: 2.4438 | LR: 2.55e-04 | 7719.4ms/step | 2.6 samples/s
|
| 649 |
+
Step 6170/11700 | Loss: 2.4556 | LR: 2.54e-04 | 7713.4ms/step | 2.6 samples/s
|
| 650 |
+
Step 6180/11700 | Loss: 2.4228 | LR: 2.53e-04 | 7712.0ms/step | 2.6 samples/s
|
| 651 |
+
Step 6190/11700 | Loss: 2.4831 | LR: 2.53e-04 | 7712.0ms/step | 2.6 samples/s
|
| 652 |
+
Step 6200/11700 | Loss: 2.6099 | LR: 2.52e-04 | 7715.7ms/step | 2.6 samples/s
|
| 653 |
+
Step 6210/11700 | Loss: 2.5924 | LR: 2.51e-04 | 7716.3ms/step | 2.6 samples/s
|
| 654 |
+
Step 6220/11700 | Loss: 2.5394 | LR: 2.50e-04 | 7715.8ms/step | 2.6 samples/s
|
| 655 |
+
Step 6230/11700 | Loss: 2.6469 | LR: 2.50e-04 | 7716.3ms/step | 2.6 samples/s
|
| 656 |
+
Step 6240/11700 | Loss: 2.5055 | LR: 2.49e-04 | 7712.0ms/step | 2.6 samples/s
|
| 657 |
+
Step 6250/11700 | Loss: 2.5434 | LR: 2.48e-04 | 7713.8ms/step | 2.6 samples/s
|
| 658 |
+
Validation at step 6250: Loss = 2.5216, Perplexity = 12.45
|
| 659 |
+
Step 6260/11700 | Loss: 2.4382 | LR: 2.47e-04 | 19204.4ms/step | 1.0 samples/s
|
| 660 |
+
Step 6270/11700 | Loss: 2.5341 | LR: 2.47e-04 | 7718.2ms/step | 2.6 samples/s
|
| 661 |
+
Step 6280/11700 | Loss: 2.5142 | LR: 2.46e-04 | 7715.3ms/step | 2.6 samples/s
|
| 662 |
+
Step 6290/11700 | Loss: 2.4916 | LR: 2.45e-04 | 7712.8ms/step | 2.6 samples/s
|
| 663 |
+
Step 6300/11700 | Loss: 2.4999 | LR: 2.45e-04 | 7714.6ms/step | 2.6 samples/s
|
| 664 |
+
Step 6310/11700 | Loss: 2.4935 | LR: 2.44e-04 | 7714.0ms/step | 2.6 samples/s
|
| 665 |
+
Step 6320/11700 | Loss: 2.5593 | LR: 2.43e-04 | 7718.7ms/step | 2.6 samples/s
|
| 666 |
+
Step 6330/11700 | Loss: 2.6215 | LR: 2.42e-04 | 7719.5ms/step | 2.6 samples/s
|
| 667 |
+
Step 6340/11700 | Loss: 2.5353 | LR: 2.42e-04 | 7717.8ms/step | 2.6 samples/s
|
| 668 |
+
Step 6350/11700 | Loss: 2.5418 | LR: 2.41e-04 | 7717.2ms/step | 2.6 samples/s
|
| 669 |
+
Step 6360/11700 | Loss: 2.4434 | LR: 2.40e-04 | 7716.7ms/step | 2.6 samples/s
|
| 670 |
+
Step 6370/11700 | Loss: 2.4257 | LR: 2.40e-04 | 7718.7ms/step | 2.6 samples/s
|
| 671 |
+
Step 6380/11700 | Loss: 2.3961 | LR: 2.39e-04 | 7720.2ms/step | 2.6 samples/s
|
| 672 |
+
Step 6390/11700 | Loss: 2.5280 | LR: 2.38e-04 | 7719.7ms/step | 2.6 samples/s
|
| 673 |
+
Step 6400/11700 | Loss: 2.5042 | LR: 2.37e-04 | 7727.0ms/step | 2.6 samples/s
|
| 674 |
+
Step 6410/11700 | Loss: 2.4333 | LR: 2.37e-04 | 7723.5ms/step | 2.6 samples/s
|
| 675 |
+
Step 6420/11700 | Loss: 2.4309 | LR: 2.36e-04 | 7716.7ms/step | 2.6 samples/s
|
| 676 |
+
Step 6430/11700 | Loss: 2.3267 | LR: 2.35e-04 | 7721.7ms/step | 2.6 samples/s
|
| 677 |
+
Step 6440/11700 | Loss: 2.4984 | LR: 2.35e-04 | 7719.0ms/step | 2.6 samples/s
|
| 678 |
+
Step 6450/11700 | Loss: 2.4384 | LR: 2.34e-04 | 7719.6ms/step | 2.6 samples/s
|
| 679 |
+
Step 6460/11700 | Loss: 2.5700 | LR: 2.33e-04 | 7719.8ms/step | 2.6 samples/s
|
| 680 |
+
Step 6470/11700 | Loss: 2.4347 | LR: 2.32e-04 | 7724.7ms/step | 2.6 samples/s
|
| 681 |
+
Step 6480/11700 | Loss: 2.6838 | LR: 2.32e-04 | 7728.3ms/step | 2.6 samples/s
|
| 682 |
+
Step 6490/11700 | Loss: 2.4904 | LR: 2.31e-04 | 7724.9ms/step | 2.6 samples/s
|
| 683 |
+
Step 6500/11700 | Loss: 2.5242 | LR: 2.30e-04 | 7725.7ms/step | 2.6 samples/s
|
| 684 |
+
Validation at step 6500: Loss = 2.5119, Perplexity = 12.33
|
| 685 |
+
Step 6510/11700 | Loss: 2.5039 | LR: 2.30e-04 | 19238.7ms/step | 1.0 samples/s
|
| 686 |
+
Step 6520/11700 | Loss: 2.5041 | LR: 2.29e-04 | 7719.7ms/step | 2.6 samples/s
|
| 687 |
+
Step 6530/11700 | Loss: 2.4219 | LR: 2.28e-04 | 7721.1ms/step | 2.6 samples/s
|
| 688 |
+
Step 6540/11700 | Loss: 2.5747 | LR: 2.27e-04 | 7719.5ms/step | 2.6 samples/s
|
| 689 |
+
Step 6550/11700 | Loss: 2.4757 | LR: 2.27e-04 | 7718.3ms/step | 2.6 samples/s
|
| 690 |
+
Step 6560/11700 | Loss: 2.4263 | LR: 2.26e-04 | 7719.5ms/step | 2.6 samples/s
|
| 691 |
+
Step 6570/11700 | Loss: 2.3809 | LR: 2.25e-04 | 7717.9ms/step | 2.6 samples/s
|
| 692 |
+
Step 6580/11700 | Loss: 2.5411 | LR: 2.25e-04 | 7721.1ms/step | 2.6 samples/s
|
| 693 |
+
Step 6590/11700 | Loss: 2.5220 | LR: 2.24e-04 | 7723.8ms/step | 2.6 samples/s
|
| 694 |
+
Step 6600/11700 | Loss: 2.5348 | LR: 2.23e-04 | 7723.2ms/step | 2.6 samples/s
|
| 695 |
+
Step 6610/11700 | Loss: 2.6072 | LR: 2.22e-04 | 7720.4ms/step | 2.6 samples/s
|
| 696 |
+
Step 6620/11700 | Loss: 2.4071 | LR: 2.22e-04 | 7717.0ms/step | 2.6 samples/s
|
| 697 |
+
Step 6630/11700 | Loss: 2.5141 | LR: 2.21e-04 | 7715.0ms/step | 2.6 samples/s
|
| 698 |
+
Step 6640/11700 | Loss: 2.5221 | LR: 2.20e-04 | 7719.8ms/step | 2.6 samples/s
|
| 699 |
+
Step 6650/11700 | Loss: 2.4546 | LR: 2.20e-04 | 7718.4ms/step | 2.6 samples/s
|
| 700 |
+
Step 6660/11700 | Loss: 2.4780 | LR: 2.19e-04 | 7719.0ms/step | 2.6 samples/s
|
| 701 |
+
Step 6670/11700 | Loss: 2.4723 | LR: 2.18e-04 | 7718.1ms/step | 2.6 samples/s
|
| 702 |
+
Step 6680/11700 | Loss: 2.4691 | LR: 2.17e-04 | 7718.7ms/step | 2.6 samples/s
|
| 703 |
+
Step 6690/11700 | Loss: 2.5591 | LR: 2.17e-04 | 7721.0ms/step | 2.6 samples/s
|
| 704 |
+
Step 6700/11700 | Loss: 2.4978 | LR: 2.16e-04 | 7716.3ms/step | 2.6 samples/s
|
| 705 |
+
Step 6710/11700 | Loss: 2.5105 | LR: 2.15e-04 | 7718.2ms/step | 2.6 samples/s
|
| 706 |
+
Step 6720/11700 | Loss: 2.3939 | LR: 2.15e-04 | 7720.0ms/step | 2.6 samples/s
|
| 707 |
+
Step 6730/11700 | Loss: 2.4516 | LR: 2.14e-04 | 7717.1ms/step | 2.6 samples/s
|
| 708 |
+
Step 6740/11700 | Loss: 2.4693 | LR: 2.13e-04 | 7717.7ms/step | 2.6 samples/s
|
| 709 |
+
Step 6750/11700 | Loss: 2.4520 | LR: 2.12e-04 | 7718.4ms/step | 2.6 samples/s
|
| 710 |
+
Validation at step 6750: Loss = 2.5026, Perplexity = 12.21
|
| 711 |
+
Step 6760/11700 | Loss: 2.5251 | LR: 2.12e-04 | 19224.0ms/step | 1.0 samples/s
|
| 712 |
+
Step 6770/11700 | Loss: 2.4921 | LR: 2.11e-04 | 7743.5ms/step | 2.6 samples/s
|
| 713 |
+
Step 6780/11700 | Loss: 2.4322 | LR: 2.10e-04 | 7713.6ms/step | 2.6 samples/s
|
| 714 |
+
Step 6790/11700 | Loss: 2.4377 | LR: 2.10e-04 | 7716.6ms/step | 2.6 samples/s
|
| 715 |
+
Step 6800/11700 | Loss: 2.5184 | LR: 2.09e-04 | 7726.5ms/step | 2.6 samples/s
|
| 716 |
+
Step 6810/11700 | Loss: 2.6861 | LR: 2.08e-04 | 7729.2ms/step | 2.6 samples/s
|
| 717 |
+
Step 6820/11700 | Loss: 2.5266 | LR: 2.08e-04 | 7721.5ms/step | 2.6 samples/s
|
| 718 |
+
Step 6830/11700 | Loss: 2.4372 | LR: 2.07e-04 | 7718.9ms/step | 2.6 samples/s
|
| 719 |
+
Step 6840/11700 | Loss: 2.4566 | LR: 2.06e-04 | 7720.1ms/step | 2.6 samples/s
|
| 720 |
+
Step 6850/11700 | Loss: 2.5301 | LR: 2.05e-04 | 7718.6ms/step | 2.6 samples/s
|
| 721 |
+
Step 6860/11700 | Loss: 2.4702 | LR: 2.05e-04 | 7718.3ms/step | 2.6 samples/s
|
| 722 |
+
Step 6870/11700 | Loss: 2.4578 | LR: 2.04e-04 | 7719.0ms/step | 2.6 samples/s
|
| 723 |
+
Step 6880/11700 | Loss: 2.4594 | LR: 2.03e-04 | 7721.2ms/step | 2.6 samples/s
|
| 724 |
+
Step 6890/11700 | Loss: 2.4803 | LR: 2.03e-04 | 7716.5ms/step | 2.6 samples/s
|
| 725 |
+
Step 6900/11700 | Loss: 2.6036 | LR: 2.02e-04 | 7716.6ms/step | 2.6 samples/s
|
| 726 |
+
Step 6910/11700 | Loss: 2.5710 | LR: 2.01e-04 | 7721.1ms/step | 2.6 samples/s
|
| 727 |
+
Step 6920/11700 | Loss: 2.4275 | LR: 2.00e-04 | 7707.9ms/step | 2.6 samples/s
|
| 728 |
+
Step 6930/11700 | Loss: 2.3760 | LR: 2.00e-04 | 7717.7ms/step | 2.6 samples/s
|
| 729 |
+
Step 6940/11700 | Loss: 2.5863 | LR: 1.99e-04 | 7721.1ms/step | 2.6 samples/s
|
| 730 |
+
Step 6950/11700 | Loss: 2.5347 | LR: 1.98e-04 | 7715.2ms/step | 2.6 samples/s
|
| 731 |
+
Step 6960/11700 | Loss: 2.4838 | LR: 1.98e-04 | 7722.0ms/step | 2.6 samples/s
|
| 732 |
+
Step 6970/11700 | Loss: 2.3307 | LR: 1.97e-04 | 7717.2ms/step | 2.6 samples/s
|
| 733 |
+
Step 6980/11700 | Loss: 2.4919 | LR: 1.96e-04 | 7721.4ms/step | 2.6 samples/s
|
| 734 |
+
Step 6990/11700 | Loss: 2.5594 | LR: 1.96e-04 | 7726.0ms/step | 2.6 samples/s
|
| 735 |
+
Step 7000/11700 | Loss: 2.4925 | LR: 1.95e-04 | 7715.5ms/step | 2.6 samples/s
|
| 736 |
+
Validation at step 7000: Loss = 2.4948, Perplexity = 12.12
|
| 737 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-7000
|
| 738 |
+
Step 7010/11700 | Loss: 2.5951 | LR: 1.94e-04 | 19539.0ms/step | 1.0 samples/s
|
| 739 |
+
Step 7020/11700 | Loss: 2.4756 | LR: 1.93e-04 | 7711.3ms/step | 2.6 samples/s
|
| 740 |
+
Step 7030/11700 | Loss: 2.5686 | LR: 1.93e-04 | 7711.3ms/step | 2.6 samples/s
|
| 741 |
+
Step 7040/11700 | Loss: 2.5451 | LR: 1.92e-04 | 7710.7ms/step | 2.6 samples/s
|
| 742 |
+
Step 7050/11700 | Loss: 2.4170 | LR: 1.91e-04 | 7714.8ms/step | 2.6 samples/s
|
| 743 |
+
Step 7060/11700 | Loss: 2.5385 | LR: 1.91e-04 | 7719.0ms/step | 2.6 samples/s
|
| 744 |
+
Step 7070/11700 | Loss: 2.4664 | LR: 1.90e-04 | 7714.3ms/step | 2.6 samples/s
|
| 745 |
+
Step 7080/11700 | Loss: 2.4809 | LR: 1.89e-04 | 7709.3ms/step | 2.6 samples/s
|
| 746 |
+
Step 7090/11700 | Loss: 2.4835 | LR: 1.89e-04 | 7706.2ms/step | 2.6 samples/s
|
| 747 |
+
Step 7100/11700 | Loss: 2.5223 | LR: 1.88e-04 | 7716.8ms/step | 2.6 samples/s
|
| 748 |
+
Step 7110/11700 | Loss: 2.4628 | LR: 1.87e-04 | 7722.5ms/step | 2.6 samples/s
|
| 749 |
+
Step 7120/11700 | Loss: 2.4072 | LR: 1.87e-04 | 7719.9ms/step | 2.6 samples/s
|
| 750 |
+
Step 7130/11700 | Loss: 2.4150 | LR: 1.86e-04 | 7706.4ms/step | 2.6 samples/s
|
| 751 |
+
Step 7140/11700 | Loss: 2.4664 | LR: 1.85e-04 | 7693.0ms/step | 2.6 samples/s
|
| 752 |
+
Step 7150/11700 | Loss: 2.5466 | LR: 1.84e-04 | 7693.7ms/step | 2.6 samples/s
|
| 753 |
+
Step 7160/11700 | Loss: 2.4603 | LR: 1.84e-04 | 7696.9ms/step | 2.6 samples/s
|
| 754 |
+
Step 7170/11700 | Loss: 2.4982 | LR: 1.83e-04 | 7702.0ms/step | 2.6 samples/s
|
| 755 |
+
Step 7180/11700 | Loss: 2.4075 | LR: 1.82e-04 | 7708.9ms/step | 2.6 samples/s
|
| 756 |
+
Step 7190/11700 | Loss: 2.4080 | LR: 1.82e-04 | 7700.2ms/step | 2.6 samples/s
|
| 757 |
+
Step 7200/11700 | Loss: 2.3734 | LR: 1.81e-04 | 7693.0ms/step | 2.6 samples/s
|
| 758 |
+
Step 7210/11700 | Loss: 2.3176 | LR: 1.80e-04 | 7685.0ms/step | 2.6 samples/s
|
| 759 |
+
Step 7220/11700 | Loss: 2.4558 | LR: 1.80e-04 | 7698.7ms/step | 2.6 samples/s
|
| 760 |
+
Step 7230/11700 | Loss: 2.5001 | LR: 1.79e-04 | 7706.8ms/step | 2.6 samples/s
|
| 761 |
+
Step 7240/11700 | Loss: 2.4581 | LR: 1.78e-04 | 7702.3ms/step | 2.6 samples/s
|
| 762 |
+
Step 7250/11700 | Loss: 2.3292 | LR: 1.78e-04 | 7700.4ms/step | 2.6 samples/s
|
| 763 |
+
Validation at step 7250: Loss = 2.4871, Perplexity = 12.03
|
| 764 |
+
Step 7260/11700 | Loss: 2.4478 | LR: 1.77e-04 | 19164.4ms/step | 1.0 samples/s
|
| 765 |
+
Step 7270/11700 | Loss: 2.5017 | LR: 1.76e-04 | 7697.6ms/step | 2.6 samples/s
|
| 766 |
+
Step 7280/11700 | Loss: 2.5066 | LR: 1.75e-04 | 7694.7ms/step | 2.6 samples/s
|
| 767 |
+
Step 7290/11700 | Loss: 2.5016 | LR: 1.75e-04 | 7684.6ms/step | 2.6 samples/s
|
| 768 |
+
Step 7300/11700 | Loss: 2.4233 | LR: 1.74e-04 | 7694.4ms/step | 2.6 samples/s
|
| 769 |
+
Step 7310/11700 | Loss: 2.5473 | LR: 1.73e-04 | 7707.5ms/step | 2.6 samples/s
|
| 770 |
+
Step 7320/11700 | Loss: 2.4460 | LR: 1.73e-04 | 7695.5ms/step | 2.6 samples/s
|
| 771 |
+
Step 7330/11700 | Loss: 2.5148 | LR: 1.72e-04 | 7691.5ms/step | 2.6 samples/s
|
| 772 |
+
Step 7340/11700 | Loss: 2.4542 | LR: 1.71e-04 | 7700.5ms/step | 2.6 samples/s
|
| 773 |
+
Step 7350/11700 | Loss: 2.4371 | LR: 1.71e-04 | 7677.0ms/step | 2.6 samples/s
|
| 774 |
+
Step 7360/11700 | Loss: 2.3903 | LR: 1.70e-04 | 7708.5ms/step | 2.6 samples/s
|
| 775 |
+
Step 7370/11700 | Loss: 2.4494 | LR: 1.69e-04 | 7694.6ms/step | 2.6 samples/s
|
| 776 |
+
Step 7380/11700 | Loss: 2.4878 | LR: 1.69e-04 | 7691.3ms/step | 2.6 samples/s
|
| 777 |
+
Step 7390/11700 | Loss: 2.4546 | LR: 1.68e-04 | 7687.6ms/step | 2.6 samples/s
|
| 778 |
+
Step 7400/11700 | Loss: 2.4886 | LR: 1.67e-04 | 7686.8ms/step | 2.6 samples/s
|
| 779 |
+
Step 7410/11700 | Loss: 2.4113 | LR: 1.67e-04 | 7713.6ms/step | 2.6 samples/s
|
| 780 |
+
Step 7420/11700 | Loss: 2.3948 | LR: 1.66e-04 | 7720.0ms/step | 2.6 samples/s
|
| 781 |
+
Step 7430/11700 | Loss: 2.4066 | LR: 1.65e-04 | 7718.2ms/step | 2.6 samples/s
|
| 782 |
+
Step 7440/11700 | Loss: 2.5102 | LR: 1.65e-04 | 7719.8ms/step | 2.6 samples/s
|
| 783 |
+
Step 7450/11700 | Loss: 2.4899 | LR: 1.64e-04 | 7721.0ms/step | 2.6 samples/s
|
| 784 |
+
Step 7460/11700 | Loss: 2.4973 | LR: 1.63e-04 | 7721.1ms/step | 2.6 samples/s
|
| 785 |
+
Step 7470/11700 | Loss: 2.4489 | LR: 1.63e-04 | 7721.0ms/step | 2.6 samples/s
|
| 786 |
+
Step 7480/11700 | Loss: 2.5814 | LR: 1.62e-04 | 7717.2ms/step | 2.6 samples/s
|
| 787 |
+
Step 7490/11700 | Loss: 2.4169 | LR: 1.61e-04 | 7721.4ms/step | 2.6 samples/s
|
| 788 |
+
Step 7500/11700 | Loss: 2.4069 | LR: 1.61e-04 | 7719.0ms/step | 2.6 samples/s
|
| 789 |
+
Validation at step 7500: Loss = 2.4813, Perplexity = 11.96
|
| 790 |
+
Step 7510/11700 | Loss: 2.4913 | LR: 1.60e-04 | 19217.7ms/step | 1.0 samples/s
|
| 791 |
+
Step 7520/11700 | Loss: 2.5149 | LR: 1.59e-04 | 7720.4ms/step | 2.6 samples/s
|
| 792 |
+
Step 7530/11700 | Loss: 2.4184 | LR: 1.59e-04 | 7717.1ms/step | 2.6 samples/s
|
| 793 |
+
Step 7540/11700 | Loss: 2.4970 | LR: 1.58e-04 | 7740.1ms/step | 2.6 samples/s
|
| 794 |
+
Step 7550/11700 | Loss: 2.4429 | LR: 1.57e-04 | 7734.1ms/step | 2.6 samples/s
|
| 795 |
+
Step 7560/11700 | Loss: 2.5030 | LR: 1.57e-04 | 7734.3ms/step | 2.6 samples/s
|
| 796 |
+
Step 7570/11700 | Loss: 2.4626 | LR: 1.56e-04 | 7731.1ms/step | 2.6 samples/s
|
| 797 |
+
Step 7580/11700 | Loss: 2.4510 | LR: 1.55e-04 | 7733.4ms/step | 2.6 samples/s
|
| 798 |
+
Step 7590/11700 | Loss: 2.4319 | LR: 1.55e-04 | 7734.4ms/step | 2.6 samples/s
|
| 799 |
+
Step 7600/11700 | Loss: 2.4878 | LR: 1.54e-04 | 7735.9ms/step | 2.6 samples/s
|
| 800 |
+
Step 7610/11700 | Loss: 2.5823 | LR: 1.53e-04 | 7734.6ms/step | 2.6 samples/s
|
| 801 |
+
Step 7620/11700 | Loss: 2.5496 | LR: 1.53e-04 | 7730.4ms/step | 2.6 samples/s
|
| 802 |
+
Step 7630/11700 | Loss: 2.4238 | LR: 1.52e-04 | 7734.4ms/step | 2.6 samples/s
|
| 803 |
+
Step 7640/11700 | Loss: 2.4581 | LR: 1.51e-04 | 7734.3ms/step | 2.6 samples/s
|
| 804 |
+
Step 7650/11700 | Loss: 2.4939 | LR: 1.51e-04 | 7739.1ms/step | 2.6 samples/s
|
| 805 |
+
Step 7660/11700 | Loss: 2.4428 | LR: 1.50e-04 | 7739.5ms/step | 2.6 samples/s
|
| 806 |
+
Step 7670/11700 | Loss: 2.3800 | LR: 1.49e-04 | 7726.1ms/step | 2.6 samples/s
|
| 807 |
+
Step 7680/11700 | Loss: 2.4427 | LR: 1.49e-04 | 7724.5ms/step | 2.6 samples/s
|
| 808 |
+
Step 7690/11700 | Loss: 2.5223 | LR: 1.48e-04 | 7724.8ms/step | 2.6 samples/s
|
| 809 |
+
Step 7700/11700 | Loss: 2.5864 | LR: 1.47e-04 | 7722.5ms/step | 2.6 samples/s
|
| 810 |
+
Step 7710/11700 | Loss: 2.3891 | LR: 1.47e-04 | 7720.2ms/step | 2.6 samples/s
|
| 811 |
+
Step 7720/11700 | Loss: 2.4900 | LR: 1.46e-04 | 7720.7ms/step | 2.6 samples/s
|
| 812 |
+
Step 7730/11700 | Loss: 2.3844 | LR: 1.45e-04 | 7719.6ms/step | 2.6 samples/s
|
| 813 |
+
Step 7740/11700 | Loss: 2.5431 | LR: 1.45e-04 | 7719.3ms/step | 2.6 samples/s
|
| 814 |
+
Step 7750/11700 | Loss: 2.4592 | LR: 1.44e-04 | 7718.1ms/step | 2.6 samples/s
|
| 815 |
+
Validation at step 7750: Loss = 2.4746, Perplexity = 11.88
|
| 816 |
+
Step 7760/11700 | Loss: 2.5756 | LR: 1.43e-04 | 19210.4ms/step | 1.0 samples/s
|
| 817 |
+
Step 7770/11700 | Loss: 2.4398 | LR: 1.43e-04 | 7719.4ms/step | 2.6 samples/s
|
| 818 |
+
Step 7780/11700 | Loss: 2.5353 | LR: 1.42e-04 | 7716.0ms/step | 2.6 samples/s
|
| 819 |
+
Step 7790/11700 | Loss: 2.4361 | LR: 1.41e-04 | 7715.3ms/step | 2.6 samples/s
|
| 820 |
+
Step 7800/11700 | Loss: 2.5205 | LR: 1.41e-04 | 7717.2ms/step | 2.6 samples/s
|
| 821 |
+
Step 7810/11700 | Loss: 2.3626 | LR: 1.40e-04 | 7717.3ms/step | 2.6 samples/s
|
| 822 |
+
Step 7820/11700 | Loss: 2.4066 | LR: 1.40e-04 | 7720.3ms/step | 2.6 samples/s
|
| 823 |
+
Step 7830/11700 | Loss: 2.4102 | LR: 1.39e-04 | 7717.1ms/step | 2.6 samples/s
|
| 824 |
+
Step 7840/11700 | Loss: 2.4712 | LR: 1.38e-04 | 7716.8ms/step | 2.6 samples/s
|
| 825 |
+
Step 7850/11700 | Loss: 2.4915 | LR: 1.38e-04 | 7714.8ms/step | 2.6 samples/s
|
| 826 |
+
Step 7860/11700 | Loss: 2.4880 | LR: 1.37e-04 | 7709.1ms/step | 2.6 samples/s
|
| 827 |
+
Step 7870/11700 | Loss: 2.5291 | LR: 1.36e-04 | 7714.9ms/step | 2.6 samples/s
|
| 828 |
+
Step 7880/11700 | Loss: 2.4851 | LR: 1.36e-04 | 7715.0ms/step | 2.6 samples/s
|
| 829 |
+
Step 7890/11700 | Loss: 2.4420 | LR: 1.35e-04 | 7714.3ms/step | 2.6 samples/s
|
| 830 |
+
Step 7900/11700 | Loss: 2.5435 | LR: 1.34e-04 | 7716.0ms/step | 2.6 samples/s
|
| 831 |
+
Step 7910/11700 | Loss: 2.3228 | LR: 1.34e-04 | 7701.1ms/step | 2.6 samples/s
|
| 832 |
+
Step 7920/11700 | Loss: 2.4256 | LR: 1.33e-04 | 7714.3ms/step | 2.6 samples/s
|
| 833 |
+
Step 7930/11700 | Loss: 2.4364 | LR: 1.33e-04 | 7713.0ms/step | 2.6 samples/s
|
| 834 |
+
Step 7940/11700 | Loss: 2.4865 | LR: 1.32e-04 | 7718.7ms/step | 2.6 samples/s
|
| 835 |
+
Step 7950/11700 | Loss: 2.5094 | LR: 1.31e-04 | 7717.2ms/step | 2.6 samples/s
|
| 836 |
+
Step 7960/11700 | Loss: 2.4356 | LR: 1.31e-04 | 7713.1ms/step | 2.6 samples/s
|
| 837 |
+
Step 7970/11700 | Loss: 2.3839 | LR: 1.30e-04 | 7715.1ms/step | 2.6 samples/s
|
| 838 |
+
Step 7980/11700 | Loss: 2.5378 | LR: 1.29e-04 | 7715.1ms/step | 2.6 samples/s
|
| 839 |
+
Step 7990/11700 | Loss: 2.5144 | LR: 1.29e-04 | 7715.0ms/step | 2.6 samples/s
|
| 840 |
+
Step 8000/11700 | Loss: 2.3971 | LR: 1.28e-04 | 7715.0ms/step | 2.6 samples/s
|
| 841 |
+
Validation at step 8000: Loss = 2.4698, Perplexity = 11.82
|
| 842 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-8000
|
| 843 |
+
Step 8010/11700 | Loss: 2.4850 | LR: 1.27e-04 | 19515.8ms/step | 1.0 samples/s
|
| 844 |
+
Step 8020/11700 | Loss: 2.3603 | LR: 1.27e-04 | 7713.5ms/step | 2.6 samples/s
|
| 845 |
+
Step 8030/11700 | Loss: 2.4499 | LR: 1.26e-04 | 7718.7ms/step | 2.6 samples/s
|
| 846 |
+
Step 8040/11700 | Loss: 2.4566 | LR: 1.26e-04 | 7709.5ms/step | 2.6 samples/s
|
| 847 |
+
Step 8050/11700 | Loss: 2.4197 | LR: 1.25e-04 | 7712.4ms/step | 2.6 samples/s
|
| 848 |
+
Step 8060/11700 | Loss: 2.4474 | LR: 1.24e-04 | 7718.4ms/step | 2.6 samples/s
|
| 849 |
+
Step 8070/11700 | Loss: 2.5019 | LR: 1.24e-04 | 7716.1ms/step | 2.6 samples/s
|
| 850 |
+
Step 8080/11700 | Loss: 2.4997 | LR: 1.23e-04 | 7715.7ms/step | 2.6 samples/s
|
| 851 |
+
Step 8090/11700 | Loss: 2.4310 | LR: 1.23e-04 | 7712.9ms/step | 2.6 samples/s
|
| 852 |
+
Step 8100/11700 | Loss: 2.4676 | LR: 1.22e-04 | 7710.4ms/step | 2.6 samples/s
|
| 853 |
+
Step 8110/11700 | Loss: 2.4154 | LR: 1.21e-04 | 7715.4ms/step | 2.6 samples/s
|
| 854 |
+
Step 8120/11700 | Loss: 2.3953 | LR: 1.21e-04 | 7716.1ms/step | 2.6 samples/s
|
| 855 |
+
Step 8130/11700 | Loss: 2.3738 | LR: 1.20e-04 | 7714.3ms/step | 2.6 samples/s
|
| 856 |
+
Step 8140/11700 | Loss: 2.4277 | LR: 1.19e-04 | 7716.4ms/step | 2.6 samples/s
|
| 857 |
+
Step 8150/11700 | Loss: 2.3127 | LR: 1.19e-04 | 7718.4ms/step | 2.6 samples/s
|
| 858 |
+
Step 8160/11700 | Loss: 2.5410 | LR: 1.18e-04 | 7717.1ms/step | 2.6 samples/s
|
| 859 |
+
Step 8170/11700 | Loss: 2.4374 | LR: 1.18e-04 | 7716.1ms/step | 2.6 samples/s
|
| 860 |
+
Step 8180/11700 | Loss: 2.4965 | LR: 1.17e-04 | 7715.4ms/step | 2.6 samples/s
|
| 861 |
+
Step 8190/11700 | Loss: 2.4010 | LR: 1.16e-04 | 7714.0ms/step | 2.6 samples/s
|
| 862 |
+
Step 8200/11700 | Loss: 2.4532 | LR: 1.16e-04 | 7718.5ms/step | 2.6 samples/s
|
| 863 |
+
Step 8210/11700 | Loss: 2.4864 | LR: 1.15e-04 | 7721.4ms/step | 2.6 samples/s
|
| 864 |
+
Step 8220/11700 | Loss: 2.4864 | LR: 1.15e-04 | 7715.7ms/step | 2.6 samples/s
|
| 865 |
+
Step 8230/11700 | Loss: 2.4691 | LR: 1.14e-04 | 7719.3ms/step | 2.6 samples/s
|
| 866 |
+
Step 8240/11700 | Loss: 2.4398 | LR: 1.13e-04 | 7720.5ms/step | 2.6 samples/s
|
| 867 |
+
Step 8250/11700 | Loss: 2.4693 | LR: 1.13e-04 | 7717.7ms/step | 2.6 samples/s
|
| 868 |
+
Validation at step 8250: Loss = 2.4657, Perplexity = 11.77
|
| 869 |
+
Step 8260/11700 | Loss: 2.4069 | LR: 1.12e-04 | 19196.9ms/step | 1.0 samples/s
|
| 870 |
+
Step 8270/11700 | Loss: 2.4163 | LR: 1.12e-04 | 7710.0ms/step | 2.6 samples/s
|
| 871 |
+
Step 8280/11700 | Loss: 2.5645 | LR: 1.11e-04 | 7715.8ms/step | 2.6 samples/s
|
| 872 |
+
Step 8290/11700 | Loss: 2.4268 | LR: 1.10e-04 | 7715.4ms/step | 2.6 samples/s
|
| 873 |
+
Step 8300/11700 | Loss: 2.4787 | LR: 1.10e-04 | 7717.3ms/step | 2.6 samples/s
|
| 874 |
+
Step 8310/11700 | Loss: 2.4113 | LR: 1.09e-04 | 7720.3ms/step | 2.6 samples/s
|
| 875 |
+
Step 8320/11700 | Loss: 2.4966 | LR: 1.09e-04 | 7717.2ms/step | 2.6 samples/s
|
| 876 |
+
Step 8330/11700 | Loss: 2.5032 | LR: 1.08e-04 | 7717.6ms/step | 2.6 samples/s
|
| 877 |
+
Step 8340/11700 | Loss: 2.4684 | LR: 1.07e-04 | 7716.7ms/step | 2.6 samples/s
|
| 878 |
+
Step 8350/11700 | Loss: 2.4815 | LR: 1.07e-04 | 7717.8ms/step | 2.6 samples/s
|
| 879 |
+
Step 8360/11700 | Loss: 2.4228 | LR: 1.06e-04 | 7719.4ms/step | 2.6 samples/s
|
| 880 |
+
Step 8370/11700 | Loss: 2.4132 | LR: 1.06e-04 | 7716.8ms/step | 2.6 samples/s
|
| 881 |
+
Step 8380/11700 | Loss: 2.4461 | LR: 1.05e-04 | 7717.7ms/step | 2.6 samples/s
|
| 882 |
+
Step 8390/11700 | Loss: 2.4526 | LR: 1.05e-04 | 7714.4ms/step | 2.6 samples/s
|
| 883 |
+
Step 8400/11700 | Loss: 2.5131 | LR: 1.04e-04 | 7716.0ms/step | 2.6 samples/s
|
| 884 |
+
Step 8410/11700 | Loss: 2.6352 | LR: 1.03e-04 | 7719.3ms/step | 2.6 samples/s
|
| 885 |
+
Step 8420/11700 | Loss: 2.3927 | LR: 1.03e-04 | 7719.4ms/step | 2.6 samples/s
|
| 886 |
+
Step 8430/11700 | Loss: 2.5133 | LR: 1.02e-04 | 7712.8ms/step | 2.6 samples/s
|
| 887 |
+
Step 8440/11700 | Loss: 2.5112 | LR: 1.02e-04 | 7714.7ms/step | 2.6 samples/s
|
| 888 |
+
Step 8450/11700 | Loss: 2.4550 | LR: 1.01e-04 | 7712.6ms/step | 2.6 samples/s
|
| 889 |
+
Step 8460/11700 | Loss: 2.4440 | LR: 1.00e-04 | 7712.0ms/step | 2.6 samples/s
|
| 890 |
+
Step 8470/11700 | Loss: 2.4398 | LR: 9.99e-05 | 7713.1ms/step | 2.6 samples/s
|
| 891 |
+
Step 8480/11700 | Loss: 2.4927 | LR: 9.93e-05 | 7713.1ms/step | 2.6 samples/s
|
| 892 |
+
Step 8490/11700 | Loss: 2.4441 | LR: 9.87e-05 | 7711.9ms/step | 2.6 samples/s
|
| 893 |
+
Step 8500/11700 | Loss: 2.3084 | LR: 9.82e-05 | 7711.9ms/step | 2.6 samples/s
|
| 894 |
+
Validation at step 8500: Loss = 2.4619, Perplexity = 11.73
|
| 895 |
+
Step 8510/11700 | Loss: 2.4107 | LR: 9.76e-05 | 19197.0ms/step | 1.0 samples/s
|
| 896 |
+
Step 8520/11700 | Loss: 2.4575 | LR: 9.70e-05 | 7715.0ms/step | 2.6 samples/s
|
| 897 |
+
Step 8530/11700 | Loss: 2.4824 | LR: 9.65e-05 | 7715.9ms/step | 2.6 samples/s
|
| 898 |
+
Step 8540/11700 | Loss: 2.4529 | LR: 9.59e-05 | 7716.4ms/step | 2.6 samples/s
|
| 899 |
+
Step 8550/11700 | Loss: 2.3431 | LR: 9.53e-05 | 7714.6ms/step | 2.6 samples/s
|
| 900 |
+
Step 8560/11700 | Loss: 2.4540 | LR: 9.48e-05 | 7715.2ms/step | 2.6 samples/s
|
| 901 |
+
Step 8570/11700 | Loss: 2.4456 | LR: 9.42e-05 | 7714.1ms/step | 2.6 samples/s
|
| 902 |
+
Step 8580/11700 | Loss: 2.4315 | LR: 9.36e-05 | 7715.5ms/step | 2.6 samples/s
|
| 903 |
+
Step 8590/11700 | Loss: 2.4020 | LR: 9.31e-05 | 7712.5ms/step | 2.6 samples/s
|
| 904 |
+
Step 8600/11700 | Loss: 2.4731 | LR: 9.25e-05 | 7713.3ms/step | 2.6 samples/s
|
| 905 |
+
Step 8610/11700 | Loss: 2.4483 | LR: 9.20e-05 | 7711.9ms/step | 2.6 samples/s
|
| 906 |
+
Step 8620/11700 | Loss: 2.5025 | LR: 9.14e-05 | 7711.8ms/step | 2.6 samples/s
|
| 907 |
+
Step 8630/11700 | Loss: 2.4756 | LR: 9.09e-05 | 7709.3ms/step | 2.6 samples/s
|
| 908 |
+
Step 8640/11700 | Loss: 2.4122 | LR: 9.03e-05 | 7714.8ms/step | 2.6 samples/s
|
| 909 |
+
Step 8650/11700 | Loss: 2.3700 | LR: 8.98e-05 | 7714.9ms/step | 2.6 samples/s
|
| 910 |
+
Step 8660/11700 | Loss: 2.4331 | LR: 8.92e-05 | 7713.7ms/step | 2.6 samples/s
|
| 911 |
+
Step 8670/11700 | Loss: 2.5247 | LR: 8.87e-05 | 7717.6ms/step | 2.6 samples/s
|
| 912 |
+
Step 8680/11700 | Loss: 2.4024 | LR: 8.81e-05 | 7720.5ms/step | 2.6 samples/s
|
| 913 |
+
Step 8690/11700 | Loss: 2.5264 | LR: 8.76e-05 | 7711.9ms/step | 2.6 samples/s
|
| 914 |
+
Step 8700/11700 | Loss: 2.4512 | LR: 8.70e-05 | 7714.0ms/step | 2.6 samples/s
|
| 915 |
+
Step 8710/11700 | Loss: 2.3176 | LR: 8.65e-05 | 7719.7ms/step | 2.6 samples/s
|
| 916 |
+
Step 8720/11700 | Loss: 2.3917 | LR: 8.59e-05 | 7716.5ms/step | 2.6 samples/s
|
| 917 |
+
Step 8730/11700 | Loss: 2.4389 | LR: 8.54e-05 | 7715.6ms/step | 2.6 samples/s
|
| 918 |
+
Step 8740/11700 | Loss: 2.4680 | LR: 8.49e-05 | 7719.2ms/step | 2.6 samples/s
|
| 919 |
+
Step 8750/11700 | Loss: 2.4745 | LR: 8.43e-05 | 7712.8ms/step | 2.6 samples/s
|
| 920 |
+
Validation at step 8750: Loss = 2.4591, Perplexity = 11.69
|
| 921 |
+
Step 8760/11700 | Loss: 2.3831 | LR: 8.38e-05 | 19199.5ms/step | 1.0 samples/s
|
| 922 |
+
Step 8770/11700 | Loss: 2.4408 | LR: 8.33e-05 | 7713.5ms/step | 2.6 samples/s
|
| 923 |
+
Step 8780/11700 | Loss: 2.4396 | LR: 8.27e-05 | 7715.7ms/step | 2.6 samples/s
|
| 924 |
+
Step 8790/11700 | Loss: 2.4786 | LR: 8.22e-05 | 7711.9ms/step | 2.6 samples/s
|
| 925 |
+
Step 8800/11700 | Loss: 2.5072 | LR: 8.17e-05 | 7709.5ms/step | 2.6 samples/s
|
| 926 |
+
Step 8810/11700 | Loss: 2.3616 | LR: 8.11e-05 | 7713.7ms/step | 2.6 samples/s
|
| 927 |
+
Step 8820/11700 | Loss: 2.3570 | LR: 8.06e-05 | 7714.7ms/step | 2.6 samples/s
|
| 928 |
+
Step 8830/11700 | Loss: 2.4162 | LR: 8.01e-05 | 7717.3ms/step | 2.6 samples/s
|
| 929 |
+
Step 8840/11700 | Loss: 2.4973 | LR: 7.95e-05 | 7714.8ms/step | 2.6 samples/s
|
| 930 |
+
Step 8850/11700 | Loss: 2.4642 | LR: 7.90e-05 | 7718.8ms/step | 2.6 samples/s
|
| 931 |
+
Step 8860/11700 | Loss: 2.3433 | LR: 7.85e-05 | 7717.8ms/step | 2.6 samples/s
|
| 932 |
+
Step 8870/11700 | Loss: 2.3987 | LR: 7.80e-05 | 7715.1ms/step | 2.6 samples/s
|
| 933 |
+
Step 8880/11700 | Loss: 2.4850 | LR: 7.75e-05 | 7712.5ms/step | 2.6 samples/s
|
| 934 |
+
Step 8890/11700 | Loss: 2.4459 | LR: 7.69e-05 | 7703.5ms/step | 2.6 samples/s
|
| 935 |
+
Step 8900/11700 | Loss: 2.4248 | LR: 7.64e-05 | 7709.0ms/step | 2.6 samples/s
|
| 936 |
+
Step 8910/11700 | Loss: 2.5221 | LR: 7.59e-05 | 7716.6ms/step | 2.6 samples/s
|
| 937 |
+
Step 8920/11700 | Loss: 2.5006 | LR: 7.54e-05 | 7717.7ms/step | 2.6 samples/s
|
| 938 |
+
Step 8930/11700 | Loss: 2.3848 | LR: 7.49e-05 | 7718.9ms/step | 2.6 samples/s
|
| 939 |
+
Step 8940/11700 | Loss: 2.4845 | LR: 7.44e-05 | 7717.5ms/step | 2.6 samples/s
|
| 940 |
+
Step 8950/11700 | Loss: 2.4151 | LR: 7.39e-05 | 7717.1ms/step | 2.6 samples/s
|
| 941 |
+
Step 8960/11700 | Loss: 2.4237 | LR: 7.34e-05 | 7716.7ms/step | 2.6 samples/s
|
| 942 |
+
Step 8970/11700 | Loss: 2.4320 | LR: 7.28e-05 | 7715.3ms/step | 2.6 samples/s
|
| 943 |
+
Step 8980/11700 | Loss: 2.4220 | LR: 7.23e-05 | 7717.9ms/step | 2.6 samples/s
|
| 944 |
+
Step 8990/11700 | Loss: 2.4595 | LR: 7.18e-05 | 7714.8ms/step | 2.6 samples/s
|
| 945 |
+
Step 9000/11700 | Loss: 2.4368 | LR: 7.13e-05 | 7718.6ms/step | 2.6 samples/s
|
| 946 |
+
Validation at step 9000: Loss = 2.4569, Perplexity = 11.67
|
| 947 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-9000
|
| 948 |
+
Step 9010/11700 | Loss: 2.5245 | LR: 7.08e-05 | 19577.9ms/step | 1.0 samples/s
|
| 949 |
+
Step 9020/11700 | Loss: 2.4373 | LR: 7.03e-05 | 7714.5ms/step | 2.6 samples/s
|
| 950 |
+
Step 9030/11700 | Loss: 2.4009 | LR: 6.98e-05 | 7713.1ms/step | 2.6 samples/s
|
| 951 |
+
Step 9040/11700 | Loss: 2.3753 | LR: 6.93e-05 | 7719.5ms/step | 2.6 samples/s
|
| 952 |
+
Step 9050/11700 | Loss: 2.4707 | LR: 6.88e-05 | 7717.4ms/step | 2.6 samples/s
|
| 953 |
+
Step 9060/11700 | Loss: 2.4850 | LR: 6.83e-05 | 7718.3ms/step | 2.6 samples/s
|
| 954 |
+
Step 9070/11700 | Loss: 2.4983 | LR: 6.79e-05 | 7715.6ms/step | 2.6 samples/s
|
| 955 |
+
Step 9080/11700 | Loss: 2.3945 | LR: 6.74e-05 | 7713.1ms/step | 2.6 samples/s
|
| 956 |
+
Step 9090/11700 | Loss: 2.3810 | LR: 6.69e-05 | 7713.7ms/step | 2.6 samples/s
|
| 957 |
+
Step 9100/11700 | Loss: 2.4681 | LR: 6.64e-05 | 7711.3ms/step | 2.6 samples/s
|
| 958 |
+
Step 9110/11700 | Loss: 2.4675 | LR: 6.59e-05 | 7714.4ms/step | 2.6 samples/s
|
| 959 |
+
Step 9120/11700 | Loss: 2.4984 | LR: 6.54e-05 | 7714.1ms/step | 2.6 samples/s
|
| 960 |
+
Step 9130/11700 | Loss: 2.3642 | LR: 6.49e-05 | 7712.9ms/step | 2.6 samples/s
|
| 961 |
+
Step 9140/11700 | Loss: 2.3425 | LR: 6.45e-05 | 7702.6ms/step | 2.6 samples/s
|
| 962 |
+
Step 9150/11700 | Loss: 2.4541 | LR: 6.40e-05 | 7710.3ms/step | 2.6 samples/s
|
| 963 |
+
Step 9160/11700 | Loss: 2.5430 | LR: 6.35e-05 | 7701.9ms/step | 2.6 samples/s
|
| 964 |
+
Step 9170/11700 | Loss: 2.4931 | LR: 6.30e-05 | 7701.6ms/step | 2.6 samples/s
|
| 965 |
+
Step 9180/11700 | Loss: 2.4295 | LR: 6.25e-05 | 7708.4ms/step | 2.6 samples/s
|
| 966 |
+
Step 9190/11700 | Loss: 2.4519 | LR: 6.21e-05 | 7704.1ms/step | 2.6 samples/s
|
| 967 |
+
Step 9200/11700 | Loss: 2.3347 | LR: 6.16e-05 | 7705.3ms/step | 2.6 samples/s
|
| 968 |
+
Step 9210/11700 | Loss: 2.4398 | LR: 6.11e-05 | 7701.5ms/step | 2.6 samples/s
|
| 969 |
+
Step 9220/11700 | Loss: 2.5223 | LR: 6.07e-05 | 7700.6ms/step | 2.6 samples/s
|
| 970 |
+
Step 9230/11700 | Loss: 2.4295 | LR: 6.02e-05 | 7702.9ms/step | 2.6 samples/s
|
| 971 |
+
Step 9240/11700 | Loss: 2.4156 | LR: 5.97e-05 | 7706.3ms/step | 2.6 samples/s
|
| 972 |
+
Step 9250/11700 | Loss: 2.4470 | LR: 5.93e-05 | 7702.8ms/step | 2.6 samples/s
|
| 973 |
+
Validation at step 9250: Loss = 2.4552, Perplexity = 11.65
|
| 974 |
+
Step 9260/11700 | Loss: 2.3702 | LR: 5.88e-05 | 19170.8ms/step | 1.0 samples/s
|
| 975 |
+
Step 9270/11700 | Loss: 2.4628 | LR: 5.83e-05 | 7704.4ms/step | 2.6 samples/s
|
| 976 |
+
Step 9280/11700 | Loss: 2.3184 | LR: 5.79e-05 | 7703.1ms/step | 2.6 samples/s
|
| 977 |
+
Step 9290/11700 | Loss: 2.4067 | LR: 5.74e-05 | 7709.9ms/step | 2.6 samples/s
|
| 978 |
+
Step 9300/11700 | Loss: 2.4617 | LR: 5.70e-05 | 7709.7ms/step | 2.6 samples/s
|
| 979 |
+
Step 9310/11700 | Loss: 2.3570 | LR: 5.65e-05 | 7705.7ms/step | 2.6 samples/s
|
| 980 |
+
Step 9320/11700 | Loss: 2.3999 | LR: 5.61e-05 | 7705.7ms/step | 2.6 samples/s
|
| 981 |
+
Step 9330/11700 | Loss: 2.3953 | LR: 5.56e-05 | 7705.0ms/step | 2.6 samples/s
|
| 982 |
+
Step 9340/11700 | Loss: 2.5060 | LR: 5.52e-05 | 7703.4ms/step | 2.6 samples/s
|
| 983 |
+
Step 9350/11700 | Loss: 2.3944 | LR: 5.47e-05 | 7709.5ms/step | 2.6 samples/s
|
| 984 |
+
Step 9360/11700 | Loss: 2.4264 | LR: 5.43e-05 | 7706.7ms/step | 2.6 samples/s
|
| 985 |
+
Step 9370/11700 | Loss: 2.4433 | LR: 5.38e-05 | 7703.1ms/step | 2.6 samples/s
|
| 986 |
+
Step 9380/11700 | Loss: 2.5465 | LR: 5.34e-05 | 7703.7ms/step | 2.6 samples/s
|
| 987 |
+
Step 9390/11700 | Loss: 2.4533 | LR: 5.29e-05 | 7712.2ms/step | 2.6 samples/s
|
| 988 |
+
Step 9400/11700 | Loss: 2.4302 | LR: 5.25e-05 | 7705.8ms/step | 2.6 samples/s
|
| 989 |
+
Step 9410/11700 | Loss: 2.4456 | LR: 5.20e-05 | 7709.3ms/step | 2.6 samples/s
|
| 990 |
+
Step 9420/11700 | Loss: 2.5089 | LR: 5.16e-05 | 7701.2ms/step | 2.6 samples/s
|
| 991 |
+
Step 9430/11700 | Loss: 2.3192 | LR: 5.12e-05 | 7705.0ms/step | 2.6 samples/s
|
| 992 |
+
Step 9440/11700 | Loss: 2.5359 | LR: 5.07e-05 | 7707.6ms/step | 2.6 samples/s
|
| 993 |
+
Step 9450/11700 | Loss: 2.4783 | LR: 5.03e-05 | 7705.3ms/step | 2.6 samples/s
|
| 994 |
+
Step 9460/11700 | Loss: 2.4782 | LR: 4.99e-05 | 7700.9ms/step | 2.6 samples/s
|
| 995 |
+
Step 9470/11700 | Loss: 2.4535 | LR: 4.94e-05 | 7705.7ms/step | 2.6 samples/s
|
| 996 |
+
Step 9480/11700 | Loss: 2.4118 | LR: 4.90e-05 | 7709.5ms/step | 2.6 samples/s
|
| 997 |
+
Step 9490/11700 | Loss: 2.5626 | LR: 4.86e-05 | 7705.8ms/step | 2.6 samples/s
|
| 998 |
+
Step 9500/11700 | Loss: 2.3689 | LR: 4.82e-05 | 7706.0ms/step | 2.6 samples/s
|
| 999 |
+
Validation at step 9500: Loss = 2.4539, Perplexity = 11.63
|
| 1000 |
+
Step 9510/11700 | Loss: 2.4562 | LR: 4.77e-05 | 19181.5ms/step | 1.0 samples/s
|
| 1001 |
+
Step 9520/11700 | Loss: 2.4096 | LR: 4.73e-05 | 7700.4ms/step | 2.6 samples/s
|
| 1002 |
+
Step 9530/11700 | Loss: 2.4124 | LR: 4.69e-05 | 7707.2ms/step | 2.6 samples/s
|
| 1003 |
+
Step 9540/11700 | Loss: 2.3747 | LR: 4.65e-05 | 7705.6ms/step | 2.6 samples/s
|
| 1004 |
+
Step 9550/11700 | Loss: 2.5802 | LR: 4.61e-05 | 7707.5ms/step | 2.6 samples/s
|
| 1005 |
+
Step 9560/11700 | Loss: 2.3599 | LR: 4.57e-05 | 7704.1ms/step | 2.6 samples/s
|
| 1006 |
+
Step 9570/11700 | Loss: 2.3970 | LR: 4.52e-05 | 7703.9ms/step | 2.6 samples/s
|
| 1007 |
+
Step 9580/11700 | Loss: 2.5173 | LR: 4.48e-05 | 7699.5ms/step | 2.6 samples/s
|
| 1008 |
+
Step 9590/11700 | Loss: 2.4486 | LR: 4.44e-05 | 7701.1ms/step | 2.6 samples/s
|
| 1009 |
+
Step 9600/11700 | Loss: 2.4590 | LR: 4.40e-05 | 7705.4ms/step | 2.6 samples/s
|
| 1010 |
+
Step 9610/11700 | Loss: 2.4581 | LR: 4.36e-05 | 7706.9ms/step | 2.6 samples/s
|
| 1011 |
+
Step 9620/11700 | Loss: 2.4706 | LR: 4.32e-05 | 7707.0ms/step | 2.6 samples/s
|
| 1012 |
+
Step 9630/11700 | Loss: 2.4179 | LR: 4.28e-05 | 7707.2ms/step | 2.6 samples/s
|
| 1013 |
+
Step 9640/11700 | Loss: 2.4679 | LR: 4.24e-05 | 7706.1ms/step | 2.6 samples/s
|
| 1014 |
+
Step 9650/11700 | Loss: 2.5466 | LR: 4.20e-05 | 7702.6ms/step | 2.6 samples/s
|
| 1015 |
+
Step 9660/11700 | Loss: 2.5507 | LR: 4.16e-05 | 7692.1ms/step | 2.6 samples/s
|
| 1016 |
+
Step 9670/11700 | Loss: 2.4491 | LR: 4.12e-05 | 7704.8ms/step | 2.6 samples/s
|
| 1017 |
+
Step 9680/11700 | Loss: 2.2707 | LR: 4.08e-05 | 7704.1ms/step | 2.6 samples/s
|
| 1018 |
+
Step 9690/11700 | Loss: 2.5089 | LR: 4.04e-05 | 7705.2ms/step | 2.6 samples/s
|
| 1019 |
+
Step 9700/11700 | Loss: 2.3789 | LR: 4.00e-05 | 7714.7ms/step | 2.6 samples/s
|
| 1020 |
+
Step 9710/11700 | Loss: 2.4302 | LR: 3.97e-05 | 7700.1ms/step | 2.6 samples/s
|
| 1021 |
+
Step 9720/11700 | Loss: 2.5051 | LR: 3.93e-05 | 7705.1ms/step | 2.6 samples/s
|
| 1022 |
+
Step 9730/11700 | Loss: 2.3471 | LR: 3.89e-05 | 7701.3ms/step | 2.6 samples/s
|
| 1023 |
+
Step 9740/11700 | Loss: 2.5185 | LR: 3.85e-05 | 7704.3ms/step | 2.6 samples/s
|
| 1024 |
+
Step 9750/11700 | Loss: 2.4419 | LR: 3.81e-05 | 7705.8ms/step | 2.6 samples/s
|
| 1025 |
+
Validation at step 9750: Loss = 2.4530, Perplexity = 11.62
|
| 1026 |
+
Step 9760/11700 | Loss: 2.4270 | LR: 3.77e-05 | 19178.0ms/step | 1.0 samples/s
|
| 1027 |
+
Step 9770/11700 | Loss: 2.3702 | LR: 3.74e-05 | 7708.2ms/step | 2.6 samples/s
|
| 1028 |
+
Step 9780/11700 | Loss: 2.5654 | LR: 3.70e-05 | 7707.7ms/step | 2.6 samples/s
|
| 1029 |
+
Step 9790/11700 | Loss: 2.4232 | LR: 3.66e-05 | 7705.6ms/step | 2.6 samples/s
|
| 1030 |
+
Step 9800/11700 | Loss: 2.3719 | LR: 3.62e-05 | 7706.7ms/step | 2.6 samples/s
|
| 1031 |
+
Step 9810/11700 | Loss: 2.4899 | LR: 3.59e-05 | 7703.6ms/step | 2.6 samples/s
|
| 1032 |
+
Step 9820/11700 | Loss: 2.3372 | LR: 3.55e-05 | 7709.2ms/step | 2.6 samples/s
|
| 1033 |
+
Step 9830/11700 | Loss: 2.3593 | LR: 3.51e-05 | 7707.3ms/step | 2.6 samples/s
|
| 1034 |
+
Step 9840/11700 | Loss: 2.3922 | LR: 3.48e-05 | 7704.5ms/step | 2.6 samples/s
|
| 1035 |
+
Step 9850/11700 | Loss: 2.4685 | LR: 3.44e-05 | 7707.5ms/step | 2.6 samples/s
|
| 1036 |
+
Step 9860/11700 | Loss: 2.4474 | LR: 3.40e-05 | 7705.0ms/step | 2.6 samples/s
|
| 1037 |
+
Step 9870/11700 | Loss: 2.3950 | LR: 3.37e-05 | 7703.5ms/step | 2.6 samples/s
|
| 1038 |
+
Step 9880/11700 | Loss: 2.4353 | LR: 3.33e-05 | 7703.4ms/step | 2.6 samples/s
|
| 1039 |
+
Step 9890/11700 | Loss: 2.3419 | LR: 3.30e-05 | 7701.9ms/step | 2.6 samples/s
|
| 1040 |
+
Step 9900/11700 | Loss: 2.4758 | LR: 3.26e-05 | 7705.1ms/step | 2.6 samples/s
|
| 1041 |
+
Step 9910/11700 | Loss: 2.4559 | LR: 3.22e-05 | 7704.9ms/step | 2.6 samples/s
|
| 1042 |
+
Step 9920/11700 | Loss: 2.4227 | LR: 3.19e-05 | 7702.6ms/step | 2.6 samples/s
|
| 1043 |
+
Step 9930/11700 | Loss: 2.5049 | LR: 3.15e-05 | 7689.1ms/step | 2.6 samples/s
|
| 1044 |
+
Step 9940/11700 | Loss: 2.3164 | LR: 3.12e-05 | 7703.9ms/step | 2.6 samples/s
|
| 1045 |
+
Step 9950/11700 | Loss: 2.4259 | LR: 3.09e-05 | 7708.7ms/step | 2.6 samples/s
|
| 1046 |
+
Step 9960/11700 | Loss: 2.3728 | LR: 3.05e-05 | 7709.1ms/step | 2.6 samples/s
|
| 1047 |
+
Step 9970/11700 | Loss: 2.4496 | LR: 3.02e-05 | 7701.2ms/step | 2.6 samples/s
|
| 1048 |
+
Step 9980/11700 | Loss: 2.3795 | LR: 2.98e-05 | 7708.4ms/step | 2.6 samples/s
|
| 1049 |
+
Step 9990/11700 | Loss: 2.3333 | LR: 2.95e-05 | 7705.2ms/step | 2.6 samples/s
|
| 1050 |
+
Step 10000/11700 | Loss: 2.4480 | LR: 2.92e-05 | 7709.8ms/step | 2.6 samples/s
|
| 1051 |
+
Validation at step 10000: Loss = 2.4525, Perplexity = 11.62
|
| 1052 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-10000
|
| 1053 |
+
Step 10010/11700 | Loss: 2.5449 | LR: 2.88e-05 | 19552.1ms/step | 1.0 samples/s
|
| 1054 |
+
Step 10020/11700 | Loss: 2.5302 | LR: 2.85e-05 | 7691.8ms/step | 2.6 samples/s
|
| 1055 |
+
Step 10030/11700 | Loss: 2.4305 | LR: 2.82e-05 | 7696.4ms/step | 2.6 samples/s
|
| 1056 |
+
Step 10040/11700 | Loss: 2.4814 | LR: 2.78e-05 | 7699.7ms/step | 2.6 samples/s
|
| 1057 |
+
Step 10050/11700 | Loss: 2.5176 | LR: 2.75e-05 | 7696.9ms/step | 2.6 samples/s
|
| 1058 |
+
Step 10060/11700 | Loss: 2.3921 | LR: 2.72e-05 | 7703.1ms/step | 2.6 samples/s
|
| 1059 |
+
Step 10070/11700 | Loss: 2.5107 | LR: 2.68e-05 | 7701.8ms/step | 2.6 samples/s
|
| 1060 |
+
Step 10080/11700 | Loss: 2.3864 | LR: 2.65e-05 | 7701.3ms/step | 2.6 samples/s
|
| 1061 |
+
Step 10090/11700 | Loss: 2.4569 | LR: 2.62e-05 | 7693.0ms/step | 2.6 samples/s
|
| 1062 |
+
Step 10100/11700 | Loss: 2.4196 | LR: 2.59e-05 | 7697.7ms/step | 2.6 samples/s
|
| 1063 |
+
Step 10110/11700 | Loss: 2.4901 | LR: 2.56e-05 | 7700.3ms/step | 2.6 samples/s
|
| 1064 |
+
Step 10120/11700 | Loss: 2.3582 | LR: 2.52e-05 | 7696.8ms/step | 2.6 samples/s
|
| 1065 |
+
Step 10130/11700 | Loss: 2.5255 | LR: 2.49e-05 | 7702.4ms/step | 2.6 samples/s
|
| 1066 |
+
Step 10140/11700 | Loss: 2.5584 | LR: 2.46e-05 | 7702.0ms/step | 2.6 samples/s
|
| 1067 |
+
Step 10150/11700 | Loss: 2.4694 | LR: 2.43e-05 | 7698.0ms/step | 2.6 samples/s
|
| 1068 |
+
Step 10160/11700 | Loss: 2.4539 | LR: 2.40e-05 | 7702.9ms/step | 2.6 samples/s
|
| 1069 |
+
Step 10170/11700 | Loss: 2.4373 | LR: 2.37e-05 | 7698.0ms/step | 2.6 samples/s
|
| 1070 |
+
Step 10180/11700 | Loss: 2.4605 | LR: 2.34e-05 | 7703.0ms/step | 2.6 samples/s
|
| 1071 |
+
Step 10190/11700 | Loss: 2.3669 | LR: 2.31e-05 | 7707.6ms/step | 2.6 samples/s
|
| 1072 |
+
Step 10200/11700 | Loss: 2.3933 | LR: 2.28e-05 | 7701.2ms/step | 2.6 samples/s
|
| 1073 |
+
Step 10210/11700 | Loss: 2.4884 | LR: 2.25e-05 | 7698.1ms/step | 2.6 samples/s
|
| 1074 |
+
Step 10220/11700 | Loss: 2.4461 | LR: 2.22e-05 | 7702.0ms/step | 2.6 samples/s
|
| 1075 |
+
Step 10230/11700 | Loss: 2.5493 | LR: 2.19e-05 | 7698.2ms/step | 2.6 samples/s
|
| 1076 |
+
Step 10240/11700 | Loss: 2.5371 | LR: 2.16e-05 | 7698.4ms/step | 2.6 samples/s
|
| 1077 |
+
Step 10250/11700 | Loss: 2.4860 | LR: 2.13e-05 | 7701.3ms/step | 2.6 samples/s
|
| 1078 |
+
Validation at step 10250: Loss = 2.4521, Perplexity = 11.61
|
| 1079 |
+
Step 10260/11700 | Loss: 2.4937 | LR: 2.10e-05 | 19166.4ms/step | 1.0 samples/s
|
| 1080 |
+
Step 10270/11700 | Loss: 2.3966 | LR: 2.07e-05 | 7704.5ms/step | 2.6 samples/s
|
| 1081 |
+
Step 10280/11700 | Loss: 2.4944 | LR: 2.05e-05 | 7704.7ms/step | 2.6 samples/s
|
| 1082 |
+
Step 10290/11700 | Loss: 2.4788 | LR: 2.02e-05 | 7702.7ms/step | 2.6 samples/s
|
| 1083 |
+
Step 10300/11700 | Loss: 2.3318 | LR: 1.99e-05 | 7700.3ms/step | 2.6 samples/s
|
| 1084 |
+
Step 10310/11700 | Loss: 2.4386 | LR: 1.96e-05 | 7703.1ms/step | 2.6 samples/s
|
| 1085 |
+
Step 10320/11700 | Loss: 2.3847 | LR: 1.93e-05 | 7703.5ms/step | 2.6 samples/s
|
| 1086 |
+
Step 10330/11700 | Loss: 2.3758 | LR: 1.91e-05 | 7709.9ms/step | 2.6 samples/s
|
| 1087 |
+
Step 10340/11700 | Loss: 2.5451 | LR: 1.88e-05 | 7705.0ms/step | 2.6 samples/s
|
| 1088 |
+
Step 10350/11700 | Loss: 2.4757 | LR: 1.85e-05 | 7702.7ms/step | 2.6 samples/s
|
| 1089 |
+
Step 10360/11700 | Loss: 2.4714 | LR: 1.82e-05 | 7701.8ms/step | 2.6 samples/s
|
| 1090 |
+
Step 10370/11700 | Loss: 2.4522 | LR: 1.80e-05 | 7703.3ms/step | 2.6 samples/s
|
| 1091 |
+
Step 10380/11700 | Loss: 2.2890 | LR: 1.77e-05 | 7701.1ms/step | 2.6 samples/s
|
| 1092 |
+
Step 10390/11700 | Loss: 2.5852 | LR: 1.75e-05 | 7708.2ms/step | 2.6 samples/s
|
| 1093 |
+
Step 10400/11700 | Loss: 2.4939 | LR: 1.72e-05 | 7711.8ms/step | 2.6 samples/s
|
| 1094 |
+
Step 10410/11700 | Loss: 2.4434 | LR: 1.69e-05 | 7705.9ms/step | 2.6 samples/s
|
| 1095 |
+
Step 10420/11700 | Loss: 2.4220 | LR: 1.67e-05 | 7687.9ms/step | 2.6 samples/s
|
| 1096 |
+
Step 10430/11700 | Loss: 2.4157 | LR: 1.64e-05 | 7699.2ms/step | 2.6 samples/s
|
| 1097 |
+
Step 10440/11700 | Loss: 2.3882 | LR: 1.62e-05 | 7698.7ms/step | 2.6 samples/s
|
| 1098 |
+
Step 10450/11700 | Loss: 2.4650 | LR: 1.59e-05 | 7698.8ms/step | 2.6 samples/s
|
| 1099 |
+
Step 10460/11700 | Loss: 2.4885 | LR: 1.57e-05 | 7703.7ms/step | 2.6 samples/s
|
| 1100 |
+
Step 10470/11700 | Loss: 2.3756 | LR: 1.54e-05 | 7702.4ms/step | 2.6 samples/s
|
| 1101 |
+
Step 10480/11700 | Loss: 2.4160 | LR: 1.52e-05 | 7702.5ms/step | 2.6 samples/s
|
| 1102 |
+
Step 10490/11700 | Loss: 2.3577 | LR: 1.49e-05 | 7704.5ms/step | 2.6 samples/s
|
| 1103 |
+
Step 10500/11700 | Loss: 2.4465 | LR: 1.47e-05 | 7701.0ms/step | 2.6 samples/s
|
| 1104 |
+
Validation at step 10500: Loss = 2.4520, Perplexity = 11.61
|
| 1105 |
+
Step 10510/11700 | Loss: 2.5701 | LR: 1.44e-05 | 19163.5ms/step | 1.0 samples/s
|
| 1106 |
+
Step 10520/11700 | Loss: 2.4714 | LR: 1.42e-05 | 7703.5ms/step | 2.6 samples/s
|
| 1107 |
+
Step 10530/11700 | Loss: 2.4579 | LR: 1.40e-05 | 7701.7ms/step | 2.6 samples/s
|
| 1108 |
+
Step 10540/11700 | Loss: 2.3073 | LR: 1.37e-05 | 7706.4ms/step | 2.6 samples/s
|
| 1109 |
+
Step 10550/11700 | Loss: 2.4841 | LR: 1.35e-05 | 7698.4ms/step | 2.6 samples/s
|
| 1110 |
+
Step 10560/11700 | Loss: 2.5833 | LR: 1.33e-05 | 7701.0ms/step | 2.6 samples/s
|
| 1111 |
+
Step 10570/11700 | Loss: 2.3392 | LR: 1.30e-05 | 7699.8ms/step | 2.6 samples/s
|
| 1112 |
+
Step 10580/11700 | Loss: 2.5154 | LR: 1.28e-05 | 7704.2ms/step | 2.6 samples/s
|
| 1113 |
+
Step 10590/11700 | Loss: 2.3793 | LR: 1.26e-05 | 7700.2ms/step | 2.6 samples/s
|
| 1114 |
+
Step 10600/11700 | Loss: 2.4676 | LR: 1.23e-05 | 7700.5ms/step | 2.6 samples/s
|
| 1115 |
+
Step 10610/11700 | Loss: 2.4372 | LR: 1.21e-05 | 7701.3ms/step | 2.6 samples/s
|
| 1116 |
+
Step 10620/11700 | Loss: 2.3563 | LR: 1.19e-05 | 7700.9ms/step | 2.6 samples/s
|
| 1117 |
+
Step 10630/11700 | Loss: 2.4241 | LR: 1.17e-05 | 7702.5ms/step | 2.6 samples/s
|
| 1118 |
+
Step 10640/11700 | Loss: 2.4080 | LR: 1.15e-05 | 7703.0ms/step | 2.6 samples/s
|
| 1119 |
+
Step 10650/11700 | Loss: 2.4611 | LR: 1.13e-05 | 7701.0ms/step | 2.6 samples/s
|
| 1120 |
+
Step 10660/11700 | Loss: 2.4137 | LR: 1.10e-05 | 7707.3ms/step | 2.6 samples/s
|
| 1121 |
+
Step 10670/11700 | Loss: 2.4009 | LR: 1.08e-05 | 7693.8ms/step | 2.6 samples/s
|
| 1122 |
+
Step 10680/11700 | Loss: 2.4768 | LR: 1.06e-05 | 7704.4ms/step | 2.6 samples/s
|
| 1123 |
+
Step 10690/11700 | Loss: 2.4650 | LR: 1.04e-05 | 7702.5ms/step | 2.6 samples/s
|
| 1124 |
+
Step 10700/11700 | Loss: 2.4336 | LR: 1.02e-05 | 7701.1ms/step | 2.6 samples/s
|
| 1125 |
+
Step 10710/11700 | Loss: 2.4666 | LR: 1.00e-05 | 7698.4ms/step | 2.6 samples/s
|
| 1126 |
+
Step 10720/11700 | Loss: 2.4017 | LR: 9.82e-06 | 7701.6ms/step | 2.6 samples/s
|
| 1127 |
+
Step 10730/11700 | Loss: 2.5031 | LR: 9.62e-06 | 7700.1ms/step | 2.6 samples/s
|
| 1128 |
+
Step 10740/11700 | Loss: 2.3314 | LR: 9.42e-06 | 7703.4ms/step | 2.6 samples/s
|
| 1129 |
+
Step 10750/11700 | Loss: 2.3598 | LR: 9.23e-06 | 7702.5ms/step | 2.6 samples/s
|
| 1130 |
+
Validation at step 10750: Loss = 2.4519, Perplexity = 11.61
|
| 1131 |
+
Step 10760/11700 | Loss: 2.5284 | LR: 9.04e-06 | 19168.2ms/step | 1.0 samples/s
|
| 1132 |
+
Step 10770/11700 | Loss: 2.5031 | LR: 8.85e-06 | 7700.8ms/step | 2.6 samples/s
|
| 1133 |
+
Step 10780/11700 | Loss: 2.5317 | LR: 8.66e-06 | 7705.3ms/step | 2.6 samples/s
|
| 1134 |
+
Step 10790/11700 | Loss: 2.4418 | LR: 8.47e-06 | 7702.1ms/step | 2.6 samples/s
|
| 1135 |
+
Step 10800/11700 | Loss: 2.5358 | LR: 8.29e-06 | 7704.5ms/step | 2.6 samples/s
|
| 1136 |
+
Step 10810/11700 | Loss: 2.4331 | LR: 8.11e-06 | 7696.4ms/step | 2.6 samples/s
|
| 1137 |
+
Step 10820/11700 | Loss: 2.5209 | LR: 7.93e-06 | 7697.7ms/step | 2.6 samples/s
|
| 1138 |
+
Step 10830/11700 | Loss: 2.4200 | LR: 7.75e-06 | 7693.1ms/step | 2.6 samples/s
|
| 1139 |
+
Step 10840/11700 | Loss: 2.4018 | LR: 7.57e-06 | 7695.6ms/step | 2.6 samples/s
|
| 1140 |
+
Step 10850/11700 | Loss: 2.3528 | LR: 7.40e-06 | 7701.4ms/step | 2.6 samples/s
|
| 1141 |
+
Step 10860/11700 | Loss: 2.3533 | LR: 7.22e-06 | 7706.6ms/step | 2.6 samples/s
|
| 1142 |
+
Step 10870/11700 | Loss: 2.3819 | LR: 7.05e-06 | 7700.9ms/step | 2.6 samples/s
|
| 1143 |
+
Step 10880/11700 | Loss: 2.3614 | LR: 6.89e-06 | 7702.0ms/step | 2.6 samples/s
|
| 1144 |
+
Step 10890/11700 | Loss: 2.3476 | LR: 6.72e-06 | 7698.6ms/step | 2.6 samples/s
|
| 1145 |
+
Step 10900/11700 | Loss: 2.4531 | LR: 6.56e-06 | 7698.2ms/step | 2.6 samples/s
|
| 1146 |
+
Step 10910/11700 | Loss: 2.4933 | LR: 6.39e-06 | 7698.2ms/step | 2.6 samples/s
|
| 1147 |
+
Step 10920/11700 | Loss: 2.4169 | LR: 6.23e-06 | 7708.5ms/step | 2.6 samples/s
|
| 1148 |
+
Step 10930/11700 | Loss: 2.3385 | LR: 6.08e-06 | 7701.6ms/step | 2.6 samples/s
|
| 1149 |
+
Step 10940/11700 | Loss: 2.3606 | LR: 5.92e-06 | 7706.6ms/step | 2.6 samples/s
|
| 1150 |
+
Step 10950/11700 | Loss: 2.3672 | LR: 5.77e-06 | 7702.0ms/step | 2.6 samples/s
|
| 1151 |
+
Step 10960/11700 | Loss: 2.5175 | LR: 5.61e-06 | 7703.5ms/step | 2.6 samples/s
|
| 1152 |
+
Step 10970/11700 | Loss: 2.4739 | LR: 5.46e-06 | 7702.7ms/step | 2.6 samples/s
|
| 1153 |
+
Step 10980/11700 | Loss: 2.4702 | LR: 5.31e-06 | 7703.9ms/step | 2.6 samples/s
|
| 1154 |
+
Step 10990/11700 | Loss: 2.4089 | LR: 5.17e-06 | 7704.0ms/step | 2.6 samples/s
|
| 1155 |
+
Step 11000/11700 | Loss: 2.3884 | LR: 5.02e-06 | 7702.3ms/step | 2.6 samples/s
|
| 1156 |
+
Validation at step 11000: Loss = 2.4518, Perplexity = 11.61
|
| 1157 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-11000
|
| 1158 |
+
Step 11010/11700 | Loss: 2.3783 | LR: 4.88e-06 | 19544.6ms/step | 1.0 samples/s
|
| 1159 |
+
Step 11020/11700 | Loss: 2.4470 | LR: 4.74e-06 | 7696.6ms/step | 2.6 samples/s
|
| 1160 |
+
Step 11030/11700 | Loss: 2.4809 | LR: 4.60e-06 | 7695.9ms/step | 2.6 samples/s
|
| 1161 |
+
Step 11040/11700 | Loss: 2.4720 | LR: 4.47e-06 | 7704.9ms/step | 2.6 samples/s
|
| 1162 |
+
Step 11050/11700 | Loss: 2.4942 | LR: 4.33e-06 | 7692.8ms/step | 2.6 samples/s
|
| 1163 |
+
Step 11060/11700 | Loss: 2.4278 | LR: 4.20e-06 | 7702.2ms/step | 2.6 samples/s
|
| 1164 |
+
Step 11070/11700 | Loss: 2.3902 | LR: 4.07e-06 | 7700.8ms/step | 2.6 samples/s
|
| 1165 |
+
Step 11080/11700 | Loss: 2.4501 | LR: 3.94e-06 | 7700.2ms/step | 2.6 samples/s
|
| 1166 |
+
Step 11090/11700 | Loss: 2.4349 | LR: 3.82e-06 | 7700.8ms/step | 2.6 samples/s
|
| 1167 |
+
Step 11100/11700 | Loss: 2.4803 | LR: 3.69e-06 | 7699.3ms/step | 2.6 samples/s
|
| 1168 |
+
Step 11110/11700 | Loss: 2.4260 | LR: 3.57e-06 | 7704.3ms/step | 2.6 samples/s
|
| 1169 |
+
Step 11120/11700 | Loss: 2.4703 | LR: 3.45e-06 | 7701.3ms/step | 2.6 samples/s
|
| 1170 |
+
Step 11130/11700 | Loss: 2.5178 | LR: 3.34e-06 | 7698.0ms/step | 2.6 samples/s
|
| 1171 |
+
Step 11140/11700 | Loss: 2.4209 | LR: 3.22e-06 | 7702.9ms/step | 2.6 samples/s
|
| 1172 |
+
Step 11150/11700 | Loss: 2.4434 | LR: 3.11e-06 | 7704.5ms/step | 2.6 samples/s
|
| 1173 |
+
Step 11160/11700 | Loss: 2.3901 | LR: 2.99e-06 | 7702.4ms/step | 2.6 samples/s
|
| 1174 |
+
Step 11170/11700 | Loss: 2.5031 | LR: 2.88e-06 | 7705.7ms/step | 2.6 samples/s
|
| 1175 |
+
Step 11180/11700 | Loss: 2.4145 | LR: 2.78e-06 | 7705.6ms/step | 2.6 samples/s
|
| 1176 |
+
Step 11190/11700 | Loss: 2.4535 | LR: 2.67e-06 | 7710.9ms/step | 2.6 samples/s
|
| 1177 |
+
Step 11200/11700 | Loss: 2.4476 | LR: 2.57e-06 | 7715.9ms/step | 2.6 samples/s
|
| 1178 |
+
Step 11210/11700 | Loss: 2.4964 | LR: 2.47e-06 | 7718.2ms/step | 2.6 samples/s
|
| 1179 |
+
Step 11220/11700 | Loss: 2.3891 | LR: 2.37e-06 | 7715.4ms/step | 2.6 samples/s
|
| 1180 |
+
Step 11230/11700 | Loss: 2.3164 | LR: 2.27e-06 | 7710.0ms/step | 2.6 samples/s
|
| 1181 |
+
Step 11240/11700 | Loss: 2.3842 | LR: 2.17e-06 | 7711.1ms/step | 2.6 samples/s
|
| 1182 |
+
Step 11250/11700 | Loss: 2.4000 | LR: 2.08e-06 | 7718.7ms/step | 2.6 samples/s
|
| 1183 |
+
Validation at step 11250: Loss = 2.4518, Perplexity = 11.61
|
| 1184 |
+
Step 11260/11700 | Loss: 2.4325 | LR: 1.99e-06 | 19201.8ms/step | 1.0 samples/s
|
| 1185 |
+
Step 11270/11700 | Loss: 2.4776 | LR: 1.90e-06 | 7714.6ms/step | 2.6 samples/s
|
| 1186 |
+
Step 11280/11700 | Loss: 2.4181 | LR: 1.81e-06 | 7710.4ms/step | 2.6 samples/s
|
| 1187 |
+
Step 11290/11700 | Loss: 2.4872 | LR: 1.73e-06 | 7714.8ms/step | 2.6 samples/s
|
| 1188 |
+
Step 11300/11700 | Loss: 2.4546 | LR: 1.64e-06 | 7717.0ms/step | 2.6 samples/s
|
| 1189 |
+
Step 11310/11700 | Loss: 2.4231 | LR: 1.56e-06 | 7713.8ms/step | 2.6 samples/s
|
| 1190 |
+
Step 11320/11700 | Loss: 2.4529 | LR: 1.48e-06 | 7709.7ms/step | 2.6 samples/s
|
| 1191 |
+
Step 11330/11700 | Loss: 2.3795 | LR: 1.41e-06 | 7709.8ms/step | 2.6 samples/s
|
| 1192 |
+
Step 11340/11700 | Loss: 2.3568 | LR: 1.33e-06 | 7709.2ms/step | 2.6 samples/s
|
| 1193 |
+
Step 11350/11700 | Loss: 2.4806 | LR: 1.26e-06 | 7710.8ms/step | 2.6 samples/s
|
| 1194 |
+
Step 11360/11700 | Loss: 2.4726 | LR: 1.19e-06 | 7707.4ms/step | 2.6 samples/s
|
| 1195 |
+
Step 11370/11700 | Loss: 2.4328 | LR: 1.12e-06 | 7716.5ms/step | 2.6 samples/s
|
| 1196 |
+
Step 11380/11700 | Loss: 2.4193 | LR: 1.05e-06 | 7713.0ms/step | 2.6 samples/s
|
| 1197 |
+
Step 11390/11700 | Loss: 2.4377 | LR: 9.88e-07 | 7711.1ms/step | 2.6 samples/s
|
| 1198 |
+
Step 11400/11700 | Loss: 2.4647 | LR: 9.25e-07 | 7710.0ms/step | 2.6 samples/s
|
| 1199 |
+
Step 11410/11700 | Loss: 2.4407 | LR: 8.65e-07 | 7709.6ms/step | 2.6 samples/s
|
| 1200 |
+
Step 11420/11700 | Loss: 2.2900 | LR: 8.06e-07 | 7708.8ms/step | 2.6 samples/s
|
| 1201 |
+
Step 11430/11700 | Loss: 2.3918 | LR: 7.50e-07 | 7713.3ms/step | 2.6 samples/s
|
| 1202 |
+
Step 11440/11700 | Loss: 2.4817 | LR: 6.95e-07 | 7714.7ms/step | 2.6 samples/s
|
| 1203 |
+
Step 11450/11700 | Loss: 2.3282 | LR: 6.43e-07 | 7709.0ms/step | 2.6 samples/s
|
| 1204 |
+
Step 11460/11700 | Loss: 2.4531 | LR: 5.92e-07 | 7710.0ms/step | 2.6 samples/s
|
| 1205 |
+
Step 11470/11700 | Loss: 2.4988 | LR: 5.44e-07 | 7712.0ms/step | 2.6 samples/s
|
| 1206 |
+
Step 11480/11700 | Loss: 2.3685 | LR: 4.98e-07 | 7718.0ms/step | 2.6 samples/s
|
| 1207 |
+
Step 11490/11700 | Loss: 2.4695 | LR: 4.54e-07 | 7716.6ms/step | 2.6 samples/s
|
| 1208 |
+
Step 11500/11700 | Loss: 2.4040 | LR: 4.11e-07 | 7715.3ms/step | 2.6 samples/s
|
| 1209 |
+
Validation at step 11500: Loss = 2.4518, Perplexity = 11.61
|
| 1210 |
+
Step 11510/11700 | Loss: 2.3895 | LR: 3.71e-07 | 19186.4ms/step | 1.0 samples/s
|
| 1211 |
+
Step 11520/11700 | Loss: 2.4635 | LR: 3.33e-07 | 7710.6ms/step | 2.6 samples/s
|
| 1212 |
+
Step 11530/11700 | Loss: 2.4084 | LR: 2.97e-07 | 7713.4ms/step | 2.6 samples/s
|
| 1213 |
+
Step 11540/11700 | Loss: 2.3849 | LR: 2.63e-07 | 7715.6ms/step | 2.6 samples/s
|
| 1214 |
+
Step 11550/11700 | Loss: 2.5060 | LR: 2.31e-07 | 7717.8ms/step | 2.6 samples/s
|
| 1215 |
+
Step 11560/11700 | Loss: 2.4039 | LR: 2.02e-07 | 7710.8ms/step | 2.6 samples/s
|
| 1216 |
+
Step 11570/11700 | Loss: 2.5163 | LR: 1.74e-07 | 7717.5ms/step | 2.6 samples/s
|
| 1217 |
+
Step 11580/11700 | Loss: 2.3533 | LR: 1.48e-07 | 7716.9ms/step | 2.6 samples/s
|
| 1218 |
+
Step 11590/11700 | Loss: 2.4703 | LR: 1.24e-07 | 7713.2ms/step | 2.6 samples/s
|
| 1219 |
+
Step 11600/11700 | Loss: 2.4717 | LR: 1.03e-07 | 7709.7ms/step | 2.6 samples/s
|
| 1220 |
+
Step 11610/11700 | Loss: 2.4736 | LR: 8.33e-08 | 7718.2ms/step | 2.6 samples/s
|
| 1221 |
+
Step 11620/11700 | Loss: 2.3893 | LR: 6.58e-08 | 7713.9ms/step | 2.6 samples/s
|
| 1222 |
+
Step 11630/11700 | Loss: 2.3372 | LR: 5.04e-08 | 7712.0ms/step | 2.6 samples/s
|
| 1223 |
+
Step 11640/11700 | Loss: 2.4724 | LR: 3.70e-08 | 7716.3ms/step | 2.6 samples/s
|
| 1224 |
+
Step 11650/11700 | Loss: 2.5234 | LR: 2.57e-08 | 7715.2ms/step | 2.6 samples/s
|
| 1225 |
+
Step 11660/11700 | Loss: 2.2979 | LR: 1.65e-08 | 7710.8ms/step | 2.6 samples/s
|
| 1226 |
+
Step 11670/11700 | Loss: 2.4527 | LR: 9.26e-09 | 7713.1ms/step | 2.6 samples/s
|
| 1227 |
+
Step 11680/11700 | Loss: 2.4643 | LR: 4.12e-09 | 7710.3ms/step | 2.6 samples/s
|
| 1228 |
+
Step 11690/11700 | Loss: 2.3694 | LR: 1.03e-09 | 7712.7ms/step | 2.6 samples/s
|
| 1229 |
+
Step 11700/11700 | Loss: 2.4336 | LR: 0.00e+00 | 7715.1ms/step | 2.6 samples/s
|
| 1230 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756/checkpoint-11700
|
epfl_final/SparseAtt/wandb/run-20260707_035814-kxni9nyw/files/requirements.txt
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
epfl_final/SparseAtt/wandb/run-20260707_035814-kxni9nyw/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-07-07T03:58:14.345278Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"configs/ladder30k/fixed_s1.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/TokenPrune.git",
|
| 14 |
+
"commit": "88298c7a418622530ca5426b22b1e5ebd10ca0cb"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "h100-train-4-gpu-0-0",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 23 |
+
"gpu_count": 4,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "9374931091456",
|
| 27 |
+
"used": "4873970139136"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1623005732864"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-85844f64-d144-1c84-306d-f88b5f232a77"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 43 |
+
"memoryTotal": "85520809984",
|
| 44 |
+
"cudaCores": 16896,
|
| 45 |
+
"architecture": "Hopper",
|
| 46 |
+
"uuid": "GPU-4e4c2343-4f5b-a2ff-312e-03d402a27803"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 50 |
+
"memoryTotal": "85520809984",
|
| 51 |
+
"cudaCores": 16896,
|
| 52 |
+
"architecture": "Hopper",
|
| 53 |
+
"uuid": "GPU-fbf0ccd1-8bd4-5f57-6e74-6c0cf7eb70c9"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 57 |
+
"memoryTotal": "85520809984",
|
| 58 |
+
"cudaCores": 16896,
|
| 59 |
+
"architecture": "Hopper",
|
| 60 |
+
"uuid": "GPU-410cba62-f098-3454-c960-ebc55bd081af"
|
| 61 |
+
}
|
| 62 |
+
],
|
| 63 |
+
"cudaVersion": "13.0",
|
| 64 |
+
"writerId": "m8mrvef218tbffx2zbh913ea7l1zv5ur"
|
| 65 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260707_035814-kxni9nyw/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"eval/dense_interval_end":-1,"train/learning_rate":0,"eval/dense_interval_active":0,"eval/perplexity":11.60939888152088,"eval/loss":2.451815018445253,"_step":11700,"train/epoch":0.3122998078155029,"eval/uses_sparse_attention":1,"_runtime":95655.544992404,"speed/ms_per_step":7715.112900733948,"_timestamp":1.7834923475622954e+09,"speed/samples_per_sec":2.5923146242095014,"eval/dense_interval_start":-1,"eval/uses_dense_attention":0,"speed/steps_per_sec":0.12961573121047507,"_wandb":{"runtime":95655},"train/global_step":11700,"train/loss":2.433622387424111}
|
epfl_final/SparseAtt/wandb/run-20260707_035814-kxni9nyw/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260707_035814-kxni9nyw/logs/debug.log
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-07-07 03:58:14,366 INFO MainThread:2505 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-07-07 03:58:14,366 INFO MainThread:2505 [wandb_setup.py:_flush():81] Configure stats pid to 2505
|
| 3 |
+
2026-07-07 03:58:14,367 INFO MainThread:2505 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-07-07 03:58:14,367 INFO MainThread:2505 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260707_035814-kxni9nyw/logs/debug.log
|
| 5 |
+
2026-07-07 03:58:14,367 INFO MainThread:2505 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260707_035814-kxni9nyw/logs/debug-internal.log
|
| 6 |
+
2026-07-07 03:58:14,367 INFO MainThread:2505 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-07-07 03:58:14,367 INFO MainThread:2505 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/longctx30k/tokenized', 'val_path': None, 'max_seq_length': 30720, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s1/run_20260707_035756', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 11700, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 20, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m fixed s1 30k', 'wandb_tags': ['qwen3', '365m', '30k', 'scaleladder', 'fixed', 'seed1'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 120, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_30k_fixed_s1', 'sparse_mask_seed': 1, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-07-07 03:58:14,367 INFO MainThread:2505 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-07-07 03:58:14,624 INFO MainThread:2505 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-07-07 03:58:14,957 INFO MainThread:2505 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-07-07 03:58:14,959 INFO MainThread:2505 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-07-07 03:58:14,976 INFO MainThread:2505 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-07-07 03:58:15,396 INFO MainThread:2505 [wandb_init.py:init():1056] starting run threads in backend
|
| 15 |
+
2026-07-07 03:58:15,634 INFO MainThread:2505 [wandb_run.py:_console_start():2554] atexit reg
|
| 16 |
+
2026-07-07 03:58:15,634 INFO MainThread:2505 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 17 |
+
2026-07-07 03:58:15,635 INFO MainThread:2505 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 18 |
+
2026-07-07 03:58:15,635 INFO MainThread:2505 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 19 |
+
2026-07-07 03:58:15,638 INFO MainThread:2505 [wandb_init.py:init():1094] run started, returning control to user process
|
| 20 |
+
2026-07-08 06:32:30,939 INFO MainThread:2505 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/kxni9nyw
|
| 21 |
+
2026-07-08 06:32:30,940 INFO MainThread:2505 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
|
| 22 |
+
2026-07-08 06:32:30,940 INFO MainThread:2505 [wandb_run.py:_restore():2502] restore
|
| 23 |
+
2026-07-08 06:32:30,940 INFO MainThread:2505 [wandb_run.py:_restore():2508] restore done
|
| 24 |
+
2026-07-08 06:32:32,193 INFO MainThread:2505 [wandb_run.py:_footer_sync_info():3889] logging synced files
|
epfl_final/SparseAtt/wandb/run-20260707_035814-kxni9nyw/run-kxni9nyw.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:85e052ccddb9a5e81ed5dfa45c6b065c52391bd5a87e17aa7bd4250c3036ad3d
|
| 3 |
+
size 30302130
|
epfl_final/SparseAtt/wandb/run-20260707_035818-m10lf13v/files/config.yaml
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.26.1
|
| 4 |
+
e:
|
| 5 |
+
ryjy1nj9s7a4cu1orvx223hpmj42juni:
|
| 6 |
+
args:
|
| 7 |
+
- --config
|
| 8 |
+
- configs/ladder30k/fixed_s2.json
|
| 9 |
+
codePath: train.py
|
| 10 |
+
codePathLocal: train.py
|
| 11 |
+
cpu_count: 96
|
| 12 |
+
cpu_count_logical: 192
|
| 13 |
+
cudaVersion: "13.0"
|
| 14 |
+
disk:
|
| 15 |
+
/:
|
| 16 |
+
total: "9374931091456"
|
| 17 |
+
used: "4873970163712"
|
| 18 |
+
email: ruben.navasardyan@mail.utoronto.ca
|
| 19 |
+
executable: /mloscratch/homes/navasard/conda/bin/python
|
| 20 |
+
git:
|
| 21 |
+
commit: 88298c7a418622530ca5426b22b1e5ebd10ca0cb
|
| 22 |
+
remote: https://github.com/rubentium/TokenPrune.git
|
| 23 |
+
gpu: NVIDIA H100 80GB HBM3
|
| 24 |
+
gpu_count: 4
|
| 25 |
+
gpu_nvidia:
|
| 26 |
+
- architecture: Hopper
|
| 27 |
+
cudaCores: 16896
|
| 28 |
+
memoryTotal: "85520809984"
|
| 29 |
+
name: NVIDIA H100 80GB HBM3
|
| 30 |
+
uuid: GPU-85844f64-d144-1c84-306d-f88b5f232a77
|
| 31 |
+
- architecture: Hopper
|
| 32 |
+
cudaCores: 16896
|
| 33 |
+
memoryTotal: "85520809984"
|
| 34 |
+
name: NVIDIA H100 80GB HBM3
|
| 35 |
+
uuid: GPU-4e4c2343-4f5b-a2ff-312e-03d402a27803
|
| 36 |
+
- architecture: Hopper
|
| 37 |
+
cudaCores: 16896
|
| 38 |
+
memoryTotal: "85520809984"
|
| 39 |
+
name: NVIDIA H100 80GB HBM3
|
| 40 |
+
uuid: GPU-fbf0ccd1-8bd4-5f57-6e74-6c0cf7eb70c9
|
| 41 |
+
- architecture: Hopper
|
| 42 |
+
cudaCores: 16896
|
| 43 |
+
memoryTotal: "85520809984"
|
| 44 |
+
name: NVIDIA H100 80GB HBM3
|
| 45 |
+
uuid: GPU-410cba62-f098-3454-c960-ebc55bd081af
|
| 46 |
+
host: h100-train-4-gpu-0-0
|
| 47 |
+
memory:
|
| 48 |
+
total: "1623005732864"
|
| 49 |
+
os: Linux-6.8.0-124-generic-x86_64-with-glibc2.35
|
| 50 |
+
program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
|
| 51 |
+
python: CPython 3.13.13
|
| 52 |
+
root: /mloscratch/homes/navasard/sparse/SparseAtt
|
| 53 |
+
startedAt: "2026-07-07T03:58:18.687564Z"
|
| 54 |
+
writerId: ryjy1nj9s7a4cu1orvx223hpmj42juni
|
| 55 |
+
m: []
|
| 56 |
+
python_version: 3.13.13
|
| 57 |
+
t:
|
| 58 |
+
"1":
|
| 59 |
+
- 1
|
| 60 |
+
- 5
|
| 61 |
+
- 11
|
| 62 |
+
- 49
|
| 63 |
+
- 51
|
| 64 |
+
- 53
|
| 65 |
+
- 71
|
| 66 |
+
"2":
|
| 67 |
+
- 1
|
| 68 |
+
- 5
|
| 69 |
+
- 11
|
| 70 |
+
- 49
|
| 71 |
+
- 51
|
| 72 |
+
- 53
|
| 73 |
+
- 71
|
| 74 |
+
"3":
|
| 75 |
+
- 2
|
| 76 |
+
- 13
|
| 77 |
+
- 15
|
| 78 |
+
- 16
|
| 79 |
+
- 61
|
| 80 |
+
"4": 3.13.13
|
| 81 |
+
"5": 0.26.1
|
| 82 |
+
"6": 5.7.0
|
| 83 |
+
"12": 0.26.1
|
| 84 |
+
"13": linux-x86_64
|
| 85 |
+
attention_type:
|
| 86 |
+
value: sparse_stochastic
|
| 87 |
+
checkpoint_activations:
|
| 88 |
+
value: false
|
| 89 |
+
dense_mode:
|
| 90 |
+
value: false
|
| 91 |
+
dense_step_intervals:
|
| 92 |
+
value: null
|
| 93 |
+
eval_batches:
|
| 94 |
+
value: 50
|
| 95 |
+
eval_steps:
|
| 96 |
+
value: 250
|
| 97 |
+
force_new_run:
|
| 98 |
+
value: true
|
| 99 |
+
gradient_accumulation_steps:
|
| 100 |
+
value: 20
|
| 101 |
+
head_dim:
|
| 102 |
+
value: null
|
| 103 |
+
hidden_size:
|
| 104 |
+
value: 768
|
| 105 |
+
init_from_scratch:
|
| 106 |
+
value: true
|
| 107 |
+
intermediate_size:
|
| 108 |
+
value: 3072
|
| 109 |
+
learning_rate:
|
| 110 |
+
value: 0.0005
|
| 111 |
+
logging_steps:
|
| 112 |
+
value: 10
|
| 113 |
+
lr_scheduler:
|
| 114 |
+
value: cosine
|
| 115 |
+
max_grad_norm:
|
| 116 |
+
value: 1
|
| 117 |
+
max_seq_length:
|
| 118 |
+
value: 30720
|
| 119 |
+
max_steps:
|
| 120 |
+
value: 11700
|
| 121 |
+
mixed_precision:
|
| 122 |
+
value: bf16
|
| 123 |
+
model_dtype:
|
| 124 |
+
value: bfloat16
|
| 125 |
+
model_path:
|
| 126 |
+
value: /mloscratch/homes/navasard/qwen3_06b_base
|
| 127 |
+
num_attention_heads:
|
| 128 |
+
value: 6
|
| 129 |
+
num_epochs:
|
| 130 |
+
value: 1
|
| 131 |
+
num_hidden_layers:
|
| 132 |
+
value: 28
|
| 133 |
+
num_key_value_heads:
|
| 134 |
+
value: 2
|
| 135 |
+
num_workers:
|
| 136 |
+
value: 8
|
| 137 |
+
output_dir:
|
| 138 |
+
value: /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759
|
| 139 |
+
per_device_batch_size:
|
| 140 |
+
value: 1
|
| 141 |
+
resume_from_checkpoint:
|
| 142 |
+
value: null
|
| 143 |
+
save_steps:
|
| 144 |
+
value: 1000
|
| 145 |
+
seed:
|
| 146 |
+
value: 42
|
| 147 |
+
sparse_block_size:
|
| 148 |
+
value: 128
|
| 149 |
+
sparse_layer_indices:
|
| 150 |
+
value: null
|
| 151 |
+
sparse_mask_cache_dir:
|
| 152 |
+
value: /mloscratch/homes/navasard/sparse/mask_cache_30k_fixed_s2
|
| 153 |
+
sparse_mask_pool_size:
|
| 154 |
+
value: 1
|
| 155 |
+
sparse_mask_seed:
|
| 156 |
+
value: 2
|
| 157 |
+
sparse_num_random_blocks:
|
| 158 |
+
value: 120
|
| 159 |
+
sparse_val_dense_compare:
|
| 160 |
+
value: false
|
| 161 |
+
strategy:
|
| 162 |
+
value: ddp
|
| 163 |
+
train_path:
|
| 164 |
+
value: /mloscratch/homes/navasard/data/longctx30k/tokenized
|
| 165 |
+
val_path:
|
| 166 |
+
value: null
|
| 167 |
+
verbose:
|
| 168 |
+
value: true
|
| 169 |
+
wandb_enabled:
|
| 170 |
+
value: true
|
| 171 |
+
wandb_name:
|
| 172 |
+
value: 365m fixed s2 30k
|
| 173 |
+
wandb_project:
|
| 174 |
+
value: stochastic_sparsity
|
| 175 |
+
wandb_tags:
|
| 176 |
+
value:
|
| 177 |
+
- qwen3
|
| 178 |
+
- 365m
|
| 179 |
+
- 30k
|
| 180 |
+
- scaleladder
|
| 181 |
+
- fixed
|
| 182 |
+
- seed2
|
| 183 |
+
warmup_steps:
|
| 184 |
+
value: 750
|
| 185 |
+
weight_decay:
|
| 186 |
+
value: 0.01
|
epfl_final/SparseAtt/wandb/run-20260707_035818-m10lf13v/files/output.log
ADDED
|
@@ -0,0 +1,1230 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Starting training...
|
| 2 |
+
Switching attention mode for step 1: sparse_stochastic (configured sparse attention)
|
| 3 |
+
Step 10/11700 | Loss: 12.0483 | LR: 6.67e-06 | 12124.8ms/step | 1.6 samples/s
|
| 4 |
+
Step 20/11700 | Loss: 11.9634 | LR: 1.33e-05 | 7730.6ms/step | 2.6 samples/s
|
| 5 |
+
Step 30/11700 | Loss: 11.7003 | LR: 2.00e-05 | 7733.9ms/step | 2.6 samples/s
|
| 6 |
+
Step 40/11700 | Loss: 11.1857 | LR: 2.67e-05 | 7713.5ms/step | 2.6 samples/s
|
| 7 |
+
Step 50/11700 | Loss: 10.8576 | LR: 3.33e-05 | 7699.7ms/step | 2.6 samples/s
|
| 8 |
+
Step 60/11700 | Loss: 10.5731 | LR: 4.00e-05 | 7690.4ms/step | 2.6 samples/s
|
| 9 |
+
Step 70/11700 | Loss: 10.2941 | LR: 4.67e-05 | 7684.3ms/step | 2.6 samples/s
|
| 10 |
+
Step 80/11700 | Loss: 10.0259 | LR: 5.33e-05 | 7677.4ms/step | 2.6 samples/s
|
| 11 |
+
Step 90/11700 | Loss: 9.7345 | LR: 6.00e-05 | 7674.1ms/step | 2.6 samples/s
|
| 12 |
+
Step 100/11700 | Loss: 9.3523 | LR: 6.67e-05 | 7686.2ms/step | 2.6 samples/s
|
| 13 |
+
Step 110/11700 | Loss: 8.8819 | LR: 7.33e-05 | 7686.3ms/step | 2.6 samples/s
|
| 14 |
+
Step 120/11700 | Loss: 8.3306 | LR: 8.00e-05 | 7687.3ms/step | 2.6 samples/s
|
| 15 |
+
Step 130/11700 | Loss: 7.8669 | LR: 8.67e-05 | 7693.2ms/step | 2.6 samples/s
|
| 16 |
+
Step 140/11700 | Loss: 7.5868 | LR: 9.33e-05 | 7697.2ms/step | 2.6 samples/s
|
| 17 |
+
Step 150/11700 | Loss: 7.3864 | LR: 1.00e-04 | 7703.8ms/step | 2.6 samples/s
|
| 18 |
+
Step 160/11700 | Loss: 7.1825 | LR: 1.07e-04 | 7714.1ms/step | 2.6 samples/s
|
| 19 |
+
Step 170/11700 | Loss: 7.0674 | LR: 1.13e-04 | 7722.6ms/step | 2.6 samples/s
|
| 20 |
+
Step 180/11700 | Loss: 6.9281 | LR: 1.20e-04 | 7727.1ms/step | 2.6 samples/s
|
| 21 |
+
Step 190/11700 | Loss: 6.8349 | LR: 1.27e-04 | 7734.0ms/step | 2.6 samples/s
|
| 22 |
+
Step 200/11700 | Loss: 6.6655 | LR: 1.33e-04 | 7735.5ms/step | 2.6 samples/s
|
| 23 |
+
Step 210/11700 | Loss: 6.5757 | LR: 1.40e-04 | 7739.9ms/step | 2.6 samples/s
|
| 24 |
+
Step 220/11700 | Loss: 6.4278 | LR: 1.47e-04 | 7745.3ms/step | 2.6 samples/s
|
| 25 |
+
Step 230/11700 | Loss: 6.3634 | LR: 1.53e-04 | 7744.0ms/step | 2.6 samples/s
|
| 26 |
+
Step 240/11700 | Loss: 6.2138 | LR: 1.60e-04 | 7742.8ms/step | 2.6 samples/s
|
| 27 |
+
Step 250/11700 | Loss: 6.2299 | LR: 1.67e-04 | 7745.0ms/step | 2.6 samples/s
|
| 28 |
+
Validation at step 250: Loss = 6.0940, Perplexity = 443.21
|
| 29 |
+
Step 260/11700 | Loss: 6.1446 | LR: 1.73e-04 | 19932.3ms/step | 1.0 samples/s
|
| 30 |
+
Step 270/11700 | Loss: 6.0133 | LR: 1.80e-04 | 7789.6ms/step | 2.6 samples/s
|
| 31 |
+
Step 280/11700 | Loss: 5.9732 | LR: 1.87e-04 | 7795.7ms/step | 2.6 samples/s
|
| 32 |
+
Step 290/11700 | Loss: 5.8510 | LR: 1.93e-04 | 7793.1ms/step | 2.6 samples/s
|
| 33 |
+
Step 300/11700 | Loss: 5.8357 | LR: 2.00e-04 | 7812.2ms/step | 2.6 samples/s
|
| 34 |
+
Step 310/11700 | Loss: 5.8228 | LR: 2.07e-04 | 7782.8ms/step | 2.6 samples/s
|
| 35 |
+
Step 320/11700 | Loss: 5.7001 | LR: 2.13e-04 | 7783.7ms/step | 2.6 samples/s
|
| 36 |
+
Step 330/11700 | Loss: 5.5870 | LR: 2.20e-04 | 7798.1ms/step | 2.6 samples/s
|
| 37 |
+
Step 340/11700 | Loss: 5.5375 | LR: 2.27e-04 | 7797.9ms/step | 2.6 samples/s
|
| 38 |
+
Step 350/11700 | Loss: 5.3756 | LR: 2.33e-04 | 7781.6ms/step | 2.6 samples/s
|
| 39 |
+
Step 360/11700 | Loss: 5.4584 | LR: 2.40e-04 | 7792.3ms/step | 2.6 samples/s
|
| 40 |
+
Step 370/11700 | Loss: 5.4884 | LR: 2.47e-04 | 7796.5ms/step | 2.6 samples/s
|
| 41 |
+
Step 380/11700 | Loss: 5.4716 | LR: 2.53e-04 | 7780.1ms/step | 2.6 samples/s
|
| 42 |
+
Step 390/11700 | Loss: 5.4301 | LR: 2.60e-04 | 7787.4ms/step | 2.6 samples/s
|
| 43 |
+
Step 400/11700 | Loss: 5.2668 | LR: 2.67e-04 | 7798.2ms/step | 2.6 samples/s
|
| 44 |
+
Step 410/11700 | Loss: 5.2921 | LR: 2.73e-04 | 7798.0ms/step | 2.6 samples/s
|
| 45 |
+
Step 420/11700 | Loss: 5.2471 | LR: 2.80e-04 | 7795.8ms/step | 2.6 samples/s
|
| 46 |
+
Step 430/11700 | Loss: 5.2181 | LR: 2.87e-04 | 7787.1ms/step | 2.6 samples/s
|
| 47 |
+
Step 440/11700 | Loss: 5.2177 | LR: 2.93e-04 | 7795.5ms/step | 2.6 samples/s
|
| 48 |
+
Step 450/11700 | Loss: 5.1271 | LR: 3.00e-04 | 7799.0ms/step | 2.6 samples/s
|
| 49 |
+
Step 460/11700 | Loss: 5.1526 | LR: 3.07e-04 | 7792.4ms/step | 2.6 samples/s
|
| 50 |
+
Step 470/11700 | Loss: 5.1211 | LR: 3.13e-04 | 7798.8ms/step | 2.6 samples/s
|
| 51 |
+
Step 480/11700 | Loss: 5.1265 | LR: 3.20e-04 | 7790.0ms/step | 2.6 samples/s
|
| 52 |
+
Step 490/11700 | Loss: 5.0241 | LR: 3.27e-04 | 7788.2ms/step | 2.6 samples/s
|
| 53 |
+
Step 500/11700 | Loss: 4.9728 | LR: 3.33e-04 | 7799.0ms/step | 2.6 samples/s
|
| 54 |
+
Validation at step 500: Loss = 4.9183, Perplexity = 136.76
|
| 55 |
+
Step 510/11700 | Loss: 4.8970 | LR: 3.40e-04 | 19399.3ms/step | 1.0 samples/s
|
| 56 |
+
Step 520/11700 | Loss: 4.9103 | LR: 3.47e-04 | 7790.7ms/step | 2.6 samples/s
|
| 57 |
+
Step 530/11700 | Loss: 4.8397 | LR: 3.53e-04 | 7797.1ms/step | 2.6 samples/s
|
| 58 |
+
Step 540/11700 | Loss: 4.7412 | LR: 3.60e-04 | 7794.5ms/step | 2.6 samples/s
|
| 59 |
+
Step 550/11700 | Loss: 4.7552 | LR: 3.67e-04 | 7803.4ms/step | 2.6 samples/s
|
| 60 |
+
Step 560/11700 | Loss: 4.7564 | LR: 3.73e-04 | 7808.8ms/step | 2.6 samples/s
|
| 61 |
+
Step 570/11700 | Loss: 4.5380 | LR: 3.80e-04 | 7803.9ms/step | 2.6 samples/s
|
| 62 |
+
Step 580/11700 | Loss: 4.5698 | LR: 3.87e-04 | 7795.9ms/step | 2.6 samples/s
|
| 63 |
+
Step 590/11700 | Loss: 4.5068 | LR: 3.93e-04 | 7792.7ms/step | 2.6 samples/s
|
| 64 |
+
Step 600/11700 | Loss: 4.5625 | LR: 4.00e-04 | 7801.5ms/step | 2.6 samples/s
|
| 65 |
+
Step 610/11700 | Loss: 4.5315 | LR: 4.07e-04 | 7803.1ms/step | 2.6 samples/s
|
| 66 |
+
Step 620/11700 | Loss: 4.5185 | LR: 4.13e-04 | 7801.5ms/step | 2.6 samples/s
|
| 67 |
+
Step 630/11700 | Loss: 4.3512 | LR: 4.20e-04 | 7807.5ms/step | 2.6 samples/s
|
| 68 |
+
Step 640/11700 | Loss: 4.4416 | LR: 4.27e-04 | 7795.8ms/step | 2.6 samples/s
|
| 69 |
+
Step 650/11700 | Loss: 4.3219 | LR: 4.33e-04 | 7792.0ms/step | 2.6 samples/s
|
| 70 |
+
Step 660/11700 | Loss: 4.4264 | LR: 4.40e-04 | 7804.0ms/step | 2.6 samples/s
|
| 71 |
+
Step 670/11700 | Loss: 4.3093 | LR: 4.47e-04 | 7798.1ms/step | 2.6 samples/s
|
| 72 |
+
Step 680/11700 | Loss: 4.2234 | LR: 4.53e-04 | 7803.6ms/step | 2.6 samples/s
|
| 73 |
+
Step 690/11700 | Loss: 4.0791 | LR: 4.60e-04 | 7801.1ms/step | 2.6 samples/s
|
| 74 |
+
Step 700/11700 | Loss: 4.2298 | LR: 4.67e-04 | 7799.7ms/step | 2.6 samples/s
|
| 75 |
+
Step 710/11700 | Loss: 4.2181 | LR: 4.73e-04 | 7800.5ms/step | 2.6 samples/s
|
| 76 |
+
Step 720/11700 | Loss: 4.1002 | LR: 4.80e-04 | 7801.8ms/step | 2.6 samples/s
|
| 77 |
+
Step 730/11700 | Loss: 3.9669 | LR: 4.87e-04 | 7805.9ms/step | 2.6 samples/s
|
| 78 |
+
Step 740/11700 | Loss: 4.1025 | LR: 4.93e-04 | 7798.7ms/step | 2.6 samples/s
|
| 79 |
+
Step 750/11700 | Loss: 4.0185 | LR: 5.00e-04 | 7804.9ms/step | 2.6 samples/s
|
| 80 |
+
Validation at step 750: Loss = 3.9681, Perplexity = 52.89
|
| 81 |
+
Step 760/11700 | Loss: 4.0503 | LR: 5.00e-04 | 19394.1ms/step | 1.0 samples/s
|
| 82 |
+
Step 770/11700 | Loss: 3.8120 | LR: 5.00e-04 | 7799.2ms/step | 2.6 samples/s
|
| 83 |
+
Step 780/11700 | Loss: 4.0163 | LR: 5.00e-04 | 7795.1ms/step | 2.6 samples/s
|
| 84 |
+
Step 790/11700 | Loss: 3.7902 | LR: 5.00e-04 | 7793.5ms/step | 2.6 samples/s
|
| 85 |
+
Step 800/11700 | Loss: 3.7888 | LR: 5.00e-04 | 7801.3ms/step | 2.6 samples/s
|
| 86 |
+
Step 810/11700 | Loss: 3.8159 | LR: 5.00e-04 | 7808.5ms/step | 2.6 samples/s
|
| 87 |
+
Step 820/11700 | Loss: 3.7995 | LR: 5.00e-04 | 7794.3ms/step | 2.6 samples/s
|
| 88 |
+
Step 830/11700 | Loss: 3.7938 | LR: 5.00e-04 | 7795.7ms/step | 2.6 samples/s
|
| 89 |
+
Step 840/11700 | Loss: 3.8030 | LR: 5.00e-04 | 7809.7ms/step | 2.6 samples/s
|
| 90 |
+
Step 850/11700 | Loss: 3.8463 | LR: 5.00e-04 | 7796.9ms/step | 2.6 samples/s
|
| 91 |
+
Step 860/11700 | Loss: 3.6814 | LR: 5.00e-04 | 7801.2ms/step | 2.6 samples/s
|
| 92 |
+
Step 870/11700 | Loss: 3.7150 | LR: 5.00e-04 | 7794.7ms/step | 2.6 samples/s
|
| 93 |
+
Step 880/11700 | Loss: 3.7608 | LR: 5.00e-04 | 7801.3ms/step | 2.6 samples/s
|
| 94 |
+
Step 890/11700 | Loss: 3.6078 | LR: 5.00e-04 | 7803.5ms/step | 2.6 samples/s
|
| 95 |
+
Step 900/11700 | Loss: 3.5381 | LR: 5.00e-04 | 7808.3ms/step | 2.6 samples/s
|
| 96 |
+
Step 910/11700 | Loss: 3.7214 | LR: 5.00e-04 | 7799.8ms/step | 2.6 samples/s
|
| 97 |
+
Step 920/11700 | Loss: 3.6001 | LR: 5.00e-04 | 7808.0ms/step | 2.6 samples/s
|
| 98 |
+
Step 930/11700 | Loss: 3.5864 | LR: 5.00e-04 | 7806.1ms/step | 2.6 samples/s
|
| 99 |
+
Step 940/11700 | Loss: 3.6561 | LR: 5.00e-04 | 7814.9ms/step | 2.6 samples/s
|
| 100 |
+
Step 950/11700 | Loss: 3.4872 | LR: 5.00e-04 | 7801.6ms/step | 2.6 samples/s
|
| 101 |
+
Step 960/11700 | Loss: 3.5553 | LR: 5.00e-04 | 7797.7ms/step | 2.6 samples/s
|
| 102 |
+
Step 970/11700 | Loss: 3.6223 | LR: 5.00e-04 | 7798.4ms/step | 2.6 samples/s
|
| 103 |
+
Step 980/11700 | Loss: 3.3539 | LR: 4.99e-04 | 7808.4ms/step | 2.6 samples/s
|
| 104 |
+
Step 990/11700 | Loss: 3.5634 | LR: 4.99e-04 | 7795.5ms/step | 2.6 samples/s
|
| 105 |
+
Step 1000/11700 | Loss: 3.4551 | LR: 4.99e-04 | 7808.4ms/step | 2.6 samples/s
|
| 106 |
+
Validation at step 1000: Loss = 3.5078, Perplexity = 33.38
|
| 107 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-1000
|
| 108 |
+
Step 1010/11700 | Loss: 3.5678 | LR: 4.99e-04 | 19714.2ms/step | 1.0 samples/s
|
| 109 |
+
Step 1020/11700 | Loss: 3.4669 | LR: 4.99e-04 | 7794.8ms/step | 2.6 samples/s
|
| 110 |
+
Step 1030/11700 | Loss: 3.4822 | LR: 4.99e-04 | 7795.1ms/step | 2.6 samples/s
|
| 111 |
+
Step 1040/11700 | Loss: 3.5082 | LR: 4.99e-04 | 7801.2ms/step | 2.6 samples/s
|
| 112 |
+
Step 1050/11700 | Loss: 3.4285 | LR: 4.99e-04 | 7815.4ms/step | 2.6 samples/s
|
| 113 |
+
Step 1060/11700 | Loss: 3.4857 | LR: 4.99e-04 | 7795.8ms/step | 2.6 samples/s
|
| 114 |
+
Step 1070/11700 | Loss: 3.4423 | LR: 4.99e-04 | 7808.3ms/step | 2.6 samples/s
|
| 115 |
+
Step 1080/11700 | Loss: 3.3041 | LR: 4.99e-04 | 7797.5ms/step | 2.6 samples/s
|
| 116 |
+
Step 1090/11700 | Loss: 3.3595 | LR: 4.99e-04 | 7804.3ms/step | 2.6 samples/s
|
| 117 |
+
Step 1100/11700 | Loss: 3.5647 | LR: 4.99e-04 | 7799.9ms/step | 2.6 samples/s
|
| 118 |
+
Step 1110/11700 | Loss: 3.5133 | LR: 4.99e-04 | 7800.2ms/step | 2.6 samples/s
|
| 119 |
+
Step 1120/11700 | Loss: 3.3988 | LR: 4.99e-04 | 7800.6ms/step | 2.6 samples/s
|
| 120 |
+
Step 1130/11700 | Loss: 3.4001 | LR: 4.99e-04 | 7796.6ms/step | 2.6 samples/s
|
| 121 |
+
Step 1140/11700 | Loss: 3.1744 | LR: 4.98e-04 | 7805.7ms/step | 2.6 samples/s
|
| 122 |
+
Step 1150/11700 | Loss: 3.3319 | LR: 4.98e-04 | 7808.8ms/step | 2.6 samples/s
|
| 123 |
+
Step 1160/11700 | Loss: 3.3401 | LR: 4.98e-04 | 7789.7ms/step | 2.6 samples/s
|
| 124 |
+
Step 1170/11700 | Loss: 3.3070 | LR: 4.98e-04 | 7798.5ms/step | 2.6 samples/s
|
| 125 |
+
Step 1180/11700 | Loss: 3.2484 | LR: 4.98e-04 | 7796.8ms/step | 2.6 samples/s
|
| 126 |
+
Step 1190/11700 | Loss: 3.2085 | LR: 4.98e-04 | 7806.3ms/step | 2.6 samples/s
|
| 127 |
+
Step 1200/11700 | Loss: 3.1442 | LR: 4.98e-04 | 7808.7ms/step | 2.6 samples/s
|
| 128 |
+
Step 1210/11700 | Loss: 3.4724 | LR: 4.98e-04 | 7798.6ms/step | 2.6 samples/s
|
| 129 |
+
Step 1220/11700 | Loss: 3.2654 | LR: 4.98e-04 | 7802.8ms/step | 2.6 samples/s
|
| 130 |
+
Step 1230/11700 | Loss: 3.3053 | LR: 4.98e-04 | 7791.3ms/step | 2.6 samples/s
|
| 131 |
+
Step 1240/11700 | Loss: 3.2461 | LR: 4.98e-04 | 7802.7ms/step | 2.6 samples/s
|
| 132 |
+
Step 1250/11700 | Loss: 3.2277 | LR: 4.97e-04 | 7800.7ms/step | 2.6 samples/s
|
| 133 |
+
Validation at step 1250: Loss = 3.2741, Perplexity = 26.42
|
| 134 |
+
Step 1260/11700 | Loss: 3.3033 | LR: 4.97e-04 | 19401.9ms/step | 1.0 samples/s
|
| 135 |
+
Step 1270/11700 | Loss: 3.1401 | LR: 4.97e-04 | 7799.0ms/step | 2.6 samples/s
|
| 136 |
+
Step 1280/11700 | Loss: 3.2463 | LR: 4.97e-04 | 7811.2ms/step | 2.6 samples/s
|
| 137 |
+
Step 1290/11700 | Loss: 3.2376 | LR: 4.97e-04 | 7808.9ms/step | 2.6 samples/s
|
| 138 |
+
Step 1300/11700 | Loss: 3.2451 | LR: 4.97e-04 | 7810.3ms/step | 2.6 samples/s
|
| 139 |
+
Step 1310/11700 | Loss: 3.2959 | LR: 4.97e-04 | 7795.3ms/step | 2.6 samples/s
|
| 140 |
+
Step 1320/11700 | Loss: 3.2567 | LR: 4.97e-04 | 7801.7ms/step | 2.6 samples/s
|
| 141 |
+
Step 1330/11700 | Loss: 3.1817 | LR: 4.97e-04 | 7805.7ms/step | 2.6 samples/s
|
| 142 |
+
Step 1340/11700 | Loss: 3.2302 | LR: 4.96e-04 | 7808.4ms/step | 2.6 samples/s
|
| 143 |
+
Step 1350/11700 | Loss: 3.2422 | LR: 4.96e-04 | 7798.2ms/step | 2.6 samples/s
|
| 144 |
+
Step 1360/11700 | Loss: 3.2010 | LR: 4.96e-04 | 7791.6ms/step | 2.6 samples/s
|
| 145 |
+
Step 1370/11700 | Loss: 3.1433 | LR: 4.96e-04 | 7799.6ms/step | 2.6 samples/s
|
| 146 |
+
Step 1380/11700 | Loss: 3.0110 | LR: 4.96e-04 | 7800.7ms/step | 2.6 samples/s
|
| 147 |
+
Step 1390/11700 | Loss: 3.1734 | LR: 4.96e-04 | 7805.8ms/step | 2.6 samples/s
|
| 148 |
+
Step 1400/11700 | Loss: 3.2579 | LR: 4.96e-04 | 7797.5ms/step | 2.6 samples/s
|
| 149 |
+
Step 1410/11700 | Loss: 3.1981 | LR: 4.96e-04 | 7800.5ms/step | 2.6 samples/s
|
| 150 |
+
Step 1420/11700 | Loss: 3.0974 | LR: 4.95e-04 | 7795.3ms/step | 2.6 samples/s
|
| 151 |
+
Step 1430/11700 | Loss: 3.1448 | LR: 4.95e-04 | 7787.0ms/step | 2.6 samples/s
|
| 152 |
+
Step 1440/11700 | Loss: 3.1359 | LR: 4.95e-04 | 7786.0ms/step | 2.6 samples/s
|
| 153 |
+
Step 1450/11700 | Loss: 3.0600 | LR: 4.95e-04 | 7786.8ms/step | 2.6 samples/s
|
| 154 |
+
Step 1460/11700 | Loss: 3.1119 | LR: 4.95e-04 | 7789.0ms/step | 2.6 samples/s
|
| 155 |
+
Step 1470/11700 | Loss: 3.1236 | LR: 4.95e-04 | 7803.7ms/step | 2.6 samples/s
|
| 156 |
+
Step 1480/11700 | Loss: 3.0541 | LR: 4.95e-04 | 7790.0ms/step | 2.6 samples/s
|
| 157 |
+
Step 1490/11700 | Loss: 3.2103 | LR: 4.94e-04 | 7787.2ms/step | 2.6 samples/s
|
| 158 |
+
Step 1500/11700 | Loss: 3.0483 | LR: 4.94e-04 | 7793.7ms/step | 2.6 samples/s
|
| 159 |
+
Validation at step 1500: Loss = 3.1282, Perplexity = 22.83
|
| 160 |
+
Step 1510/11700 | Loss: 3.1229 | LR: 4.94e-04 | 19416.2ms/step | 1.0 samples/s
|
| 161 |
+
Step 1520/11700 | Loss: 3.1079 | LR: 4.94e-04 | 7785.2ms/step | 2.6 samples/s
|
| 162 |
+
Step 1530/11700 | Loss: 3.1122 | LR: 4.94e-04 | 7786.4ms/step | 2.6 samples/s
|
| 163 |
+
Step 1540/11700 | Loss: 3.2660 | LR: 4.94e-04 | 7786.7ms/step | 2.6 samples/s
|
| 164 |
+
Step 1550/11700 | Loss: 3.1147 | LR: 4.93e-04 | 7787.0ms/step | 2.6 samples/s
|
| 165 |
+
Step 1560/11700 | Loss: 3.1368 | LR: 4.93e-04 | 7783.2ms/step | 2.6 samples/s
|
| 166 |
+
Step 1570/11700 | Loss: 3.1234 | LR: 4.93e-04 | 7793.1ms/step | 2.6 samples/s
|
| 167 |
+
Step 1580/11700 | Loss: 3.0872 | LR: 4.93e-04 | 7789.4ms/step | 2.6 samples/s
|
| 168 |
+
Step 1590/11700 | Loss: 3.2243 | LR: 4.93e-04 | 7789.7ms/step | 2.6 samples/s
|
| 169 |
+
Step 1600/11700 | Loss: 3.1110 | LR: 4.93e-04 | 7792.0ms/step | 2.6 samples/s
|
| 170 |
+
Step 1610/11700 | Loss: 3.1547 | LR: 4.92e-04 | 7790.0ms/step | 2.6 samples/s
|
| 171 |
+
Step 1620/11700 | Loss: 3.0310 | LR: 4.92e-04 | 7794.5ms/step | 2.6 samples/s
|
| 172 |
+
Step 1630/11700 | Loss: 2.9387 | LR: 4.92e-04 | 7791.8ms/step | 2.6 samples/s
|
| 173 |
+
Step 1640/11700 | Loss: 3.1185 | LR: 4.92e-04 | 7790.7ms/step | 2.6 samples/s
|
| 174 |
+
Step 1650/11700 | Loss: 3.0533 | LR: 4.92e-04 | 7790.0ms/step | 2.6 samples/s
|
| 175 |
+
Step 1660/11700 | Loss: 3.0873 | LR: 4.92e-04 | 7792.7ms/step | 2.6 samples/s
|
| 176 |
+
Step 1670/11700 | Loss: 3.2119 | LR: 4.91e-04 | 7789.2ms/step | 2.6 samples/s
|
| 177 |
+
Step 1680/11700 | Loss: 3.0581 | LR: 4.91e-04 | 7790.0ms/step | 2.6 samples/s
|
| 178 |
+
Step 1690/11700 | Loss: 2.9176 | LR: 4.91e-04 | 7790.8ms/step | 2.6 samples/s
|
| 179 |
+
Step 1700/11700 | Loss: 2.9755 | LR: 4.91e-04 | 7790.0ms/step | 2.6 samples/s
|
| 180 |
+
Step 1710/11700 | Loss: 3.1080 | LR: 4.91e-04 | 7789.2ms/step | 2.6 samples/s
|
| 181 |
+
Step 1720/11700 | Loss: 3.0318 | LR: 4.90e-04 | 7790.1ms/step | 2.6 samples/s
|
| 182 |
+
Step 1730/11700 | Loss: 3.0640 | LR: 4.90e-04 | 7789.7ms/step | 2.6 samples/s
|
| 183 |
+
Step 1740/11700 | Loss: 2.9609 | LR: 4.90e-04 | 7789.0ms/step | 2.6 samples/s
|
| 184 |
+
Step 1750/11700 | Loss: 2.9304 | LR: 4.90e-04 | 7790.0ms/step | 2.6 samples/s
|
| 185 |
+
Validation at step 1750: Loss = 3.0505, Perplexity = 21.13
|
| 186 |
+
Step 1760/11700 | Loss: 3.0282 | LR: 4.90e-04 | 19367.9ms/step | 1.0 samples/s
|
| 187 |
+
Step 1770/11700 | Loss: 3.0279 | LR: 4.89e-04 | 7793.3ms/step | 2.6 samples/s
|
| 188 |
+
Step 1780/11700 | Loss: 3.0518 | LR: 4.89e-04 | 7790.9ms/step | 2.6 samples/s
|
| 189 |
+
Step 1790/11700 | Loss: 3.0216 | LR: 4.89e-04 | 7791.1ms/step | 2.6 samples/s
|
| 190 |
+
Step 1800/11700 | Loss: 3.0367 | LR: 4.89e-04 | 7789.9ms/step | 2.6 samples/s
|
| 191 |
+
Step 1810/11700 | Loss: 3.0937 | LR: 4.89e-04 | 7789.0ms/step | 2.6 samples/s
|
| 192 |
+
Step 1820/11700 | Loss: 3.0198 | LR: 4.88e-04 | 7789.0ms/step | 2.6 samples/s
|
| 193 |
+
Step 1830/11700 | Loss: 2.9920 | LR: 4.88e-04 | 7791.9ms/step | 2.6 samples/s
|
| 194 |
+
Step 1840/11700 | Loss: 2.9863 | LR: 4.88e-04 | 7790.1ms/step | 2.6 samples/s
|
| 195 |
+
Step 1850/11700 | Loss: 3.0090 | LR: 4.88e-04 | 7790.3ms/step | 2.6 samples/s
|
| 196 |
+
Step 1860/11700 | Loss: 2.9850 | LR: 4.87e-04 | 7786.5ms/step | 2.6 samples/s
|
| 197 |
+
Step 1870/11700 | Loss: 2.9133 | LR: 4.87e-04 | 7785.1ms/step | 2.6 samples/s
|
| 198 |
+
Step 1880/11700 | Loss: 2.9351 | LR: 4.87e-04 | 7785.4ms/step | 2.6 samples/s
|
| 199 |
+
Step 1890/11700 | Loss: 3.0574 | LR: 4.87e-04 | 7787.3ms/step | 2.6 samples/s
|
| 200 |
+
Step 1900/11700 | Loss: 3.0048 | LR: 4.87e-04 | 7781.6ms/step | 2.6 samples/s
|
| 201 |
+
Step 1910/11700 | Loss: 3.0309 | LR: 4.86e-04 | 7783.2ms/step | 2.6 samples/s
|
| 202 |
+
Step 1920/11700 | Loss: 2.8259 | LR: 4.86e-04 | 7783.2ms/step | 2.6 samples/s
|
| 203 |
+
Step 1930/11700 | Loss: 3.0181 | LR: 4.86e-04 | 7782.0ms/step | 2.6 samples/s
|
| 204 |
+
Step 1940/11700 | Loss: 2.9504 | LR: 4.86e-04 | 7781.8ms/step | 2.6 samples/s
|
| 205 |
+
Step 1950/11700 | Loss: 2.8694 | LR: 4.85e-04 | 7784.8ms/step | 2.6 samples/s
|
| 206 |
+
Step 1960/11700 | Loss: 2.9380 | LR: 4.85e-04 | 7789.4ms/step | 2.6 samples/s
|
| 207 |
+
Step 1970/11700 | Loss: 2.9598 | LR: 4.85e-04 | 7793.2ms/step | 2.6 samples/s
|
| 208 |
+
Step 1980/11700 | Loss: 2.9388 | LR: 4.85e-04 | 7788.9ms/step | 2.6 samples/s
|
| 209 |
+
Step 1990/11700 | Loss: 2.8826 | LR: 4.84e-04 | 7793.5ms/step | 2.6 samples/s
|
| 210 |
+
Step 2000/11700 | Loss: 3.0925 | LR: 4.84e-04 | 7797.4ms/step | 2.6 samples/s
|
| 211 |
+
Validation at step 2000: Loss = 2.9627, Perplexity = 19.35
|
| 212 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-2000
|
| 213 |
+
Step 2010/11700 | Loss: 2.9809 | LR: 4.84e-04 | 19661.1ms/step | 1.0 samples/s
|
| 214 |
+
Step 2020/11700 | Loss: 3.0177 | LR: 4.84e-04 | 7783.3ms/step | 2.6 samples/s
|
| 215 |
+
Step 2030/11700 | Loss: 2.9192 | LR: 4.83e-04 | 7787.2ms/step | 2.6 samples/s
|
| 216 |
+
Step 2040/11700 | Loss: 2.9645 | LR: 4.83e-04 | 7790.2ms/step | 2.6 samples/s
|
| 217 |
+
Step 2050/11700 | Loss: 2.8349 | LR: 4.83e-04 | 7790.6ms/step | 2.6 samples/s
|
| 218 |
+
Step 2060/11700 | Loss: 2.9050 | LR: 4.83e-04 | 7791.0ms/step | 2.6 samples/s
|
| 219 |
+
Step 2070/11700 | Loss: 2.9080 | LR: 4.82e-04 | 7788.1ms/step | 2.6 samples/s
|
| 220 |
+
Step 2080/11700 | Loss: 2.8856 | LR: 4.82e-04 | 7794.9ms/step | 2.6 samples/s
|
| 221 |
+
Step 2090/11700 | Loss: 2.9159 | LR: 4.82e-04 | 7789.7ms/step | 2.6 samples/s
|
| 222 |
+
Step 2100/11700 | Loss: 3.0044 | LR: 4.81e-04 | 7790.2ms/step | 2.6 samples/s
|
| 223 |
+
Step 2110/11700 | Loss: 2.9992 | LR: 4.81e-04 | 7794.1ms/step | 2.6 samples/s
|
| 224 |
+
Step 2120/11700 | Loss: 2.9365 | LR: 4.81e-04 | 7790.7ms/step | 2.6 samples/s
|
| 225 |
+
Step 2130/11700 | Loss: 2.9314 | LR: 4.81e-04 | 7790.5ms/step | 2.6 samples/s
|
| 226 |
+
Step 2140/11700 | Loss: 3.0457 | LR: 4.80e-04 | 7784.8ms/step | 2.6 samples/s
|
| 227 |
+
Step 2150/11700 | Loss: 2.9629 | LR: 4.80e-04 | 7788.3ms/step | 2.6 samples/s
|
| 228 |
+
Step 2160/11700 | Loss: 2.9219 | LR: 4.80e-04 | 7790.7ms/step | 2.6 samples/s
|
| 229 |
+
Step 2170/11700 | Loss: 2.8508 | LR: 4.80e-04 | 7795.5ms/step | 2.6 samples/s
|
| 230 |
+
Step 2180/11700 | Loss: 2.8844 | LR: 4.79e-04 | 7790.5ms/step | 2.6 samples/s
|
| 231 |
+
Step 2190/11700 | Loss: 2.8776 | LR: 4.79e-04 | 7794.0ms/step | 2.6 samples/s
|
| 232 |
+
Step 2200/11700 | Loss: 2.9411 | LR: 4.79e-04 | 7791.9ms/step | 2.6 samples/s
|
| 233 |
+
Step 2210/11700 | Loss: 2.9755 | LR: 4.78e-04 | 7787.1ms/step | 2.6 samples/s
|
| 234 |
+
Step 2220/11700 | Loss: 2.7846 | LR: 4.78e-04 | 7789.3ms/step | 2.6 samples/s
|
| 235 |
+
Step 2230/11700 | Loss: 2.8206 | LR: 4.78e-04 | 7788.0ms/step | 2.6 samples/s
|
| 236 |
+
Step 2240/11700 | Loss: 2.8053 | LR: 4.78e-04 | 7791.6ms/step | 2.6 samples/s
|
| 237 |
+
Step 2250/11700 | Loss: 2.9920 | LR: 4.77e-04 | 7792.1ms/step | 2.6 samples/s
|
| 238 |
+
Validation at step 2250: Loss = 2.8970, Perplexity = 18.12
|
| 239 |
+
Step 2260/11700 | Loss: 2.7742 | LR: 4.77e-04 | 19373.9ms/step | 1.0 samples/s
|
| 240 |
+
Step 2270/11700 | Loss: 2.9844 | LR: 4.77e-04 | 7793.7ms/step | 2.6 samples/s
|
| 241 |
+
Step 2280/11700 | Loss: 2.9035 | LR: 4.76e-04 | 7790.1ms/step | 2.6 samples/s
|
| 242 |
+
Step 2290/11700 | Loss: 2.9236 | LR: 4.76e-04 | 7790.4ms/step | 2.6 samples/s
|
| 243 |
+
Step 2300/11700 | Loss: 2.8607 | LR: 4.76e-04 | 7788.2ms/step | 2.6 samples/s
|
| 244 |
+
Step 2310/11700 | Loss: 2.8478 | LR: 4.75e-04 | 7786.6ms/step | 2.6 samples/s
|
| 245 |
+
Step 2320/11700 | Loss: 2.9960 | LR: 4.75e-04 | 7787.2ms/step | 2.6 samples/s
|
| 246 |
+
Step 2330/11700 | Loss: 2.8204 | LR: 4.75e-04 | 7787.6ms/step | 2.6 samples/s
|
| 247 |
+
Step 2340/11700 | Loss: 2.8540 | LR: 4.74e-04 | 7788.9ms/step | 2.6 samples/s
|
| 248 |
+
Step 2350/11700 | Loss: 2.7768 | LR: 4.74e-04 | 7785.2ms/step | 2.6 samples/s
|
| 249 |
+
Step 2360/11700 | Loss: 2.7951 | LR: 4.74e-04 | 7788.9ms/step | 2.6 samples/s
|
| 250 |
+
Step 2370/11700 | Loss: 2.7778 | LR: 4.73e-04 | 7786.7ms/step | 2.6 samples/s
|
| 251 |
+
Step 2380/11700 | Loss: 2.7728 | LR: 4.73e-04 | 7789.7ms/step | 2.6 samples/s
|
| 252 |
+
Step 2390/11700 | Loss: 2.9198 | LR: 4.73e-04 | 7787.4ms/step | 2.6 samples/s
|
| 253 |
+
Step 2400/11700 | Loss: 2.8568 | LR: 4.73e-04 | 7790.4ms/step | 2.6 samples/s
|
| 254 |
+
Step 2410/11700 | Loss: 2.8731 | LR: 4.72e-04 | 7791.7ms/step | 2.6 samples/s
|
| 255 |
+
Step 2420/11700 | Loss: 2.7317 | LR: 4.72e-04 | 7791.8ms/step | 2.6 samples/s
|
| 256 |
+
Step 2430/11700 | Loss: 2.9058 | LR: 4.72e-04 | 7789.4ms/step | 2.6 samples/s
|
| 257 |
+
Step 2440/11700 | Loss: 2.9292 | LR: 4.71e-04 | 7789.3ms/step | 2.6 samples/s
|
| 258 |
+
Step 2450/11700 | Loss: 2.7653 | LR: 4.71e-04 | 7786.2ms/step | 2.6 samples/s
|
| 259 |
+
Step 2460/11700 | Loss: 2.8779 | LR: 4.71e-04 | 7790.1ms/step | 2.6 samples/s
|
| 260 |
+
Step 2470/11700 | Loss: 2.8823 | LR: 4.70e-04 | 7789.6ms/step | 2.6 samples/s
|
| 261 |
+
Step 2480/11700 | Loss: 2.8120 | LR: 4.70e-04 | 7784.7ms/step | 2.6 samples/s
|
| 262 |
+
Step 2490/11700 | Loss: 2.9037 | LR: 4.69e-04 | 7784.2ms/step | 2.6 samples/s
|
| 263 |
+
Step 2500/11700 | Loss: 2.7833 | LR: 4.69e-04 | 7816.2ms/step | 2.6 samples/s
|
| 264 |
+
Validation at step 2500: Loss = 2.8503, Perplexity = 17.29
|
| 265 |
+
Step 2510/11700 | Loss: 2.7603 | LR: 4.69e-04 | 19407.0ms/step | 1.0 samples/s
|
| 266 |
+
Step 2520/11700 | Loss: 2.9160 | LR: 4.68e-04 | 7785.2ms/step | 2.6 samples/s
|
| 267 |
+
Step 2530/11700 | Loss: 2.8090 | LR: 4.68e-04 | 7801.7ms/step | 2.6 samples/s
|
| 268 |
+
Step 2540/11700 | Loss: 2.7422 | LR: 4.68e-04 | 7794.4ms/step | 2.6 samples/s
|
| 269 |
+
Step 2550/11700 | Loss: 2.9117 | LR: 4.67e-04 | 7785.7ms/step | 2.6 samples/s
|
| 270 |
+
Step 2560/11700 | Loss: 2.8538 | LR: 4.67e-04 | 7782.2ms/step | 2.6 samples/s
|
| 271 |
+
Step 2570/11700 | Loss: 2.8179 | LR: 4.67e-04 | 7782.5ms/step | 2.6 samples/s
|
| 272 |
+
Step 2580/11700 | Loss: 2.8388 | LR: 4.66e-04 | 7785.6ms/step | 2.6 samples/s
|
| 273 |
+
Step 2590/11700 | Loss: 2.7792 | LR: 4.66e-04 | 7779.4ms/step | 2.6 samples/s
|
| 274 |
+
Step 2600/11700 | Loss: 2.8060 | LR: 4.66e-04 | 7778.4ms/step | 2.6 samples/s
|
| 275 |
+
Step 2610/11700 | Loss: 2.7331 | LR: 4.65e-04 | 7776.5ms/step | 2.6 samples/s
|
| 276 |
+
Step 2620/11700 | Loss: 2.7939 | LR: 4.65e-04 | 7780.9ms/step | 2.6 samples/s
|
| 277 |
+
Step 2630/11700 | Loss: 2.7359 | LR: 4.65e-04 | 7785.4ms/step | 2.6 samples/s
|
| 278 |
+
Step 2640/11700 | Loss: 2.8422 | LR: 4.64e-04 | 7784.6ms/step | 2.6 samples/s
|
| 279 |
+
Step 2650/11700 | Loss: 2.9054 | LR: 4.64e-04 | 7787.8ms/step | 2.6 samples/s
|
| 280 |
+
Step 2660/11700 | Loss: 2.8373 | LR: 4.63e-04 | 7783.0ms/step | 2.6 samples/s
|
| 281 |
+
Step 2670/11700 | Loss: 2.7785 | LR: 4.63e-04 | 7785.0ms/step | 2.6 samples/s
|
| 282 |
+
Step 2680/11700 | Loss: 2.8231 | LR: 4.63e-04 | 7785.0ms/step | 2.6 samples/s
|
| 283 |
+
Step 2690/11700 | Loss: 2.9284 | LR: 4.62e-04 | 7783.3ms/step | 2.6 samples/s
|
| 284 |
+
Step 2700/11700 | Loss: 2.8220 | LR: 4.62e-04 | 7784.6ms/step | 2.6 samples/s
|
| 285 |
+
Step 2710/11700 | Loss: 2.8176 | LR: 4.62e-04 | 7786.1ms/step | 2.6 samples/s
|
| 286 |
+
Step 2720/11700 | Loss: 2.8246 | LR: 4.61e-04 | 7783.6ms/step | 2.6 samples/s
|
| 287 |
+
Step 2730/11700 | Loss: 2.7267 | LR: 4.61e-04 | 7785.5ms/step | 2.6 samples/s
|
| 288 |
+
Step 2740/11700 | Loss: 2.8178 | LR: 4.60e-04 | 7787.6ms/step | 2.6 samples/s
|
| 289 |
+
Step 2750/11700 | Loss: 2.8689 | LR: 4.60e-04 | 7785.5ms/step | 2.6 samples/s
|
| 290 |
+
Validation at step 2750: Loss = 2.8061, Perplexity = 16.55
|
| 291 |
+
Step 2760/11700 | Loss: 2.6175 | LR: 4.60e-04 | 19366.7ms/step | 1.0 samples/s
|
| 292 |
+
Step 2770/11700 | Loss: 2.7706 | LR: 4.59e-04 | 7789.1ms/step | 2.6 samples/s
|
| 293 |
+
Step 2780/11700 | Loss: 2.7836 | LR: 4.59e-04 | 7782.8ms/step | 2.6 samples/s
|
| 294 |
+
Step 2790/11700 | Loss: 2.8283 | LR: 4.58e-04 | 7784.0ms/step | 2.6 samples/s
|
| 295 |
+
Step 2800/11700 | Loss: 2.7333 | LR: 4.58e-04 | 7785.5ms/step | 2.6 samples/s
|
| 296 |
+
Step 2810/11700 | Loss: 2.8317 | LR: 4.58e-04 | 7785.2ms/step | 2.6 samples/s
|
| 297 |
+
Step 2820/11700 | Loss: 2.8161 | LR: 4.57e-04 | 7787.3ms/step | 2.6 samples/s
|
| 298 |
+
Step 2830/11700 | Loss: 2.7831 | LR: 4.57e-04 | 7786.2ms/step | 2.6 samples/s
|
| 299 |
+
Step 2840/11700 | Loss: 2.8587 | LR: 4.56e-04 | 7785.0ms/step | 2.6 samples/s
|
| 300 |
+
Step 2850/11700 | Loss: 2.8937 | LR: 4.56e-04 | 7785.4ms/step | 2.6 samples/s
|
| 301 |
+
Step 2860/11700 | Loss: 2.7747 | LR: 4.56e-04 | 7787.2ms/step | 2.6 samples/s
|
| 302 |
+
Step 2870/11700 | Loss: 2.8488 | LR: 4.55e-04 | 7784.2ms/step | 2.6 samples/s
|
| 303 |
+
Step 2880/11700 | Loss: 2.5763 | LR: 4.55e-04 | 7783.3ms/step | 2.6 samples/s
|
| 304 |
+
Step 2890/11700 | Loss: 2.8293 | LR: 4.54e-04 | 7783.5ms/step | 2.6 samples/s
|
| 305 |
+
Step 2900/11700 | Loss: 2.7750 | LR: 4.54e-04 | 7785.0ms/step | 2.6 samples/s
|
| 306 |
+
Step 2910/11700 | Loss: 2.8464 | LR: 4.54e-04 | 7784.0ms/step | 2.6 samples/s
|
| 307 |
+
Step 2920/11700 | Loss: 2.7725 | LR: 4.53e-04 | 7786.0ms/step | 2.6 samples/s
|
| 308 |
+
Step 2930/11700 | Loss: 2.8552 | LR: 4.53e-04 | 7784.0ms/step | 2.6 samples/s
|
| 309 |
+
Step 2940/11700 | Loss: 2.7844 | LR: 4.52e-04 | 7782.8ms/step | 2.6 samples/s
|
| 310 |
+
Step 2950/11700 | Loss: 2.8336 | LR: 4.52e-04 | 7781.0ms/step | 2.6 samples/s
|
| 311 |
+
Step 2960/11700 | Loss: 2.7274 | LR: 4.51e-04 | 7783.2ms/step | 2.6 samples/s
|
| 312 |
+
Step 2970/11700 | Loss: 2.6788 | LR: 4.51e-04 | 7794.4ms/step | 2.6 samples/s
|
| 313 |
+
Step 2980/11700 | Loss: 2.7615 | LR: 4.51e-04 | 7793.8ms/step | 2.6 samples/s
|
| 314 |
+
Step 2990/11700 | Loss: 2.6929 | LR: 4.50e-04 | 7801.3ms/step | 2.6 samples/s
|
| 315 |
+
Step 3000/11700 | Loss: 2.7701 | LR: 4.50e-04 | 7794.1ms/step | 2.6 samples/s
|
| 316 |
+
Validation at step 3000: Loss = 2.7740, Perplexity = 16.02
|
| 317 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-3000
|
| 318 |
+
Step 3010/11700 | Loss: 2.7411 | LR: 4.49e-04 | 19676.9ms/step | 1.0 samples/s
|
| 319 |
+
Step 3020/11700 | Loss: 2.8868 | LR: 4.49e-04 | 7795.1ms/step | 2.6 samples/s
|
| 320 |
+
Step 3030/11700 | Loss: 2.7223 | LR: 4.48e-04 | 7794.7ms/step | 2.6 samples/s
|
| 321 |
+
Step 3040/11700 | Loss: 2.8285 | LR: 4.48e-04 | 7792.4ms/step | 2.6 samples/s
|
| 322 |
+
Step 3050/11700 | Loss: 2.7628 | LR: 4.48e-04 | 7800.1ms/step | 2.6 samples/s
|
| 323 |
+
Step 3060/11700 | Loss: 2.9437 | LR: 4.47e-04 | 7808.6ms/step | 2.6 samples/s
|
| 324 |
+
Step 3070/11700 | Loss: 2.7199 | LR: 4.47e-04 | 7812.9ms/step | 2.6 samples/s
|
| 325 |
+
Step 3080/11700 | Loss: 2.7179 | LR: 4.46e-04 | 7812.6ms/step | 2.6 samples/s
|
| 326 |
+
Step 3090/11700 | Loss: 2.7264 | LR: 4.46e-04 | 7812.1ms/step | 2.6 samples/s
|
| 327 |
+
Step 3100/11700 | Loss: 2.8584 | LR: 4.45e-04 | 7817.1ms/step | 2.6 samples/s
|
| 328 |
+
Step 3110/11700 | Loss: 2.6370 | LR: 4.45e-04 | 7814.9ms/step | 2.6 samples/s
|
| 329 |
+
Step 3120/11700 | Loss: 2.7122 | LR: 4.44e-04 | 7814.4ms/step | 2.6 samples/s
|
| 330 |
+
Step 3130/11700 | Loss: 2.7483 | LR: 4.44e-04 | 7813.7ms/step | 2.6 samples/s
|
| 331 |
+
Step 3140/11700 | Loss: 2.7006 | LR: 4.43e-04 | 7817.5ms/step | 2.6 samples/s
|
| 332 |
+
Step 3150/11700 | Loss: 2.7190 | LR: 4.43e-04 | 7814.2ms/step | 2.6 samples/s
|
| 333 |
+
Step 3160/11700 | Loss: 2.7208 | LR: 4.43e-04 | 7816.5ms/step | 2.6 samples/s
|
| 334 |
+
Step 3170/11700 | Loss: 2.8133 | LR: 4.42e-04 | 7827.0ms/step | 2.6 samples/s
|
| 335 |
+
Step 3180/11700 | Loss: 2.7785 | LR: 4.42e-04 | 7817.1ms/step | 2.6 samples/s
|
| 336 |
+
Step 3190/11700 | Loss: 2.6510 | LR: 4.41e-04 | 7817.0ms/step | 2.6 samples/s
|
| 337 |
+
Step 3200/11700 | Loss: 2.6789 | LR: 4.41e-04 | 7815.7ms/step | 2.6 samples/s
|
| 338 |
+
Step 3210/11700 | Loss: 2.7041 | LR: 4.40e-04 | 7814.9ms/step | 2.6 samples/s
|
| 339 |
+
Step 3220/11700 | Loss: 2.7256 | LR: 4.40e-04 | 7812.1ms/step | 2.6 samples/s
|
| 340 |
+
Step 3230/11700 | Loss: 2.6738 | LR: 4.39e-04 | 7815.5ms/step | 2.6 samples/s
|
| 341 |
+
Step 3240/11700 | Loss: 2.7417 | LR: 4.39e-04 | 7815.8ms/step | 2.6 samples/s
|
| 342 |
+
Step 3250/11700 | Loss: 2.8304 | LR: 4.38e-04 | 7811.9ms/step | 2.6 samples/s
|
| 343 |
+
Validation at step 3250: Loss = 2.7376, Perplexity = 15.45
|
| 344 |
+
Step 3260/11700 | Loss: 2.7471 | LR: 4.38e-04 | 19442.5ms/step | 1.0 samples/s
|
| 345 |
+
Step 3270/11700 | Loss: 2.6905 | LR: 4.37e-04 | 7808.9ms/step | 2.6 samples/s
|
| 346 |
+
Step 3280/11700 | Loss: 2.7481 | LR: 4.37e-04 | 7809.9ms/step | 2.6 samples/s
|
| 347 |
+
Step 3290/11700 | Loss: 2.6938 | LR: 4.37e-04 | 7810.0ms/step | 2.6 samples/s
|
| 348 |
+
Step 3300/11700 | Loss: 2.7568 | LR: 4.36e-04 | 7814.8ms/step | 2.6 samples/s
|
| 349 |
+
Step 3310/11700 | Loss: 2.7561 | LR: 4.36e-04 | 7812.8ms/step | 2.6 samples/s
|
| 350 |
+
Step 3320/11700 | Loss: 2.6733 | LR: 4.35e-04 | 7811.3ms/step | 2.6 samples/s
|
| 351 |
+
Step 3330/11700 | Loss: 2.6597 | LR: 4.35e-04 | 7805.7ms/step | 2.6 samples/s
|
| 352 |
+
Step 3340/11700 | Loss: 2.7964 | LR: 4.34e-04 | 7805.2ms/step | 2.6 samples/s
|
| 353 |
+
Step 3350/11700 | Loss: 2.6977 | LR: 4.34e-04 | 7801.7ms/step | 2.6 samples/s
|
| 354 |
+
Step 3360/11700 | Loss: 2.7787 | LR: 4.33e-04 | 7801.8ms/step | 2.6 samples/s
|
| 355 |
+
Step 3370/11700 | Loss: 2.6637 | LR: 4.33e-04 | 7802.9ms/step | 2.6 samples/s
|
| 356 |
+
Step 3380/11700 | Loss: 2.7256 | LR: 4.32e-04 | 7804.8ms/step | 2.6 samples/s
|
| 357 |
+
Step 3390/11700 | Loss: 2.6890 | LR: 4.32e-04 | 7805.7ms/step | 2.6 samples/s
|
| 358 |
+
Step 3400/11700 | Loss: 2.7992 | LR: 4.31e-04 | 7808.2ms/step | 2.6 samples/s
|
| 359 |
+
Step 3410/11700 | Loss: 2.7400 | LR: 4.31e-04 | 7806.0ms/step | 2.6 samples/s
|
| 360 |
+
Step 3420/11700 | Loss: 2.5689 | LR: 4.30e-04 | 7810.9ms/step | 2.6 samples/s
|
| 361 |
+
Step 3430/11700 | Loss: 2.7553 | LR: 4.30e-04 | 7821.2ms/step | 2.6 samples/s
|
| 362 |
+
Step 3440/11700 | Loss: 2.6094 | LR: 4.29e-04 | 7818.0ms/step | 2.6 samples/s
|
| 363 |
+
Step 3450/11700 | Loss: 2.7223 | LR: 4.29e-04 | 7817.1ms/step | 2.6 samples/s
|
| 364 |
+
Step 3460/11700 | Loss: 2.7058 | LR: 4.28e-04 | 7816.4ms/step | 2.6 samples/s
|
| 365 |
+
Step 3470/11700 | Loss: 2.5958 | LR: 4.28e-04 | 7822.1ms/step | 2.6 samples/s
|
| 366 |
+
Step 3480/11700 | Loss: 2.5197 | LR: 4.27e-04 | 7820.6ms/step | 2.6 samples/s
|
| 367 |
+
Step 3490/11700 | Loss: 2.8477 | LR: 4.27e-04 | 7814.5ms/step | 2.6 samples/s
|
| 368 |
+
Step 3500/11700 | Loss: 2.7588 | LR: 4.26e-04 | 7813.9ms/step | 2.6 samples/s
|
| 369 |
+
Validation at step 3500: Loss = 2.7077, Perplexity = 14.99
|
| 370 |
+
Step 3510/11700 | Loss: 2.7405 | LR: 4.26e-04 | 19417.3ms/step | 1.0 samples/s
|
| 371 |
+
Step 3520/11700 | Loss: 2.7293 | LR: 4.25e-04 | 7802.7ms/step | 2.6 samples/s
|
| 372 |
+
Step 3530/11700 | Loss: 2.7394 | LR: 4.25e-04 | 7811.2ms/step | 2.6 samples/s
|
| 373 |
+
Step 3540/11700 | Loss: 2.7461 | LR: 4.24e-04 | 7815.2ms/step | 2.6 samples/s
|
| 374 |
+
Step 3550/11700 | Loss: 2.6650 | LR: 4.24e-04 | 7813.1ms/step | 2.6 samples/s
|
| 375 |
+
Step 3560/11700 | Loss: 2.8531 | LR: 4.23e-04 | 7811.6ms/step | 2.6 samples/s
|
| 376 |
+
Step 3570/11700 | Loss: 2.7221 | LR: 4.23e-04 | 7816.0ms/step | 2.6 samples/s
|
| 377 |
+
Step 3580/11700 | Loss: 2.7368 | LR: 4.22e-04 | 7815.5ms/step | 2.6 samples/s
|
| 378 |
+
Step 3590/11700 | Loss: 2.6369 | LR: 4.22e-04 | 7814.4ms/step | 2.6 samples/s
|
| 379 |
+
Step 3600/11700 | Loss: 2.6327 | LR: 4.21e-04 | 7814.3ms/step | 2.6 samples/s
|
| 380 |
+
Step 3610/11700 | Loss: 2.7634 | LR: 4.20e-04 | 7815.1ms/step | 2.6 samples/s
|
| 381 |
+
Step 3620/11700 | Loss: 2.6312 | LR: 4.20e-04 | 7813.2ms/step | 2.6 samples/s
|
| 382 |
+
Step 3630/11700 | Loss: 2.6417 | LR: 4.19e-04 | 7792.0ms/step | 2.6 samples/s
|
| 383 |
+
Step 3640/11700 | Loss: 2.5898 | LR: 4.19e-04 | 7776.9ms/step | 2.6 samples/s
|
| 384 |
+
Step 3650/11700 | Loss: 2.7195 | LR: 4.18e-04 | 7778.9ms/step | 2.6 samples/s
|
| 385 |
+
Step 3660/11700 | Loss: 2.6868 | LR: 4.18e-04 | 7779.1ms/step | 2.6 samples/s
|
| 386 |
+
Step 3670/11700 | Loss: 2.6408 | LR: 4.17e-04 | 7778.0ms/step | 2.6 samples/s
|
| 387 |
+
Step 3680/11700 | Loss: 2.7360 | LR: 4.17e-04 | 7778.2ms/step | 2.6 samples/s
|
| 388 |
+
Step 3690/11700 | Loss: 2.6718 | LR: 4.16e-04 | 7777.4ms/step | 2.6 samples/s
|
| 389 |
+
Step 3700/11700 | Loss: 2.7135 | LR: 4.16e-04 | 7769.5ms/step | 2.6 samples/s
|
| 390 |
+
Step 3710/11700 | Loss: 2.6116 | LR: 4.15e-04 | 7771.8ms/step | 2.6 samples/s
|
| 391 |
+
Step 3720/11700 | Loss: 2.6384 | LR: 4.15e-04 | 7774.0ms/step | 2.6 samples/s
|
| 392 |
+
Step 3730/11700 | Loss: 2.6904 | LR: 4.14e-04 | 7773.4ms/step | 2.6 samples/s
|
| 393 |
+
Step 3740/11700 | Loss: 2.6396 | LR: 4.14e-04 | 7771.9ms/step | 2.6 samples/s
|
| 394 |
+
Step 3750/11700 | Loss: 2.7130 | LR: 4.13e-04 | 7771.9ms/step | 2.6 samples/s
|
| 395 |
+
Validation at step 3750: Loss = 2.6804, Perplexity = 14.59
|
| 396 |
+
Step 3760/11700 | Loss: 2.7227 | LR: 4.12e-04 | 19324.2ms/step | 1.0 samples/s
|
| 397 |
+
Step 3770/11700 | Loss: 2.7143 | LR: 4.12e-04 | 7774.8ms/step | 2.6 samples/s
|
| 398 |
+
Step 3780/11700 | Loss: 2.6673 | LR: 4.11e-04 | 7772.8ms/step | 2.6 samples/s
|
| 399 |
+
Step 3790/11700 | Loss: 2.8107 | LR: 4.11e-04 | 7772.2ms/step | 2.6 samples/s
|
| 400 |
+
Step 3800/11700 | Loss: 2.6996 | LR: 4.10e-04 | 7772.2ms/step | 2.6 samples/s
|
| 401 |
+
Step 3810/11700 | Loss: 2.6574 | LR: 4.10e-04 | 7772.5ms/step | 2.6 samples/s
|
| 402 |
+
Step 3820/11700 | Loss: 2.6426 | LR: 4.09e-04 | 7774.0ms/step | 2.6 samples/s
|
| 403 |
+
Step 3830/11700 | Loss: 2.6652 | LR: 4.09e-04 | 7772.8ms/step | 2.6 samples/s
|
| 404 |
+
Step 3840/11700 | Loss: 2.5947 | LR: 4.08e-04 | 7771.1ms/step | 2.6 samples/s
|
| 405 |
+
Step 3850/11700 | Loss: 2.7118 | LR: 4.07e-04 | 7772.0ms/step | 2.6 samples/s
|
| 406 |
+
Step 3860/11700 | Loss: 2.6307 | LR: 4.07e-04 | 7774.0ms/step | 2.6 samples/s
|
| 407 |
+
Step 3870/11700 | Loss: 2.6647 | LR: 4.06e-04 | 7773.7ms/step | 2.6 samples/s
|
| 408 |
+
Step 3880/11700 | Loss: 2.5481 | LR: 4.06e-04 | 7773.2ms/step | 2.6 samples/s
|
| 409 |
+
Step 3890/11700 | Loss: 2.7217 | LR: 4.05e-04 | 7773.6ms/step | 2.6 samples/s
|
| 410 |
+
Step 3900/11700 | Loss: 2.5928 | LR: 4.05e-04 | 7775.4ms/step | 2.6 samples/s
|
| 411 |
+
Step 3910/11700 | Loss: 2.6814 | LR: 4.04e-04 | 7773.9ms/step | 2.6 samples/s
|
| 412 |
+
Step 3920/11700 | Loss: 2.5886 | LR: 4.04e-04 | 7771.1ms/step | 2.6 samples/s
|
| 413 |
+
Step 3930/11700 | Loss: 2.6458 | LR: 4.03e-04 | 7760.7ms/step | 2.6 samples/s
|
| 414 |
+
Step 3940/11700 | Loss: 2.7853 | LR: 4.02e-04 | 7773.0ms/step | 2.6 samples/s
|
| 415 |
+
Step 3950/11700 | Loss: 2.6102 | LR: 4.02e-04 | 7773.3ms/step | 2.6 samples/s
|
| 416 |
+
Step 3960/11700 | Loss: 2.6625 | LR: 4.01e-04 | 7771.7ms/step | 2.6 samples/s
|
| 417 |
+
Step 3970/11700 | Loss: 2.6422 | LR: 4.01e-04 | 7775.3ms/step | 2.6 samples/s
|
| 418 |
+
Step 3980/11700 | Loss: 2.7038 | LR: 4.00e-04 | 7774.4ms/step | 2.6 samples/s
|
| 419 |
+
Step 3990/11700 | Loss: 2.6862 | LR: 4.00e-04 | 7768.7ms/step | 2.6 samples/s
|
| 420 |
+
Step 4000/11700 | Loss: 2.7374 | LR: 3.99e-04 | 7770.1ms/step | 2.6 samples/s
|
| 421 |
+
Validation at step 4000: Loss = 2.6626, Perplexity = 14.33
|
| 422 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-4000
|
| 423 |
+
Step 4010/11700 | Loss: 2.5564 | LR: 3.98e-04 | 19617.7ms/step | 1.0 samples/s
|
| 424 |
+
Step 4020/11700 | Loss: 2.6545 | LR: 3.98e-04 | 7765.9ms/step | 2.6 samples/s
|
| 425 |
+
Step 4030/11700 | Loss: 2.5487 | LR: 3.97e-04 | 7767.2ms/step | 2.6 samples/s
|
| 426 |
+
Step 4040/11700 | Loss: 2.6881 | LR: 3.97e-04 | 7770.2ms/step | 2.6 samples/s
|
| 427 |
+
Step 4050/11700 | Loss: 2.6830 | LR: 3.96e-04 | 7770.2ms/step | 2.6 samples/s
|
| 428 |
+
Step 4060/11700 | Loss: 2.5570 | LR: 3.95e-04 | 7772.1ms/step | 2.6 samples/s
|
| 429 |
+
Step 4070/11700 | Loss: 2.7404 | LR: 3.95e-04 | 7772.5ms/step | 2.6 samples/s
|
| 430 |
+
Step 4080/11700 | Loss: 2.6298 | LR: 3.94e-04 | 7770.6ms/step | 2.6 samples/s
|
| 431 |
+
Step 4090/11700 | Loss: 2.6052 | LR: 3.94e-04 | 7764.1ms/step | 2.6 samples/s
|
| 432 |
+
Step 4100/11700 | Loss: 2.6464 | LR: 3.93e-04 | 7769.4ms/step | 2.6 samples/s
|
| 433 |
+
Step 4110/11700 | Loss: 2.6319 | LR: 3.93e-04 | 7770.8ms/step | 2.6 samples/s
|
| 434 |
+
Step 4120/11700 | Loss: 2.7015 | LR: 3.92e-04 | 7770.0ms/step | 2.6 samples/s
|
| 435 |
+
Step 4130/11700 | Loss: 2.5645 | LR: 3.91e-04 | 7771.6ms/step | 2.6 samples/s
|
| 436 |
+
Step 4140/11700 | Loss: 2.6128 | LR: 3.91e-04 | 7769.1ms/step | 2.6 samples/s
|
| 437 |
+
Step 4150/11700 | Loss: 2.5576 | LR: 3.90e-04 | 7770.0ms/step | 2.6 samples/s
|
| 438 |
+
Step 4160/11700 | Loss: 2.6320 | LR: 3.90e-04 | 7771.9ms/step | 2.6 samples/s
|
| 439 |
+
Step 4170/11700 | Loss: 2.6890 | LR: 3.89e-04 | 7770.3ms/step | 2.6 samples/s
|
| 440 |
+
Step 4180/11700 | Loss: 2.6293 | LR: 3.88e-04 | 7769.5ms/step | 2.6 samples/s
|
| 441 |
+
Step 4190/11700 | Loss: 2.6602 | LR: 3.88e-04 | 7771.0ms/step | 2.6 samples/s
|
| 442 |
+
Step 4200/11700 | Loss: 2.5753 | LR: 3.87e-04 | 7771.3ms/step | 2.6 samples/s
|
| 443 |
+
Step 4210/11700 | Loss: 2.6791 | LR: 3.87e-04 | 7767.4ms/step | 2.6 samples/s
|
| 444 |
+
Step 4220/11700 | Loss: 2.6452 | LR: 3.86e-04 | 7767.9ms/step | 2.6 samples/s
|
| 445 |
+
Step 4230/11700 | Loss: 2.5682 | LR: 3.85e-04 | 7770.0ms/step | 2.6 samples/s
|
| 446 |
+
Step 4240/11700 | Loss: 2.6828 | LR: 3.85e-04 | 7771.0ms/step | 2.6 samples/s
|
| 447 |
+
Step 4250/11700 | Loss: 2.7766 | LR: 3.84e-04 | 7767.1ms/step | 2.6 samples/s
|
| 448 |
+
Validation at step 4250: Loss = 2.6456, Perplexity = 14.09
|
| 449 |
+
Step 4260/11700 | Loss: 2.7427 | LR: 3.84e-04 | 19297.1ms/step | 1.0 samples/s
|
| 450 |
+
Step 4270/11700 | Loss: 2.5905 | LR: 3.83e-04 | 7765.6ms/step | 2.6 samples/s
|
| 451 |
+
Step 4280/11700 | Loss: 2.6221 | LR: 3.82e-04 | 7769.3ms/step | 2.6 samples/s
|
| 452 |
+
Step 4290/11700 | Loss: 2.6287 | LR: 3.82e-04 | 7769.8ms/step | 2.6 samples/s
|
| 453 |
+
Step 4300/11700 | Loss: 2.6821 | LR: 3.81e-04 | 7768.7ms/step | 2.6 samples/s
|
| 454 |
+
Step 4310/11700 | Loss: 2.6061 | LR: 3.81e-04 | 7770.6ms/step | 2.6 samples/s
|
| 455 |
+
Step 4320/11700 | Loss: 2.5827 | LR: 3.80e-04 | 7769.7ms/step | 2.6 samples/s
|
| 456 |
+
Step 4330/11700 | Loss: 2.5930 | LR: 3.79e-04 | 7768.8ms/step | 2.6 samples/s
|
| 457 |
+
Step 4340/11700 | Loss: 2.6633 | LR: 3.79e-04 | 7768.1ms/step | 2.6 samples/s
|
| 458 |
+
Step 4350/11700 | Loss: 2.7383 | LR: 3.78e-04 | 7767.7ms/step | 2.6 samples/s
|
| 459 |
+
Step 4360/11700 | Loss: 2.5752 | LR: 3.77e-04 | 7768.7ms/step | 2.6 samples/s
|
| 460 |
+
Step 4370/11700 | Loss: 2.6121 | LR: 3.77e-04 | 7767.8ms/step | 2.6 samples/s
|
| 461 |
+
Step 4380/11700 | Loss: 2.5713 | LR: 3.76e-04 | 7767.4ms/step | 2.6 samples/s
|
| 462 |
+
Step 4390/11700 | Loss: 2.6445 | LR: 3.76e-04 | 7770.1ms/step | 2.6 samples/s
|
| 463 |
+
Step 4400/11700 | Loss: 2.6299 | LR: 3.75e-04 | 7770.1ms/step | 2.6 samples/s
|
| 464 |
+
Step 4410/11700 | Loss: 2.6507 | LR: 3.74e-04 | 7766.4ms/step | 2.6 samples/s
|
| 465 |
+
Step 4420/11700 | Loss: 2.6375 | LR: 3.74e-04 | 7769.8ms/step | 2.6 samples/s
|
| 466 |
+
Step 4430/11700 | Loss: 2.6646 | LR: 3.73e-04 | 7769.3ms/step | 2.6 samples/s
|
| 467 |
+
Step 4440/11700 | Loss: 2.6654 | LR: 3.73e-04 | 7768.4ms/step | 2.6 samples/s
|
| 468 |
+
Step 4450/11700 | Loss: 2.5990 | LR: 3.72e-04 | 7766.4ms/step | 2.6 samples/s
|
| 469 |
+
Step 4460/11700 | Loss: 2.7188 | LR: 3.71e-04 | 7764.9ms/step | 2.6 samples/s
|
| 470 |
+
Step 4470/11700 | Loss: 2.5219 | LR: 3.71e-04 | 7770.3ms/step | 2.6 samples/s
|
| 471 |
+
Step 4480/11700 | Loss: 2.5990 | LR: 3.70e-04 | 7769.1ms/step | 2.6 samples/s
|
| 472 |
+
Step 4490/11700 | Loss: 2.6111 | LR: 3.69e-04 | 7773.5ms/step | 2.6 samples/s
|
| 473 |
+
Step 4500/11700 | Loss: 2.6242 | LR: 3.69e-04 | 7768.8ms/step | 2.6 samples/s
|
| 474 |
+
Validation at step 4500: Loss = 2.6277, Perplexity = 13.84
|
| 475 |
+
Step 4510/11700 | Loss: 2.5234 | LR: 3.68e-04 | 19313.3ms/step | 1.0 samples/s
|
| 476 |
+
Step 4520/11700 | Loss: 2.6524 | LR: 3.67e-04 | 7821.0ms/step | 2.6 samples/s
|
| 477 |
+
Step 4530/11700 | Loss: 2.7433 | LR: 3.67e-04 | 7825.0ms/step | 2.6 samples/s
|
| 478 |
+
Step 4540/11700 | Loss: 2.6663 | LR: 3.66e-04 | 7835.9ms/step | 2.6 samples/s
|
| 479 |
+
Step 4550/11700 | Loss: 2.5982 | LR: 3.66e-04 | 7831.0ms/step | 2.6 samples/s
|
| 480 |
+
Step 4560/11700 | Loss: 2.5687 | LR: 3.65e-04 | 7843.5ms/step | 2.5 samples/s
|
| 481 |
+
Step 4570/11700 | Loss: 2.5915 | LR: 3.64e-04 | 7804.2ms/step | 2.6 samples/s
|
| 482 |
+
Step 4580/11700 | Loss: 2.5595 | LR: 3.64e-04 | 7779.2ms/step | 2.6 samples/s
|
| 483 |
+
Step 4590/11700 | Loss: 2.7110 | LR: 3.63e-04 | 7774.3ms/step | 2.6 samples/s
|
| 484 |
+
Step 4600/11700 | Loss: 2.4597 | LR: 3.62e-04 | 7795.6ms/step | 2.6 samples/s
|
| 485 |
+
Step 4610/11700 | Loss: 2.5113 | LR: 3.62e-04 | 7771.1ms/step | 2.6 samples/s
|
| 486 |
+
Step 4620/11700 | Loss: 2.6229 | LR: 3.61e-04 | 7788.1ms/step | 2.6 samples/s
|
| 487 |
+
Step 4630/11700 | Loss: 2.6296 | LR: 3.60e-04 | 7831.2ms/step | 2.6 samples/s
|
| 488 |
+
Step 4640/11700 | Loss: 2.5770 | LR: 3.60e-04 | 7817.1ms/step | 2.6 samples/s
|
| 489 |
+
Step 4650/11700 | Loss: 2.5732 | LR: 3.59e-04 | 7832.6ms/step | 2.6 samples/s
|
| 490 |
+
Step 4660/11700 | Loss: 2.6424 | LR: 3.59e-04 | 7792.2ms/step | 2.6 samples/s
|
| 491 |
+
Step 4670/11700 | Loss: 2.5610 | LR: 3.58e-04 | 7785.6ms/step | 2.6 samples/s
|
| 492 |
+
Step 4680/11700 | Loss: 2.6483 | LR: 3.57e-04 | 7798.0ms/step | 2.6 samples/s
|
| 493 |
+
Step 4690/11700 | Loss: 2.5449 | LR: 3.57e-04 | 7801.0ms/step | 2.6 samples/s
|
| 494 |
+
Step 4700/11700 | Loss: 2.5464 | LR: 3.56e-04 | 7800.2ms/step | 2.6 samples/s
|
| 495 |
+
Step 4710/11700 | Loss: 2.7074 | LR: 3.55e-04 | 7800.7ms/step | 2.6 samples/s
|
| 496 |
+
Step 4720/11700 | Loss: 2.5725 | LR: 3.55e-04 | 7805.1ms/step | 2.6 samples/s
|
| 497 |
+
Step 4730/11700 | Loss: 2.6332 | LR: 3.54e-04 | 7803.3ms/step | 2.6 samples/s
|
| 498 |
+
Step 4740/11700 | Loss: 2.6430 | LR: 3.53e-04 | 7806.7ms/step | 2.6 samples/s
|
| 499 |
+
Step 4750/11700 | Loss: 2.5887 | LR: 3.53e-04 | 7798.8ms/step | 2.6 samples/s
|
| 500 |
+
Validation at step 4750: Loss = 2.6070, Perplexity = 13.56
|
| 501 |
+
Step 4760/11700 | Loss: 2.6835 | LR: 3.52e-04 | 19397.2ms/step | 1.0 samples/s
|
| 502 |
+
Step 4770/11700 | Loss: 2.6405 | LR: 3.51e-04 | 7808.6ms/step | 2.6 samples/s
|
| 503 |
+
Step 4780/11700 | Loss: 2.6052 | LR: 3.51e-04 | 7806.5ms/step | 2.6 samples/s
|
| 504 |
+
Step 4790/11700 | Loss: 2.5807 | LR: 3.50e-04 | 7800.5ms/step | 2.6 samples/s
|
| 505 |
+
Step 4800/11700 | Loss: 2.5236 | LR: 3.49e-04 | 7801.4ms/step | 2.6 samples/s
|
| 506 |
+
Step 4810/11700 | Loss: 2.4949 | LR: 3.49e-04 | 7805.7ms/step | 2.6 samples/s
|
| 507 |
+
Step 4820/11700 | Loss: 2.5963 | LR: 3.48e-04 | 7800.5ms/step | 2.6 samples/s
|
| 508 |
+
Step 4830/11700 | Loss: 2.6169 | LR: 3.47e-04 | 7801.7ms/step | 2.6 samples/s
|
| 509 |
+
Step 4840/11700 | Loss: 2.6136 | LR: 3.47e-04 | 7805.0ms/step | 2.6 samples/s
|
| 510 |
+
Step 4850/11700 | Loss: 2.6221 | LR: 3.46e-04 | 7804.0ms/step | 2.6 samples/s
|
| 511 |
+
Step 4860/11700 | Loss: 2.5662 | LR: 3.45e-04 | 7801.1ms/step | 2.6 samples/s
|
| 512 |
+
Step 4870/11700 | Loss: 2.6917 | LR: 3.45e-04 | 7802.8ms/step | 2.6 samples/s
|
| 513 |
+
Step 4880/11700 | Loss: 2.6209 | LR: 3.44e-04 | 7802.0ms/step | 2.6 samples/s
|
| 514 |
+
Step 4890/11700 | Loss: 2.6831 | LR: 3.43e-04 | 7804.2ms/step | 2.6 samples/s
|
| 515 |
+
Step 4900/11700 | Loss: 2.6238 | LR: 3.43e-04 | 7797.8ms/step | 2.6 samples/s
|
| 516 |
+
Step 4910/11700 | Loss: 2.5729 | LR: 3.42e-04 | 7789.4ms/step | 2.6 samples/s
|
| 517 |
+
Step 4920/11700 | Loss: 2.5437 | LR: 3.41e-04 | 7799.5ms/step | 2.6 samples/s
|
| 518 |
+
Step 4930/11700 | Loss: 2.5115 | LR: 3.41e-04 | 7801.3ms/step | 2.6 samples/s
|
| 519 |
+
Step 4940/11700 | Loss: 2.6362 | LR: 3.40e-04 | 7801.5ms/step | 2.6 samples/s
|
| 520 |
+
Step 4950/11700 | Loss: 2.4540 | LR: 3.39e-04 | 7801.8ms/step | 2.6 samples/s
|
| 521 |
+
Step 4960/11700 | Loss: 2.6159 | LR: 3.39e-04 | 7800.7ms/step | 2.6 samples/s
|
| 522 |
+
Step 4970/11700 | Loss: 2.6017 | LR: 3.38e-04 | 7802.5ms/step | 2.6 samples/s
|
| 523 |
+
Step 4980/11700 | Loss: 2.7127 | LR: 3.37e-04 | 7792.0ms/step | 2.6 samples/s
|
| 524 |
+
Step 4990/11700 | Loss: 2.5300 | LR: 3.37e-04 | 7799.8ms/step | 2.6 samples/s
|
| 525 |
+
Step 5000/11700 | Loss: 2.6930 | LR: 3.36e-04 | 7800.4ms/step | 2.6 samples/s
|
| 526 |
+
Validation at step 5000: Loss = 2.5904, Perplexity = 13.33
|
| 527 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-5000
|
| 528 |
+
Step 5010/11700 | Loss: 2.6921 | LR: 3.35e-04 | 19668.7ms/step | 1.0 samples/s
|
| 529 |
+
Step 5020/11700 | Loss: 2.6735 | LR: 3.35e-04 | 7799.0ms/step | 2.6 samples/s
|
| 530 |
+
Step 5030/11700 | Loss: 2.6672 | LR: 3.34e-04 | 7799.1ms/step | 2.6 samples/s
|
| 531 |
+
Step 5040/11700 | Loss: 2.5005 | LR: 3.33e-04 | 7800.2ms/step | 2.6 samples/s
|
| 532 |
+
Step 5050/11700 | Loss: 2.4407 | LR: 3.33e-04 | 7800.9ms/step | 2.6 samples/s
|
| 533 |
+
Step 5060/11700 | Loss: 2.6105 | LR: 3.32e-04 | 7800.9ms/step | 2.6 samples/s
|
| 534 |
+
Step 5070/11700 | Loss: 2.5694 | LR: 3.31e-04 | 7799.0ms/step | 2.6 samples/s
|
| 535 |
+
Step 5080/11700 | Loss: 2.7023 | LR: 3.31e-04 | 7800.1ms/step | 2.6 samples/s
|
| 536 |
+
Step 5090/11700 | Loss: 2.6046 | LR: 3.30e-04 | 7800.1ms/step | 2.6 samples/s
|
| 537 |
+
Step 5100/11700 | Loss: 2.7009 | LR: 3.29e-04 | 7798.5ms/step | 2.6 samples/s
|
| 538 |
+
Step 5110/11700 | Loss: 2.6003 | LR: 3.29e-04 | 7783.4ms/step | 2.6 samples/s
|
| 539 |
+
Step 5120/11700 | Loss: 2.5508 | LR: 3.28e-04 | 7779.4ms/step | 2.6 samples/s
|
| 540 |
+
Step 5130/11700 | Loss: 2.4662 | LR: 3.27e-04 | 7792.7ms/step | 2.6 samples/s
|
| 541 |
+
Step 5140/11700 | Loss: 2.5191 | LR: 3.27e-04 | 7800.6ms/step | 2.6 samples/s
|
| 542 |
+
Step 5150/11700 | Loss: 2.5877 | LR: 3.26e-04 | 7800.6ms/step | 2.6 samples/s
|
| 543 |
+
Step 5160/11700 | Loss: 2.5329 | LR: 3.25e-04 | 7810.2ms/step | 2.6 samples/s
|
| 544 |
+
Step 5170/11700 | Loss: 2.5402 | LR: 3.25e-04 | 7812.4ms/step | 2.6 samples/s
|
| 545 |
+
Step 5180/11700 | Loss: 2.5251 | LR: 3.24e-04 | 7809.8ms/step | 2.6 samples/s
|
| 546 |
+
Step 5190/11700 | Loss: 2.5973 | LR: 3.23e-04 | 7818.7ms/step | 2.6 samples/s
|
| 547 |
+
Step 5200/11700 | Loss: 2.4316 | LR: 3.22e-04 | 7834.8ms/step | 2.6 samples/s
|
| 548 |
+
Step 5210/11700 | Loss: 2.5473 | LR: 3.22e-04 | 7820.7ms/step | 2.6 samples/s
|
| 549 |
+
Step 5220/11700 | Loss: 2.6009 | LR: 3.21e-04 | 7821.2ms/step | 2.6 samples/s
|
| 550 |
+
Step 5230/11700 | Loss: 2.6767 | LR: 3.20e-04 | 7825.7ms/step | 2.6 samples/s
|
| 551 |
+
Step 5240/11700 | Loss: 2.5516 | LR: 3.20e-04 | 7833.1ms/step | 2.6 samples/s
|
| 552 |
+
Step 5250/11700 | Loss: 2.5910 | LR: 3.19e-04 | 7828.2ms/step | 2.6 samples/s
|
| 553 |
+
Validation at step 5250: Loss = 2.5729, Perplexity = 13.10
|
| 554 |
+
Step 5260/11700 | Loss: 2.5837 | LR: 3.18e-04 | 19482.4ms/step | 1.0 samples/s
|
| 555 |
+
Step 5270/11700 | Loss: 2.5638 | LR: 3.18e-04 | 7825.4ms/step | 2.6 samples/s
|
| 556 |
+
Step 5280/11700 | Loss: 2.5207 | LR: 3.17e-04 | 7825.7ms/step | 2.6 samples/s
|
| 557 |
+
Step 5290/11700 | Loss: 2.5263 | LR: 3.16e-04 | 7816.7ms/step | 2.6 samples/s
|
| 558 |
+
Step 5300/11700 | Loss: 2.5433 | LR: 3.16e-04 | 7825.7ms/step | 2.6 samples/s
|
| 559 |
+
Step 5310/11700 | Loss: 2.5335 | LR: 3.15e-04 | 7814.3ms/step | 2.6 samples/s
|
| 560 |
+
Step 5320/11700 | Loss: 2.5086 | LR: 3.14e-04 | 7822.3ms/step | 2.6 samples/s
|
| 561 |
+
Step 5330/11700 | Loss: 2.4975 | LR: 3.13e-04 | 7835.3ms/step | 2.6 samples/s
|
| 562 |
+
Step 5340/11700 | Loss: 2.5322 | LR: 3.13e-04 | 7829.6ms/step | 2.6 samples/s
|
| 563 |
+
Step 5350/11700 | Loss: 2.5290 | LR: 3.12e-04 | 7826.0ms/step | 2.6 samples/s
|
| 564 |
+
Step 5360/11700 | Loss: 2.4582 | LR: 3.11e-04 | 7826.4ms/step | 2.6 samples/s
|
| 565 |
+
Step 5370/11700 | Loss: 2.5644 | LR: 3.11e-04 | 7821.5ms/step | 2.6 samples/s
|
| 566 |
+
Step 5380/11700 | Loss: 2.4625 | LR: 3.10e-04 | 7804.8ms/step | 2.6 samples/s
|
| 567 |
+
Step 5390/11700 | Loss: 2.6025 | LR: 3.09e-04 | 7822.0ms/step | 2.6 samples/s
|
| 568 |
+
Step 5400/11700 | Loss: 2.5107 | LR: 3.09e-04 | 7820.1ms/step | 2.6 samples/s
|
| 569 |
+
Step 5410/11700 | Loss: 2.4943 | LR: 3.08e-04 | 7817.2ms/step | 2.6 samples/s
|
| 570 |
+
Step 5420/11700 | Loss: 2.5665 | LR: 3.07e-04 | 7816.8ms/step | 2.6 samples/s
|
| 571 |
+
Step 5430/11700 | Loss: 2.6351 | LR: 3.07e-04 | 7819.9ms/step | 2.6 samples/s
|
| 572 |
+
Step 5440/11700 | Loss: 2.6398 | LR: 3.06e-04 | 7819.8ms/step | 2.6 samples/s
|
| 573 |
+
Step 5450/11700 | Loss: 2.5230 | LR: 3.05e-04 | 7820.9ms/step | 2.6 samples/s
|
| 574 |
+
Step 5460/11700 | Loss: 2.5624 | LR: 3.04e-04 | 7815.4ms/step | 2.6 samples/s
|
| 575 |
+
Step 5470/11700 | Loss: 2.4260 | LR: 3.04e-04 | 7819.7ms/step | 2.6 samples/s
|
| 576 |
+
Step 5480/11700 | Loss: 2.5305 | LR: 3.03e-04 | 7819.7ms/step | 2.6 samples/s
|
| 577 |
+
Step 5490/11700 | Loss: 2.6940 | LR: 3.02e-04 | 7813.4ms/step | 2.6 samples/s
|
| 578 |
+
Step 5500/11700 | Loss: 2.5992 | LR: 3.02e-04 | 7792.6ms/step | 2.6 samples/s
|
| 579 |
+
Validation at step 5500: Loss = 2.5577, Perplexity = 12.91
|
| 580 |
+
Step 5510/11700 | Loss: 2.4369 | LR: 3.01e-04 | 19413.8ms/step | 1.0 samples/s
|
| 581 |
+
Step 5520/11700 | Loss: 2.4177 | LR: 3.00e-04 | 7819.4ms/step | 2.6 samples/s
|
| 582 |
+
Step 5530/11700 | Loss: 2.5758 | LR: 3.00e-04 | 7825.8ms/step | 2.6 samples/s
|
| 583 |
+
Step 5540/11700 | Loss: 2.5552 | LR: 2.99e-04 | 7826.8ms/step | 2.6 samples/s
|
| 584 |
+
Step 5550/11700 | Loss: 2.5621 | LR: 2.98e-04 | 7826.4ms/step | 2.6 samples/s
|
| 585 |
+
Step 5560/11700 | Loss: 2.6024 | LR: 2.97e-04 | 7828.4ms/step | 2.6 samples/s
|
| 586 |
+
Step 5570/11700 | Loss: 2.4767 | LR: 2.97e-04 | 7828.7ms/step | 2.6 samples/s
|
| 587 |
+
Step 5580/11700 | Loss: 2.5686 | LR: 2.96e-04 | 7828.6ms/step | 2.6 samples/s
|
| 588 |
+
Step 5590/11700 | Loss: 2.5936 | LR: 2.95e-04 | 7826.6ms/step | 2.6 samples/s
|
| 589 |
+
Step 5600/11700 | Loss: 2.5386 | LR: 2.95e-04 | 7827.9ms/step | 2.6 samples/s
|
| 590 |
+
Step 5610/11700 | Loss: 2.3961 | LR: 2.94e-04 | 7827.8ms/step | 2.6 samples/s
|
| 591 |
+
Step 5620/11700 | Loss: 2.5558 | LR: 2.93e-04 | 7827.2ms/step | 2.6 samples/s
|
| 592 |
+
Step 5630/11700 | Loss: 2.4468 | LR: 2.92e-04 | 7825.5ms/step | 2.6 samples/s
|
| 593 |
+
Step 5640/11700 | Loss: 2.5780 | LR: 2.92e-04 | 7827.2ms/step | 2.6 samples/s
|
| 594 |
+
Step 5650/11700 | Loss: 2.6305 | LR: 2.91e-04 | 7827.8ms/step | 2.6 samples/s
|
| 595 |
+
Step 5660/11700 | Loss: 2.5379 | LR: 2.90e-04 | 7826.4ms/step | 2.6 samples/s
|
| 596 |
+
Step 5670/11700 | Loss: 2.5231 | LR: 2.90e-04 | 7824.9ms/step | 2.6 samples/s
|
| 597 |
+
Step 5680/11700 | Loss: 2.5948 | LR: 2.89e-04 | 7826.6ms/step | 2.6 samples/s
|
| 598 |
+
Step 5690/11700 | Loss: 2.5084 | LR: 2.88e-04 | 7828.2ms/step | 2.6 samples/s
|
| 599 |
+
Step 5700/11700 | Loss: 2.5294 | LR: 2.88e-04 | 7827.5ms/step | 2.6 samples/s
|
| 600 |
+
Step 5710/11700 | Loss: 2.5041 | LR: 2.87e-04 | 7827.7ms/step | 2.6 samples/s
|
| 601 |
+
Step 5720/11700 | Loss: 2.4807 | LR: 2.86e-04 | 7827.5ms/step | 2.6 samples/s
|
| 602 |
+
Step 5730/11700 | Loss: 2.5959 | LR: 2.85e-04 | 7822.6ms/step | 2.6 samples/s
|
| 603 |
+
Step 5740/11700 | Loss: 2.4633 | LR: 2.85e-04 | 7820.7ms/step | 2.6 samples/s
|
| 604 |
+
Step 5750/11700 | Loss: 2.4992 | LR: 2.84e-04 | 7821.6ms/step | 2.6 samples/s
|
| 605 |
+
Validation at step 5750: Loss = 2.5464, Perplexity = 12.76
|
| 606 |
+
Step 5760/11700 | Loss: 2.5367 | LR: 2.83e-04 | 19466.7ms/step | 1.0 samples/s
|
| 607 |
+
Step 5770/11700 | Loss: 2.6648 | LR: 2.83e-04 | 7822.6ms/step | 2.6 samples/s
|
| 608 |
+
Step 5780/11700 | Loss: 2.5031 | LR: 2.82e-04 | 7814.1ms/step | 2.6 samples/s
|
| 609 |
+
Step 5790/11700 | Loss: 2.6283 | LR: 2.81e-04 | 7803.9ms/step | 2.6 samples/s
|
| 610 |
+
Step 5800/11700 | Loss: 2.5959 | LR: 2.80e-04 | 7808.1ms/step | 2.6 samples/s
|
| 611 |
+
Step 5810/11700 | Loss: 2.5697 | LR: 2.80e-04 | 7811.0ms/step | 2.6 samples/s
|
| 612 |
+
Step 5820/11700 | Loss: 2.4641 | LR: 2.79e-04 | 7814.7ms/step | 2.6 samples/s
|
| 613 |
+
Step 5830/11700 | Loss: 2.5155 | LR: 2.78e-04 | 7812.6ms/step | 2.6 samples/s
|
| 614 |
+
Step 5840/11700 | Loss: 2.5064 | LR: 2.78e-04 | 7813.7ms/step | 2.6 samples/s
|
| 615 |
+
Step 5850/11700 | Loss: 2.5130 | LR: 2.77e-04 | 7821.1ms/step | 2.6 samples/s
|
| 616 |
+
Step 5860/11700 | Loss: 2.5860 | LR: 2.76e-04 | 7813.8ms/step | 2.6 samples/s
|
| 617 |
+
Step 5870/11700 | Loss: 2.5945 | LR: 2.75e-04 | 7804.9ms/step | 2.6 samples/s
|
| 618 |
+
Step 5880/11700 | Loss: 2.5727 | LR: 2.75e-04 | 7811.4ms/step | 2.6 samples/s
|
| 619 |
+
Step 5890/11700 | Loss: 2.5089 | LR: 2.74e-04 | 7800.9ms/step | 2.6 samples/s
|
| 620 |
+
Step 5900/11700 | Loss: 2.5219 | LR: 2.73e-04 | 7816.9ms/step | 2.6 samples/s
|
| 621 |
+
Step 5910/11700 | Loss: 2.5700 | LR: 2.73e-04 | 7823.1ms/step | 2.6 samples/s
|
| 622 |
+
Step 5920/11700 | Loss: 2.4796 | LR: 2.72e-04 | 7825.0ms/step | 2.6 samples/s
|
| 623 |
+
Step 5930/11700 | Loss: 2.5087 | LR: 2.71e-04 | 7820.4ms/step | 2.6 samples/s
|
| 624 |
+
Step 5940/11700 | Loss: 2.4384 | LR: 2.70e-04 | 7805.5ms/step | 2.6 samples/s
|
| 625 |
+
Step 5950/11700 | Loss: 2.6062 | LR: 2.70e-04 | 7798.7ms/step | 2.6 samples/s
|
| 626 |
+
Step 5960/11700 | Loss: 2.6311 | LR: 2.69e-04 | 7799.0ms/step | 2.6 samples/s
|
| 627 |
+
Step 5970/11700 | Loss: 2.5213 | LR: 2.68e-04 | 7799.7ms/step | 2.6 samples/s
|
| 628 |
+
Step 5980/11700 | Loss: 2.5393 | LR: 2.68e-04 | 7800.1ms/step | 2.6 samples/s
|
| 629 |
+
Step 5990/11700 | Loss: 2.5306 | LR: 2.67e-04 | 7798.4ms/step | 2.6 samples/s
|
| 630 |
+
Step 6000/11700 | Loss: 2.5261 | LR: 2.66e-04 | 7802.0ms/step | 2.6 samples/s
|
| 631 |
+
Validation at step 6000: Loss = 2.5339, Perplexity = 12.60
|
| 632 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-6000
|
| 633 |
+
Step 6010/11700 | Loss: 2.5800 | LR: 2.65e-04 | 19724.1ms/step | 1.0 samples/s
|
| 634 |
+
Step 6020/11700 | Loss: 2.4430 | LR: 2.65e-04 | 7804.9ms/step | 2.6 samples/s
|
| 635 |
+
Step 6030/11700 | Loss: 2.5022 | LR: 2.64e-04 | 7804.9ms/step | 2.6 samples/s
|
| 636 |
+
Step 6040/11700 | Loss: 2.4668 | LR: 2.63e-04 | 7810.1ms/step | 2.6 samples/s
|
| 637 |
+
Step 6050/11700 | Loss: 2.4693 | LR: 2.63e-04 | 7805.1ms/step | 2.6 samples/s
|
| 638 |
+
Step 6060/11700 | Loss: 2.5706 | LR: 2.62e-04 | 7807.2ms/step | 2.6 samples/s
|
| 639 |
+
Step 6070/11700 | Loss: 2.5613 | LR: 2.61e-04 | 7807.5ms/step | 2.6 samples/s
|
| 640 |
+
Step 6080/11700 | Loss: 2.5921 | LR: 2.60e-04 | 7803.2ms/step | 2.6 samples/s
|
| 641 |
+
Step 6090/11700 | Loss: 2.4603 | LR: 2.60e-04 | 7799.6ms/step | 2.6 samples/s
|
| 642 |
+
Step 6100/11700 | Loss: 2.5337 | LR: 2.59e-04 | 7805.8ms/step | 2.6 samples/s
|
| 643 |
+
Step 6110/11700 | Loss: 2.4657 | LR: 2.58e-04 | 7804.7ms/step | 2.6 samples/s
|
| 644 |
+
Step 6120/11700 | Loss: 2.5920 | LR: 2.58e-04 | 7803.7ms/step | 2.6 samples/s
|
| 645 |
+
Step 6130/11700 | Loss: 2.5223 | LR: 2.57e-04 | 7803.0ms/step | 2.6 samples/s
|
| 646 |
+
Step 6140/11700 | Loss: 2.5337 | LR: 2.56e-04 | 7810.0ms/step | 2.6 samples/s
|
| 647 |
+
Step 6150/11700 | Loss: 2.5059 | LR: 2.55e-04 | 7806.4ms/step | 2.6 samples/s
|
| 648 |
+
Step 6160/11700 | Loss: 2.4447 | LR: 2.55e-04 | 7809.7ms/step | 2.6 samples/s
|
| 649 |
+
Step 6170/11700 | Loss: 2.4559 | LR: 2.54e-04 | 7802.9ms/step | 2.6 samples/s
|
| 650 |
+
Step 6180/11700 | Loss: 2.4236 | LR: 2.53e-04 | 7801.0ms/step | 2.6 samples/s
|
| 651 |
+
Step 6190/11700 | Loss: 2.4851 | LR: 2.53e-04 | 7804.4ms/step | 2.6 samples/s
|
| 652 |
+
Step 6200/11700 | Loss: 2.6110 | LR: 2.52e-04 | 7806.8ms/step | 2.6 samples/s
|
| 653 |
+
Step 6210/11700 | Loss: 2.5930 | LR: 2.51e-04 | 7806.8ms/step | 2.6 samples/s
|
| 654 |
+
Step 6220/11700 | Loss: 2.5394 | LR: 2.50e-04 | 7814.3ms/step | 2.6 samples/s
|
| 655 |
+
Step 6230/11700 | Loss: 2.6473 | LR: 2.50e-04 | 7806.7ms/step | 2.6 samples/s
|
| 656 |
+
Step 6240/11700 | Loss: 2.5061 | LR: 2.49e-04 | 7799.9ms/step | 2.6 samples/s
|
| 657 |
+
Step 6250/11700 | Loss: 2.5439 | LR: 2.48e-04 | 7806.9ms/step | 2.6 samples/s
|
| 658 |
+
Validation at step 6250: Loss = 2.5226, Perplexity = 12.46
|
| 659 |
+
Step 6260/11700 | Loss: 2.4395 | LR: 2.47e-04 | 19422.1ms/step | 1.0 samples/s
|
| 660 |
+
Step 6270/11700 | Loss: 2.5352 | LR: 2.47e-04 | 7796.3ms/step | 2.6 samples/s
|
| 661 |
+
Step 6280/11700 | Loss: 2.5144 | LR: 2.46e-04 | 7793.5ms/step | 2.6 samples/s
|
| 662 |
+
Step 6290/11700 | Loss: 2.4924 | LR: 2.45e-04 | 7803.4ms/step | 2.6 samples/s
|
| 663 |
+
Step 6300/11700 | Loss: 2.5009 | LR: 2.45e-04 | 7811.3ms/step | 2.6 samples/s
|
| 664 |
+
Step 6310/11700 | Loss: 2.4944 | LR: 2.44e-04 | 7814.9ms/step | 2.6 samples/s
|
| 665 |
+
Step 6320/11700 | Loss: 2.5597 | LR: 2.43e-04 | 7824.6ms/step | 2.6 samples/s
|
| 666 |
+
Step 6330/11700 | Loss: 2.6233 | LR: 2.42e-04 | 7823.0ms/step | 2.6 samples/s
|
| 667 |
+
Step 6340/11700 | Loss: 2.5349 | LR: 2.42e-04 | 7816.6ms/step | 2.6 samples/s
|
| 668 |
+
Step 6350/11700 | Loss: 2.5434 | LR: 2.41e-04 | 7814.5ms/step | 2.6 samples/s
|
| 669 |
+
Step 6360/11700 | Loss: 2.4347 | LR: 2.40e-04 | 7814.6ms/step | 2.6 samples/s
|
| 670 |
+
Step 6370/11700 | Loss: 2.4254 | LR: 2.40e-04 | 7823.4ms/step | 2.6 samples/s
|
| 671 |
+
Step 6380/11700 | Loss: 2.3966 | LR: 2.39e-04 | 7820.4ms/step | 2.6 samples/s
|
| 672 |
+
Step 6390/11700 | Loss: 2.5298 | LR: 2.38e-04 | 7821.7ms/step | 2.6 samples/s
|
| 673 |
+
Step 6400/11700 | Loss: 2.5052 | LR: 2.37e-04 | 7824.5ms/step | 2.6 samples/s
|
| 674 |
+
Step 6410/11700 | Loss: 2.4332 | LR: 2.37e-04 | 7827.2ms/step | 2.6 samples/s
|
| 675 |
+
Step 6420/11700 | Loss: 2.4320 | LR: 2.36e-04 | 7819.5ms/step | 2.6 samples/s
|
| 676 |
+
Step 6430/11700 | Loss: 2.3275 | LR: 2.35e-04 | 7817.7ms/step | 2.6 samples/s
|
| 677 |
+
Step 6440/11700 | Loss: 2.4999 | LR: 2.35e-04 | 7825.5ms/step | 2.6 samples/s
|
| 678 |
+
Step 6450/11700 | Loss: 2.4388 | LR: 2.34e-04 | 7825.4ms/step | 2.6 samples/s
|
| 679 |
+
Step 6460/11700 | Loss: 2.5712 | LR: 2.33e-04 | 7822.4ms/step | 2.6 samples/s
|
| 680 |
+
Step 6470/11700 | Loss: 2.4351 | LR: 2.32e-04 | 7812.8ms/step | 2.6 samples/s
|
| 681 |
+
Step 6480/11700 | Loss: 2.6843 | LR: 2.32e-04 | 7816.0ms/step | 2.6 samples/s
|
| 682 |
+
Step 6490/11700 | Loss: 2.4916 | LR: 2.31e-04 | 7818.0ms/step | 2.6 samples/s
|
| 683 |
+
Step 6500/11700 | Loss: 2.5258 | LR: 2.30e-04 | 7811.3ms/step | 2.6 samples/s
|
| 684 |
+
Validation at step 6500: Loss = 2.5124, Perplexity = 12.33
|
| 685 |
+
Step 6510/11700 | Loss: 2.5044 | LR: 2.30e-04 | 19437.9ms/step | 1.0 samples/s
|
| 686 |
+
Step 6520/11700 | Loss: 2.5051 | LR: 2.29e-04 | 7811.8ms/step | 2.6 samples/s
|
| 687 |
+
Step 6530/11700 | Loss: 2.4231 | LR: 2.28e-04 | 7814.4ms/step | 2.6 samples/s
|
| 688 |
+
Step 6540/11700 | Loss: 2.5749 | LR: 2.27e-04 | 7811.4ms/step | 2.6 samples/s
|
| 689 |
+
Step 6550/11700 | Loss: 2.4755 | LR: 2.27e-04 | 7813.0ms/step | 2.6 samples/s
|
| 690 |
+
Step 6560/11700 | Loss: 2.4271 | LR: 2.26e-04 | 7803.8ms/step | 2.6 samples/s
|
| 691 |
+
Step 6570/11700 | Loss: 2.3821 | LR: 2.25e-04 | 7808.8ms/step | 2.6 samples/s
|
| 692 |
+
Step 6580/11700 | Loss: 2.5421 | LR: 2.25e-04 | 7820.2ms/step | 2.6 samples/s
|
| 693 |
+
Step 6590/11700 | Loss: 2.5226 | LR: 2.24e-04 | 7814.6ms/step | 2.6 samples/s
|
| 694 |
+
Step 6600/11700 | Loss: 2.5348 | LR: 2.23e-04 | 7822.2ms/step | 2.6 samples/s
|
| 695 |
+
Step 6610/11700 | Loss: 2.6074 | LR: 2.22e-04 | 7824.0ms/step | 2.6 samples/s
|
| 696 |
+
Step 6620/11700 | Loss: 2.4071 | LR: 2.22e-04 | 7823.6ms/step | 2.6 samples/s
|
| 697 |
+
Step 6630/11700 | Loss: 2.5150 | LR: 2.21e-04 | 7822.6ms/step | 2.6 samples/s
|
| 698 |
+
Step 6640/11700 | Loss: 2.5228 | LR: 2.20e-04 | 7808.7ms/step | 2.6 samples/s
|
| 699 |
+
Step 6650/11700 | Loss: 2.4555 | LR: 2.20e-04 | 7817.0ms/step | 2.6 samples/s
|
| 700 |
+
Step 6660/11700 | Loss: 2.4791 | LR: 2.19e-04 | 7821.1ms/step | 2.6 samples/s
|
| 701 |
+
Step 6670/11700 | Loss: 2.4724 | LR: 2.18e-04 | 7819.3ms/step | 2.6 samples/s
|
| 702 |
+
Step 6680/11700 | Loss: 2.4702 | LR: 2.17e-04 | 7820.7ms/step | 2.6 samples/s
|
| 703 |
+
Step 6690/11700 | Loss: 2.5599 | LR: 2.17e-04 | 7825.7ms/step | 2.6 samples/s
|
| 704 |
+
Step 6700/11700 | Loss: 2.4991 | LR: 2.16e-04 | 7825.2ms/step | 2.6 samples/s
|
| 705 |
+
Step 6710/11700 | Loss: 2.5114 | LR: 2.15e-04 | 7814.8ms/step | 2.6 samples/s
|
| 706 |
+
Step 6720/11700 | Loss: 2.3938 | LR: 2.15e-04 | 7809.6ms/step | 2.6 samples/s
|
| 707 |
+
Step 6730/11700 | Loss: 2.4530 | LR: 2.14e-04 | 7784.3ms/step | 2.6 samples/s
|
| 708 |
+
Step 6740/11700 | Loss: 2.4708 | LR: 2.13e-04 | 7792.0ms/step | 2.6 samples/s
|
| 709 |
+
Step 6750/11700 | Loss: 2.4540 | LR: 2.12e-04 | 7822.4ms/step | 2.6 samples/s
|
| 710 |
+
Validation at step 6750: Loss = 2.5047, Perplexity = 12.24
|
| 711 |
+
Step 6760/11700 | Loss: 2.5274 | LR: 2.12e-04 | 19428.3ms/step | 1.0 samples/s
|
| 712 |
+
Step 6770/11700 | Loss: 2.4930 | LR: 2.11e-04 | 7792.0ms/step | 2.6 samples/s
|
| 713 |
+
Step 6780/11700 | Loss: 2.4326 | LR: 2.10e-04 | 7789.1ms/step | 2.6 samples/s
|
| 714 |
+
Step 6790/11700 | Loss: 2.4376 | LR: 2.10e-04 | 7788.4ms/step | 2.6 samples/s
|
| 715 |
+
Step 6800/11700 | Loss: 2.5184 | LR: 2.09e-04 | 7790.7ms/step | 2.6 samples/s
|
| 716 |
+
Step 6810/11700 | Loss: 2.6868 | LR: 2.08e-04 | 7785.0ms/step | 2.6 samples/s
|
| 717 |
+
Step 6820/11700 | Loss: 2.5269 | LR: 2.08e-04 | 7781.8ms/step | 2.6 samples/s
|
| 718 |
+
Step 6830/11700 | Loss: 2.4382 | LR: 2.07e-04 | 7783.3ms/step | 2.6 samples/s
|
| 719 |
+
Step 6840/11700 | Loss: 2.4574 | LR: 2.06e-04 | 7781.7ms/step | 2.6 samples/s
|
| 720 |
+
Step 6850/11700 | Loss: 2.5312 | LR: 2.05e-04 | 7790.3ms/step | 2.6 samples/s
|
| 721 |
+
Step 6860/11700 | Loss: 2.4709 | LR: 2.05e-04 | 7784.6ms/step | 2.6 samples/s
|
| 722 |
+
Step 6870/11700 | Loss: 2.4593 | LR: 2.04e-04 | 7784.5ms/step | 2.6 samples/s
|
| 723 |
+
Step 6880/11700 | Loss: 2.4602 | LR: 2.03e-04 | 7784.8ms/step | 2.6 samples/s
|
| 724 |
+
Step 6890/11700 | Loss: 2.4811 | LR: 2.03e-04 | 7784.6ms/step | 2.6 samples/s
|
| 725 |
+
Step 6900/11700 | Loss: 2.6042 | LR: 2.02e-04 | 7785.0ms/step | 2.6 samples/s
|
| 726 |
+
Step 6910/11700 | Loss: 2.5719 | LR: 2.01e-04 | 7786.7ms/step | 2.6 samples/s
|
| 727 |
+
Step 6920/11700 | Loss: 2.4285 | LR: 2.00e-04 | 7784.4ms/step | 2.6 samples/s
|
| 728 |
+
Step 6930/11700 | Loss: 2.3769 | LR: 2.00e-04 | 7782.7ms/step | 2.6 samples/s
|
| 729 |
+
Step 6940/11700 | Loss: 2.5880 | LR: 1.99e-04 | 7781.0ms/step | 2.6 samples/s
|
| 730 |
+
Step 6950/11700 | Loss: 2.5362 | LR: 1.98e-04 | 7782.7ms/step | 2.6 samples/s
|
| 731 |
+
Step 6960/11700 | Loss: 2.4841 | LR: 1.98e-04 | 7780.0ms/step | 2.6 samples/s
|
| 732 |
+
Step 6970/11700 | Loss: 2.3318 | LR: 1.97e-04 | 7782.7ms/step | 2.6 samples/s
|
| 733 |
+
Step 6980/11700 | Loss: 2.4924 | LR: 1.96e-04 | 7779.4ms/step | 2.6 samples/s
|
| 734 |
+
Step 6990/11700 | Loss: 2.5604 | LR: 1.96e-04 | 7780.4ms/step | 2.6 samples/s
|
| 735 |
+
Step 7000/11700 | Loss: 2.4931 | LR: 1.95e-04 | 7783.1ms/step | 2.6 samples/s
|
| 736 |
+
Validation at step 7000: Loss = 2.4955, Perplexity = 12.13
|
| 737 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-7000
|
| 738 |
+
Step 7010/11700 | Loss: 2.5957 | LR: 1.94e-04 | 19639.1ms/step | 1.0 samples/s
|
| 739 |
+
Step 7020/11700 | Loss: 2.4764 | LR: 1.93e-04 | 7776.9ms/step | 2.6 samples/s
|
| 740 |
+
Step 7030/11700 | Loss: 2.5694 | LR: 1.93e-04 | 7784.2ms/step | 2.6 samples/s
|
| 741 |
+
Step 7040/11700 | Loss: 2.5455 | LR: 1.92e-04 | 7785.4ms/step | 2.6 samples/s
|
| 742 |
+
Step 7050/11700 | Loss: 2.4175 | LR: 1.91e-04 | 7788.1ms/step | 2.6 samples/s
|
| 743 |
+
Step 7060/11700 | Loss: 2.5392 | LR: 1.91e-04 | 7771.8ms/step | 2.6 samples/s
|
| 744 |
+
Step 7070/11700 | Loss: 2.4667 | LR: 1.90e-04 | 7781.7ms/step | 2.6 samples/s
|
| 745 |
+
Step 7080/11700 | Loss: 2.4816 | LR: 1.89e-04 | 7797.6ms/step | 2.6 samples/s
|
| 746 |
+
Step 7090/11700 | Loss: 2.4846 | LR: 1.89e-04 | 7804.4ms/step | 2.6 samples/s
|
| 747 |
+
Step 7100/11700 | Loss: 2.5226 | LR: 1.88e-04 | 7812.6ms/step | 2.6 samples/s
|
| 748 |
+
Step 7110/11700 | Loss: 2.4640 | LR: 1.87e-04 | 7820.1ms/step | 2.6 samples/s
|
| 749 |
+
Step 7120/11700 | Loss: 2.4084 | LR: 1.87e-04 | 7798.4ms/step | 2.6 samples/s
|
| 750 |
+
Step 7130/11700 | Loss: 2.4072 | LR: 1.86e-04 | 7790.7ms/step | 2.6 samples/s
|
| 751 |
+
Step 7140/11700 | Loss: 2.4666 | LR: 1.85e-04 | 7796.5ms/step | 2.6 samples/s
|
| 752 |
+
Step 7150/11700 | Loss: 2.5472 | LR: 1.84e-04 | 7817.5ms/step | 2.6 samples/s
|
| 753 |
+
Step 7160/11700 | Loss: 2.4615 | LR: 1.84e-04 | 7813.7ms/step | 2.6 samples/s
|
| 754 |
+
Step 7170/11700 | Loss: 2.4994 | LR: 1.83e-04 | 7809.9ms/step | 2.6 samples/s
|
| 755 |
+
Step 7180/11700 | Loss: 2.4072 | LR: 1.82e-04 | 7800.1ms/step | 2.6 samples/s
|
| 756 |
+
Step 7190/11700 | Loss: 2.4095 | LR: 1.82e-04 | 7794.5ms/step | 2.6 samples/s
|
| 757 |
+
Step 7200/11700 | Loss: 2.3742 | LR: 1.81e-04 | 7819.6ms/step | 2.6 samples/s
|
| 758 |
+
Step 7210/11700 | Loss: 2.3187 | LR: 1.80e-04 | 7812.0ms/step | 2.6 samples/s
|
| 759 |
+
Step 7220/11700 | Loss: 2.4567 | LR: 1.80e-04 | 7803.6ms/step | 2.6 samples/s
|
| 760 |
+
Step 7230/11700 | Loss: 2.5009 | LR: 1.79e-04 | 7796.9ms/step | 2.6 samples/s
|
| 761 |
+
Step 7240/11700 | Loss: 2.4589 | LR: 1.78e-04 | 7801.2ms/step | 2.6 samples/s
|
| 762 |
+
Step 7250/11700 | Loss: 2.3306 | LR: 1.78e-04 | 7802.5ms/step | 2.6 samples/s
|
| 763 |
+
Validation at step 7250: Loss = 2.4880, Perplexity = 12.04
|
| 764 |
+
Step 7260/11700 | Loss: 2.4482 | LR: 1.77e-04 | 19411.7ms/step | 1.0 samples/s
|
| 765 |
+
Step 7270/11700 | Loss: 2.5023 | LR: 1.76e-04 | 7807.0ms/step | 2.6 samples/s
|
| 766 |
+
Step 7280/11700 | Loss: 2.5073 | LR: 1.75e-04 | 7800.9ms/step | 2.6 samples/s
|
| 767 |
+
Step 7290/11700 | Loss: 2.5019 | LR: 1.75e-04 | 7802.9ms/step | 2.6 samples/s
|
| 768 |
+
Step 7300/11700 | Loss: 2.4238 | LR: 1.74e-04 | 7802.8ms/step | 2.6 samples/s
|
| 769 |
+
Step 7310/11700 | Loss: 2.5480 | LR: 1.73e-04 | 7799.3ms/step | 2.6 samples/s
|
| 770 |
+
Step 7320/11700 | Loss: 2.4474 | LR: 1.73e-04 | 7799.4ms/step | 2.6 samples/s
|
| 771 |
+
Step 7330/11700 | Loss: 2.5160 | LR: 1.72e-04 | 7816.7ms/step | 2.6 samples/s
|
| 772 |
+
Step 7340/11700 | Loss: 2.4545 | LR: 1.71e-04 | 7840.5ms/step | 2.6 samples/s
|
| 773 |
+
Step 7350/11700 | Loss: 2.4382 | LR: 1.71e-04 | 7824.5ms/step | 2.6 samples/s
|
| 774 |
+
Step 7360/11700 | Loss: 2.3920 | LR: 1.70e-04 | 7815.3ms/step | 2.6 samples/s
|
| 775 |
+
Step 7370/11700 | Loss: 2.4508 | LR: 1.69e-04 | 7842.8ms/step | 2.6 samples/s
|
| 776 |
+
Step 7380/11700 | Loss: 2.4892 | LR: 1.69e-04 | 7848.6ms/step | 2.5 samples/s
|
| 777 |
+
Step 7390/11700 | Loss: 2.4554 | LR: 1.68e-04 | 7847.3ms/step | 2.5 samples/s
|
| 778 |
+
Step 7400/11700 | Loss: 2.4894 | LR: 1.67e-04 | 7848.3ms/step | 2.5 samples/s
|
| 779 |
+
Step 7410/11700 | Loss: 2.4119 | LR: 1.67e-04 | 7826.7ms/step | 2.6 samples/s
|
| 780 |
+
Step 7420/11700 | Loss: 2.3951 | LR: 1.66e-04 | 7822.0ms/step | 2.6 samples/s
|
| 781 |
+
Step 7430/11700 | Loss: 2.4082 | LR: 1.65e-04 | 7843.1ms/step | 2.5 samples/s
|
| 782 |
+
Step 7440/11700 | Loss: 2.5110 | LR: 1.65e-04 | 7850.1ms/step | 2.5 samples/s
|
| 783 |
+
Step 7450/11700 | Loss: 2.4915 | LR: 1.64e-04 | 7850.3ms/step | 2.5 samples/s
|
| 784 |
+
Step 7460/11700 | Loss: 2.4979 | LR: 1.63e-04 | 7846.9ms/step | 2.5 samples/s
|
| 785 |
+
Step 7470/11700 | Loss: 2.4507 | LR: 1.63e-04 | 7824.8ms/step | 2.6 samples/s
|
| 786 |
+
Step 7480/11700 | Loss: 2.5832 | LR: 1.62e-04 | 7810.2ms/step | 2.6 samples/s
|
| 787 |
+
Step 7490/11700 | Loss: 2.4159 | LR: 1.61e-04 | 7837.1ms/step | 2.6 samples/s
|
| 788 |
+
Step 7500/11700 | Loss: 2.4072 | LR: 1.61e-04 | 7842.2ms/step | 2.6 samples/s
|
| 789 |
+
Validation at step 7500: Loss = 2.4818, Perplexity = 11.96
|
| 790 |
+
Step 7510/11700 | Loss: 2.4921 | LR: 1.60e-04 | 19529.6ms/step | 1.0 samples/s
|
| 791 |
+
Step 7520/11700 | Loss: 2.5159 | LR: 1.59e-04 | 7817.7ms/step | 2.6 samples/s
|
| 792 |
+
Step 7530/11700 | Loss: 2.4188 | LR: 1.59e-04 | 7830.7ms/step | 2.6 samples/s
|
| 793 |
+
Step 7540/11700 | Loss: 2.4978 | LR: 1.58e-04 | 7841.5ms/step | 2.6 samples/s
|
| 794 |
+
Step 7550/11700 | Loss: 2.4444 | LR: 1.57e-04 | 7844.4ms/step | 2.5 samples/s
|
| 795 |
+
Step 7560/11700 | Loss: 2.5038 | LR: 1.57e-04 | 7847.1ms/step | 2.5 samples/s
|
| 796 |
+
Step 7570/11700 | Loss: 2.4642 | LR: 1.56e-04 | 7848.9ms/step | 2.5 samples/s
|
| 797 |
+
Step 7580/11700 | Loss: 2.4506 | LR: 1.55e-04 | 7823.4ms/step | 2.6 samples/s
|
| 798 |
+
Step 7590/11700 | Loss: 2.4324 | LR: 1.55e-04 | 7809.9ms/step | 2.6 samples/s
|
| 799 |
+
Step 7600/11700 | Loss: 2.4882 | LR: 1.54e-04 | 7840.8ms/step | 2.6 samples/s
|
| 800 |
+
Step 7610/11700 | Loss: 2.5838 | LR: 1.53e-04 | 7845.5ms/step | 2.5 samples/s
|
| 801 |
+
Step 7620/11700 | Loss: 2.5510 | LR: 1.53e-04 | 7847.4ms/step | 2.5 samples/s
|
| 802 |
+
Step 7630/11700 | Loss: 2.4245 | LR: 1.52e-04 | 7842.1ms/step | 2.6 samples/s
|
| 803 |
+
Step 7640/11700 | Loss: 2.4589 | LR: 1.51e-04 | 7836.6ms/step | 2.6 samples/s
|
| 804 |
+
Step 7650/11700 | Loss: 2.4951 | LR: 1.51e-04 | 7801.7ms/step | 2.6 samples/s
|
| 805 |
+
Step 7660/11700 | Loss: 2.4432 | LR: 1.50e-04 | 7822.1ms/step | 2.6 samples/s
|
| 806 |
+
Step 7670/11700 | Loss: 2.3815 | LR: 1.49e-04 | 7842.9ms/step | 2.6 samples/s
|
| 807 |
+
Step 7680/11700 | Loss: 2.4447 | LR: 1.49e-04 | 7848.1ms/step | 2.5 samples/s
|
| 808 |
+
Step 7690/11700 | Loss: 2.5231 | LR: 1.48e-04 | 7849.1ms/step | 2.5 samples/s
|
| 809 |
+
Step 7700/11700 | Loss: 2.5877 | LR: 1.47e-04 | 7849.0ms/step | 2.5 samples/s
|
| 810 |
+
Step 7710/11700 | Loss: 2.3894 | LR: 1.47e-04 | 7820.8ms/step | 2.6 samples/s
|
| 811 |
+
Step 7720/11700 | Loss: 2.4913 | LR: 1.46e-04 | 7798.0ms/step | 2.6 samples/s
|
| 812 |
+
Step 7730/11700 | Loss: 2.3849 | LR: 1.45e-04 | 7822.2ms/step | 2.6 samples/s
|
| 813 |
+
Step 7740/11700 | Loss: 2.5442 | LR: 1.45e-04 | 7841.8ms/step | 2.6 samples/s
|
| 814 |
+
Step 7750/11700 | Loss: 2.4609 | LR: 1.44e-04 | 7847.4ms/step | 2.5 samples/s
|
| 815 |
+
Validation at step 7750: Loss = 2.4755, Perplexity = 11.89
|
| 816 |
+
Step 7760/11700 | Loss: 2.5763 | LR: 1.43e-04 | 19523.8ms/step | 1.0 samples/s
|
| 817 |
+
Step 7770/11700 | Loss: 2.4406 | LR: 1.43e-04 | 7800.5ms/step | 2.6 samples/s
|
| 818 |
+
Step 7780/11700 | Loss: 2.5355 | LR: 1.42e-04 | 7806.2ms/step | 2.6 samples/s
|
| 819 |
+
Step 7790/11700 | Loss: 2.4365 | LR: 1.41e-04 | 7836.1ms/step | 2.6 samples/s
|
| 820 |
+
Step 7800/11700 | Loss: 2.5216 | LR: 1.41e-04 | 7839.5ms/step | 2.6 samples/s
|
| 821 |
+
Step 7810/11700 | Loss: 2.3641 | LR: 1.40e-04 | 7836.4ms/step | 2.6 samples/s
|
| 822 |
+
Step 7820/11700 | Loss: 2.4079 | LR: 1.40e-04 | 7840.9ms/step | 2.6 samples/s
|
| 823 |
+
Step 7830/11700 | Loss: 2.4113 | LR: 1.39e-04 | 7802.0ms/step | 2.6 samples/s
|
| 824 |
+
Step 7840/11700 | Loss: 2.4726 | LR: 1.38e-04 | 7797.8ms/step | 2.6 samples/s
|
| 825 |
+
Step 7850/11700 | Loss: 2.4930 | LR: 1.38e-04 | 7823.5ms/step | 2.6 samples/s
|
| 826 |
+
Step 7860/11700 | Loss: 2.4896 | LR: 1.37e-04 | 7838.7ms/step | 2.6 samples/s
|
| 827 |
+
Step 7870/11700 | Loss: 2.5296 | LR: 1.36e-04 | 7839.1ms/step | 2.6 samples/s
|
| 828 |
+
Step 7880/11700 | Loss: 2.4859 | LR: 1.36e-04 | 7841.4ms/step | 2.6 samples/s
|
| 829 |
+
Step 7890/11700 | Loss: 2.4428 | LR: 1.35e-04 | 7843.0ms/step | 2.6 samples/s
|
| 830 |
+
Step 7900/11700 | Loss: 2.5441 | LR: 1.34e-04 | 7816.2ms/step | 2.6 samples/s
|
| 831 |
+
Step 7910/11700 | Loss: 2.3239 | LR: 1.34e-04 | 7823.0ms/step | 2.6 samples/s
|
| 832 |
+
Step 7920/11700 | Loss: 2.4270 | LR: 1.33e-04 | 7837.9ms/step | 2.6 samples/s
|
| 833 |
+
Step 7930/11700 | Loss: 2.4368 | LR: 1.33e-04 | 7840.7ms/step | 2.6 samples/s
|
| 834 |
+
Step 7940/11700 | Loss: 2.4876 | LR: 1.32e-04 | 7824.9ms/step | 2.6 samples/s
|
| 835 |
+
Step 7950/11700 | Loss: 2.5097 | LR: 1.31e-04 | 7837.0ms/step | 2.6 samples/s
|
| 836 |
+
Step 7960/11700 | Loss: 2.4358 | LR: 1.31e-04 | 7823.6ms/step | 2.6 samples/s
|
| 837 |
+
Step 7970/11700 | Loss: 2.3846 | LR: 1.30e-04 | 7811.3ms/step | 2.6 samples/s
|
| 838 |
+
Step 7980/11700 | Loss: 2.5386 | LR: 1.29e-04 | 7838.9ms/step | 2.6 samples/s
|
| 839 |
+
Step 7990/11700 | Loss: 2.5148 | LR: 1.29e-04 | 7847.3ms/step | 2.5 samples/s
|
| 840 |
+
Step 8000/11700 | Loss: 2.3986 | LR: 1.28e-04 | 7846.4ms/step | 2.5 samples/s
|
| 841 |
+
Validation at step 8000: Loss = 2.4708, Perplexity = 11.83
|
| 842 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-8000
|
| 843 |
+
Step 8010/11700 | Loss: 2.4861 | LR: 1.27e-04 | 19781.3ms/step | 1.0 samples/s
|
| 844 |
+
Step 8020/11700 | Loss: 2.3606 | LR: 1.27e-04 | 7836.4ms/step | 2.6 samples/s
|
| 845 |
+
Step 8030/11700 | Loss: 2.4509 | LR: 1.26e-04 | 7840.8ms/step | 2.6 samples/s
|
| 846 |
+
Step 8040/11700 | Loss: 2.4581 | LR: 1.26e-04 | 7838.8ms/step | 2.6 samples/s
|
| 847 |
+
Step 8050/11700 | Loss: 2.4202 | LR: 1.25e-04 | 7838.6ms/step | 2.6 samples/s
|
| 848 |
+
Step 8060/11700 | Loss: 2.4478 | LR: 1.24e-04 | 7838.5ms/step | 2.6 samples/s
|
| 849 |
+
Step 8070/11700 | Loss: 2.5031 | LR: 1.24e-04 | 7812.9ms/step | 2.6 samples/s
|
| 850 |
+
Step 8080/11700 | Loss: 2.5007 | LR: 1.23e-04 | 7826.1ms/step | 2.6 samples/s
|
| 851 |
+
Step 8090/11700 | Loss: 2.4323 | LR: 1.23e-04 | 7851.9ms/step | 2.5 samples/s
|
| 852 |
+
Step 8100/11700 | Loss: 2.4683 | LR: 1.22e-04 | 7853.8ms/step | 2.5 samples/s
|
| 853 |
+
Step 8110/11700 | Loss: 2.4158 | LR: 1.21e-04 | 7836.3ms/step | 2.6 samples/s
|
| 854 |
+
Step 8120/11700 | Loss: 2.3962 | LR: 1.21e-04 | 7835.9ms/step | 2.6 samples/s
|
| 855 |
+
Step 8130/11700 | Loss: 2.3748 | LR: 1.20e-04 | 7809.5ms/step | 2.6 samples/s
|
| 856 |
+
Step 8140/11700 | Loss: 2.4292 | LR: 1.19e-04 | 7816.4ms/step | 2.6 samples/s
|
| 857 |
+
Step 8150/11700 | Loss: 2.3144 | LR: 1.19e-04 | 7830.2ms/step | 2.6 samples/s
|
| 858 |
+
Step 8160/11700 | Loss: 2.5424 | LR: 1.18e-04 | 7836.9ms/step | 2.6 samples/s
|
| 859 |
+
Step 8170/11700 | Loss: 2.4382 | LR: 1.18e-04 | 7837.4ms/step | 2.6 samples/s
|
| 860 |
+
Step 8180/11700 | Loss: 2.4804 | LR: 1.17e-04 | 7840.1ms/step | 2.6 samples/s
|
| 861 |
+
Step 8190/11700 | Loss: 2.4022 | LR: 1.16e-04 | 7803.9ms/step | 2.6 samples/s
|
| 862 |
+
Step 8200/11700 | Loss: 2.4545 | LR: 1.16e-04 | 7809.3ms/step | 2.6 samples/s
|
| 863 |
+
Step 8210/11700 | Loss: 2.4871 | LR: 1.15e-04 | 7827.0ms/step | 2.6 samples/s
|
| 864 |
+
Step 8220/11700 | Loss: 2.4882 | LR: 1.15e-04 | 7832.7ms/step | 2.6 samples/s
|
| 865 |
+
Step 8230/11700 | Loss: 2.4703 | LR: 1.14e-04 | 7835.8ms/step | 2.6 samples/s
|
| 866 |
+
Step 8240/11700 | Loss: 2.4400 | LR: 1.13e-04 | 7838.6ms/step | 2.6 samples/s
|
| 867 |
+
Step 8250/11700 | Loss: 2.4703 | LR: 1.13e-04 | 7816.9ms/step | 2.6 samples/s
|
| 868 |
+
Validation at step 8250: Loss = 2.4666, Perplexity = 11.78
|
| 869 |
+
Step 8260/11700 | Loss: 2.4082 | LR: 1.12e-04 | 19447.1ms/step | 1.0 samples/s
|
| 870 |
+
Step 8270/11700 | Loss: 2.4175 | LR: 1.12e-04 | 7834.0ms/step | 2.6 samples/s
|
| 871 |
+
Step 8280/11700 | Loss: 2.5655 | LR: 1.11e-04 | 7830.9ms/step | 2.6 samples/s
|
| 872 |
+
Step 8290/11700 | Loss: 2.4276 | LR: 1.10e-04 | 7829.4ms/step | 2.6 samples/s
|
| 873 |
+
Step 8300/11700 | Loss: 2.4794 | LR: 1.10e-04 | 7799.0ms/step | 2.6 samples/s
|
| 874 |
+
Step 8310/11700 | Loss: 2.4122 | LR: 1.09e-04 | 7820.3ms/step | 2.6 samples/s
|
| 875 |
+
Step 8320/11700 | Loss: 2.4975 | LR: 1.09e-04 | 7836.8ms/step | 2.6 samples/s
|
| 876 |
+
Step 8330/11700 | Loss: 2.5047 | LR: 1.08e-04 | 7836.2ms/step | 2.6 samples/s
|
| 877 |
+
Step 8340/11700 | Loss: 2.4698 | LR: 1.07e-04 | 7830.4ms/step | 2.6 samples/s
|
| 878 |
+
Step 8350/11700 | Loss: 2.4818 | LR: 1.07e-04 | 7831.5ms/step | 2.6 samples/s
|
| 879 |
+
Step 8360/11700 | Loss: 2.4248 | LR: 1.06e-04 | 7800.0ms/step | 2.6 samples/s
|
| 880 |
+
Step 8370/11700 | Loss: 2.4140 | LR: 1.06e-04 | 7809.9ms/step | 2.6 samples/s
|
| 881 |
+
Step 8380/11700 | Loss: 2.4472 | LR: 1.05e-04 | 7823.3ms/step | 2.6 samples/s
|
| 882 |
+
Step 8390/11700 | Loss: 2.4532 | LR: 1.05e-04 | 7826.0ms/step | 2.6 samples/s
|
| 883 |
+
Step 8400/11700 | Loss: 2.5144 | LR: 1.04e-04 | 7824.4ms/step | 2.6 samples/s
|
| 884 |
+
Step 8410/11700 | Loss: 2.6364 | LR: 1.03e-04 | 7820.5ms/step | 2.6 samples/s
|
| 885 |
+
Step 8420/11700 | Loss: 2.3937 | LR: 1.03e-04 | 7806.8ms/step | 2.6 samples/s
|
| 886 |
+
Step 8430/11700 | Loss: 2.5142 | LR: 1.02e-04 | 7813.2ms/step | 2.6 samples/s
|
| 887 |
+
Step 8440/11700 | Loss: 2.5117 | LR: 1.02e-04 | 7827.9ms/step | 2.6 samples/s
|
| 888 |
+
Step 8450/11700 | Loss: 2.4557 | LR: 1.01e-04 | 7833.2ms/step | 2.6 samples/s
|
| 889 |
+
Step 8460/11700 | Loss: 2.4446 | LR: 1.00e-04 | 7834.4ms/step | 2.6 samples/s
|
| 890 |
+
Step 8470/11700 | Loss: 2.4416 | LR: 9.99e-05 | 7830.1ms/step | 2.6 samples/s
|
| 891 |
+
Step 8480/11700 | Loss: 2.4932 | LR: 9.93e-05 | 7819.8ms/step | 2.6 samples/s
|
| 892 |
+
Step 8490/11700 | Loss: 2.4453 | LR: 9.87e-05 | 7807.3ms/step | 2.6 samples/s
|
| 893 |
+
Step 8500/11700 | Loss: 2.3093 | LR: 9.82e-05 | 7828.3ms/step | 2.6 samples/s
|
| 894 |
+
Validation at step 8500: Loss = 2.4629, Perplexity = 11.74
|
| 895 |
+
Step 8510/11700 | Loss: 2.4116 | LR: 9.76e-05 | 19460.0ms/step | 1.0 samples/s
|
| 896 |
+
Step 8520/11700 | Loss: 2.4585 | LR: 9.70e-05 | 7827.4ms/step | 2.6 samples/s
|
| 897 |
+
Step 8530/11700 | Loss: 2.4830 | LR: 9.65e-05 | 7792.6ms/step | 2.6 samples/s
|
| 898 |
+
Step 8540/11700 | Loss: 2.4533 | LR: 9.59e-05 | 7822.9ms/step | 2.6 samples/s
|
| 899 |
+
Step 8550/11700 | Loss: 2.3446 | LR: 9.53e-05 | 7850.0ms/step | 2.5 samples/s
|
| 900 |
+
Step 8560/11700 | Loss: 2.4550 | LR: 9.48e-05 | 7836.0ms/step | 2.6 samples/s
|
| 901 |
+
Step 8570/11700 | Loss: 2.4470 | LR: 9.42e-05 | 7836.3ms/step | 2.6 samples/s
|
| 902 |
+
Step 8580/11700 | Loss: 2.4328 | LR: 9.36e-05 | 7839.3ms/step | 2.6 samples/s
|
| 903 |
+
Step 8590/11700 | Loss: 2.4029 | LR: 9.31e-05 | 7814.1ms/step | 2.6 samples/s
|
| 904 |
+
Step 8600/11700 | Loss: 2.4745 | LR: 9.25e-05 | 7818.3ms/step | 2.6 samples/s
|
| 905 |
+
Step 8610/11700 | Loss: 2.4489 | LR: 9.20e-05 | 7849.9ms/step | 2.5 samples/s
|
| 906 |
+
Step 8620/11700 | Loss: 2.5033 | LR: 9.14e-05 | 7853.1ms/step | 2.5 samples/s
|
| 907 |
+
Step 8630/11700 | Loss: 2.4756 | LR: 9.09e-05 | 7848.5ms/step | 2.5 samples/s
|
| 908 |
+
Step 8640/11700 | Loss: 2.4132 | LR: 9.03e-05 | 7851.5ms/step | 2.5 samples/s
|
| 909 |
+
Step 8650/11700 | Loss: 2.3711 | LR: 8.98e-05 | 7831.9ms/step | 2.6 samples/s
|
| 910 |
+
Step 8660/11700 | Loss: 2.4346 | LR: 8.92e-05 | 7826.4ms/step | 2.6 samples/s
|
| 911 |
+
Step 8670/11700 | Loss: 2.5258 | LR: 8.87e-05 | 7844.9ms/step | 2.5 samples/s
|
| 912 |
+
Step 8680/11700 | Loss: 2.4037 | LR: 8.81e-05 | 7841.5ms/step | 2.6 samples/s
|
| 913 |
+
Step 8690/11700 | Loss: 2.5263 | LR: 8.76e-05 | 7842.6ms/step | 2.6 samples/s
|
| 914 |
+
Step 8700/11700 | Loss: 2.4520 | LR: 8.70e-05 | 7841.8ms/step | 2.6 samples/s
|
| 915 |
+
Step 8710/11700 | Loss: 2.3174 | LR: 8.65e-05 | 7821.9ms/step | 2.6 samples/s
|
| 916 |
+
Step 8720/11700 | Loss: 2.3938 | LR: 8.59e-05 | 7814.0ms/step | 2.6 samples/s
|
| 917 |
+
Step 8730/11700 | Loss: 2.4405 | LR: 8.54e-05 | 7835.5ms/step | 2.6 samples/s
|
| 918 |
+
Step 8740/11700 | Loss: 2.4696 | LR: 8.49e-05 | 7839.7ms/step | 2.6 samples/s
|
| 919 |
+
Step 8750/11700 | Loss: 2.4759 | LR: 8.43e-05 | 7842.3ms/step | 2.6 samples/s
|
| 920 |
+
Validation at step 8750: Loss = 2.4600, Perplexity = 11.70
|
| 921 |
+
Step 8760/11700 | Loss: 2.3842 | LR: 8.38e-05 | 19494.0ms/step | 1.0 samples/s
|
| 922 |
+
Step 8770/11700 | Loss: 2.4418 | LR: 8.33e-05 | 7824.6ms/step | 2.6 samples/s
|
| 923 |
+
Step 8780/11700 | Loss: 2.4403 | LR: 8.27e-05 | 7833.5ms/step | 2.6 samples/s
|
| 924 |
+
Step 8790/11700 | Loss: 2.4803 | LR: 8.22e-05 | 7835.2ms/step | 2.6 samples/s
|
| 925 |
+
Step 8800/11700 | Loss: 2.5085 | LR: 8.17e-05 | 7822.0ms/step | 2.6 samples/s
|
| 926 |
+
Step 8810/11700 | Loss: 2.3625 | LR: 8.11e-05 | 7844.5ms/step | 2.5 samples/s
|
| 927 |
+
Step 8820/11700 | Loss: 2.3582 | LR: 8.06e-05 | 7823.1ms/step | 2.6 samples/s
|
| 928 |
+
Step 8830/11700 | Loss: 2.4177 | LR: 8.01e-05 | 7823.8ms/step | 2.6 samples/s
|
| 929 |
+
Step 8840/11700 | Loss: 2.4982 | LR: 7.95e-05 | 7839.1ms/step | 2.6 samples/s
|
| 930 |
+
Step 8850/11700 | Loss: 2.4652 | LR: 7.90e-05 | 7845.4ms/step | 2.5 samples/s
|
| 931 |
+
Step 8860/11700 | Loss: 2.3449 | LR: 7.85e-05 | 7843.1ms/step | 2.6 samples/s
|
| 932 |
+
Step 8870/11700 | Loss: 2.4001 | LR: 7.80e-05 | 7841.8ms/step | 2.6 samples/s
|
| 933 |
+
Step 8880/11700 | Loss: 2.4856 | LR: 7.75e-05 | 7836.2ms/step | 2.6 samples/s
|
| 934 |
+
Step 8890/11700 | Loss: 2.4474 | LR: 7.69e-05 | 7818.8ms/step | 2.6 samples/s
|
| 935 |
+
Step 8900/11700 | Loss: 2.4257 | LR: 7.64e-05 | 7839.8ms/step | 2.6 samples/s
|
| 936 |
+
Step 8910/11700 | Loss: 2.5230 | LR: 7.59e-05 | 7839.1ms/step | 2.6 samples/s
|
| 937 |
+
Step 8920/11700 | Loss: 2.5016 | LR: 7.54e-05 | 7838.7ms/step | 2.6 samples/s
|
| 938 |
+
Step 8930/11700 | Loss: 2.3857 | LR: 7.49e-05 | 7837.3ms/step | 2.6 samples/s
|
| 939 |
+
Step 8940/11700 | Loss: 2.4854 | LR: 7.44e-05 | 7832.9ms/step | 2.6 samples/s
|
| 940 |
+
Step 8950/11700 | Loss: 2.4159 | LR: 7.39e-05 | 7824.4ms/step | 2.6 samples/s
|
| 941 |
+
Step 8960/11700 | Loss: 2.4243 | LR: 7.34e-05 | 7840.8ms/step | 2.6 samples/s
|
| 942 |
+
Step 8970/11700 | Loss: 2.4332 | LR: 7.28e-05 | 7843.6ms/step | 2.5 samples/s
|
| 943 |
+
Step 8980/11700 | Loss: 2.4225 | LR: 7.23e-05 | 7844.5ms/step | 2.5 samples/s
|
| 944 |
+
Step 8990/11700 | Loss: 2.4607 | LR: 7.18e-05 | 7840.6ms/step | 2.6 samples/s
|
| 945 |
+
Step 9000/11700 | Loss: 2.4372 | LR: 7.13e-05 | 7831.1ms/step | 2.6 samples/s
|
| 946 |
+
Validation at step 9000: Loss = 2.4578, Perplexity = 11.68
|
| 947 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-9000
|
| 948 |
+
Step 9010/11700 | Loss: 2.5252 | LR: 7.08e-05 | 19756.1ms/step | 1.0 samples/s
|
| 949 |
+
Step 9020/11700 | Loss: 2.4386 | LR: 7.03e-05 | 7840.1ms/step | 2.6 samples/s
|
| 950 |
+
Step 9030/11700 | Loss: 2.4032 | LR: 6.98e-05 | 7839.9ms/step | 2.6 samples/s
|
| 951 |
+
Step 9040/11700 | Loss: 2.3762 | LR: 6.93e-05 | 7830.1ms/step | 2.6 samples/s
|
| 952 |
+
Step 9050/11700 | Loss: 2.4711 | LR: 6.88e-05 | 7816.1ms/step | 2.6 samples/s
|
| 953 |
+
Step 9060/11700 | Loss: 2.4854 | LR: 6.83e-05 | 7808.8ms/step | 2.6 samples/s
|
| 954 |
+
Step 9070/11700 | Loss: 2.4995 | LR: 6.79e-05 | 7819.8ms/step | 2.6 samples/s
|
| 955 |
+
Step 9080/11700 | Loss: 2.3963 | LR: 6.74e-05 | 7837.3ms/step | 2.6 samples/s
|
| 956 |
+
Step 9090/11700 | Loss: 2.3819 | LR: 6.69e-05 | 7813.3ms/step | 2.6 samples/s
|
| 957 |
+
Step 9100/11700 | Loss: 2.4695 | LR: 6.64e-05 | 7829.1ms/step | 2.6 samples/s
|
| 958 |
+
Step 9110/11700 | Loss: 2.4687 | LR: 6.59e-05 | 7804.9ms/step | 2.6 samples/s
|
| 959 |
+
Step 9120/11700 | Loss: 2.4992 | LR: 6.54e-05 | 7796.5ms/step | 2.6 samples/s
|
| 960 |
+
Step 9130/11700 | Loss: 2.3654 | LR: 6.49e-05 | 7824.6ms/step | 2.6 samples/s
|
| 961 |
+
Step 9140/11700 | Loss: 2.3425 | LR: 6.45e-05 | 7824.9ms/step | 2.6 samples/s
|
| 962 |
+
Step 9150/11700 | Loss: 2.4551 | LR: 6.40e-05 | 7826.9ms/step | 2.6 samples/s
|
| 963 |
+
Step 9160/11700 | Loss: 2.5439 | LR: 6.35e-05 | 7828.3ms/step | 2.6 samples/s
|
| 964 |
+
Step 9170/11700 | Loss: 2.4943 | LR: 6.30e-05 | 7814.7ms/step | 2.6 samples/s
|
| 965 |
+
Step 9180/11700 | Loss: 2.4300 | LR: 6.25e-05 | 7801.3ms/step | 2.6 samples/s
|
| 966 |
+
Step 9190/11700 | Loss: 2.4526 | LR: 6.21e-05 | 7820.7ms/step | 2.6 samples/s
|
| 967 |
+
Step 9200/11700 | Loss: 2.3352 | LR: 6.16e-05 | 7829.4ms/step | 2.6 samples/s
|
| 968 |
+
Step 9210/11700 | Loss: 2.4410 | LR: 6.11e-05 | 7833.1ms/step | 2.6 samples/s
|
| 969 |
+
Step 9220/11700 | Loss: 2.5236 | LR: 6.07e-05 | 7829.3ms/step | 2.6 samples/s
|
| 970 |
+
Step 9230/11700 | Loss: 2.4305 | LR: 6.02e-05 | 7817.5ms/step | 2.6 samples/s
|
| 971 |
+
Step 9240/11700 | Loss: 2.4172 | LR: 5.97e-05 | 7804.1ms/step | 2.6 samples/s
|
| 972 |
+
Step 9250/11700 | Loss: 2.4483 | LR: 5.93e-05 | 7811.2ms/step | 2.6 samples/s
|
| 973 |
+
Validation at step 9250: Loss = 2.4562, Perplexity = 11.66
|
| 974 |
+
Step 9260/11700 | Loss: 2.3719 | LR: 5.88e-05 | 19454.7ms/step | 1.0 samples/s
|
| 975 |
+
Step 9270/11700 | Loss: 2.4650 | LR: 5.83e-05 | 7817.8ms/step | 2.6 samples/s
|
| 976 |
+
Step 9280/11700 | Loss: 2.3196 | LR: 5.79e-05 | 7806.8ms/step | 2.6 samples/s
|
| 977 |
+
Step 9290/11700 | Loss: 2.4079 | LR: 5.74e-05 | 7803.8ms/step | 2.6 samples/s
|
| 978 |
+
Step 9300/11700 | Loss: 2.4630 | LR: 5.70e-05 | 7820.1ms/step | 2.6 samples/s
|
| 979 |
+
Step 9310/11700 | Loss: 2.3573 | LR: 5.65e-05 | 7822.7ms/step | 2.6 samples/s
|
| 980 |
+
Step 9320/11700 | Loss: 2.3999 | LR: 5.61e-05 | 7831.6ms/step | 2.6 samples/s
|
| 981 |
+
Step 9330/11700 | Loss: 2.3964 | LR: 5.56e-05 | 7826.3ms/step | 2.6 samples/s
|
| 982 |
+
Step 9340/11700 | Loss: 2.5077 | LR: 5.52e-05 | 7813.2ms/step | 2.6 samples/s
|
| 983 |
+
Step 9350/11700 | Loss: 2.3960 | LR: 5.47e-05 | 7786.7ms/step | 2.6 samples/s
|
| 984 |
+
Step 9360/11700 | Loss: 2.4275 | LR: 5.43e-05 | 7813.5ms/step | 2.6 samples/s
|
| 985 |
+
Step 9370/11700 | Loss: 2.4449 | LR: 5.38e-05 | 7830.9ms/step | 2.6 samples/s
|
| 986 |
+
Step 9380/11700 | Loss: 2.5467 | LR: 5.34e-05 | 7829.0ms/step | 2.6 samples/s
|
| 987 |
+
Step 9390/11700 | Loss: 2.4543 | LR: 5.29e-05 | 7823.4ms/step | 2.6 samples/s
|
| 988 |
+
Step 9400/11700 | Loss: 2.4320 | LR: 5.25e-05 | 7832.8ms/step | 2.6 samples/s
|
| 989 |
+
Step 9410/11700 | Loss: 2.4471 | LR: 5.20e-05 | 7809.4ms/step | 2.6 samples/s
|
| 990 |
+
Step 9420/11700 | Loss: 2.5099 | LR: 5.16e-05 | 7786.3ms/step | 2.6 samples/s
|
| 991 |
+
Step 9430/11700 | Loss: 2.3209 | LR: 5.12e-05 | 7802.1ms/step | 2.6 samples/s
|
| 992 |
+
Step 9440/11700 | Loss: 2.5371 | LR: 5.07e-05 | 7826.2ms/step | 2.6 samples/s
|
| 993 |
+
Step 9450/11700 | Loss: 2.4793 | LR: 5.03e-05 | 7827.8ms/step | 2.6 samples/s
|
| 994 |
+
Step 9460/11700 | Loss: 2.4794 | LR: 4.99e-05 | 7821.2ms/step | 2.6 samples/s
|
| 995 |
+
Step 9470/11700 | Loss: 2.4543 | LR: 4.94e-05 | 7824.1ms/step | 2.6 samples/s
|
| 996 |
+
Step 9480/11700 | Loss: 2.4128 | LR: 4.90e-05 | 7801.3ms/step | 2.6 samples/s
|
| 997 |
+
Step 9490/11700 | Loss: 2.5642 | LR: 4.86e-05 | 7780.8ms/step | 2.6 samples/s
|
| 998 |
+
Step 9500/11700 | Loss: 2.3701 | LR: 4.82e-05 | 7802.5ms/step | 2.6 samples/s
|
| 999 |
+
Validation at step 9500: Loss = 2.4549, Perplexity = 11.65
|
| 1000 |
+
Step 9510/11700 | Loss: 2.4572 | LR: 4.77e-05 | 19479.3ms/step | 1.0 samples/s
|
| 1001 |
+
Step 9520/11700 | Loss: 2.4111 | LR: 4.73e-05 | 7830.7ms/step | 2.6 samples/s
|
| 1002 |
+
Step 9530/11700 | Loss: 2.4135 | LR: 4.69e-05 | 7827.3ms/step | 2.6 samples/s
|
| 1003 |
+
Step 9540/11700 | Loss: 2.3756 | LR: 4.65e-05 | 7804.0ms/step | 2.6 samples/s
|
| 1004 |
+
Step 9550/11700 | Loss: 2.5816 | LR: 4.61e-05 | 7786.5ms/step | 2.6 samples/s
|
| 1005 |
+
Step 9560/11700 | Loss: 2.3608 | LR: 4.57e-05 | 7816.6ms/step | 2.6 samples/s
|
| 1006 |
+
Step 9570/11700 | Loss: 2.3989 | LR: 4.52e-05 | 7824.6ms/step | 2.6 samples/s
|
| 1007 |
+
Step 9580/11700 | Loss: 2.5185 | LR: 4.48e-05 | 7827.2ms/step | 2.6 samples/s
|
| 1008 |
+
Step 9590/11700 | Loss: 2.4499 | LR: 4.44e-05 | 7840.9ms/step | 2.6 samples/s
|
| 1009 |
+
Step 9600/11700 | Loss: 2.4595 | LR: 4.40e-05 | 7815.7ms/step | 2.6 samples/s
|
| 1010 |
+
Step 9610/11700 | Loss: 2.4592 | LR: 4.36e-05 | 7799.0ms/step | 2.6 samples/s
|
| 1011 |
+
Step 9620/11700 | Loss: 2.4717 | LR: 4.32e-05 | 7785.9ms/step | 2.6 samples/s
|
| 1012 |
+
Step 9630/11700 | Loss: 2.4191 | LR: 4.28e-05 | 7817.8ms/step | 2.6 samples/s
|
| 1013 |
+
Step 9640/11700 | Loss: 2.4690 | LR: 4.24e-05 | 7828.6ms/step | 2.6 samples/s
|
| 1014 |
+
Step 9650/11700 | Loss: 2.5474 | LR: 4.20e-05 | 7832.4ms/step | 2.6 samples/s
|
| 1015 |
+
Step 9660/11700 | Loss: 2.5480 | LR: 4.16e-05 | 7827.3ms/step | 2.6 samples/s
|
| 1016 |
+
Step 9670/11700 | Loss: 2.4500 | LR: 4.12e-05 | 7825.9ms/step | 2.6 samples/s
|
| 1017 |
+
Step 9680/11700 | Loss: 2.2714 | LR: 4.08e-05 | 7804.5ms/step | 2.6 samples/s
|
| 1018 |
+
Step 9690/11700 | Loss: 2.5100 | LR: 4.04e-05 | 7795.9ms/step | 2.6 samples/s
|
| 1019 |
+
Step 9700/11700 | Loss: 2.3795 | LR: 4.00e-05 | 7810.2ms/step | 2.6 samples/s
|
| 1020 |
+
Step 9710/11700 | Loss: 2.4311 | LR: 3.97e-05 | 7811.8ms/step | 2.6 samples/s
|
| 1021 |
+
Step 9720/11700 | Loss: 2.5051 | LR: 3.93e-05 | 7818.1ms/step | 2.6 samples/s
|
| 1022 |
+
Step 9730/11700 | Loss: 2.3479 | LR: 3.89e-05 | 7811.2ms/step | 2.6 samples/s
|
| 1023 |
+
Step 9740/11700 | Loss: 2.5195 | LR: 3.85e-05 | 7812.2ms/step | 2.6 samples/s
|
| 1024 |
+
Step 9750/11700 | Loss: 2.4428 | LR: 3.81e-05 | 7797.0ms/step | 2.6 samples/s
|
| 1025 |
+
Validation at step 9750: Loss = 2.4540, Perplexity = 11.64
|
| 1026 |
+
Step 9760/11700 | Loss: 2.4279 | LR: 3.77e-05 | 19405.4ms/step | 1.0 samples/s
|
| 1027 |
+
Step 9770/11700 | Loss: 2.3718 | LR: 3.74e-05 | 7812.0ms/step | 2.6 samples/s
|
| 1028 |
+
Step 9780/11700 | Loss: 2.5666 | LR: 3.70e-05 | 7809.7ms/step | 2.6 samples/s
|
| 1029 |
+
Step 9790/11700 | Loss: 2.4250 | LR: 3.66e-05 | 7810.4ms/step | 2.6 samples/s
|
| 1030 |
+
Step 9800/11700 | Loss: 2.3728 | LR: 3.62e-05 | 7807.4ms/step | 2.6 samples/s
|
| 1031 |
+
Step 9810/11700 | Loss: 2.4898 | LR: 3.59e-05 | 7780.9ms/step | 2.6 samples/s
|
| 1032 |
+
Step 9820/11700 | Loss: 2.3392 | LR: 3.55e-05 | 7803.6ms/step | 2.6 samples/s
|
| 1033 |
+
Step 9830/11700 | Loss: 2.3605 | LR: 3.51e-05 | 7813.8ms/step | 2.6 samples/s
|
| 1034 |
+
Step 9840/11700 | Loss: 2.3936 | LR: 3.48e-05 | 7820.1ms/step | 2.6 samples/s
|
| 1035 |
+
Step 9850/11700 | Loss: 2.4693 | LR: 3.44e-05 | 7817.4ms/step | 2.6 samples/s
|
| 1036 |
+
Step 9860/11700 | Loss: 2.4479 | LR: 3.40e-05 | 7828.5ms/step | 2.6 samples/s
|
| 1037 |
+
Step 9870/11700 | Loss: 2.3956 | LR: 3.37e-05 | 7808.9ms/step | 2.6 samples/s
|
| 1038 |
+
Step 9880/11700 | Loss: 2.4360 | LR: 3.33e-05 | 7799.8ms/step | 2.6 samples/s
|
| 1039 |
+
Step 9890/11700 | Loss: 2.3433 | LR: 3.30e-05 | 7812.5ms/step | 2.6 samples/s
|
| 1040 |
+
Step 9900/11700 | Loss: 2.4769 | LR: 3.26e-05 | 7809.2ms/step | 2.6 samples/s
|
| 1041 |
+
Step 9910/11700 | Loss: 2.4580 | LR: 3.22e-05 | 7804.5ms/step | 2.6 samples/s
|
| 1042 |
+
Step 9920/11700 | Loss: 2.4231 | LR: 3.19e-05 | 7804.8ms/step | 2.6 samples/s
|
| 1043 |
+
Step 9930/11700 | Loss: 2.5058 | LR: 3.15e-05 | 7804.8ms/step | 2.6 samples/s
|
| 1044 |
+
Step 9940/11700 | Loss: 2.3176 | LR: 3.12e-05 | 7792.5ms/step | 2.6 samples/s
|
| 1045 |
+
Step 9950/11700 | Loss: 2.4275 | LR: 3.09e-05 | 7793.8ms/step | 2.6 samples/s
|
| 1046 |
+
Step 9960/11700 | Loss: 2.3738 | LR: 3.05e-05 | 7813.8ms/step | 2.6 samples/s
|
| 1047 |
+
Step 9970/11700 | Loss: 2.4504 | LR: 3.02e-05 | 7815.7ms/step | 2.6 samples/s
|
| 1048 |
+
Step 9980/11700 | Loss: 2.3802 | LR: 2.98e-05 | 7815.7ms/step | 2.6 samples/s
|
| 1049 |
+
Step 9990/11700 | Loss: 2.3340 | LR: 2.95e-05 | 7800.6ms/step | 2.6 samples/s
|
| 1050 |
+
Step 10000/11700 | Loss: 2.4497 | LR: 2.92e-05 | 7803.8ms/step | 2.6 samples/s
|
| 1051 |
+
Validation at step 10000: Loss = 2.4535, Perplexity = 11.63
|
| 1052 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-10000
|
| 1053 |
+
Step 10010/11700 | Loss: 2.5456 | LR: 2.88e-05 | 19682.6ms/step | 1.0 samples/s
|
| 1054 |
+
Step 10020/11700 | Loss: 2.5323 | LR: 2.85e-05 | 7814.9ms/step | 2.6 samples/s
|
| 1055 |
+
Step 10030/11700 | Loss: 2.4306 | LR: 2.82e-05 | 7815.8ms/step | 2.6 samples/s
|
| 1056 |
+
Step 10040/11700 | Loss: 2.4823 | LR: 2.78e-05 | 7818.1ms/step | 2.6 samples/s
|
| 1057 |
+
Step 10050/11700 | Loss: 2.5193 | LR: 2.75e-05 | 7814.2ms/step | 2.6 samples/s
|
| 1058 |
+
Step 10060/11700 | Loss: 2.3934 | LR: 2.72e-05 | 7793.2ms/step | 2.6 samples/s
|
| 1059 |
+
Step 10070/11700 | Loss: 2.5112 | LR: 2.68e-05 | 7811.9ms/step | 2.6 samples/s
|
| 1060 |
+
Step 10080/11700 | Loss: 2.3874 | LR: 2.65e-05 | 7816.0ms/step | 2.6 samples/s
|
| 1061 |
+
Step 10090/11700 | Loss: 2.4573 | LR: 2.62e-05 | 7812.8ms/step | 2.6 samples/s
|
| 1062 |
+
Step 10100/11700 | Loss: 2.4197 | LR: 2.59e-05 | 7817.8ms/step | 2.6 samples/s
|
| 1063 |
+
Step 10110/11700 | Loss: 2.4912 | LR: 2.56e-05 | 7817.3ms/step | 2.6 samples/s
|
| 1064 |
+
Step 10120/11700 | Loss: 2.3584 | LR: 2.52e-05 | 7802.4ms/step | 2.6 samples/s
|
| 1065 |
+
Step 10130/11700 | Loss: 2.5266 | LR: 2.49e-05 | 7791.2ms/step | 2.6 samples/s
|
| 1066 |
+
Step 10140/11700 | Loss: 2.5596 | LR: 2.46e-05 | 7806.7ms/step | 2.6 samples/s
|
| 1067 |
+
Step 10150/11700 | Loss: 2.4701 | LR: 2.43e-05 | 7824.1ms/step | 2.6 samples/s
|
| 1068 |
+
Step 10160/11700 | Loss: 2.4547 | LR: 2.40e-05 | 7825.3ms/step | 2.6 samples/s
|
| 1069 |
+
Step 10170/11700 | Loss: 2.4399 | LR: 2.37e-05 | 7824.2ms/step | 2.6 samples/s
|
| 1070 |
+
Step 10180/11700 | Loss: 2.4609 | LR: 2.34e-05 | 7826.2ms/step | 2.6 samples/s
|
| 1071 |
+
Step 10190/11700 | Loss: 2.3682 | LR: 2.31e-05 | 7800.8ms/step | 2.6 samples/s
|
| 1072 |
+
Step 10200/11700 | Loss: 2.3947 | LR: 2.28e-05 | 7784.5ms/step | 2.6 samples/s
|
| 1073 |
+
Step 10210/11700 | Loss: 2.4897 | LR: 2.25e-05 | 7806.1ms/step | 2.6 samples/s
|
| 1074 |
+
Step 10220/11700 | Loss: 2.4468 | LR: 2.22e-05 | 7831.9ms/step | 2.6 samples/s
|
| 1075 |
+
Step 10230/11700 | Loss: 2.5497 | LR: 2.19e-05 | 7831.4ms/step | 2.6 samples/s
|
| 1076 |
+
Step 10240/11700 | Loss: 2.5382 | LR: 2.16e-05 | 7821.0ms/step | 2.6 samples/s
|
| 1077 |
+
Step 10250/11700 | Loss: 2.4871 | LR: 2.13e-05 | 7822.7ms/step | 2.6 samples/s
|
| 1078 |
+
Validation at step 10250: Loss = 2.4531, Perplexity = 11.62
|
| 1079 |
+
Step 10260/11700 | Loss: 2.4944 | LR: 2.10e-05 | 19408.1ms/step | 1.0 samples/s
|
| 1080 |
+
Step 10270/11700 | Loss: 2.3971 | LR: 2.07e-05 | 7834.7ms/step | 2.6 samples/s
|
| 1081 |
+
Step 10280/11700 | Loss: 2.4948 | LR: 2.05e-05 | 7846.4ms/step | 2.5 samples/s
|
| 1082 |
+
Step 10290/11700 | Loss: 2.4799 | LR: 2.02e-05 | 7829.6ms/step | 2.6 samples/s
|
| 1083 |
+
Step 10300/11700 | Loss: 2.3335 | LR: 1.99e-05 | 7798.0ms/step | 2.6 samples/s
|
| 1084 |
+
Step 10310/11700 | Loss: 2.4395 | LR: 1.96e-05 | 7797.0ms/step | 2.6 samples/s
|
| 1085 |
+
Step 10320/11700 | Loss: 2.3861 | LR: 1.93e-05 | 7805.2ms/step | 2.6 samples/s
|
| 1086 |
+
Step 10330/11700 | Loss: 2.3779 | LR: 1.91e-05 | 7809.0ms/step | 2.6 samples/s
|
| 1087 |
+
Step 10340/11700 | Loss: 2.5466 | LR: 1.88e-05 | 7822.0ms/step | 2.6 samples/s
|
| 1088 |
+
Step 10350/11700 | Loss: 2.4763 | LR: 1.85e-05 | 7814.6ms/step | 2.6 samples/s
|
| 1089 |
+
Step 10360/11700 | Loss: 2.4718 | LR: 1.82e-05 | 7816.5ms/step | 2.6 samples/s
|
| 1090 |
+
Step 10370/11700 | Loss: 2.4531 | LR: 1.80e-05 | 7802.3ms/step | 2.6 samples/s
|
| 1091 |
+
Step 10380/11700 | Loss: 2.2895 | LR: 1.77e-05 | 7800.3ms/step | 2.6 samples/s
|
| 1092 |
+
Step 10390/11700 | Loss: 2.5854 | LR: 1.75e-05 | 7816.6ms/step | 2.6 samples/s
|
| 1093 |
+
Step 10400/11700 | Loss: 2.4952 | LR: 1.72e-05 | 7817.7ms/step | 2.6 samples/s
|
| 1094 |
+
Step 10410/11700 | Loss: 2.4445 | LR: 1.69e-05 | 7817.3ms/step | 2.6 samples/s
|
| 1095 |
+
Step 10420/11700 | Loss: 2.4225 | LR: 1.67e-05 | 7814.2ms/step | 2.6 samples/s
|
| 1096 |
+
Step 10430/11700 | Loss: 2.4169 | LR: 1.64e-05 | 7804.5ms/step | 2.6 samples/s
|
| 1097 |
+
Step 10440/11700 | Loss: 2.3892 | LR: 1.62e-05 | 7797.6ms/step | 2.6 samples/s
|
| 1098 |
+
Step 10450/11700 | Loss: 2.4652 | LR: 1.59e-05 | 7810.2ms/step | 2.6 samples/s
|
| 1099 |
+
Step 10460/11700 | Loss: 2.4902 | LR: 1.57e-05 | 7811.6ms/step | 2.6 samples/s
|
| 1100 |
+
Step 10470/11700 | Loss: 2.3788 | LR: 1.54e-05 | 7824.4ms/step | 2.6 samples/s
|
| 1101 |
+
Step 10480/11700 | Loss: 2.4171 | LR: 1.52e-05 | 7820.0ms/step | 2.6 samples/s
|
| 1102 |
+
Step 10490/11700 | Loss: 2.3586 | LR: 1.49e-05 | 7802.7ms/step | 2.6 samples/s
|
| 1103 |
+
Step 10500/11700 | Loss: 2.4476 | LR: 1.47e-05 | 7800.1ms/step | 2.6 samples/s
|
| 1104 |
+
Validation at step 10500: Loss = 2.4530, Perplexity = 11.62
|
| 1105 |
+
Step 10510/11700 | Loss: 2.5713 | LR: 1.44e-05 | 19439.0ms/step | 1.0 samples/s
|
| 1106 |
+
Step 10520/11700 | Loss: 2.4727 | LR: 1.42e-05 | 7810.5ms/step | 2.6 samples/s
|
| 1107 |
+
Step 10530/11700 | Loss: 2.4588 | LR: 1.40e-05 | 7814.1ms/step | 2.6 samples/s
|
| 1108 |
+
Step 10540/11700 | Loss: 2.3084 | LR: 1.37e-05 | 7798.2ms/step | 2.6 samples/s
|
| 1109 |
+
Step 10550/11700 | Loss: 2.4850 | LR: 1.35e-05 | 7797.2ms/step | 2.6 samples/s
|
| 1110 |
+
Step 10560/11700 | Loss: 2.5836 | LR: 1.33e-05 | 7816.0ms/step | 2.6 samples/s
|
| 1111 |
+
Step 10570/11700 | Loss: 2.3408 | LR: 1.30e-05 | 7815.3ms/step | 2.6 samples/s
|
| 1112 |
+
Step 10580/11700 | Loss: 2.5168 | LR: 1.28e-05 | 7812.5ms/step | 2.6 samples/s
|
| 1113 |
+
Step 10590/11700 | Loss: 2.3799 | LR: 1.26e-05 | 7810.5ms/step | 2.6 samples/s
|
| 1114 |
+
Step 10600/11700 | Loss: 2.4678 | LR: 1.23e-05 | 7799.4ms/step | 2.6 samples/s
|
| 1115 |
+
Step 10610/11700 | Loss: 2.4375 | LR: 1.21e-05 | 7801.0ms/step | 2.6 samples/s
|
| 1116 |
+
Step 10620/11700 | Loss: 2.3548 | LR: 1.19e-05 | 7813.3ms/step | 2.6 samples/s
|
| 1117 |
+
Step 10630/11700 | Loss: 2.4255 | LR: 1.17e-05 | 7821.9ms/step | 2.6 samples/s
|
| 1118 |
+
Step 10640/11700 | Loss: 2.4092 | LR: 1.15e-05 | 7842.8ms/step | 2.6 samples/s
|
| 1119 |
+
Step 10650/11700 | Loss: 2.4631 | LR: 1.13e-05 | 7807.2ms/step | 2.6 samples/s
|
| 1120 |
+
Step 10660/11700 | Loss: 2.4145 | LR: 1.10e-05 | 7801.7ms/step | 2.6 samples/s
|
| 1121 |
+
Step 10670/11700 | Loss: 2.4016 | LR: 1.08e-05 | 7810.0ms/step | 2.6 samples/s
|
| 1122 |
+
Step 10680/11700 | Loss: 2.4779 | LR: 1.06e-05 | 7815.9ms/step | 2.6 samples/s
|
| 1123 |
+
Step 10690/11700 | Loss: 2.4662 | LR: 1.04e-05 | 7831.6ms/step | 2.6 samples/s
|
| 1124 |
+
Step 10700/11700 | Loss: 2.4350 | LR: 1.02e-05 | 7806.8ms/step | 2.6 samples/s
|
| 1125 |
+
Step 10710/11700 | Loss: 2.4678 | LR: 1.00e-05 | 7809.8ms/step | 2.6 samples/s
|
| 1126 |
+
Step 10720/11700 | Loss: 2.4028 | LR: 9.82e-06 | 7798.6ms/step | 2.6 samples/s
|
| 1127 |
+
Step 10730/11700 | Loss: 2.5046 | LR: 9.62e-06 | 7796.7ms/step | 2.6 samples/s
|
| 1128 |
+
Step 10740/11700 | Loss: 2.3321 | LR: 9.42e-06 | 7813.9ms/step | 2.6 samples/s
|
| 1129 |
+
Step 10750/11700 | Loss: 2.3613 | LR: 9.23e-06 | 7826.0ms/step | 2.6 samples/s
|
| 1130 |
+
Validation at step 10750: Loss = 2.4529, Perplexity = 11.62
|
| 1131 |
+
Step 10760/11700 | Loss: 2.5294 | LR: 9.04e-06 | 19452.0ms/step | 1.0 samples/s
|
| 1132 |
+
Step 10770/11700 | Loss: 2.5040 | LR: 8.85e-06 | 7793.2ms/step | 2.6 samples/s
|
| 1133 |
+
Step 10780/11700 | Loss: 2.5327 | LR: 8.66e-06 | 7805.7ms/step | 2.6 samples/s
|
| 1134 |
+
Step 10790/11700 | Loss: 2.4431 | LR: 8.47e-06 | 7827.0ms/step | 2.6 samples/s
|
| 1135 |
+
Step 10800/11700 | Loss: 2.5364 | LR: 8.29e-06 | 7828.3ms/step | 2.6 samples/s
|
| 1136 |
+
Step 10810/11700 | Loss: 2.4342 | LR: 8.11e-06 | 7834.6ms/step | 2.6 samples/s
|
| 1137 |
+
Step 10820/11700 | Loss: 2.5222 | LR: 7.93e-06 | 7830.3ms/step | 2.6 samples/s
|
| 1138 |
+
Step 10830/11700 | Loss: 2.4207 | LR: 7.75e-06 | 7808.6ms/step | 2.6 samples/s
|
| 1139 |
+
Step 10840/11700 | Loss: 2.4038 | LR: 7.57e-06 | 7807.6ms/step | 2.6 samples/s
|
| 1140 |
+
Step 10850/11700 | Loss: 2.3546 | LR: 7.40e-06 | 7812.2ms/step | 2.6 samples/s
|
| 1141 |
+
Step 10860/11700 | Loss: 2.3549 | LR: 7.22e-06 | 7815.5ms/step | 2.6 samples/s
|
| 1142 |
+
Step 10870/11700 | Loss: 2.3830 | LR: 7.05e-06 | 7812.9ms/step | 2.6 samples/s
|
| 1143 |
+
Step 10880/11700 | Loss: 2.3626 | LR: 6.89e-06 | 7814.8ms/step | 2.6 samples/s
|
| 1144 |
+
Step 10890/11700 | Loss: 2.3491 | LR: 6.72e-06 | 7800.4ms/step | 2.6 samples/s
|
| 1145 |
+
Step 10900/11700 | Loss: 2.4538 | LR: 6.56e-06 | 7783.8ms/step | 2.6 samples/s
|
| 1146 |
+
Step 10910/11700 | Loss: 2.4947 | LR: 6.39e-06 | 7801.9ms/step | 2.6 samples/s
|
| 1147 |
+
Step 10920/11700 | Loss: 2.4177 | LR: 6.23e-06 | 7814.5ms/step | 2.6 samples/s
|
| 1148 |
+
Step 10930/11700 | Loss: 2.3395 | LR: 6.08e-06 | 7813.6ms/step | 2.6 samples/s
|
| 1149 |
+
Step 10940/11700 | Loss: 2.3622 | LR: 5.92e-06 | 7817.3ms/step | 2.6 samples/s
|
| 1150 |
+
Step 10950/11700 | Loss: 2.3689 | LR: 5.77e-06 | 7828.4ms/step | 2.6 samples/s
|
| 1151 |
+
Step 10960/11700 | Loss: 2.5181 | LR: 5.61e-06 | 7799.8ms/step | 2.6 samples/s
|
| 1152 |
+
Step 10970/11700 | Loss: 2.4746 | LR: 5.46e-06 | 7788.4ms/step | 2.6 samples/s
|
| 1153 |
+
Step 10980/11700 | Loss: 2.4712 | LR: 5.31e-06 | 7807.2ms/step | 2.6 samples/s
|
| 1154 |
+
Step 10990/11700 | Loss: 2.4102 | LR: 5.17e-06 | 7812.9ms/step | 2.6 samples/s
|
| 1155 |
+
Step 11000/11700 | Loss: 2.3902 | LR: 5.02e-06 | 7819.4ms/step | 2.6 samples/s
|
| 1156 |
+
Validation at step 11000: Loss = 2.4528, Perplexity = 11.62
|
| 1157 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-11000
|
| 1158 |
+
Step 11010/11700 | Loss: 2.3791 | LR: 4.88e-06 | 19706.2ms/step | 1.0 samples/s
|
| 1159 |
+
Step 11020/11700 | Loss: 2.4472 | LR: 4.74e-06 | 7786.0ms/step | 2.6 samples/s
|
| 1160 |
+
Step 11030/11700 | Loss: 2.4814 | LR: 4.60e-06 | 7810.3ms/step | 2.6 samples/s
|
| 1161 |
+
Step 11040/11700 | Loss: 2.4730 | LR: 4.47e-06 | 7827.5ms/step | 2.6 samples/s
|
| 1162 |
+
Step 11050/11700 | Loss: 2.4961 | LR: 4.33e-06 | 7830.2ms/step | 2.6 samples/s
|
| 1163 |
+
Step 11060/11700 | Loss: 2.4295 | LR: 4.20e-06 | 7840.8ms/step | 2.6 samples/s
|
| 1164 |
+
Step 11070/11700 | Loss: 2.3912 | LR: 4.07e-06 | 7842.4ms/step | 2.6 samples/s
|
| 1165 |
+
Step 11080/11700 | Loss: 2.4507 | LR: 3.94e-06 | 7811.8ms/step | 2.6 samples/s
|
| 1166 |
+
Step 11090/11700 | Loss: 2.4363 | LR: 3.82e-06 | 7802.4ms/step | 2.6 samples/s
|
| 1167 |
+
Step 11100/11700 | Loss: 2.4816 | LR: 3.69e-06 | 7829.0ms/step | 2.6 samples/s
|
| 1168 |
+
Step 11110/11700 | Loss: 2.4271 | LR: 3.57e-06 | 7839.9ms/step | 2.6 samples/s
|
| 1169 |
+
Step 11120/11700 | Loss: 2.4717 | LR: 3.45e-06 | 7841.2ms/step | 2.6 samples/s
|
| 1170 |
+
Step 11130/11700 | Loss: 2.5182 | LR: 3.34e-06 | 7841.1ms/step | 2.6 samples/s
|
| 1171 |
+
Step 11140/11700 | Loss: 2.4229 | LR: 3.22e-06 | 7836.4ms/step | 2.6 samples/s
|
| 1172 |
+
Step 11150/11700 | Loss: 2.4447 | LR: 3.11e-06 | 7810.2ms/step | 2.6 samples/s
|
| 1173 |
+
Step 11160/11700 | Loss: 2.3918 | LR: 2.99e-06 | 7835.1ms/step | 2.6 samples/s
|
| 1174 |
+
Step 11170/11700 | Loss: 2.5040 | LR: 2.88e-06 | 7843.6ms/step | 2.5 samples/s
|
| 1175 |
+
Step 11180/11700 | Loss: 2.4160 | LR: 2.78e-06 | 7841.7ms/step | 2.6 samples/s
|
| 1176 |
+
Step 11190/11700 | Loss: 2.4543 | LR: 2.67e-06 | 7840.5ms/step | 2.6 samples/s
|
| 1177 |
+
Step 11200/11700 | Loss: 2.4486 | LR: 2.57e-06 | 7830.9ms/step | 2.6 samples/s
|
| 1178 |
+
Step 11210/11700 | Loss: 2.4977 | LR: 2.47e-06 | 7804.6ms/step | 2.6 samples/s
|
| 1179 |
+
Step 11220/11700 | Loss: 2.3903 | LR: 2.37e-06 | 7833.4ms/step | 2.6 samples/s
|
| 1180 |
+
Step 11230/11700 | Loss: 2.3175 | LR: 2.27e-06 | 7835.9ms/step | 2.6 samples/s
|
| 1181 |
+
Step 11240/11700 | Loss: 2.3858 | LR: 2.17e-06 | 7830.1ms/step | 2.6 samples/s
|
| 1182 |
+
Step 11250/11700 | Loss: 2.4011 | LR: 2.08e-06 | 7842.4ms/step | 2.6 samples/s
|
| 1183 |
+
Validation at step 11250: Loss = 2.4528, Perplexity = 11.62
|
| 1184 |
+
Step 11260/11700 | Loss: 2.4335 | LR: 1.99e-06 | 19462.0ms/step | 1.0 samples/s
|
| 1185 |
+
Step 11270/11700 | Loss: 2.4790 | LR: 1.90e-06 | 7836.4ms/step | 2.6 samples/s
|
| 1186 |
+
Step 11280/11700 | Loss: 2.4196 | LR: 1.81e-06 | 7835.9ms/step | 2.6 samples/s
|
| 1187 |
+
Step 11290/11700 | Loss: 2.4880 | LR: 1.73e-06 | 7839.4ms/step | 2.6 samples/s
|
| 1188 |
+
Step 11300/11700 | Loss: 2.4554 | LR: 1.64e-06 | 7844.1ms/step | 2.5 samples/s
|
| 1189 |
+
Step 11310/11700 | Loss: 2.4237 | LR: 1.56e-06 | 7815.3ms/step | 2.6 samples/s
|
| 1190 |
+
Step 11320/11700 | Loss: 2.4546 | LR: 1.48e-06 | 7818.5ms/step | 2.6 samples/s
|
| 1191 |
+
Step 11330/11700 | Loss: 2.3807 | LR: 1.41e-06 | 7836.7ms/step | 2.6 samples/s
|
| 1192 |
+
Step 11340/11700 | Loss: 2.3584 | LR: 1.33e-06 | 7842.3ms/step | 2.6 samples/s
|
| 1193 |
+
Step 11350/11700 | Loss: 2.4818 | LR: 1.26e-06 | 7842.9ms/step | 2.6 samples/s
|
| 1194 |
+
Step 11360/11700 | Loss: 2.4741 | LR: 1.19e-06 | 7843.0ms/step | 2.6 samples/s
|
| 1195 |
+
Step 11370/11700 | Loss: 2.4346 | LR: 1.12e-06 | 7822.3ms/step | 2.6 samples/s
|
| 1196 |
+
Step 11380/11700 | Loss: 2.4208 | LR: 1.05e-06 | 7819.6ms/step | 2.6 samples/s
|
| 1197 |
+
Step 11390/11700 | Loss: 2.4392 | LR: 9.88e-07 | 7839.1ms/step | 2.6 samples/s
|
| 1198 |
+
Step 11400/11700 | Loss: 2.4661 | LR: 9.25e-07 | 7839.2ms/step | 2.6 samples/s
|
| 1199 |
+
Step 11410/11700 | Loss: 2.4414 | LR: 8.65e-07 | 7839.8ms/step | 2.6 samples/s
|
| 1200 |
+
Step 11420/11700 | Loss: 2.2899 | LR: 8.06e-07 | 7841.5ms/step | 2.6 samples/s
|
| 1201 |
+
Step 11430/11700 | Loss: 2.3927 | LR: 7.50e-07 | 7829.4ms/step | 2.6 samples/s
|
| 1202 |
+
Step 11440/11700 | Loss: 2.4829 | LR: 6.95e-07 | 7818.5ms/step | 2.6 samples/s
|
| 1203 |
+
Step 11450/11700 | Loss: 2.3293 | LR: 6.43e-07 | 7843.5ms/step | 2.5 samples/s
|
| 1204 |
+
Step 11460/11700 | Loss: 2.4548 | LR: 5.92e-07 | 7845.1ms/step | 2.5 samples/s
|
| 1205 |
+
Step 11470/11700 | Loss: 2.4996 | LR: 5.44e-07 | 7841.1ms/step | 2.6 samples/s
|
| 1206 |
+
Step 11480/11700 | Loss: 2.3698 | LR: 4.98e-07 | 7842.8ms/step | 2.6 samples/s
|
| 1207 |
+
Step 11490/11700 | Loss: 2.4704 | LR: 4.54e-07 | 7831.0ms/step | 2.6 samples/s
|
| 1208 |
+
Step 11500/11700 | Loss: 2.4046 | LR: 4.11e-07 | 7823.1ms/step | 2.6 samples/s
|
| 1209 |
+
Validation at step 11500: Loss = 2.4528, Perplexity = 11.62
|
| 1210 |
+
Step 11510/11700 | Loss: 2.3907 | LR: 3.71e-07 | 19505.9ms/step | 1.0 samples/s
|
| 1211 |
+
Step 11520/11700 | Loss: 2.4647 | LR: 3.33e-07 | 7838.0ms/step | 2.6 samples/s
|
| 1212 |
+
Step 11530/11700 | Loss: 2.4093 | LR: 2.97e-07 | 7839.8ms/step | 2.6 samples/s
|
| 1213 |
+
Step 11540/11700 | Loss: 2.3859 | LR: 2.63e-07 | 7811.4ms/step | 2.6 samples/s
|
| 1214 |
+
Step 11550/11700 | Loss: 2.5074 | LR: 2.31e-07 | 7832.2ms/step | 2.6 samples/s
|
| 1215 |
+
Step 11560/11700 | Loss: 2.4051 | LR: 2.02e-07 | 7838.6ms/step | 2.6 samples/s
|
| 1216 |
+
Step 11570/11700 | Loss: 2.5180 | LR: 1.74e-07 | 7840.0ms/step | 2.6 samples/s
|
| 1217 |
+
Step 11580/11700 | Loss: 2.3546 | LR: 1.48e-07 | 7837.6ms/step | 2.6 samples/s
|
| 1218 |
+
Step 11590/11700 | Loss: 2.4715 | LR: 1.24e-07 | 7837.4ms/step | 2.6 samples/s
|
| 1219 |
+
Step 11600/11700 | Loss: 2.4732 | LR: 1.03e-07 | 7807.9ms/step | 2.6 samples/s
|
| 1220 |
+
Step 11610/11700 | Loss: 2.4744 | LR: 8.33e-08 | 7849.0ms/step | 2.5 samples/s
|
| 1221 |
+
Step 11620/11700 | Loss: 2.3910 | LR: 6.58e-08 | 7851.8ms/step | 2.5 samples/s
|
| 1222 |
+
Step 11630/11700 | Loss: 2.3381 | LR: 5.04e-08 | 7850.5ms/step | 2.5 samples/s
|
| 1223 |
+
Step 11640/11700 | Loss: 2.4726 | LR: 3.70e-08 | 7838.2ms/step | 2.6 samples/s
|
| 1224 |
+
Step 11650/11700 | Loss: 2.5243 | LR: 2.57e-08 | 7845.3ms/step | 2.5 samples/s
|
| 1225 |
+
Step 11660/11700 | Loss: 2.2990 | LR: 1.65e-08 | 7830.4ms/step | 2.6 samples/s
|
| 1226 |
+
Step 11670/11700 | Loss: 2.4536 | LR: 9.26e-09 | 7829.5ms/step | 2.6 samples/s
|
| 1227 |
+
Step 11680/11700 | Loss: 2.4660 | LR: 4.12e-09 | 7845.8ms/step | 2.5 samples/s
|
| 1228 |
+
Step 11690/11700 | Loss: 2.3698 | LR: 1.03e-09 | 7851.4ms/step | 2.5 samples/s
|
| 1229 |
+
Step 11700/11700 | Loss: 2.4355 | LR: 0.00e+00 | 7843.0ms/step | 2.6 samples/s
|
| 1230 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759/checkpoint-11700
|
epfl_final/SparseAtt/wandb/run-20260707_035818-m10lf13v/files/requirements.txt
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
epfl_final/SparseAtt/wandb/run-20260707_035818-m10lf13v/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-07-07T03:58:18.687564Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"configs/ladder30k/fixed_s2.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/TokenPrune.git",
|
| 14 |
+
"commit": "88298c7a418622530ca5426b22b1e5ebd10ca0cb"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "h100-train-4-gpu-0-0",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 23 |
+
"gpu_count": 4,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "9374931091456",
|
| 27 |
+
"used": "4873970163712"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1623005732864"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-85844f64-d144-1c84-306d-f88b5f232a77"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 43 |
+
"memoryTotal": "85520809984",
|
| 44 |
+
"cudaCores": 16896,
|
| 45 |
+
"architecture": "Hopper",
|
| 46 |
+
"uuid": "GPU-4e4c2343-4f5b-a2ff-312e-03d402a27803"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 50 |
+
"memoryTotal": "85520809984",
|
| 51 |
+
"cudaCores": 16896,
|
| 52 |
+
"architecture": "Hopper",
|
| 53 |
+
"uuid": "GPU-fbf0ccd1-8bd4-5f57-6e74-6c0cf7eb70c9"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 57 |
+
"memoryTotal": "85520809984",
|
| 58 |
+
"cudaCores": 16896,
|
| 59 |
+
"architecture": "Hopper",
|
| 60 |
+
"uuid": "GPU-410cba62-f098-3454-c960-ebc55bd081af"
|
| 61 |
+
}
|
| 62 |
+
],
|
| 63 |
+
"cudaVersion": "13.0",
|
| 64 |
+
"writerId": "ryjy1nj9s7a4cu1orvx223hpmj42juni"
|
| 65 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260707_035818-m10lf13v/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"train/learning_rate":0,"train/epoch":0.3122998078155029,"speed/ms_per_step":7843.028926849365,"train/loss":2.4354991952702405,"_runtime":96752.604333315,"_step":11700,"eval/perplexity":11.62103124240561,"eval/uses_sparse_attention":1,"eval/uses_dense_attention":0,"speed/steps_per_sec":0.12750176103222807,"speed/samples_per_sec":2.5500352206445616,"_wandb":{"runtime":96752},"eval/loss":2.452816494680941,"eval/dense_interval_active":0,"eval/dense_interval_end":-1,"train/global_step":11700,"_timestamp":1.7834934493710976e+09,"eval/dense_interval_start":-1}
|
epfl_final/SparseAtt/wandb/run-20260707_035818-m10lf13v/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260707_035818-m10lf13v/logs/debug.log
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-07-07 03:58:18,708 INFO MainThread:2516 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-07-07 03:58:18,708 INFO MainThread:2516 [wandb_setup.py:_flush():81] Configure stats pid to 2516
|
| 3 |
+
2026-07-07 03:58:18,708 INFO MainThread:2516 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-07-07 03:58:18,708 INFO MainThread:2516 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260707_035818-m10lf13v/logs/debug.log
|
| 5 |
+
2026-07-07 03:58:18,708 INFO MainThread:2516 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260707_035818-m10lf13v/logs/debug-internal.log
|
| 6 |
+
2026-07-07 03:58:18,708 INFO MainThread:2516 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-07-07 03:58:18,708 INFO MainThread:2516 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/longctx30k/tokenized', 'val_path': None, 'max_seq_length': 30720, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l30k365m_fixed_s2/run_20260707_035759', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 11700, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 20, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m fixed s2 30k', 'wandb_tags': ['qwen3', '365m', '30k', 'scaleladder', 'fixed', 'seed2'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 120, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_30k_fixed_s2', 'sparse_mask_seed': 2, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-07-07 03:58:18,708 INFO MainThread:2516 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-07-07 03:58:18,951 INFO MainThread:2516 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-07-07 03:58:19,281 INFO MainThread:2516 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-07-07 03:58:19,283 INFO MainThread:2516 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-07-07 03:58:19,308 INFO MainThread:2516 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-07-07 03:58:19,722 INFO MainThread:2516 [wandb_init.py:init():1056] starting run threads in backend
|
| 15 |
+
2026-07-07 03:58:19,917 INFO MainThread:2516 [wandb_run.py:_console_start():2554] atexit reg
|
| 16 |
+
2026-07-07 03:58:19,917 INFO MainThread:2516 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 17 |
+
2026-07-07 03:58:19,917 INFO MainThread:2516 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 18 |
+
2026-07-07 03:58:19,917 INFO MainThread:2516 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 19 |
+
2026-07-07 03:58:19,921 INFO MainThread:2516 [wandb_init.py:init():1094] run started, returning control to user process
|
| 20 |
+
2026-07-08 06:50:52,323 INFO MainThread:2516 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/m10lf13v
|
| 21 |
+
2026-07-08 06:50:52,325 INFO MainThread:2516 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
|
| 22 |
+
2026-07-08 06:50:52,326 INFO MainThread:2516 [wandb_run.py:_restore():2502] restore
|
| 23 |
+
2026-07-08 06:50:52,326 INFO MainThread:2516 [wandb_run.py:_restore():2508] restore done
|
| 24 |
+
2026-07-08 06:50:53,961 INFO MainThread:2516 [wandb_run.py:_footer_sync_info():3889] logging synced files
|
epfl_final/SparseAtt/wandb/run-20260707_035818-m10lf13v/run-m10lf13v.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c137d2c1408bff6d5456d8d549df4706239a52cf8e3a7f8c29af7a938f99c4ad
|
| 3 |
+
size 30621605
|
epfl_final/SparseAtt/wandb/run-20260707_035828-jmi9928p/files/config.yaml
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.26.1
|
| 4 |
+
e:
|
| 5 |
+
wt58lqx6cy40hdn2zs4v7es7obp06md4:
|
| 6 |
+
args:
|
| 7 |
+
- --config
|
| 8 |
+
- configs/ladder30k/dense.json
|
| 9 |
+
codePath: train.py
|
| 10 |
+
codePathLocal: train.py
|
| 11 |
+
cpu_count: 96
|
| 12 |
+
cpu_count_logical: 192
|
| 13 |
+
cudaVersion: "13.0"
|
| 14 |
+
disk:
|
| 15 |
+
/:
|
| 16 |
+
total: "9374931091456"
|
| 17 |
+
used: "4873970147328"
|
| 18 |
+
email: ruben.navasardyan@mail.utoronto.ca
|
| 19 |
+
executable: /mloscratch/homes/navasard/conda/bin/python
|
| 20 |
+
git:
|
| 21 |
+
commit: 88298c7a418622530ca5426b22b1e5ebd10ca0cb
|
| 22 |
+
remote: https://github.com/rubentium/TokenPrune.git
|
| 23 |
+
gpu: NVIDIA H100 80GB HBM3
|
| 24 |
+
gpu_count: 4
|
| 25 |
+
gpu_nvidia:
|
| 26 |
+
- architecture: Hopper
|
| 27 |
+
cudaCores: 16896
|
| 28 |
+
memoryTotal: "85520809984"
|
| 29 |
+
name: NVIDIA H100 80GB HBM3
|
| 30 |
+
uuid: GPU-85844f64-d144-1c84-306d-f88b5f232a77
|
| 31 |
+
- architecture: Hopper
|
| 32 |
+
cudaCores: 16896
|
| 33 |
+
memoryTotal: "85520809984"
|
| 34 |
+
name: NVIDIA H100 80GB HBM3
|
| 35 |
+
uuid: GPU-4e4c2343-4f5b-a2ff-312e-03d402a27803
|
| 36 |
+
- architecture: Hopper
|
| 37 |
+
cudaCores: 16896
|
| 38 |
+
memoryTotal: "85520809984"
|
| 39 |
+
name: NVIDIA H100 80GB HBM3
|
| 40 |
+
uuid: GPU-fbf0ccd1-8bd4-5f57-6e74-6c0cf7eb70c9
|
| 41 |
+
- architecture: Hopper
|
| 42 |
+
cudaCores: 16896
|
| 43 |
+
memoryTotal: "85520809984"
|
| 44 |
+
name: NVIDIA H100 80GB HBM3
|
| 45 |
+
uuid: GPU-410cba62-f098-3454-c960-ebc55bd081af
|
| 46 |
+
host: h100-train-4-gpu-0-0
|
| 47 |
+
memory:
|
| 48 |
+
total: "1623005732864"
|
| 49 |
+
os: Linux-6.8.0-124-generic-x86_64-with-glibc2.35
|
| 50 |
+
program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
|
| 51 |
+
python: CPython 3.13.13
|
| 52 |
+
root: /mloscratch/homes/navasard/sparse/SparseAtt
|
| 53 |
+
startedAt: "2026-07-07T03:58:28.689957Z"
|
| 54 |
+
writerId: wt58lqx6cy40hdn2zs4v7es7obp06md4
|
| 55 |
+
m: []
|
| 56 |
+
python_version: 3.13.13
|
| 57 |
+
t:
|
| 58 |
+
"1":
|
| 59 |
+
- 1
|
| 60 |
+
- 5
|
| 61 |
+
- 11
|
| 62 |
+
- 49
|
| 63 |
+
- 51
|
| 64 |
+
- 53
|
| 65 |
+
- 71
|
| 66 |
+
"2":
|
| 67 |
+
- 1
|
| 68 |
+
- 5
|
| 69 |
+
- 11
|
| 70 |
+
- 49
|
| 71 |
+
- 51
|
| 72 |
+
- 53
|
| 73 |
+
- 71
|
| 74 |
+
"3":
|
| 75 |
+
- 2
|
| 76 |
+
- 13
|
| 77 |
+
- 15
|
| 78 |
+
- 16
|
| 79 |
+
- 61
|
| 80 |
+
"4": 3.13.13
|
| 81 |
+
"5": 0.26.1
|
| 82 |
+
"6": 5.7.0
|
| 83 |
+
"12": 0.26.1
|
| 84 |
+
"13": linux-x86_64
|
| 85 |
+
attention_type:
|
| 86 |
+
value: sparse_stochastic
|
| 87 |
+
checkpoint_activations:
|
| 88 |
+
value: false
|
| 89 |
+
dense_mode:
|
| 90 |
+
value: true
|
| 91 |
+
dense_step_intervals:
|
| 92 |
+
value: null
|
| 93 |
+
eval_batches:
|
| 94 |
+
value: 50
|
| 95 |
+
eval_steps:
|
| 96 |
+
value: 250
|
| 97 |
+
force_new_run:
|
| 98 |
+
value: true
|
| 99 |
+
gradient_accumulation_steps:
|
| 100 |
+
value: 20
|
| 101 |
+
head_dim:
|
| 102 |
+
value: null
|
| 103 |
+
hidden_size:
|
| 104 |
+
value: 768
|
| 105 |
+
init_from_scratch:
|
| 106 |
+
value: true
|
| 107 |
+
intermediate_size:
|
| 108 |
+
value: 3072
|
| 109 |
+
learning_rate:
|
| 110 |
+
value: 0.0005
|
| 111 |
+
logging_steps:
|
| 112 |
+
value: 10
|
| 113 |
+
lr_scheduler:
|
| 114 |
+
value: cosine
|
| 115 |
+
max_grad_norm:
|
| 116 |
+
value: 1
|
| 117 |
+
max_seq_length:
|
| 118 |
+
value: 30720
|
| 119 |
+
max_steps:
|
| 120 |
+
value: 11700
|
| 121 |
+
mixed_precision:
|
| 122 |
+
value: bf16
|
| 123 |
+
model_dtype:
|
| 124 |
+
value: bfloat16
|
| 125 |
+
model_path:
|
| 126 |
+
value: /mloscratch/homes/navasard/qwen3_06b_base
|
| 127 |
+
num_attention_heads:
|
| 128 |
+
value: 6
|
| 129 |
+
num_epochs:
|
| 130 |
+
value: 1
|
| 131 |
+
num_hidden_layers:
|
| 132 |
+
value: 28
|
| 133 |
+
num_key_value_heads:
|
| 134 |
+
value: 2
|
| 135 |
+
num_workers:
|
| 136 |
+
value: 8
|
| 137 |
+
output_dir:
|
| 138 |
+
value: /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752
|
| 139 |
+
per_device_batch_size:
|
| 140 |
+
value: 1
|
| 141 |
+
resume_from_checkpoint:
|
| 142 |
+
value: null
|
| 143 |
+
save_steps:
|
| 144 |
+
value: 1000
|
| 145 |
+
seed:
|
| 146 |
+
value: 42
|
| 147 |
+
sparse_block_size:
|
| 148 |
+
value: 128
|
| 149 |
+
sparse_layer_indices:
|
| 150 |
+
value: null
|
| 151 |
+
sparse_mask_cache_dir:
|
| 152 |
+
value: null
|
| 153 |
+
sparse_mask_pool_size:
|
| 154 |
+
value: 1
|
| 155 |
+
sparse_mask_seed:
|
| 156 |
+
value: 0
|
| 157 |
+
sparse_num_random_blocks:
|
| 158 |
+
value: 120
|
| 159 |
+
sparse_val_dense_compare:
|
| 160 |
+
value: false
|
| 161 |
+
strategy:
|
| 162 |
+
value: ddp
|
| 163 |
+
train_path:
|
| 164 |
+
value: /mloscratch/homes/navasard/data/longctx30k/tokenized
|
| 165 |
+
val_path:
|
| 166 |
+
value: null
|
| 167 |
+
verbose:
|
| 168 |
+
value: true
|
| 169 |
+
wandb_enabled:
|
| 170 |
+
value: true
|
| 171 |
+
wandb_name:
|
| 172 |
+
value: 365m dense 30k
|
| 173 |
+
wandb_project:
|
| 174 |
+
value: stochastic_sparsity
|
| 175 |
+
wandb_tags:
|
| 176 |
+
value:
|
| 177 |
+
- qwen3
|
| 178 |
+
- 365m
|
| 179 |
+
- 30k
|
| 180 |
+
- scaleladder
|
| 181 |
+
- dense
|
| 182 |
+
warmup_steps:
|
| 183 |
+
value: 750
|
| 184 |
+
weight_decay:
|
| 185 |
+
value: 0.01
|
epfl_final/SparseAtt/wandb/run-20260707_035828-jmi9928p/files/output.log
ADDED
|
@@ -0,0 +1,1230 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Starting training...
|
| 2 |
+
Switching attention mode for step 1: dense (dense step interval)
|
| 3 |
+
Step 10/11700 | Loss: 12.0484 | LR: 6.67e-06 | 13293.3ms/step | 1.5 samples/s
|
| 4 |
+
Step 20/11700 | Loss: 11.9702 | LR: 1.33e-05 | 9042.9ms/step | 2.2 samples/s
|
| 5 |
+
Step 30/11700 | Loss: 11.7177 | LR: 2.00e-05 | 9041.7ms/step | 2.2 samples/s
|
| 6 |
+
Step 40/11700 | Loss: 11.2048 | LR: 2.67e-05 | 9025.1ms/step | 2.2 samples/s
|
| 7 |
+
Step 50/11700 | Loss: 10.8659 | LR: 3.33e-05 | 9007.3ms/step | 2.2 samples/s
|
| 8 |
+
Step 60/11700 | Loss: 10.5798 | LR: 4.00e-05 | 8999.0ms/step | 2.2 samples/s
|
| 9 |
+
Step 70/11700 | Loss: 10.3002 | LR: 4.67e-05 | 8990.6ms/step | 2.2 samples/s
|
| 10 |
+
Step 80/11700 | Loss: 10.0314 | LR: 5.33e-05 | 8986.6ms/step | 2.2 samples/s
|
| 11 |
+
Step 90/11700 | Loss: 9.7412 | LR: 6.00e-05 | 8980.7ms/step | 2.2 samples/s
|
| 12 |
+
Step 100/11700 | Loss: 9.3527 | LR: 6.67e-05 | 8990.8ms/step | 2.2 samples/s
|
| 13 |
+
Step 110/11700 | Loss: 8.8550 | LR: 7.33e-05 | 8994.8ms/step | 2.2 samples/s
|
| 14 |
+
Step 120/11700 | Loss: 8.3007 | LR: 8.00e-05 | 8988.8ms/step | 2.2 samples/s
|
| 15 |
+
Step 130/11700 | Loss: 7.8459 | LR: 8.67e-05 | 8996.4ms/step | 2.2 samples/s
|
| 16 |
+
Step 140/11700 | Loss: 7.5719 | LR: 9.33e-05 | 9003.1ms/step | 2.2 samples/s
|
| 17 |
+
Step 150/11700 | Loss: 7.3697 | LR: 1.00e-04 | 9012.8ms/step | 2.2 samples/s
|
| 18 |
+
Step 160/11700 | Loss: 7.1604 | LR: 1.07e-04 | 9020.9ms/step | 2.2 samples/s
|
| 19 |
+
Step 170/11700 | Loss: 7.0293 | LR: 1.13e-04 | 9025.3ms/step | 2.2 samples/s
|
| 20 |
+
Step 180/11700 | Loss: 6.8967 | LR: 1.20e-04 | 9037.0ms/step | 2.2 samples/s
|
| 21 |
+
Step 190/11700 | Loss: 6.8259 | LR: 1.27e-04 | 9042.1ms/step | 2.2 samples/s
|
| 22 |
+
Step 200/11700 | Loss: 6.6779 | LR: 1.33e-04 | 9045.3ms/step | 2.2 samples/s
|
| 23 |
+
Step 210/11700 | Loss: 6.5893 | LR: 1.40e-04 | 9045.9ms/step | 2.2 samples/s
|
| 24 |
+
Step 220/11700 | Loss: 6.4456 | LR: 1.47e-04 | 9050.2ms/step | 2.2 samples/s
|
| 25 |
+
Step 230/11700 | Loss: 6.3800 | LR: 1.53e-04 | 9067.1ms/step | 2.2 samples/s
|
| 26 |
+
Step 240/11700 | Loss: 6.2418 | LR: 1.60e-04 | 9062.0ms/step | 2.2 samples/s
|
| 27 |
+
Step 250/11700 | Loss: 6.2527 | LR: 1.67e-04 | 9079.3ms/step | 2.2 samples/s
|
| 28 |
+
Validation at step 250: Loss = 6.1182, Perplexity = 454.06
|
| 29 |
+
Step 260/11700 | Loss: 6.1697 | LR: 1.73e-04 | 23829.3ms/step | 0.8 samples/s
|
| 30 |
+
Step 270/11700 | Loss: 6.0604 | LR: 1.80e-04 | 9080.0ms/step | 2.2 samples/s
|
| 31 |
+
Step 280/11700 | Loss: 6.0021 | LR: 1.87e-04 | 9096.9ms/step | 2.2 samples/s
|
| 32 |
+
Step 290/11700 | Loss: 5.8951 | LR: 1.93e-04 | 9085.4ms/step | 2.2 samples/s
|
| 33 |
+
Step 300/11700 | Loss: 5.8827 | LR: 2.00e-04 | 9077.2ms/step | 2.2 samples/s
|
| 34 |
+
Step 310/11700 | Loss: 5.8564 | LR: 2.07e-04 | 9093.0ms/step | 2.2 samples/s
|
| 35 |
+
Step 320/11700 | Loss: 5.7371 | LR: 2.13e-04 | 9085.5ms/step | 2.2 samples/s
|
| 36 |
+
Step 330/11700 | Loss: 5.6231 | LR: 2.20e-04 | 9076.8ms/step | 2.2 samples/s
|
| 37 |
+
Step 340/11700 | Loss: 5.5681 | LR: 2.27e-04 | 9104.3ms/step | 2.2 samples/s
|
| 38 |
+
Step 350/11700 | Loss: 5.4096 | LR: 2.33e-04 | 9096.5ms/step | 2.2 samples/s
|
| 39 |
+
Step 360/11700 | Loss: 5.4823 | LR: 2.40e-04 | 9085.0ms/step | 2.2 samples/s
|
| 40 |
+
Step 370/11700 | Loss: 5.5145 | LR: 2.47e-04 | 9080.0ms/step | 2.2 samples/s
|
| 41 |
+
Step 380/11700 | Loss: 5.4947 | LR: 2.53e-04 | 9091.2ms/step | 2.2 samples/s
|
| 42 |
+
Step 390/11700 | Loss: 5.4637 | LR: 2.60e-04 | 9079.5ms/step | 2.2 samples/s
|
| 43 |
+
Step 400/11700 | Loss: 5.3325 | LR: 2.67e-04 | 9098.3ms/step | 2.2 samples/s
|
| 44 |
+
Step 410/11700 | Loss: 5.3319 | LR: 2.73e-04 | 9088.6ms/step | 2.2 samples/s
|
| 45 |
+
Step 420/11700 | Loss: 5.2946 | LR: 2.80e-04 | 9079.4ms/step | 2.2 samples/s
|
| 46 |
+
Step 430/11700 | Loss: 5.2497 | LR: 2.87e-04 | 9119.7ms/step | 2.2 samples/s
|
| 47 |
+
Step 440/11700 | Loss: 5.2477 | LR: 2.93e-04 | 9102.9ms/step | 2.2 samples/s
|
| 48 |
+
Step 450/11700 | Loss: 5.1558 | LR: 3.00e-04 | 9085.8ms/step | 2.2 samples/s
|
| 49 |
+
Step 460/11700 | Loss: 5.1813 | LR: 3.07e-04 | 9087.6ms/step | 2.2 samples/s
|
| 50 |
+
Step 470/11700 | Loss: 5.1461 | LR: 3.13e-04 | 9095.0ms/step | 2.2 samples/s
|
| 51 |
+
Step 480/11700 | Loss: 5.1555 | LR: 3.20e-04 | 9096.5ms/step | 2.2 samples/s
|
| 52 |
+
Step 490/11700 | Loss: 5.0623 | LR: 3.27e-04 | 9117.5ms/step | 2.2 samples/s
|
| 53 |
+
Step 500/11700 | Loss: 5.0113 | LR: 3.33e-04 | 9089.6ms/step | 2.2 samples/s
|
| 54 |
+
Validation at step 500: Loss = 4.9550, Perplexity = 141.88
|
| 55 |
+
Step 510/11700 | Loss: 4.9215 | LR: 3.40e-04 | 22160.2ms/step | 0.9 samples/s
|
| 56 |
+
Step 520/11700 | Loss: 4.9355 | LR: 3.47e-04 | 9102.8ms/step | 2.2 samples/s
|
| 57 |
+
Step 530/11700 | Loss: 4.8666 | LR: 3.53e-04 | 9110.1ms/step | 2.2 samples/s
|
| 58 |
+
Step 540/11700 | Loss: 4.7775 | LR: 3.60e-04 | 9102.8ms/step | 2.2 samples/s
|
| 59 |
+
Step 550/11700 | Loss: 4.8019 | LR: 3.67e-04 | 9075.3ms/step | 2.2 samples/s
|
| 60 |
+
Step 560/11700 | Loss: 4.7867 | LR: 3.73e-04 | 9104.8ms/step | 2.2 samples/s
|
| 61 |
+
Step 570/11700 | Loss: 4.5575 | LR: 3.80e-04 | 9093.7ms/step | 2.2 samples/s
|
| 62 |
+
Step 580/11700 | Loss: 4.5903 | LR: 3.87e-04 | 9092.8ms/step | 2.2 samples/s
|
| 63 |
+
Step 590/11700 | Loss: 4.5341 | LR: 3.93e-04 | 9100.0ms/step | 2.2 samples/s
|
| 64 |
+
Step 600/11700 | Loss: 4.5988 | LR: 4.00e-04 | 9099.7ms/step | 2.2 samples/s
|
| 65 |
+
Step 610/11700 | Loss: 4.5735 | LR: 4.07e-04 | 9095.1ms/step | 2.2 samples/s
|
| 66 |
+
Step 620/11700 | Loss: 4.5788 | LR: 4.13e-04 | 9107.8ms/step | 2.2 samples/s
|
| 67 |
+
Step 630/11700 | Loss: 4.4359 | LR: 4.20e-04 | 9095.2ms/step | 2.2 samples/s
|
| 68 |
+
Step 640/11700 | Loss: 4.5140 | LR: 4.27e-04 | 9102.9ms/step | 2.2 samples/s
|
| 69 |
+
Step 650/11700 | Loss: 4.3762 | LR: 4.33e-04 | 9091.0ms/step | 2.2 samples/s
|
| 70 |
+
Step 660/11700 | Loss: 4.4714 | LR: 4.40e-04 | 9092.0ms/step | 2.2 samples/s
|
| 71 |
+
Step 670/11700 | Loss: 4.3703 | LR: 4.47e-04 | 9097.5ms/step | 2.2 samples/s
|
| 72 |
+
Step 680/11700 | Loss: 4.2817 | LR: 4.53e-04 | 9076.5ms/step | 2.2 samples/s
|
| 73 |
+
Step 690/11700 | Loss: 4.1302 | LR: 4.60e-04 | 9092.4ms/step | 2.2 samples/s
|
| 74 |
+
Step 700/11700 | Loss: 4.2661 | LR: 4.67e-04 | 9110.4ms/step | 2.2 samples/s
|
| 75 |
+
Step 710/11700 | Loss: 4.2591 | LR: 4.73e-04 | 9083.5ms/step | 2.2 samples/s
|
| 76 |
+
Step 720/11700 | Loss: 4.1482 | LR: 4.80e-04 | 9090.7ms/step | 2.2 samples/s
|
| 77 |
+
Step 730/11700 | Loss: 4.0106 | LR: 4.87e-04 | 9111.4ms/step | 2.2 samples/s
|
| 78 |
+
Step 740/11700 | Loss: 4.1462 | LR: 4.93e-04 | 9101.3ms/step | 2.2 samples/s
|
| 79 |
+
Step 750/11700 | Loss: 4.0399 | LR: 5.00e-04 | 9088.8ms/step | 2.2 samples/s
|
| 80 |
+
Validation at step 750: Loss = 3.9839, Perplexity = 53.73
|
| 81 |
+
Step 760/11700 | Loss: 4.0546 | LR: 5.00e-04 | 22234.9ms/step | 0.9 samples/s
|
| 82 |
+
Step 770/11700 | Loss: 3.8123 | LR: 5.00e-04 | 9138.5ms/step | 2.2 samples/s
|
| 83 |
+
Step 780/11700 | Loss: 4.0243 | LR: 5.00e-04 | 9137.2ms/step | 2.2 samples/s
|
| 84 |
+
Step 790/11700 | Loss: 3.7985 | LR: 5.00e-04 | 9123.2ms/step | 2.2 samples/s
|
| 85 |
+
Step 800/11700 | Loss: 3.7874 | LR: 5.00e-04 | 9138.4ms/step | 2.2 samples/s
|
| 86 |
+
Step 810/11700 | Loss: 3.8176 | LR: 5.00e-04 | 9129.4ms/step | 2.2 samples/s
|
| 87 |
+
Step 820/11700 | Loss: 3.7912 | LR: 5.00e-04 | 9130.9ms/step | 2.2 samples/s
|
| 88 |
+
Step 830/11700 | Loss: 3.7794 | LR: 5.00e-04 | 9125.9ms/step | 2.2 samples/s
|
| 89 |
+
Step 840/11700 | Loss: 3.7799 | LR: 5.00e-04 | 9133.8ms/step | 2.2 samples/s
|
| 90 |
+
Step 850/11700 | Loss: 3.8138 | LR: 5.00e-04 | 9125.5ms/step | 2.2 samples/s
|
| 91 |
+
Step 860/11700 | Loss: 3.6481 | LR: 5.00e-04 | 9129.5ms/step | 2.2 samples/s
|
| 92 |
+
Step 870/11700 | Loss: 3.6798 | LR: 5.00e-04 | 9135.2ms/step | 2.2 samples/s
|
| 93 |
+
Step 880/11700 | Loss: 3.7357 | LR: 5.00e-04 | 9114.8ms/step | 2.2 samples/s
|
| 94 |
+
Step 890/11700 | Loss: 3.5737 | LR: 5.00e-04 | 9117.9ms/step | 2.2 samples/s
|
| 95 |
+
Step 900/11700 | Loss: 3.5135 | LR: 5.00e-04 | 9103.1ms/step | 2.2 samples/s
|
| 96 |
+
Step 910/11700 | Loss: 3.6803 | LR: 5.00e-04 | 9095.5ms/step | 2.2 samples/s
|
| 97 |
+
Step 920/11700 | Loss: 3.5651 | LR: 5.00e-04 | 9104.5ms/step | 2.2 samples/s
|
| 98 |
+
Step 930/11700 | Loss: 3.5462 | LR: 5.00e-04 | 9103.9ms/step | 2.2 samples/s
|
| 99 |
+
Step 940/11700 | Loss: 3.6153 | LR: 5.00e-04 | 9104.7ms/step | 2.2 samples/s
|
| 100 |
+
Step 950/11700 | Loss: 3.4468 | LR: 5.00e-04 | 9096.4ms/step | 2.2 samples/s
|
| 101 |
+
Step 960/11700 | Loss: 3.5193 | LR: 5.00e-04 | 9103.9ms/step | 2.2 samples/s
|
| 102 |
+
Step 970/11700 | Loss: 3.5809 | LR: 5.00e-04 | 9094.9ms/step | 2.2 samples/s
|
| 103 |
+
Step 980/11700 | Loss: 3.2973 | LR: 4.99e-04 | 9115.3ms/step | 2.2 samples/s
|
| 104 |
+
Step 990/11700 | Loss: 3.5138 | LR: 4.99e-04 | 9109.0ms/step | 2.2 samples/s
|
| 105 |
+
Step 1000/11700 | Loss: 3.4018 | LR: 4.99e-04 | 9115.6ms/step | 2.2 samples/s
|
| 106 |
+
Validation at step 1000: Loss = 3.4574, Perplexity = 31.74
|
| 107 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-1000
|
| 108 |
+
Step 1010/11700 | Loss: 3.5266 | LR: 4.99e-04 | 22490.1ms/step | 0.9 samples/s
|
| 109 |
+
Step 1020/11700 | Loss: 3.4309 | LR: 4.99e-04 | 9129.2ms/step | 2.2 samples/s
|
| 110 |
+
Step 1030/11700 | Loss: 3.4492 | LR: 4.99e-04 | 9101.9ms/step | 2.2 samples/s
|
| 111 |
+
Step 1040/11700 | Loss: 3.4735 | LR: 4.99e-04 | 9105.3ms/step | 2.2 samples/s
|
| 112 |
+
Step 1050/11700 | Loss: 3.3925 | LR: 4.99e-04 | 9098.1ms/step | 2.2 samples/s
|
| 113 |
+
Step 1060/11700 | Loss: 3.4477 | LR: 4.99e-04 | 9101.3ms/step | 2.2 samples/s
|
| 114 |
+
Step 1070/11700 | Loss: 3.3923 | LR: 4.99e-04 | 9093.7ms/step | 2.2 samples/s
|
| 115 |
+
Step 1080/11700 | Loss: 3.2509 | LR: 4.99e-04 | 9116.2ms/step | 2.2 samples/s
|
| 116 |
+
Step 1090/11700 | Loss: 3.3119 | LR: 4.99e-04 | 9114.6ms/step | 2.2 samples/s
|
| 117 |
+
Step 1100/11700 | Loss: 3.5159 | LR: 4.99e-04 | 9114.6ms/step | 2.2 samples/s
|
| 118 |
+
Step 1110/11700 | Loss: 3.4564 | LR: 4.99e-04 | 9119.0ms/step | 2.2 samples/s
|
| 119 |
+
Step 1120/11700 | Loss: 3.3510 | LR: 4.99e-04 | 9120.4ms/step | 2.2 samples/s
|
| 120 |
+
Step 1130/11700 | Loss: 3.3622 | LR: 4.99e-04 | 9097.7ms/step | 2.2 samples/s
|
| 121 |
+
Step 1140/11700 | Loss: 3.1304 | LR: 4.98e-04 | 9076.4ms/step | 2.2 samples/s
|
| 122 |
+
Step 1150/11700 | Loss: 3.2823 | LR: 4.98e-04 | 9121.4ms/step | 2.2 samples/s
|
| 123 |
+
Step 1160/11700 | Loss: 3.2942 | LR: 4.98e-04 | 9099.2ms/step | 2.2 samples/s
|
| 124 |
+
Step 1170/11700 | Loss: 3.2613 | LR: 4.98e-04 | 9085.6ms/step | 2.2 samples/s
|
| 125 |
+
Step 1180/11700 | Loss: 3.1969 | LR: 4.98e-04 | 9100.9ms/step | 2.2 samples/s
|
| 126 |
+
Step 1190/11700 | Loss: 3.1582 | LR: 4.98e-04 | 9122.3ms/step | 2.2 samples/s
|
| 127 |
+
Step 1200/11700 | Loss: 3.0894 | LR: 4.98e-04 | 9101.2ms/step | 2.2 samples/s
|
| 128 |
+
Step 1210/11700 | Loss: 3.4270 | LR: 4.98e-04 | 9094.2ms/step | 2.2 samples/s
|
| 129 |
+
Step 1220/11700 | Loss: 3.2168 | LR: 4.98e-04 | 9092.8ms/step | 2.2 samples/s
|
| 130 |
+
Step 1230/11700 | Loss: 3.2558 | LR: 4.98e-04 | 9083.7ms/step | 2.2 samples/s
|
| 131 |
+
Step 1240/11700 | Loss: 3.2018 | LR: 4.98e-04 | 9082.1ms/step | 2.2 samples/s
|
| 132 |
+
Step 1250/11700 | Loss: 3.1800 | LR: 4.97e-04 | 9082.1ms/step | 2.2 samples/s
|
| 133 |
+
Validation at step 1250: Loss = 3.2275, Perplexity = 25.22
|
| 134 |
+
Step 1260/11700 | Loss: 3.2539 | LR: 4.97e-04 | 22146.9ms/step | 0.9 samples/s
|
| 135 |
+
Step 1270/11700 | Loss: 3.0820 | LR: 4.97e-04 | 9078.5ms/step | 2.2 samples/s
|
| 136 |
+
Step 1280/11700 | Loss: 3.1964 | LR: 4.97e-04 | 9078.5ms/step | 2.2 samples/s
|
| 137 |
+
Step 1290/11700 | Loss: 3.1911 | LR: 4.97e-04 | 9079.5ms/step | 2.2 samples/s
|
| 138 |
+
Step 1300/11700 | Loss: 3.1982 | LR: 4.97e-04 | 9075.3ms/step | 2.2 samples/s
|
| 139 |
+
Step 1310/11700 | Loss: 3.2471 | LR: 4.97e-04 | 9077.9ms/step | 2.2 samples/s
|
| 140 |
+
Step 1320/11700 | Loss: 3.2131 | LR: 4.97e-04 | 9078.5ms/step | 2.2 samples/s
|
| 141 |
+
Step 1330/11700 | Loss: 3.1315 | LR: 4.97e-04 | 9079.6ms/step | 2.2 samples/s
|
| 142 |
+
Step 1340/11700 | Loss: 3.1755 | LR: 4.96e-04 | 9078.4ms/step | 2.2 samples/s
|
| 143 |
+
Step 1350/11700 | Loss: 3.1888 | LR: 4.96e-04 | 9083.5ms/step | 2.2 samples/s
|
| 144 |
+
Step 1360/11700 | Loss: 3.1484 | LR: 4.96e-04 | 9089.7ms/step | 2.2 samples/s
|
| 145 |
+
Step 1370/11700 | Loss: 3.0922 | LR: 4.96e-04 | 9102.5ms/step | 2.2 samples/s
|
| 146 |
+
Step 1380/11700 | Loss: 2.9564 | LR: 4.96e-04 | 9095.3ms/step | 2.2 samples/s
|
| 147 |
+
Step 1390/11700 | Loss: 3.1233 | LR: 4.96e-04 | 9099.3ms/step | 2.2 samples/s
|
| 148 |
+
Step 1400/11700 | Loss: 3.2290 | LR: 4.96e-04 | 9094.7ms/step | 2.2 samples/s
|
| 149 |
+
Step 1410/11700 | Loss: 3.1625 | LR: 4.96e-04 | 9096.5ms/step | 2.2 samples/s
|
| 150 |
+
Step 1420/11700 | Loss: 3.0551 | LR: 4.95e-04 | 9093.0ms/step | 2.2 samples/s
|
| 151 |
+
Step 1430/11700 | Loss: 3.0973 | LR: 4.95e-04 | 9094.7ms/step | 2.2 samples/s
|
| 152 |
+
Step 1440/11700 | Loss: 3.0840 | LR: 4.95e-04 | 9095.2ms/step | 2.2 samples/s
|
| 153 |
+
Step 1450/11700 | Loss: 3.0051 | LR: 4.95e-04 | 9094.8ms/step | 2.2 samples/s
|
| 154 |
+
Step 1460/11700 | Loss: 3.0610 | LR: 4.95e-04 | 9092.2ms/step | 2.2 samples/s
|
| 155 |
+
Step 1470/11700 | Loss: 3.0834 | LR: 4.95e-04 | 9090.2ms/step | 2.2 samples/s
|
| 156 |
+
Step 1480/11700 | Loss: 3.0170 | LR: 4.95e-04 | 9091.9ms/step | 2.2 samples/s
|
| 157 |
+
Step 1490/11700 | Loss: 3.1768 | LR: 4.94e-04 | 9092.1ms/step | 2.2 samples/s
|
| 158 |
+
Step 1500/11700 | Loss: 3.0148 | LR: 4.94e-04 | 9099.8ms/step | 2.2 samples/s
|
| 159 |
+
Validation at step 1500: Loss = 3.0923, Perplexity = 22.03
|
| 160 |
+
Step 1510/11700 | Loss: 3.0810 | LR: 4.94e-04 | 22209.7ms/step | 0.9 samples/s
|
| 161 |
+
Step 1520/11700 | Loss: 3.0657 | LR: 4.94e-04 | 9097.7ms/step | 2.2 samples/s
|
| 162 |
+
Step 1530/11700 | Loss: 3.0633 | LR: 4.94e-04 | 9094.2ms/step | 2.2 samples/s
|
| 163 |
+
Step 1540/11700 | Loss: 3.2202 | LR: 4.94e-04 | 9094.2ms/step | 2.2 samples/s
|
| 164 |
+
Step 1550/11700 | Loss: 3.0666 | LR: 4.93e-04 | 9089.4ms/step | 2.2 samples/s
|
| 165 |
+
Step 1560/11700 | Loss: 3.0896 | LR: 4.93e-04 | 9089.8ms/step | 2.2 samples/s
|
| 166 |
+
Step 1570/11700 | Loss: 3.0764 | LR: 4.93e-04 | 9095.4ms/step | 2.2 samples/s
|
| 167 |
+
Step 1580/11700 | Loss: 3.0389 | LR: 4.93e-04 | 9097.6ms/step | 2.2 samples/s
|
| 168 |
+
Step 1590/11700 | Loss: 3.1677 | LR: 4.93e-04 | 9092.2ms/step | 2.2 samples/s
|
| 169 |
+
Step 1600/11700 | Loss: 3.0551 | LR: 4.93e-04 | 9091.6ms/step | 2.2 samples/s
|
| 170 |
+
Step 1610/11700 | Loss: 3.0999 | LR: 4.92e-04 | 9087.0ms/step | 2.2 samples/s
|
| 171 |
+
Step 1620/11700 | Loss: 2.9752 | LR: 4.92e-04 | 9088.2ms/step | 2.2 samples/s
|
| 172 |
+
Step 1630/11700 | Loss: 2.8849 | LR: 4.92e-04 | 9086.6ms/step | 2.2 samples/s
|
| 173 |
+
Step 1640/11700 | Loss: 3.0746 | LR: 4.92e-04 | 9086.6ms/step | 2.2 samples/s
|
| 174 |
+
Step 1650/11700 | Loss: 3.0112 | LR: 4.92e-04 | 9087.6ms/step | 2.2 samples/s
|
| 175 |
+
Step 1660/11700 | Loss: 3.0404 | LR: 4.92e-04 | 9087.7ms/step | 2.2 samples/s
|
| 176 |
+
Step 1670/11700 | Loss: 3.1605 | LR: 4.91e-04 | 9087.4ms/step | 2.2 samples/s
|
| 177 |
+
Step 1680/11700 | Loss: 3.0063 | LR: 4.91e-04 | 9089.9ms/step | 2.2 samples/s
|
| 178 |
+
Step 1690/11700 | Loss: 2.8694 | LR: 4.91e-04 | 9092.3ms/step | 2.2 samples/s
|
| 179 |
+
Step 1700/11700 | Loss: 2.9316 | LR: 4.91e-04 | 9092.5ms/step | 2.2 samples/s
|
| 180 |
+
Step 1710/11700 | Loss: 3.0661 | LR: 4.91e-04 | 9096.3ms/step | 2.2 samples/s
|
| 181 |
+
Step 1720/11700 | Loss: 2.9882 | LR: 4.90e-04 | 9101.3ms/step | 2.2 samples/s
|
| 182 |
+
Step 1730/11700 | Loss: 3.0125 | LR: 4.90e-04 | 9093.6ms/step | 2.2 samples/s
|
| 183 |
+
Step 1740/11700 | Loss: 2.9136 | LR: 4.90e-04 | 9082.7ms/step | 2.2 samples/s
|
| 184 |
+
Step 1750/11700 | Loss: 2.8846 | LR: 4.90e-04 | 9090.7ms/step | 2.2 samples/s
|
| 185 |
+
Validation at step 1750: Loss = 2.9990, Perplexity = 20.07
|
| 186 |
+
Step 1760/11700 | Loss: 2.9725 | LR: 4.90e-04 | 22211.9ms/step | 0.9 samples/s
|
| 187 |
+
Step 1770/11700 | Loss: 2.9781 | LR: 4.89e-04 | 9093.9ms/step | 2.2 samples/s
|
| 188 |
+
Step 1780/11700 | Loss: 2.9992 | LR: 4.89e-04 | 9096.7ms/step | 2.2 samples/s
|
| 189 |
+
Step 1790/11700 | Loss: 2.9722 | LR: 4.89e-04 | 9094.9ms/step | 2.2 samples/s
|
| 190 |
+
Step 1800/11700 | Loss: 2.9929 | LR: 4.89e-04 | 9095.2ms/step | 2.2 samples/s
|
| 191 |
+
Step 1810/11700 | Loss: 3.0468 | LR: 4.89e-04 | 9094.5ms/step | 2.2 samples/s
|
| 192 |
+
Step 1820/11700 | Loss: 2.9723 | LR: 4.88e-04 | 9095.7ms/step | 2.2 samples/s
|
| 193 |
+
Step 1830/11700 | Loss: 2.9482 | LR: 4.88e-04 | 9091.6ms/step | 2.2 samples/s
|
| 194 |
+
Step 1840/11700 | Loss: 2.9366 | LR: 4.88e-04 | 9094.6ms/step | 2.2 samples/s
|
| 195 |
+
Step 1850/11700 | Loss: 2.9620 | LR: 4.88e-04 | 9095.2ms/step | 2.2 samples/s
|
| 196 |
+
Step 1860/11700 | Loss: 2.9449 | LR: 4.87e-04 | 9103.7ms/step | 2.2 samples/s
|
| 197 |
+
Step 1870/11700 | Loss: 2.8820 | LR: 4.87e-04 | 9097.8ms/step | 2.2 samples/s
|
| 198 |
+
Step 1880/11700 | Loss: 2.8935 | LR: 4.87e-04 | 9098.9ms/step | 2.2 samples/s
|
| 199 |
+
Step 1890/11700 | Loss: 3.0198 | LR: 4.87e-04 | 9095.1ms/step | 2.2 samples/s
|
| 200 |
+
Step 1900/11700 | Loss: 2.9618 | LR: 4.87e-04 | 9094.4ms/step | 2.2 samples/s
|
| 201 |
+
Step 1910/11700 | Loss: 2.9861 | LR: 4.86e-04 | 9093.4ms/step | 2.2 samples/s
|
| 202 |
+
Step 1920/11700 | Loss: 2.7773 | LR: 4.86e-04 | 9096.8ms/step | 2.2 samples/s
|
| 203 |
+
Step 1930/11700 | Loss: 2.9722 | LR: 4.86e-04 | 9095.7ms/step | 2.2 samples/s
|
| 204 |
+
Step 1940/11700 | Loss: 2.9024 | LR: 4.86e-04 | 9097.5ms/step | 2.2 samples/s
|
| 205 |
+
Step 1950/11700 | Loss: 2.8189 | LR: 4.85e-04 | 9092.7ms/step | 2.2 samples/s
|
| 206 |
+
Step 1960/11700 | Loss: 2.8877 | LR: 4.85e-04 | 9096.5ms/step | 2.2 samples/s
|
| 207 |
+
Step 1970/11700 | Loss: 2.9145 | LR: 4.85e-04 | 9097.3ms/step | 2.2 samples/s
|
| 208 |
+
Step 1980/11700 | Loss: 2.8949 | LR: 4.85e-04 | 9092.3ms/step | 2.2 samples/s
|
| 209 |
+
Step 1990/11700 | Loss: 2.8350 | LR: 4.84e-04 | 9092.3ms/step | 2.2 samples/s
|
| 210 |
+
Step 2000/11700 | Loss: 3.0469 | LR: 4.84e-04 | 9093.2ms/step | 2.2 samples/s
|
| 211 |
+
Validation at step 2000: Loss = 2.9196, Perplexity = 18.53
|
| 212 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-2000
|
| 213 |
+
Step 2010/11700 | Loss: 2.9461 | LR: 4.84e-04 | 22488.6ms/step | 0.9 samples/s
|
| 214 |
+
Step 2020/11700 | Loss: 2.9795 | LR: 4.84e-04 | 9092.4ms/step | 2.2 samples/s
|
| 215 |
+
Step 2030/11700 | Loss: 2.8785 | LR: 4.83e-04 | 9091.9ms/step | 2.2 samples/s
|
| 216 |
+
Step 2040/11700 | Loss: 2.9193 | LR: 4.83e-04 | 9089.1ms/step | 2.2 samples/s
|
| 217 |
+
Step 2050/11700 | Loss: 2.7876 | LR: 4.83e-04 | 9092.7ms/step | 2.2 samples/s
|
| 218 |
+
Step 2060/11700 | Loss: 2.8521 | LR: 4.83e-04 | 9094.2ms/step | 2.2 samples/s
|
| 219 |
+
Step 2070/11700 | Loss: 2.8590 | LR: 4.82e-04 | 9094.5ms/step | 2.2 samples/s
|
| 220 |
+
Step 2080/11700 | Loss: 2.8388 | LR: 4.82e-04 | 9091.3ms/step | 2.2 samples/s
|
| 221 |
+
Step 2090/11700 | Loss: 2.8704 | LR: 4.82e-04 | 9092.5ms/step | 2.2 samples/s
|
| 222 |
+
Step 2100/11700 | Loss: 2.9612 | LR: 4.81e-04 | 9093.1ms/step | 2.2 samples/s
|
| 223 |
+
Step 2110/11700 | Loss: 2.9572 | LR: 4.81e-04 | 9093.9ms/step | 2.2 samples/s
|
| 224 |
+
Step 2120/11700 | Loss: 2.9037 | LR: 4.81e-04 | 9086.1ms/step | 2.2 samples/s
|
| 225 |
+
Step 2130/11700 | Loss: 2.8931 | LR: 4.81e-04 | 9083.2ms/step | 2.2 samples/s
|
| 226 |
+
Step 2140/11700 | Loss: 3.0030 | LR: 4.80e-04 | 9086.1ms/step | 2.2 samples/s
|
| 227 |
+
Step 2150/11700 | Loss: 2.9265 | LR: 4.80e-04 | 9085.2ms/step | 2.2 samples/s
|
| 228 |
+
Step 2160/11700 | Loss: 2.8807 | LR: 4.80e-04 | 9086.9ms/step | 2.2 samples/s
|
| 229 |
+
Step 2170/11700 | Loss: 2.8106 | LR: 4.80e-04 | 9086.8ms/step | 2.2 samples/s
|
| 230 |
+
Step 2180/11700 | Loss: 2.8408 | LR: 4.79e-04 | 9084.0ms/step | 2.2 samples/s
|
| 231 |
+
Step 2190/11700 | Loss: 2.8298 | LR: 4.79e-04 | 9085.6ms/step | 2.2 samples/s
|
| 232 |
+
Step 2200/11700 | Loss: 2.8936 | LR: 4.79e-04 | 9092.2ms/step | 2.2 samples/s
|
| 233 |
+
Step 2210/11700 | Loss: 2.9368 | LR: 4.78e-04 | 9087.8ms/step | 2.2 samples/s
|
| 234 |
+
Step 2220/11700 | Loss: 2.7463 | LR: 4.78e-04 | 9087.6ms/step | 2.2 samples/s
|
| 235 |
+
Step 2230/11700 | Loss: 2.7851 | LR: 4.78e-04 | 9086.3ms/step | 2.2 samples/s
|
| 236 |
+
Step 2240/11700 | Loss: 2.7685 | LR: 4.78e-04 | 9083.0ms/step | 2.2 samples/s
|
| 237 |
+
Step 2250/11700 | Loss: 2.9539 | LR: 4.77e-04 | 9085.8ms/step | 2.2 samples/s
|
| 238 |
+
Validation at step 2250: Loss = 2.8584, Perplexity = 17.43
|
| 239 |
+
Step 2260/11700 | Loss: 2.7345 | LR: 4.77e-04 | 22187.9ms/step | 0.9 samples/s
|
| 240 |
+
Step 2270/11700 | Loss: 2.9457 | LR: 4.77e-04 | 9085.0ms/step | 2.2 samples/s
|
| 241 |
+
Step 2280/11700 | Loss: 2.8663 | LR: 4.76e-04 | 9086.0ms/step | 2.2 samples/s
|
| 242 |
+
Step 2290/11700 | Loss: 2.8816 | LR: 4.76e-04 | 9086.2ms/step | 2.2 samples/s
|
| 243 |
+
Step 2300/11700 | Loss: 2.8199 | LR: 4.76e-04 | 9087.4ms/step | 2.2 samples/s
|
| 244 |
+
Step 2310/11700 | Loss: 2.8055 | LR: 4.75e-04 | 9089.1ms/step | 2.2 samples/s
|
| 245 |
+
Step 2320/11700 | Loss: 2.9572 | LR: 4.75e-04 | 9084.3ms/step | 2.2 samples/s
|
| 246 |
+
Step 2330/11700 | Loss: 2.7812 | LR: 4.75e-04 | 9086.9ms/step | 2.2 samples/s
|
| 247 |
+
Step 2340/11700 | Loss: 2.8155 | LR: 4.74e-04 | 9082.2ms/step | 2.2 samples/s
|
| 248 |
+
Step 2350/11700 | Loss: 2.7372 | LR: 4.74e-04 | 9088.3ms/step | 2.2 samples/s
|
| 249 |
+
Step 2360/11700 | Loss: 2.7525 | LR: 4.74e-04 | 9084.7ms/step | 2.2 samples/s
|
| 250 |
+
Step 2370/11700 | Loss: 2.7360 | LR: 4.73e-04 | 9089.1ms/step | 2.2 samples/s
|
| 251 |
+
Step 2380/11700 | Loss: 2.7325 | LR: 4.73e-04 | 9093.9ms/step | 2.2 samples/s
|
| 252 |
+
Step 2390/11700 | Loss: 2.8781 | LR: 4.73e-04 | 9088.8ms/step | 2.2 samples/s
|
| 253 |
+
Step 2400/11700 | Loss: 2.8137 | LR: 4.73e-04 | 9087.2ms/step | 2.2 samples/s
|
| 254 |
+
Step 2410/11700 | Loss: 2.8326 | LR: 4.72e-04 | 9086.2ms/step | 2.2 samples/s
|
| 255 |
+
Step 2420/11700 | Loss: 2.6931 | LR: 4.72e-04 | 9086.2ms/step | 2.2 samples/s
|
| 256 |
+
Step 2430/11700 | Loss: 2.8656 | LR: 4.72e-04 | 9087.9ms/step | 2.2 samples/s
|
| 257 |
+
Step 2440/11700 | Loss: 2.8840 | LR: 4.71e-04 | 9091.2ms/step | 2.2 samples/s
|
| 258 |
+
Step 2450/11700 | Loss: 2.7161 | LR: 4.71e-04 | 9123.6ms/step | 2.2 samples/s
|
| 259 |
+
Step 2460/11700 | Loss: 2.8401 | LR: 4.71e-04 | 9108.4ms/step | 2.2 samples/s
|
| 260 |
+
Step 2470/11700 | Loss: 2.8459 | LR: 4.70e-04 | 9085.3ms/step | 2.2 samples/s
|
| 261 |
+
Step 2480/11700 | Loss: 2.7759 | LR: 4.70e-04 | 9083.3ms/step | 2.2 samples/s
|
| 262 |
+
Step 2490/11700 | Loss: 2.8701 | LR: 4.69e-04 | 9080.6ms/step | 2.2 samples/s
|
| 263 |
+
Step 2500/11700 | Loss: 2.7407 | LR: 4.69e-04 | 9082.1ms/step | 2.2 samples/s
|
| 264 |
+
Validation at step 2500: Loss = 2.8055, Perplexity = 16.54
|
| 265 |
+
Step 2510/11700 | Loss: 2.7124 | LR: 4.69e-04 | 22211.7ms/step | 0.9 samples/s
|
| 266 |
+
Step 2520/11700 | Loss: 2.8711 | LR: 4.68e-04 | 9154.5ms/step | 2.2 samples/s
|
| 267 |
+
Step 2530/11700 | Loss: 2.7694 | LR: 4.68e-04 | 9148.9ms/step | 2.2 samples/s
|
| 268 |
+
Step 2540/11700 | Loss: 2.7032 | LR: 4.68e-04 | 9155.0ms/step | 2.2 samples/s
|
| 269 |
+
Step 2550/11700 | Loss: 2.8745 | LR: 4.67e-04 | 9143.3ms/step | 2.2 samples/s
|
| 270 |
+
Step 2560/11700 | Loss: 2.8180 | LR: 4.67e-04 | 9147.7ms/step | 2.2 samples/s
|
| 271 |
+
Step 2570/11700 | Loss: 2.7768 | LR: 4.67e-04 | 9143.9ms/step | 2.2 samples/s
|
| 272 |
+
Step 2580/11700 | Loss: 2.7983 | LR: 4.66e-04 | 9148.5ms/step | 2.2 samples/s
|
| 273 |
+
Step 2590/11700 | Loss: 2.7380 | LR: 4.66e-04 | 9147.0ms/step | 2.2 samples/s
|
| 274 |
+
Step 2600/11700 | Loss: 2.7674 | LR: 4.66e-04 | 9145.7ms/step | 2.2 samples/s
|
| 275 |
+
Step 2610/11700 | Loss: 2.6930 | LR: 4.65e-04 | 9140.9ms/step | 2.2 samples/s
|
| 276 |
+
Step 2620/11700 | Loss: 2.7529 | LR: 4.65e-04 | 9141.6ms/step | 2.2 samples/s
|
| 277 |
+
Step 2630/11700 | Loss: 2.7013 | LR: 4.65e-04 | 9142.7ms/step | 2.2 samples/s
|
| 278 |
+
Step 2640/11700 | Loss: 2.8062 | LR: 4.64e-04 | 9143.4ms/step | 2.2 samples/s
|
| 279 |
+
Step 2650/11700 | Loss: 2.8674 | LR: 4.64e-04 | 9141.0ms/step | 2.2 samples/s
|
| 280 |
+
Step 2660/11700 | Loss: 2.7997 | LR: 4.63e-04 | 9141.2ms/step | 2.2 samples/s
|
| 281 |
+
Step 2670/11700 | Loss: 2.7375 | LR: 4.63e-04 | 9142.8ms/step | 2.2 samples/s
|
| 282 |
+
Step 2680/11700 | Loss: 2.7835 | LR: 4.63e-04 | 9143.7ms/step | 2.2 samples/s
|
| 283 |
+
Step 2690/11700 | Loss: 2.8873 | LR: 4.62e-04 | 9143.4ms/step | 2.2 samples/s
|
| 284 |
+
Step 2700/11700 | Loss: 2.7802 | LR: 4.62e-04 | 9143.9ms/step | 2.2 samples/s
|
| 285 |
+
Step 2710/11700 | Loss: 2.7743 | LR: 4.62e-04 | 9144.8ms/step | 2.2 samples/s
|
| 286 |
+
Step 2720/11700 | Loss: 2.7854 | LR: 4.61e-04 | 9142.1ms/step | 2.2 samples/s
|
| 287 |
+
Step 2730/11700 | Loss: 2.6874 | LR: 4.61e-04 | 9147.5ms/step | 2.2 samples/s
|
| 288 |
+
Step 2740/11700 | Loss: 2.7754 | LR: 4.60e-04 | 9144.8ms/step | 2.2 samples/s
|
| 289 |
+
Step 2750/11700 | Loss: 2.8248 | LR: 4.60e-04 | 9149.3ms/step | 2.2 samples/s
|
| 290 |
+
Validation at step 2750: Loss = 2.7661, Perplexity = 15.90
|
| 291 |
+
Step 2760/11700 | Loss: 2.5761 | LR: 4.60e-04 | 22340.4ms/step | 0.9 samples/s
|
| 292 |
+
Step 2770/11700 | Loss: 2.7342 | LR: 4.59e-04 | 9145.3ms/step | 2.2 samples/s
|
| 293 |
+
Step 2780/11700 | Loss: 2.7355 | LR: 4.59e-04 | 9144.4ms/step | 2.2 samples/s
|
| 294 |
+
Step 2790/11700 | Loss: 2.7813 | LR: 4.58e-04 | 9145.2ms/step | 2.2 samples/s
|
| 295 |
+
Step 2800/11700 | Loss: 2.6912 | LR: 4.58e-04 | 9142.6ms/step | 2.2 samples/s
|
| 296 |
+
Step 2810/11700 | Loss: 2.7902 | LR: 4.58e-04 | 9143.8ms/step | 2.2 samples/s
|
| 297 |
+
Step 2820/11700 | Loss: 2.7803 | LR: 4.57e-04 | 9144.7ms/step | 2.2 samples/s
|
| 298 |
+
Step 2830/11700 | Loss: 2.7456 | LR: 4.57e-04 | 9146.3ms/step | 2.2 samples/s
|
| 299 |
+
Step 2840/11700 | Loss: 2.8269 | LR: 4.56e-04 | 9145.1ms/step | 2.2 samples/s
|
| 300 |
+
Step 2850/11700 | Loss: 2.8575 | LR: 4.56e-04 | 9142.7ms/step | 2.2 samples/s
|
| 301 |
+
Step 2860/11700 | Loss: 2.7335 | LR: 4.56e-04 | 9148.2ms/step | 2.2 samples/s
|
| 302 |
+
Step 2870/11700 | Loss: 2.8067 | LR: 4.55e-04 | 9147.9ms/step | 2.2 samples/s
|
| 303 |
+
Step 2880/11700 | Loss: 2.5400 | LR: 4.55e-04 | 9147.9ms/step | 2.2 samples/s
|
| 304 |
+
Step 2890/11700 | Loss: 2.7923 | LR: 4.54e-04 | 9147.3ms/step | 2.2 samples/s
|
| 305 |
+
Step 2900/11700 | Loss: 2.7327 | LR: 4.54e-04 | 9147.8ms/step | 2.2 samples/s
|
| 306 |
+
Step 2910/11700 | Loss: 2.8040 | LR: 4.54e-04 | 9147.3ms/step | 2.2 samples/s
|
| 307 |
+
Step 2920/11700 | Loss: 2.7339 | LR: 4.53e-04 | 9147.7ms/step | 2.2 samples/s
|
| 308 |
+
Step 2930/11700 | Loss: 2.8135 | LR: 4.53e-04 | 9144.5ms/step | 2.2 samples/s
|
| 309 |
+
Step 2940/11700 | Loss: 2.7454 | LR: 4.52e-04 | 9152.0ms/step | 2.2 samples/s
|
| 310 |
+
Step 2950/11700 | Loss: 2.7969 | LR: 4.52e-04 | 9152.4ms/step | 2.2 samples/s
|
| 311 |
+
Step 2960/11700 | Loss: 2.6901 | LR: 4.51e-04 | 9151.7ms/step | 2.2 samples/s
|
| 312 |
+
Step 2970/11700 | Loss: 2.6393 | LR: 4.51e-04 | 9146.9ms/step | 2.2 samples/s
|
| 313 |
+
Step 2980/11700 | Loss: 2.7212 | LR: 4.51e-04 | 9149.6ms/step | 2.2 samples/s
|
| 314 |
+
Step 2990/11700 | Loss: 2.6585 | LR: 4.50e-04 | 9149.9ms/step | 2.2 samples/s
|
| 315 |
+
Step 3000/11700 | Loss: 2.7389 | LR: 4.50e-04 | 9145.6ms/step | 2.2 samples/s
|
| 316 |
+
Validation at step 3000: Loss = 2.7428, Perplexity = 15.53
|
| 317 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-3000
|
| 318 |
+
Step 3010/11700 | Loss: 2.7068 | LR: 4.49e-04 | 22618.7ms/step | 0.9 samples/s
|
| 319 |
+
Step 3020/11700 | Loss: 2.8515 | LR: 4.49e-04 | 9147.8ms/step | 2.2 samples/s
|
| 320 |
+
Step 3030/11700 | Loss: 2.6830 | LR: 4.48e-04 | 9149.6ms/step | 2.2 samples/s
|
| 321 |
+
Step 3040/11700 | Loss: 2.7908 | LR: 4.48e-04 | 9145.6ms/step | 2.2 samples/s
|
| 322 |
+
Step 3050/11700 | Loss: 2.7245 | LR: 4.48e-04 | 9145.2ms/step | 2.2 samples/s
|
| 323 |
+
Step 3060/11700 | Loss: 2.9043 | LR: 4.47e-04 | 9148.4ms/step | 2.2 samples/s
|
| 324 |
+
Step 3070/11700 | Loss: 2.6796 | LR: 4.47e-04 | 9149.0ms/step | 2.2 samples/s
|
| 325 |
+
Step 3080/11700 | Loss: 2.6799 | LR: 4.46e-04 | 9146.6ms/step | 2.2 samples/s
|
| 326 |
+
Step 3090/11700 | Loss: 2.6879 | LR: 4.46e-04 | 9148.9ms/step | 2.2 samples/s
|
| 327 |
+
Step 3100/11700 | Loss: 2.8214 | LR: 4.45e-04 | 9149.2ms/step | 2.2 samples/s
|
| 328 |
+
Step 3110/11700 | Loss: 2.6042 | LR: 4.45e-04 | 9153.4ms/step | 2.2 samples/s
|
| 329 |
+
Step 3120/11700 | Loss: 2.6752 | LR: 4.44e-04 | 9147.7ms/step | 2.2 samples/s
|
| 330 |
+
Step 3130/11700 | Loss: 2.7148 | LR: 4.44e-04 | 9155.7ms/step | 2.2 samples/s
|
| 331 |
+
Step 3140/11700 | Loss: 2.6663 | LR: 4.43e-04 | 9151.3ms/step | 2.2 samples/s
|
| 332 |
+
Step 3150/11700 | Loss: 2.6844 | LR: 4.43e-04 | 9150.6ms/step | 2.2 samples/s
|
| 333 |
+
Step 3160/11700 | Loss: 2.6851 | LR: 4.43e-04 | 9151.3ms/step | 2.2 samples/s
|
| 334 |
+
Step 3170/11700 | Loss: 2.7741 | LR: 4.42e-04 | 9141.1ms/step | 2.2 samples/s
|
| 335 |
+
Step 3180/11700 | Loss: 2.7398 | LR: 4.42e-04 | 9144.5ms/step | 2.2 samples/s
|
| 336 |
+
Step 3190/11700 | Loss: 2.6135 | LR: 4.41e-04 | 9142.9ms/step | 2.2 samples/s
|
| 337 |
+
Step 3200/11700 | Loss: 2.6428 | LR: 4.41e-04 | 9145.1ms/step | 2.2 samples/s
|
| 338 |
+
Step 3210/11700 | Loss: 2.6670 | LR: 4.40e-04 | 9142.9ms/step | 2.2 samples/s
|
| 339 |
+
Step 3220/11700 | Loss: 2.6909 | LR: 4.40e-04 | 9145.3ms/step | 2.2 samples/s
|
| 340 |
+
Step 3230/11700 | Loss: 2.6376 | LR: 4.39e-04 | 9143.7ms/step | 2.2 samples/s
|
| 341 |
+
Step 3240/11700 | Loss: 2.7031 | LR: 4.39e-04 | 9144.2ms/step | 2.2 samples/s
|
| 342 |
+
Step 3250/11700 | Loss: 2.8316 | LR: 4.38e-04 | 9144.3ms/step | 2.2 samples/s
|
| 343 |
+
Validation at step 3250: Loss = 2.7364, Perplexity = 15.43
|
| 344 |
+
Step 3260/11700 | Loss: 2.7379 | LR: 4.38e-04 | 22347.0ms/step | 0.9 samples/s
|
| 345 |
+
Step 3270/11700 | Loss: 2.6646 | LR: 4.37e-04 | 9148.0ms/step | 2.2 samples/s
|
| 346 |
+
Step 3280/11700 | Loss: 2.7152 | LR: 4.37e-04 | 9146.4ms/step | 2.2 samples/s
|
| 347 |
+
Step 3290/11700 | Loss: 2.6564 | LR: 4.37e-04 | 9143.2ms/step | 2.2 samples/s
|
| 348 |
+
Step 3300/11700 | Loss: 2.7192 | LR: 4.36e-04 | 9143.9ms/step | 2.2 samples/s
|
| 349 |
+
Step 3310/11700 | Loss: 2.7195 | LR: 4.36e-04 | 9146.9ms/step | 2.2 samples/s
|
| 350 |
+
Step 3320/11700 | Loss: 2.6384 | LR: 4.35e-04 | 9147.9ms/step | 2.2 samples/s
|
| 351 |
+
Step 3330/11700 | Loss: 2.6238 | LR: 4.35e-04 | 9154.1ms/step | 2.2 samples/s
|
| 352 |
+
Step 3340/11700 | Loss: 2.7555 | LR: 4.34e-04 | 9159.8ms/step | 2.2 samples/s
|
| 353 |
+
Step 3350/11700 | Loss: 2.6588 | LR: 4.34e-04 | 9152.0ms/step | 2.2 samples/s
|
| 354 |
+
Step 3360/11700 | Loss: 2.7379 | LR: 4.33e-04 | 9174.9ms/step | 2.2 samples/s
|
| 355 |
+
Step 3370/11700 | Loss: 2.6215 | LR: 4.33e-04 | 9187.1ms/step | 2.2 samples/s
|
| 356 |
+
Step 3380/11700 | Loss: 2.6871 | LR: 4.32e-04 | 9213.6ms/step | 2.2 samples/s
|
| 357 |
+
Step 3390/11700 | Loss: 2.6531 | LR: 4.32e-04 | 9187.0ms/step | 2.2 samples/s
|
| 358 |
+
Step 3400/11700 | Loss: 2.7664 | LR: 4.31e-04 | 9210.0ms/step | 2.2 samples/s
|
| 359 |
+
Step 3410/11700 | Loss: 2.7072 | LR: 4.31e-04 | 9206.2ms/step | 2.2 samples/s
|
| 360 |
+
Step 3420/11700 | Loss: 2.5366 | LR: 4.30e-04 | 9188.1ms/step | 2.2 samples/s
|
| 361 |
+
Step 3430/11700 | Loss: 2.7227 | LR: 4.30e-04 | 9207.2ms/step | 2.2 samples/s
|
| 362 |
+
Step 3440/11700 | Loss: 2.5776 | LR: 4.29e-04 | 9213.5ms/step | 2.2 samples/s
|
| 363 |
+
Step 3450/11700 | Loss: 2.6877 | LR: 4.29e-04 | 9200.0ms/step | 2.2 samples/s
|
| 364 |
+
Step 3460/11700 | Loss: 2.6685 | LR: 4.28e-04 | 9192.9ms/step | 2.2 samples/s
|
| 365 |
+
Step 3470/11700 | Loss: 2.5592 | LR: 4.28e-04 | 9214.6ms/step | 2.2 samples/s
|
| 366 |
+
Step 3480/11700 | Loss: 2.4814 | LR: 4.27e-04 | 9206.7ms/step | 2.2 samples/s
|
| 367 |
+
Step 3490/11700 | Loss: 2.8105 | LR: 4.27e-04 | 9208.5ms/step | 2.2 samples/s
|
| 368 |
+
Step 3500/11700 | Loss: 2.7242 | LR: 4.26e-04 | 9217.6ms/step | 2.2 samples/s
|
| 369 |
+
Validation at step 3500: Loss = 2.6718, Perplexity = 14.47
|
| 370 |
+
Step 3510/11700 | Loss: 2.7039 | LR: 4.26e-04 | 22433.8ms/step | 0.9 samples/s
|
| 371 |
+
Step 3520/11700 | Loss: 2.6942 | LR: 4.25e-04 | 9137.8ms/step | 2.2 samples/s
|
| 372 |
+
Step 3530/11700 | Loss: 2.7028 | LR: 4.25e-04 | 9149.7ms/step | 2.2 samples/s
|
| 373 |
+
Step 3540/11700 | Loss: 2.7097 | LR: 4.24e-04 | 9145.6ms/step | 2.2 samples/s
|
| 374 |
+
Step 3550/11700 | Loss: 2.6274 | LR: 4.24e-04 | 9150.6ms/step | 2.2 samples/s
|
| 375 |
+
Step 3560/11700 | Loss: 2.8224 | LR: 4.23e-04 | 9144.4ms/step | 2.2 samples/s
|
| 376 |
+
Step 3570/11700 | Loss: 2.6953 | LR: 4.23e-04 | 9149.1ms/step | 2.2 samples/s
|
| 377 |
+
Step 3580/11700 | Loss: 2.7066 | LR: 4.22e-04 | 9155.1ms/step | 2.2 samples/s
|
| 378 |
+
Step 3590/11700 | Loss: 2.6058 | LR: 4.22e-04 | 9154.7ms/step | 2.2 samples/s
|
| 379 |
+
Step 3600/11700 | Loss: 2.6026 | LR: 4.21e-04 | 9154.7ms/step | 2.2 samples/s
|
| 380 |
+
Step 3610/11700 | Loss: 2.7301 | LR: 4.20e-04 | 9152.7ms/step | 2.2 samples/s
|
| 381 |
+
Step 3620/11700 | Loss: 2.5953 | LR: 4.20e-04 | 9172.3ms/step | 2.2 samples/s
|
| 382 |
+
Step 3630/11700 | Loss: 2.6029 | LR: 4.19e-04 | 9178.6ms/step | 2.2 samples/s
|
| 383 |
+
Step 3640/11700 | Loss: 2.5528 | LR: 4.19e-04 | 9183.8ms/step | 2.2 samples/s
|
| 384 |
+
Step 3650/11700 | Loss: 2.6825 | LR: 4.18e-04 | 9179.0ms/step | 2.2 samples/s
|
| 385 |
+
Step 3660/11700 | Loss: 2.6443 | LR: 4.18e-04 | 9175.6ms/step | 2.2 samples/s
|
| 386 |
+
Step 3670/11700 | Loss: 2.6032 | LR: 4.17e-04 | 9181.3ms/step | 2.2 samples/s
|
| 387 |
+
Step 3680/11700 | Loss: 2.6991 | LR: 4.17e-04 | 9183.1ms/step | 2.2 samples/s
|
| 388 |
+
Step 3690/11700 | Loss: 2.6377 | LR: 4.16e-04 | 9180.6ms/step | 2.2 samples/s
|
| 389 |
+
Step 3700/11700 | Loss: 2.6789 | LR: 4.16e-04 | 9180.5ms/step | 2.2 samples/s
|
| 390 |
+
Step 3710/11700 | Loss: 2.5817 | LR: 4.15e-04 | 9185.4ms/step | 2.2 samples/s
|
| 391 |
+
Step 3720/11700 | Loss: 2.5925 | LR: 4.15e-04 | 9181.9ms/step | 2.2 samples/s
|
| 392 |
+
Step 3730/11700 | Loss: 2.6544 | LR: 4.14e-04 | 9174.1ms/step | 2.2 samples/s
|
| 393 |
+
Step 3740/11700 | Loss: 2.6027 | LR: 4.14e-04 | 9181.6ms/step | 2.2 samples/s
|
| 394 |
+
Step 3750/11700 | Loss: 2.6788 | LR: 4.13e-04 | 9183.7ms/step | 2.2 samples/s
|
| 395 |
+
Validation at step 3750: Loss = 2.6482, Perplexity = 14.13
|
| 396 |
+
Step 3760/11700 | Loss: 2.6879 | LR: 4.12e-04 | 22419.5ms/step | 0.9 samples/s
|
| 397 |
+
Step 3770/11700 | Loss: 2.6824 | LR: 4.12e-04 | 9180.1ms/step | 2.2 samples/s
|
| 398 |
+
Step 3780/11700 | Loss: 2.6346 | LR: 4.11e-04 | 9174.3ms/step | 2.2 samples/s
|
| 399 |
+
Step 3790/11700 | Loss: 2.7748 | LR: 4.11e-04 | 9174.6ms/step | 2.2 samples/s
|
| 400 |
+
Step 3800/11700 | Loss: 2.6658 | LR: 4.10e-04 | 9179.2ms/step | 2.2 samples/s
|
| 401 |
+
Step 3810/11700 | Loss: 2.6197 | LR: 4.10e-04 | 9177.4ms/step | 2.2 samples/s
|
| 402 |
+
Step 3820/11700 | Loss: 2.6065 | LR: 4.09e-04 | 9183.6ms/step | 2.2 samples/s
|
| 403 |
+
Step 3830/11700 | Loss: 2.6302 | LR: 4.09e-04 | 9186.6ms/step | 2.2 samples/s
|
| 404 |
+
Step 3840/11700 | Loss: 2.5601 | LR: 4.08e-04 | 9185.5ms/step | 2.2 samples/s
|
| 405 |
+
Step 3850/11700 | Loss: 2.6774 | LR: 4.07e-04 | 9178.6ms/step | 2.2 samples/s
|
| 406 |
+
Step 3860/11700 | Loss: 2.5934 | LR: 4.07e-04 | 9179.5ms/step | 2.2 samples/s
|
| 407 |
+
Step 3870/11700 | Loss: 2.6261 | LR: 4.06e-04 | 9183.3ms/step | 2.2 samples/s
|
| 408 |
+
Step 3880/11700 | Loss: 2.5092 | LR: 4.06e-04 | 9190.1ms/step | 2.2 samples/s
|
| 409 |
+
Step 3890/11700 | Loss: 2.6865 | LR: 4.05e-04 | 9181.2ms/step | 2.2 samples/s
|
| 410 |
+
Step 3900/11700 | Loss: 2.5557 | LR: 4.05e-04 | 9181.2ms/step | 2.2 samples/s
|
| 411 |
+
Step 3910/11700 | Loss: 2.6462 | LR: 4.04e-04 | 9187.3ms/step | 2.2 samples/s
|
| 412 |
+
Step 3920/11700 | Loss: 2.5508 | LR: 4.04e-04 | 9187.3ms/step | 2.2 samples/s
|
| 413 |
+
Step 3930/11700 | Loss: 2.6112 | LR: 4.03e-04 | 9190.3ms/step | 2.2 samples/s
|
| 414 |
+
Step 3940/11700 | Loss: 2.7508 | LR: 4.02e-04 | 9187.2ms/step | 2.2 samples/s
|
| 415 |
+
Step 3950/11700 | Loss: 2.5759 | LR: 4.02e-04 | 9179.6ms/step | 2.2 samples/s
|
| 416 |
+
Step 3960/11700 | Loss: 2.6340 | LR: 4.01e-04 | 9179.3ms/step | 2.2 samples/s
|
| 417 |
+
Step 3970/11700 | Loss: 2.6121 | LR: 4.01e-04 | 9179.7ms/step | 2.2 samples/s
|
| 418 |
+
Step 3980/11700 | Loss: 2.6706 | LR: 4.00e-04 | 9182.9ms/step | 2.2 samples/s
|
| 419 |
+
Step 3990/11700 | Loss: 2.6512 | LR: 4.00e-04 | 9177.5ms/step | 2.2 samples/s
|
| 420 |
+
Step 4000/11700 | Loss: 2.7034 | LR: 3.99e-04 | 9185.0ms/step | 2.2 samples/s
|
| 421 |
+
Validation at step 4000: Loss = 2.6267, Perplexity = 13.83
|
| 422 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-4000
|
| 423 |
+
Step 4010/11700 | Loss: 2.5210 | LR: 3.98e-04 | 22710.5ms/step | 0.9 samples/s
|
| 424 |
+
Step 4020/11700 | Loss: 2.6185 | LR: 3.98e-04 | 9184.9ms/step | 2.2 samples/s
|
| 425 |
+
Step 4030/11700 | Loss: 2.5127 | LR: 3.97e-04 | 9188.5ms/step | 2.2 samples/s
|
| 426 |
+
Step 4040/11700 | Loss: 2.6520 | LR: 3.97e-04 | 9191.1ms/step | 2.2 samples/s
|
| 427 |
+
Step 4050/11700 | Loss: 2.6454 | LR: 3.96e-04 | 9178.5ms/step | 2.2 samples/s
|
| 428 |
+
Step 4060/11700 | Loss: 2.5226 | LR: 3.95e-04 | 9177.4ms/step | 2.2 samples/s
|
| 429 |
+
Step 4070/11700 | Loss: 2.7078 | LR: 3.95e-04 | 9189.6ms/step | 2.2 samples/s
|
| 430 |
+
Step 4080/11700 | Loss: 2.5944 | LR: 3.94e-04 | 9182.0ms/step | 2.2 samples/s
|
| 431 |
+
Step 4090/11700 | Loss: 2.5680 | LR: 3.94e-04 | 9186.4ms/step | 2.2 samples/s
|
| 432 |
+
Step 4100/11700 | Loss: 2.6090 | LR: 3.93e-04 | 9181.1ms/step | 2.2 samples/s
|
| 433 |
+
Step 4110/11700 | Loss: 2.5944 | LR: 3.93e-04 | 9175.6ms/step | 2.2 samples/s
|
| 434 |
+
Step 4120/11700 | Loss: 2.6656 | LR: 3.92e-04 | 9187.2ms/step | 2.2 samples/s
|
| 435 |
+
Step 4130/11700 | Loss: 2.5277 | LR: 3.91e-04 | 9163.1ms/step | 2.2 samples/s
|
| 436 |
+
Step 4140/11700 | Loss: 2.5843 | LR: 3.91e-04 | 9182.8ms/step | 2.2 samples/s
|
| 437 |
+
Step 4150/11700 | Loss: 2.5183 | LR: 3.90e-04 | 9182.2ms/step | 2.2 samples/s
|
| 438 |
+
Step 4160/11700 | Loss: 2.5944 | LR: 3.90e-04 | 9178.9ms/step | 2.2 samples/s
|
| 439 |
+
Step 4170/11700 | Loss: 2.6527 | LR: 3.89e-04 | 9181.1ms/step | 2.2 samples/s
|
| 440 |
+
Step 4180/11700 | Loss: 2.5931 | LR: 3.88e-04 | 9181.5ms/step | 2.2 samples/s
|
| 441 |
+
Step 4190/11700 | Loss: 2.6259 | LR: 3.88e-04 | 9182.0ms/step | 2.2 samples/s
|
| 442 |
+
Step 4200/11700 | Loss: 2.5414 | LR: 3.87e-04 | 9147.4ms/step | 2.2 samples/s
|
| 443 |
+
Step 4210/11700 | Loss: 2.6445 | LR: 3.87e-04 | 9161.8ms/step | 2.2 samples/s
|
| 444 |
+
Step 4220/11700 | Loss: 2.6078 | LR: 3.86e-04 | 9176.1ms/step | 2.2 samples/s
|
| 445 |
+
Step 4230/11700 | Loss: 2.5248 | LR: 3.85e-04 | 9182.6ms/step | 2.2 samples/s
|
| 446 |
+
Step 4240/11700 | Loss: 2.6356 | LR: 3.85e-04 | 9105.1ms/step | 2.2 samples/s
|
| 447 |
+
Step 4250/11700 | Loss: 2.7374 | LR: 3.84e-04 | 9091.7ms/step | 2.2 samples/s
|
| 448 |
+
Validation at step 4250: Loss = 2.6084, Perplexity = 13.58
|
| 449 |
+
Step 4260/11700 | Loss: 2.7040 | LR: 3.84e-04 | 22210.4ms/step | 0.9 samples/s
|
| 450 |
+
Step 4270/11700 | Loss: 2.5536 | LR: 3.83e-04 | 9097.8ms/step | 2.2 samples/s
|
| 451 |
+
Step 4280/11700 | Loss: 2.5853 | LR: 3.82e-04 | 9098.5ms/step | 2.2 samples/s
|
| 452 |
+
Step 4290/11700 | Loss: 2.5905 | LR: 3.82e-04 | 9109.1ms/step | 2.2 samples/s
|
| 453 |
+
Step 4300/11700 | Loss: 2.6446 | LR: 3.81e-04 | 9094.8ms/step | 2.2 samples/s
|
| 454 |
+
Step 4310/11700 | Loss: 2.5708 | LR: 3.81e-04 | 9090.3ms/step | 2.2 samples/s
|
| 455 |
+
Step 4320/11700 | Loss: 2.5501 | LR: 3.80e-04 | 9091.2ms/step | 2.2 samples/s
|
| 456 |
+
Step 4330/11700 | Loss: 2.5576 | LR: 3.79e-04 | 9088.6ms/step | 2.2 samples/s
|
| 457 |
+
Step 4340/11700 | Loss: 2.6276 | LR: 3.79e-04 | 9091.7ms/step | 2.2 samples/s
|
| 458 |
+
Step 4350/11700 | Loss: 2.7014 | LR: 3.78e-04 | 9097.6ms/step | 2.2 samples/s
|
| 459 |
+
Step 4360/11700 | Loss: 2.5363 | LR: 3.77e-04 | 9104.7ms/step | 2.2 samples/s
|
| 460 |
+
Step 4370/11700 | Loss: 2.5719 | LR: 3.77e-04 | 9099.0ms/step | 2.2 samples/s
|
| 461 |
+
Step 4380/11700 | Loss: 2.5311 | LR: 3.76e-04 | 9094.4ms/step | 2.2 samples/s
|
| 462 |
+
Step 4390/11700 | Loss: 2.6035 | LR: 3.76e-04 | 9098.4ms/step | 2.2 samples/s
|
| 463 |
+
Step 4400/11700 | Loss: 2.5941 | LR: 3.75e-04 | 9099.8ms/step | 2.2 samples/s
|
| 464 |
+
Step 4410/11700 | Loss: 2.6133 | LR: 3.74e-04 | 9100.1ms/step | 2.2 samples/s
|
| 465 |
+
Step 4420/11700 | Loss: 2.6019 | LR: 3.74e-04 | 9100.1ms/step | 2.2 samples/s
|
| 466 |
+
Step 4430/11700 | Loss: 2.6283 | LR: 3.73e-04 | 9095.8ms/step | 2.2 samples/s
|
| 467 |
+
Step 4440/11700 | Loss: 2.6294 | LR: 3.73e-04 | 9094.4ms/step | 2.2 samples/s
|
| 468 |
+
Step 4450/11700 | Loss: 2.5654 | LR: 3.72e-04 | 9087.1ms/step | 2.2 samples/s
|
| 469 |
+
Step 4460/11700 | Loss: 2.6786 | LR: 3.71e-04 | 9089.1ms/step | 2.2 samples/s
|
| 470 |
+
Step 4470/11700 | Loss: 2.4859 | LR: 3.71e-04 | 9087.2ms/step | 2.2 samples/s
|
| 471 |
+
Step 4480/11700 | Loss: 2.5625 | LR: 3.70e-04 | 9090.8ms/step | 2.2 samples/s
|
| 472 |
+
Step 4490/11700 | Loss: 2.5761 | LR: 3.69e-04 | 9086.7ms/step | 2.2 samples/s
|
| 473 |
+
Step 4500/11700 | Loss: 2.5913 | LR: 3.69e-04 | 9086.2ms/step | 2.2 samples/s
|
| 474 |
+
Validation at step 4500: Loss = 2.5905, Perplexity = 13.34
|
| 475 |
+
Step 4510/11700 | Loss: 2.4838 | LR: 3.68e-04 | 22193.3ms/step | 0.9 samples/s
|
| 476 |
+
Step 4520/11700 | Loss: 2.6065 | LR: 3.67e-04 | 9088.9ms/step | 2.2 samples/s
|
| 477 |
+
Step 4530/11700 | Loss: 2.7042 | LR: 3.67e-04 | 9088.6ms/step | 2.2 samples/s
|
| 478 |
+
Step 4540/11700 | Loss: 2.6265 | LR: 3.66e-04 | 9097.4ms/step | 2.2 samples/s
|
| 479 |
+
Step 4550/11700 | Loss: 2.5595 | LR: 3.66e-04 | 9085.1ms/step | 2.2 samples/s
|
| 480 |
+
Step 4560/11700 | Loss: 2.5309 | LR: 3.65e-04 | 9085.8ms/step | 2.2 samples/s
|
| 481 |
+
Step 4570/11700 | Loss: 2.5565 | LR: 3.64e-04 | 9091.1ms/step | 2.2 samples/s
|
| 482 |
+
Step 4580/11700 | Loss: 2.5183 | LR: 3.64e-04 | 9087.7ms/step | 2.2 samples/s
|
| 483 |
+
Step 4590/11700 | Loss: 2.6746 | LR: 3.63e-04 | 9088.0ms/step | 2.2 samples/s
|
| 484 |
+
Step 4600/11700 | Loss: 2.4194 | LR: 3.62e-04 | 9093.4ms/step | 2.2 samples/s
|
| 485 |
+
Step 4610/11700 | Loss: 2.4734 | LR: 3.62e-04 | 9099.5ms/step | 2.2 samples/s
|
| 486 |
+
Step 4620/11700 | Loss: 2.5818 | LR: 3.61e-04 | 9086.5ms/step | 2.2 samples/s
|
| 487 |
+
Step 4630/11700 | Loss: 2.5878 | LR: 3.60e-04 | 9101.8ms/step | 2.2 samples/s
|
| 488 |
+
Step 4640/11700 | Loss: 2.5371 | LR: 3.60e-04 | 9086.3ms/step | 2.2 samples/s
|
| 489 |
+
Step 4650/11700 | Loss: 2.5379 | LR: 3.59e-04 | 9087.5ms/step | 2.2 samples/s
|
| 490 |
+
Step 4660/11700 | Loss: 2.6097 | LR: 3.59e-04 | 9089.8ms/step | 2.2 samples/s
|
| 491 |
+
Step 4670/11700 | Loss: 2.5240 | LR: 3.58e-04 | 9087.5ms/step | 2.2 samples/s
|
| 492 |
+
Step 4680/11700 | Loss: 2.6091 | LR: 3.57e-04 | 9085.7ms/step | 2.2 samples/s
|
| 493 |
+
Step 4690/11700 | Loss: 2.5086 | LR: 3.57e-04 | 9087.1ms/step | 2.2 samples/s
|
| 494 |
+
Step 4700/11700 | Loss: 2.5090 | LR: 3.56e-04 | 9083.1ms/step | 2.2 samples/s
|
| 495 |
+
Step 4710/11700 | Loss: 2.6718 | LR: 3.55e-04 | 9086.3ms/step | 2.2 samples/s
|
| 496 |
+
Step 4720/11700 | Loss: 2.5350 | LR: 3.55e-04 | 9108.2ms/step | 2.2 samples/s
|
| 497 |
+
Step 4730/11700 | Loss: 2.5965 | LR: 3.54e-04 | 9115.2ms/step | 2.2 samples/s
|
| 498 |
+
Step 4740/11700 | Loss: 2.6134 | LR: 3.53e-04 | 9115.1ms/step | 2.2 samples/s
|
| 499 |
+
Step 4750/11700 | Loss: 2.5538 | LR: 3.53e-04 | 9116.7ms/step | 2.2 samples/s
|
| 500 |
+
Validation at step 4750: Loss = 2.5700, Perplexity = 13.07
|
| 501 |
+
Step 4760/11700 | Loss: 2.6443 | LR: 3.52e-04 | 22301.6ms/step | 0.9 samples/s
|
| 502 |
+
Step 4770/11700 | Loss: 2.6017 | LR: 3.51e-04 | 9123.8ms/step | 2.2 samples/s
|
| 503 |
+
Step 4780/11700 | Loss: 2.5689 | LR: 3.51e-04 | 9125.8ms/step | 2.2 samples/s
|
| 504 |
+
Step 4790/11700 | Loss: 2.5420 | LR: 3.50e-04 | 9125.8ms/step | 2.2 samples/s
|
| 505 |
+
Step 4800/11700 | Loss: 2.4844 | LR: 3.49e-04 | 9123.6ms/step | 2.2 samples/s
|
| 506 |
+
Step 4810/11700 | Loss: 2.4562 | LR: 3.49e-04 | 9124.6ms/step | 2.2 samples/s
|
| 507 |
+
Step 4820/11700 | Loss: 2.5596 | LR: 3.48e-04 | 9124.3ms/step | 2.2 samples/s
|
| 508 |
+
Step 4830/11700 | Loss: 2.5792 | LR: 3.47e-04 | 9124.6ms/step | 2.2 samples/s
|
| 509 |
+
Step 4840/11700 | Loss: 2.5768 | LR: 3.47e-04 | 9121.6ms/step | 2.2 samples/s
|
| 510 |
+
Step 4850/11700 | Loss: 2.5850 | LR: 3.46e-04 | 9122.8ms/step | 2.2 samples/s
|
| 511 |
+
Step 4860/11700 | Loss: 2.5276 | LR: 3.45e-04 | 9122.8ms/step | 2.2 samples/s
|
| 512 |
+
Step 4870/11700 | Loss: 2.6563 | LR: 3.45e-04 | 9123.2ms/step | 2.2 samples/s
|
| 513 |
+
Step 4880/11700 | Loss: 2.5872 | LR: 3.44e-04 | 9129.5ms/step | 2.2 samples/s
|
| 514 |
+
Step 4890/11700 | Loss: 2.6480 | LR: 3.43e-04 | 9125.1ms/step | 2.2 samples/s
|
| 515 |
+
Step 4900/11700 | Loss: 2.5856 | LR: 3.43e-04 | 9121.0ms/step | 2.2 samples/s
|
| 516 |
+
Step 4910/11700 | Loss: 2.5337 | LR: 3.42e-04 | 9121.0ms/step | 2.2 samples/s
|
| 517 |
+
Step 4920/11700 | Loss: 2.5076 | LR: 3.41e-04 | 9123.2ms/step | 2.2 samples/s
|
| 518 |
+
Step 4930/11700 | Loss: 2.4733 | LR: 3.41e-04 | 9127.2ms/step | 2.2 samples/s
|
| 519 |
+
Step 4940/11700 | Loss: 2.5986 | LR: 3.40e-04 | 9123.7ms/step | 2.2 samples/s
|
| 520 |
+
Step 4950/11700 | Loss: 2.4187 | LR: 3.39e-04 | 9107.5ms/step | 2.2 samples/s
|
| 521 |
+
Step 4960/11700 | Loss: 2.5819 | LR: 3.39e-04 | 9120.8ms/step | 2.2 samples/s
|
| 522 |
+
Step 4970/11700 | Loss: 2.5678 | LR: 3.38e-04 | 9122.9ms/step | 2.2 samples/s
|
| 523 |
+
Step 4980/11700 | Loss: 2.6756 | LR: 3.37e-04 | 9126.2ms/step | 2.2 samples/s
|
| 524 |
+
Step 4990/11700 | Loss: 2.4945 | LR: 3.37e-04 | 9124.5ms/step | 2.2 samples/s
|
| 525 |
+
Step 5000/11700 | Loss: 2.6561 | LR: 3.36e-04 | 9124.5ms/step | 2.2 samples/s
|
| 526 |
+
Validation at step 5000: Loss = 2.5532, Perplexity = 12.85
|
| 527 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-5000
|
| 528 |
+
Step 5010/11700 | Loss: 2.6522 | LR: 3.35e-04 | 22609.3ms/step | 0.9 samples/s
|
| 529 |
+
Step 5020/11700 | Loss: 2.6356 | LR: 3.35e-04 | 9127.6ms/step | 2.2 samples/s
|
| 530 |
+
Step 5030/11700 | Loss: 2.6305 | LR: 3.34e-04 | 9133.8ms/step | 2.2 samples/s
|
| 531 |
+
Step 5040/11700 | Loss: 2.4645 | LR: 3.33e-04 | 9131.4ms/step | 2.2 samples/s
|
| 532 |
+
Step 5050/11700 | Loss: 2.4033 | LR: 3.33e-04 | 9139.6ms/step | 2.2 samples/s
|
| 533 |
+
Step 5060/11700 | Loss: 2.5739 | LR: 3.32e-04 | 9145.8ms/step | 2.2 samples/s
|
| 534 |
+
Step 5070/11700 | Loss: 2.5333 | LR: 3.31e-04 | 9141.6ms/step | 2.2 samples/s
|
| 535 |
+
Step 5080/11700 | Loss: 2.6661 | LR: 3.31e-04 | 9135.2ms/step | 2.2 samples/s
|
| 536 |
+
Step 5090/11700 | Loss: 2.5662 | LR: 3.30e-04 | 9131.1ms/step | 2.2 samples/s
|
| 537 |
+
Step 5100/11700 | Loss: 2.6633 | LR: 3.29e-04 | 9124.5ms/step | 2.2 samples/s
|
| 538 |
+
Step 5110/11700 | Loss: 2.5641 | LR: 3.29e-04 | 9111.3ms/step | 2.2 samples/s
|
| 539 |
+
Step 5120/11700 | Loss: 2.5140 | LR: 3.28e-04 | 9112.7ms/step | 2.2 samples/s
|
| 540 |
+
Step 5130/11700 | Loss: 2.4283 | LR: 3.27e-04 | 9114.7ms/step | 2.2 samples/s
|
| 541 |
+
Step 5140/11700 | Loss: 2.4814 | LR: 3.27e-04 | 9121.6ms/step | 2.2 samples/s
|
| 542 |
+
Step 5150/11700 | Loss: 2.5493 | LR: 3.26e-04 | 9142.0ms/step | 2.2 samples/s
|
| 543 |
+
Step 5160/11700 | Loss: 2.4954 | LR: 3.25e-04 | 9138.3ms/step | 2.2 samples/s
|
| 544 |
+
Step 5170/11700 | Loss: 2.5028 | LR: 3.25e-04 | 9138.4ms/step | 2.2 samples/s
|
| 545 |
+
Step 5180/11700 | Loss: 2.4840 | LR: 3.24e-04 | 9144.3ms/step | 2.2 samples/s
|
| 546 |
+
Step 5190/11700 | Loss: 2.5557 | LR: 3.23e-04 | 9142.2ms/step | 2.2 samples/s
|
| 547 |
+
Step 5200/11700 | Loss: 2.3914 | LR: 3.22e-04 | 9146.8ms/step | 2.2 samples/s
|
| 548 |
+
Step 5210/11700 | Loss: 2.5091 | LR: 3.22e-04 | 9137.4ms/step | 2.2 samples/s
|
| 549 |
+
Step 5220/11700 | Loss: 2.5639 | LR: 3.21e-04 | 9147.2ms/step | 2.2 samples/s
|
| 550 |
+
Step 5230/11700 | Loss: 2.6396 | LR: 3.20e-04 | 9147.1ms/step | 2.2 samples/s
|
| 551 |
+
Step 5240/11700 | Loss: 2.5145 | LR: 3.20e-04 | 9140.8ms/step | 2.2 samples/s
|
| 552 |
+
Step 5250/11700 | Loss: 2.5530 | LR: 3.19e-04 | 9134.5ms/step | 2.2 samples/s
|
| 553 |
+
Validation at step 5250: Loss = 2.5364, Perplexity = 12.63
|
| 554 |
+
Step 5260/11700 | Loss: 2.5480 | LR: 3.18e-04 | 22340.4ms/step | 0.9 samples/s
|
| 555 |
+
Step 5270/11700 | Loss: 2.5259 | LR: 3.18e-04 | 9141.2ms/step | 2.2 samples/s
|
| 556 |
+
Step 5280/11700 | Loss: 2.4820 | LR: 3.17e-04 | 9136.9ms/step | 2.2 samples/s
|
| 557 |
+
Step 5290/11700 | Loss: 2.4899 | LR: 3.16e-04 | 9135.1ms/step | 2.2 samples/s
|
| 558 |
+
Step 5300/11700 | Loss: 2.5074 | LR: 3.16e-04 | 9145.1ms/step | 2.2 samples/s
|
| 559 |
+
Step 5310/11700 | Loss: 2.4973 | LR: 3.15e-04 | 9143.2ms/step | 2.2 samples/s
|
| 560 |
+
Step 5320/11700 | Loss: 2.4731 | LR: 3.14e-04 | 9142.1ms/step | 2.2 samples/s
|
| 561 |
+
Step 5330/11700 | Loss: 2.4606 | LR: 3.13e-04 | 9144.4ms/step | 2.2 samples/s
|
| 562 |
+
Step 5340/11700 | Loss: 2.4969 | LR: 3.13e-04 | 9141.1ms/step | 2.2 samples/s
|
| 563 |
+
Step 5350/11700 | Loss: 2.4974 | LR: 3.12e-04 | 9138.0ms/step | 2.2 samples/s
|
| 564 |
+
Step 5360/11700 | Loss: 2.4239 | LR: 3.11e-04 | 9143.3ms/step | 2.2 samples/s
|
| 565 |
+
Step 5370/11700 | Loss: 2.5292 | LR: 3.11e-04 | 9148.0ms/step | 2.2 samples/s
|
| 566 |
+
Step 5380/11700 | Loss: 2.4242 | LR: 3.10e-04 | 9147.3ms/step | 2.2 samples/s
|
| 567 |
+
Step 5390/11700 | Loss: 2.5637 | LR: 3.09e-04 | 9145.0ms/step | 2.2 samples/s
|
| 568 |
+
Step 5400/11700 | Loss: 2.4731 | LR: 3.09e-04 | 9145.7ms/step | 2.2 samples/s
|
| 569 |
+
Step 5410/11700 | Loss: 2.4578 | LR: 3.08e-04 | 9145.7ms/step | 2.2 samples/s
|
| 570 |
+
Step 5420/11700 | Loss: 2.5254 | LR: 3.07e-04 | 9148.2ms/step | 2.2 samples/s
|
| 571 |
+
Step 5430/11700 | Loss: 2.5997 | LR: 3.07e-04 | 9145.5ms/step | 2.2 samples/s
|
| 572 |
+
Step 5440/11700 | Loss: 2.6039 | LR: 3.06e-04 | 9141.0ms/step | 2.2 samples/s
|
| 573 |
+
Step 5450/11700 | Loss: 2.4865 | LR: 3.05e-04 | 9143.9ms/step | 2.2 samples/s
|
| 574 |
+
Step 5460/11700 | Loss: 2.5275 | LR: 3.04e-04 | 9144.4ms/step | 2.2 samples/s
|
| 575 |
+
Step 5470/11700 | Loss: 2.3889 | LR: 3.04e-04 | 9143.7ms/step | 2.2 samples/s
|
| 576 |
+
Step 5480/11700 | Loss: 2.4927 | LR: 3.03e-04 | 9146.4ms/step | 2.2 samples/s
|
| 577 |
+
Step 5490/11700 | Loss: 2.6596 | LR: 3.02e-04 | 9146.4ms/step | 2.2 samples/s
|
| 578 |
+
Step 5500/11700 | Loss: 2.5612 | LR: 3.02e-04 | 9144.6ms/step | 2.2 samples/s
|
| 579 |
+
Validation at step 5500: Loss = 2.5212, Perplexity = 12.44
|
| 580 |
+
Step 5510/11700 | Loss: 2.4003 | LR: 3.01e-04 | 22347.8ms/step | 0.9 samples/s
|
| 581 |
+
Step 5520/11700 | Loss: 2.3806 | LR: 3.00e-04 | 9147.3ms/step | 2.2 samples/s
|
| 582 |
+
Step 5530/11700 | Loss: 2.5384 | LR: 3.00e-04 | 9140.6ms/step | 2.2 samples/s
|
| 583 |
+
Step 5540/11700 | Loss: 2.5183 | LR: 2.99e-04 | 9144.0ms/step | 2.2 samples/s
|
| 584 |
+
Step 5550/11700 | Loss: 2.5236 | LR: 2.98e-04 | 9146.6ms/step | 2.2 samples/s
|
| 585 |
+
Step 5560/11700 | Loss: 2.5641 | LR: 2.97e-04 | 9142.3ms/step | 2.2 samples/s
|
| 586 |
+
Step 5570/11700 | Loss: 2.4370 | LR: 2.97e-04 | 9144.8ms/step | 2.2 samples/s
|
| 587 |
+
Step 5580/11700 | Loss: 2.5312 | LR: 2.96e-04 | 9146.5ms/step | 2.2 samples/s
|
| 588 |
+
Step 5590/11700 | Loss: 2.5564 | LR: 2.95e-04 | 9148.6ms/step | 2.2 samples/s
|
| 589 |
+
Step 5600/11700 | Loss: 2.5006 | LR: 2.95e-04 | 9145.2ms/step | 2.2 samples/s
|
| 590 |
+
Step 5610/11700 | Loss: 2.3596 | LR: 2.94e-04 | 9146.3ms/step | 2.2 samples/s
|
| 591 |
+
Step 5620/11700 | Loss: 2.5198 | LR: 2.93e-04 | 9140.5ms/step | 2.2 samples/s
|
| 592 |
+
Step 5630/11700 | Loss: 2.4093 | LR: 2.92e-04 | 9142.6ms/step | 2.2 samples/s
|
| 593 |
+
Step 5640/11700 | Loss: 2.5402 | LR: 2.92e-04 | 9146.3ms/step | 2.2 samples/s
|
| 594 |
+
Step 5650/11700 | Loss: 2.5929 | LR: 2.91e-04 | 9143.9ms/step | 2.2 samples/s
|
| 595 |
+
Step 5660/11700 | Loss: 2.4986 | LR: 2.90e-04 | 9147.2ms/step | 2.2 samples/s
|
| 596 |
+
Step 5670/11700 | Loss: 2.4853 | LR: 2.90e-04 | 9146.6ms/step | 2.2 samples/s
|
| 597 |
+
Step 5680/11700 | Loss: 2.5581 | LR: 2.89e-04 | 9144.0ms/step | 2.2 samples/s
|
| 598 |
+
Step 5690/11700 | Loss: 2.4697 | LR: 2.88e-04 | 9145.1ms/step | 2.2 samples/s
|
| 599 |
+
Step 5700/11700 | Loss: 2.4927 | LR: 2.88e-04 | 9145.0ms/step | 2.2 samples/s
|
| 600 |
+
Step 5710/11700 | Loss: 2.4671 | LR: 2.87e-04 | 9140.2ms/step | 2.2 samples/s
|
| 601 |
+
Step 5720/11700 | Loss: 2.4447 | LR: 2.86e-04 | 9144.8ms/step | 2.2 samples/s
|
| 602 |
+
Step 5730/11700 | Loss: 2.5575 | LR: 2.85e-04 | 9148.4ms/step | 2.2 samples/s
|
| 603 |
+
Step 5740/11700 | Loss: 2.4266 | LR: 2.85e-04 | 9149.9ms/step | 2.2 samples/s
|
| 604 |
+
Step 5750/11700 | Loss: 2.4603 | LR: 2.84e-04 | 9164.8ms/step | 2.2 samples/s
|
| 605 |
+
Validation at step 5750: Loss = 2.5089, Perplexity = 12.29
|
| 606 |
+
Step 5760/11700 | Loss: 2.4971 | LR: 2.83e-04 | 22359.4ms/step | 0.9 samples/s
|
| 607 |
+
Step 5770/11700 | Loss: 2.6273 | LR: 2.83e-04 | 9163.2ms/step | 2.2 samples/s
|
| 608 |
+
Step 5780/11700 | Loss: 2.4672 | LR: 2.82e-04 | 9157.2ms/step | 2.2 samples/s
|
| 609 |
+
Step 5790/11700 | Loss: 2.5947 | LR: 2.81e-04 | 9157.8ms/step | 2.2 samples/s
|
| 610 |
+
Step 5800/11700 | Loss: 2.5658 | LR: 2.80e-04 | 9143.3ms/step | 2.2 samples/s
|
| 611 |
+
Step 5810/11700 | Loss: 2.5349 | LR: 2.80e-04 | 9155.1ms/step | 2.2 samples/s
|
| 612 |
+
Step 5820/11700 | Loss: 2.4280 | LR: 2.79e-04 | 9134.7ms/step | 2.2 samples/s
|
| 613 |
+
Step 5830/11700 | Loss: 2.4763 | LR: 2.78e-04 | 9144.2ms/step | 2.2 samples/s
|
| 614 |
+
Step 5840/11700 | Loss: 2.4669 | LR: 2.78e-04 | 9142.8ms/step | 2.2 samples/s
|
| 615 |
+
Step 5850/11700 | Loss: 2.4744 | LR: 2.77e-04 | 9143.0ms/step | 2.2 samples/s
|
| 616 |
+
Step 5860/11700 | Loss: 2.5493 | LR: 2.76e-04 | 9145.0ms/step | 2.2 samples/s
|
| 617 |
+
Step 5870/11700 | Loss: 2.5632 | LR: 2.75e-04 | 9136.9ms/step | 2.2 samples/s
|
| 618 |
+
Step 5880/11700 | Loss: 2.5382 | LR: 2.75e-04 | 9144.8ms/step | 2.2 samples/s
|
| 619 |
+
Step 5890/11700 | Loss: 2.4765 | LR: 2.74e-04 | 9143.4ms/step | 2.2 samples/s
|
| 620 |
+
Step 5900/11700 | Loss: 2.4887 | LR: 2.73e-04 | 9145.4ms/step | 2.2 samples/s
|
| 621 |
+
Step 5910/11700 | Loss: 2.5337 | LR: 2.73e-04 | 9143.6ms/step | 2.2 samples/s
|
| 622 |
+
Step 5920/11700 | Loss: 2.4420 | LR: 2.72e-04 | 9144.9ms/step | 2.2 samples/s
|
| 623 |
+
Step 5930/11700 | Loss: 2.4724 | LR: 2.71e-04 | 9139.7ms/step | 2.2 samples/s
|
| 624 |
+
Step 5940/11700 | Loss: 2.3999 | LR: 2.70e-04 | 9136.9ms/step | 2.2 samples/s
|
| 625 |
+
Step 5950/11700 | Loss: 2.5692 | LR: 2.70e-04 | 9145.2ms/step | 2.2 samples/s
|
| 626 |
+
Step 5960/11700 | Loss: 2.5918 | LR: 2.69e-04 | 9144.4ms/step | 2.2 samples/s
|
| 627 |
+
Step 5970/11700 | Loss: 2.4833 | LR: 2.68e-04 | 9144.8ms/step | 2.2 samples/s
|
| 628 |
+
Step 5980/11700 | Loss: 2.5034 | LR: 2.68e-04 | 9141.5ms/step | 2.2 samples/s
|
| 629 |
+
Step 5990/11700 | Loss: 2.4954 | LR: 2.67e-04 | 9147.2ms/step | 2.2 samples/s
|
| 630 |
+
Step 6000/11700 | Loss: 2.4905 | LR: 2.66e-04 | 9143.5ms/step | 2.2 samples/s
|
| 631 |
+
Validation at step 6000: Loss = 2.4989, Perplexity = 12.17
|
| 632 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-6000
|
| 633 |
+
Step 6010/11700 | Loss: 2.5441 | LR: 2.65e-04 | 22572.9ms/step | 0.9 samples/s
|
| 634 |
+
Step 6020/11700 | Loss: 2.4072 | LR: 2.65e-04 | 9107.4ms/step | 2.2 samples/s
|
| 635 |
+
Step 6030/11700 | Loss: 2.4658 | LR: 2.64e-04 | 9102.4ms/step | 2.2 samples/s
|
| 636 |
+
Step 6040/11700 | Loss: 2.4316 | LR: 2.63e-04 | 9107.2ms/step | 2.2 samples/s
|
| 637 |
+
Step 6050/11700 | Loss: 2.4332 | LR: 2.63e-04 | 9109.8ms/step | 2.2 samples/s
|
| 638 |
+
Step 6060/11700 | Loss: 2.5352 | LR: 2.62e-04 | 9103.8ms/step | 2.2 samples/s
|
| 639 |
+
Step 6070/11700 | Loss: 2.5268 | LR: 2.61e-04 | 9086.7ms/step | 2.2 samples/s
|
| 640 |
+
Step 6080/11700 | Loss: 2.5599 | LR: 2.60e-04 | 9090.2ms/step | 2.2 samples/s
|
| 641 |
+
Step 6090/11700 | Loss: 2.4253 | LR: 2.60e-04 | 9092.1ms/step | 2.2 samples/s
|
| 642 |
+
Step 6100/11700 | Loss: 2.4976 | LR: 2.59e-04 | 9087.8ms/step | 2.2 samples/s
|
| 643 |
+
Step 6110/11700 | Loss: 2.4302 | LR: 2.58e-04 | 9111.5ms/step | 2.2 samples/s
|
| 644 |
+
Step 6120/11700 | Loss: 2.5551 | LR: 2.58e-04 | 9103.2ms/step | 2.2 samples/s
|
| 645 |
+
Step 6130/11700 | Loss: 2.4852 | LR: 2.57e-04 | 9088.5ms/step | 2.2 samples/s
|
| 646 |
+
Step 6140/11700 | Loss: 2.4975 | LR: 2.56e-04 | 9093.7ms/step | 2.2 samples/s
|
| 647 |
+
Step 6150/11700 | Loss: 2.4709 | LR: 2.55e-04 | 9098.2ms/step | 2.2 samples/s
|
| 648 |
+
Step 6160/11700 | Loss: 2.4090 | LR: 2.55e-04 | 9091.3ms/step | 2.2 samples/s
|
| 649 |
+
Step 6170/11700 | Loss: 2.4188 | LR: 2.54e-04 | 9086.8ms/step | 2.2 samples/s
|
| 650 |
+
Step 6180/11700 | Loss: 2.3879 | LR: 2.53e-04 | 9091.1ms/step | 2.2 samples/s
|
| 651 |
+
Step 6190/11700 | Loss: 2.4477 | LR: 2.53e-04 | 9104.0ms/step | 2.2 samples/s
|
| 652 |
+
Step 6200/11700 | Loss: 2.5750 | LR: 2.52e-04 | 9091.9ms/step | 2.2 samples/s
|
| 653 |
+
Step 6210/11700 | Loss: 2.5572 | LR: 2.51e-04 | 9087.3ms/step | 2.2 samples/s
|
| 654 |
+
Step 6220/11700 | Loss: 2.5046 | LR: 2.50e-04 | 9079.4ms/step | 2.2 samples/s
|
| 655 |
+
Step 6230/11700 | Loss: 2.6126 | LR: 2.50e-04 | 9095.4ms/step | 2.2 samples/s
|
| 656 |
+
Step 6240/11700 | Loss: 2.4705 | LR: 2.49e-04 | 9097.9ms/step | 2.2 samples/s
|
| 657 |
+
Step 6250/11700 | Loss: 2.5093 | LR: 2.48e-04 | 9086.6ms/step | 2.2 samples/s
|
| 658 |
+
Validation at step 6250: Loss = 2.4877, Perplexity = 12.03
|
| 659 |
+
Step 6260/11700 | Loss: 2.4028 | LR: 2.47e-04 | 22194.2ms/step | 0.9 samples/s
|
| 660 |
+
Step 6270/11700 | Loss: 2.4997 | LR: 2.47e-04 | 9095.5ms/step | 2.2 samples/s
|
| 661 |
+
Step 6280/11700 | Loss: 2.4763 | LR: 2.46e-04 | 9088.6ms/step | 2.2 samples/s
|
| 662 |
+
Step 6290/11700 | Loss: 2.4564 | LR: 2.45e-04 | 9086.5ms/step | 2.2 samples/s
|
| 663 |
+
Step 6300/11700 | Loss: 2.4645 | LR: 2.45e-04 | 9086.9ms/step | 2.2 samples/s
|
| 664 |
+
Step 6310/11700 | Loss: 2.4618 | LR: 2.44e-04 | 9089.0ms/step | 2.2 samples/s
|
| 665 |
+
Step 6320/11700 | Loss: 2.5257 | LR: 2.43e-04 | 9090.5ms/step | 2.2 samples/s
|
| 666 |
+
Step 6330/11700 | Loss: 2.5899 | LR: 2.42e-04 | 9089.1ms/step | 2.2 samples/s
|
| 667 |
+
Step 6340/11700 | Loss: 2.4999 | LR: 2.42e-04 | 9087.3ms/step | 2.2 samples/s
|
| 668 |
+
Step 6350/11700 | Loss: 2.5105 | LR: 2.41e-04 | 9092.8ms/step | 2.2 samples/s
|
| 669 |
+
Step 6360/11700 | Loss: 2.4084 | LR: 2.40e-04 | 9090.1ms/step | 2.2 samples/s
|
| 670 |
+
Step 6370/11700 | Loss: 2.3882 | LR: 2.40e-04 | 9094.9ms/step | 2.2 samples/s
|
| 671 |
+
Step 6380/11700 | Loss: 2.3609 | LR: 2.39e-04 | 9087.4ms/step | 2.2 samples/s
|
| 672 |
+
Step 6390/11700 | Loss: 2.4928 | LR: 2.38e-04 | 9089.4ms/step | 2.2 samples/s
|
| 673 |
+
Step 6400/11700 | Loss: 2.4704 | LR: 2.37e-04 | 9081.8ms/step | 2.2 samples/s
|
| 674 |
+
Step 6410/11700 | Loss: 2.3977 | LR: 2.37e-04 | 9085.0ms/step | 2.2 samples/s
|
| 675 |
+
Step 6420/11700 | Loss: 2.3963 | LR: 2.36e-04 | 9081.8ms/step | 2.2 samples/s
|
| 676 |
+
Step 6430/11700 | Loss: 2.2914 | LR: 2.35e-04 | 9085.5ms/step | 2.2 samples/s
|
| 677 |
+
Step 6440/11700 | Loss: 2.4633 | LR: 2.35e-04 | 9084.8ms/step | 2.2 samples/s
|
| 678 |
+
Step 6450/11700 | Loss: 2.4043 | LR: 2.34e-04 | 9086.9ms/step | 2.2 samples/s
|
| 679 |
+
Step 6460/11700 | Loss: 2.5353 | LR: 2.33e-04 | 9084.0ms/step | 2.2 samples/s
|
| 680 |
+
Step 6470/11700 | Loss: 2.3994 | LR: 2.32e-04 | 9086.7ms/step | 2.2 samples/s
|
| 681 |
+
Step 6480/11700 | Loss: 2.6502 | LR: 2.32e-04 | 9086.2ms/step | 2.2 samples/s
|
| 682 |
+
Step 6490/11700 | Loss: 2.4557 | LR: 2.31e-04 | 9086.7ms/step | 2.2 samples/s
|
| 683 |
+
Step 6500/11700 | Loss: 2.4896 | LR: 2.30e-04 | 9089.1ms/step | 2.2 samples/s
|
| 684 |
+
Validation at step 6500: Loss = 2.4769, Perplexity = 11.90
|
| 685 |
+
Step 6510/11700 | Loss: 2.4695 | LR: 2.30e-04 | 22226.5ms/step | 0.9 samples/s
|
| 686 |
+
Step 6520/11700 | Loss: 2.4668 | LR: 2.29e-04 | 9092.0ms/step | 2.2 samples/s
|
| 687 |
+
Step 6530/11700 | Loss: 2.3864 | LR: 2.28e-04 | 9086.2ms/step | 2.2 samples/s
|
| 688 |
+
Step 6540/11700 | Loss: 2.5386 | LR: 2.27e-04 | 9084.5ms/step | 2.2 samples/s
|
| 689 |
+
Step 6550/11700 | Loss: 2.4390 | LR: 2.27e-04 | 9086.9ms/step | 2.2 samples/s
|
| 690 |
+
Step 6560/11700 | Loss: 2.3897 | LR: 2.26e-04 | 9087.6ms/step | 2.2 samples/s
|
| 691 |
+
Step 6570/11700 | Loss: 2.3455 | LR: 2.25e-04 | 9090.0ms/step | 2.2 samples/s
|
| 692 |
+
Step 6580/11700 | Loss: 2.5081 | LR: 2.25e-04 | 9086.5ms/step | 2.2 samples/s
|
| 693 |
+
Step 6590/11700 | Loss: 2.4877 | LR: 2.24e-04 | 9086.6ms/step | 2.2 samples/s
|
| 694 |
+
Step 6600/11700 | Loss: 2.4995 | LR: 2.23e-04 | 9085.1ms/step | 2.2 samples/s
|
| 695 |
+
Step 6610/11700 | Loss: 2.5723 | LR: 2.22e-04 | 9084.4ms/step | 2.2 samples/s
|
| 696 |
+
Step 6620/11700 | Loss: 2.3717 | LR: 2.22e-04 | 9084.0ms/step | 2.2 samples/s
|
| 697 |
+
Step 6630/11700 | Loss: 2.4787 | LR: 2.21e-04 | 9088.2ms/step | 2.2 samples/s
|
| 698 |
+
Step 6640/11700 | Loss: 2.4856 | LR: 2.20e-04 | 9083.6ms/step | 2.2 samples/s
|
| 699 |
+
Step 6650/11700 | Loss: 2.4182 | LR: 2.20e-04 | 9088.5ms/step | 2.2 samples/s
|
| 700 |
+
Step 6660/11700 | Loss: 2.4429 | LR: 2.19e-04 | 9083.3ms/step | 2.2 samples/s
|
| 701 |
+
Step 6670/11700 | Loss: 2.4375 | LR: 2.18e-04 | 9089.8ms/step | 2.2 samples/s
|
| 702 |
+
Step 6680/11700 | Loss: 2.4325 | LR: 2.17e-04 | 9084.9ms/step | 2.2 samples/s
|
| 703 |
+
Step 6690/11700 | Loss: 2.5237 | LR: 2.17e-04 | 9086.3ms/step | 2.2 samples/s
|
| 704 |
+
Step 6700/11700 | Loss: 2.4622 | LR: 2.16e-04 | 9082.0ms/step | 2.2 samples/s
|
| 705 |
+
Step 6710/11700 | Loss: 2.4756 | LR: 2.15e-04 | 9102.0ms/step | 2.2 samples/s
|
| 706 |
+
Step 6720/11700 | Loss: 2.3583 | LR: 2.15e-04 | 9086.0ms/step | 2.2 samples/s
|
| 707 |
+
Step 6730/11700 | Loss: 2.4169 | LR: 2.14e-04 | 9088.4ms/step | 2.2 samples/s
|
| 708 |
+
Step 6740/11700 | Loss: 2.4360 | LR: 2.13e-04 | 9090.0ms/step | 2.2 samples/s
|
| 709 |
+
Step 6750/11700 | Loss: 2.4182 | LR: 2.12e-04 | 9089.1ms/step | 2.2 samples/s
|
| 710 |
+
Validation at step 6750: Loss = 2.4716, Perplexity = 11.84
|
| 711 |
+
Step 6760/11700 | Loss: 2.4926 | LR: 2.12e-04 | 22201.2ms/step | 0.9 samples/s
|
| 712 |
+
Step 6770/11700 | Loss: 2.4568 | LR: 2.11e-04 | 9086.8ms/step | 2.2 samples/s
|
| 713 |
+
Step 6780/11700 | Loss: 2.3963 | LR: 2.10e-04 | 9087.4ms/step | 2.2 samples/s
|
| 714 |
+
Step 6790/11700 | Loss: 2.4010 | LR: 2.10e-04 | 9095.0ms/step | 2.2 samples/s
|
| 715 |
+
Step 6800/11700 | Loss: 2.4824 | LR: 2.09e-04 | 9089.1ms/step | 2.2 samples/s
|
| 716 |
+
Step 6810/11700 | Loss: 2.6517 | LR: 2.08e-04 | 9086.0ms/step | 2.2 samples/s
|
| 717 |
+
Step 6820/11700 | Loss: 2.4906 | LR: 2.08e-04 | 9081.7ms/step | 2.2 samples/s
|
| 718 |
+
Step 6830/11700 | Loss: 2.4017 | LR: 2.07e-04 | 9085.8ms/step | 2.2 samples/s
|
| 719 |
+
Step 6840/11700 | Loss: 2.4221 | LR: 2.06e-04 | 9093.4ms/step | 2.2 samples/s
|
| 720 |
+
Step 6850/11700 | Loss: 2.4960 | LR: 2.05e-04 | 9090.2ms/step | 2.2 samples/s
|
| 721 |
+
Step 6860/11700 | Loss: 2.4352 | LR: 2.05e-04 | 9089.5ms/step | 2.2 samples/s
|
| 722 |
+
Step 6870/11700 | Loss: 2.4228 | LR: 2.04e-04 | 9081.3ms/step | 2.2 samples/s
|
| 723 |
+
Step 6880/11700 | Loss: 2.4238 | LR: 2.03e-04 | 9085.0ms/step | 2.2 samples/s
|
| 724 |
+
Step 6890/11700 | Loss: 2.4464 | LR: 2.03e-04 | 9087.8ms/step | 2.2 samples/s
|
| 725 |
+
Step 6900/11700 | Loss: 2.5689 | LR: 2.02e-04 | 9085.3ms/step | 2.2 samples/s
|
| 726 |
+
Step 6910/11700 | Loss: 2.5364 | LR: 2.01e-04 | 9084.3ms/step | 2.2 samples/s
|
| 727 |
+
Step 6920/11700 | Loss: 2.3927 | LR: 2.00e-04 | 9083.8ms/step | 2.2 samples/s
|
| 728 |
+
Step 6930/11700 | Loss: 2.3413 | LR: 2.00e-04 | 9086.5ms/step | 2.2 samples/s
|
| 729 |
+
Step 6940/11700 | Loss: 2.5533 | LR: 1.99e-04 | 9087.6ms/step | 2.2 samples/s
|
| 730 |
+
Step 6950/11700 | Loss: 2.5008 | LR: 1.98e-04 | 9086.5ms/step | 2.2 samples/s
|
| 731 |
+
Step 6960/11700 | Loss: 2.4483 | LR: 1.98e-04 | 9089.1ms/step | 2.2 samples/s
|
| 732 |
+
Step 6970/11700 | Loss: 2.2961 | LR: 1.97e-04 | 9089.6ms/step | 2.2 samples/s
|
| 733 |
+
Step 6980/11700 | Loss: 2.4580 | LR: 1.96e-04 | 9082.8ms/step | 2.2 samples/s
|
| 734 |
+
Step 6990/11700 | Loss: 2.5262 | LR: 1.96e-04 | 9084.6ms/step | 2.2 samples/s
|
| 735 |
+
Step 7000/11700 | Loss: 2.4571 | LR: 1.95e-04 | 9089.4ms/step | 2.2 samples/s
|
| 736 |
+
Validation at step 7000: Loss = 2.4603, Perplexity = 11.71
|
| 737 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-7000
|
| 738 |
+
Step 7010/11700 | Loss: 2.5595 | LR: 1.94e-04 | 22535.3ms/step | 0.9 samples/s
|
| 739 |
+
Step 7020/11700 | Loss: 2.4402 | LR: 1.93e-04 | 9139.3ms/step | 2.2 samples/s
|
| 740 |
+
Step 7030/11700 | Loss: 2.5336 | LR: 1.93e-04 | 9140.2ms/step | 2.2 samples/s
|
| 741 |
+
Step 7040/11700 | Loss: 2.5102 | LR: 1.92e-04 | 9143.1ms/step | 2.2 samples/s
|
| 742 |
+
Step 7050/11700 | Loss: 2.3820 | LR: 1.91e-04 | 9141.9ms/step | 2.2 samples/s
|
| 743 |
+
Step 7060/11700 | Loss: 2.5037 | LR: 1.91e-04 | 9143.0ms/step | 2.2 samples/s
|
| 744 |
+
Step 7070/11700 | Loss: 2.4297 | LR: 1.90e-04 | 9143.6ms/step | 2.2 samples/s
|
| 745 |
+
Step 7080/11700 | Loss: 2.4468 | LR: 1.89e-04 | 9144.4ms/step | 2.2 samples/s
|
| 746 |
+
Step 7090/11700 | Loss: 2.4500 | LR: 1.89e-04 | 9145.8ms/step | 2.2 samples/s
|
| 747 |
+
Step 7100/11700 | Loss: 2.4864 | LR: 1.88e-04 | 9142.5ms/step | 2.2 samples/s
|
| 748 |
+
Step 7110/11700 | Loss: 2.4266 | LR: 1.87e-04 | 9142.3ms/step | 2.2 samples/s
|
| 749 |
+
Step 7120/11700 | Loss: 2.3727 | LR: 1.87e-04 | 9138.2ms/step | 2.2 samples/s
|
| 750 |
+
Step 7130/11700 | Loss: 2.3702 | LR: 1.86e-04 | 9145.9ms/step | 2.2 samples/s
|
| 751 |
+
Step 7140/11700 | Loss: 2.4294 | LR: 1.85e-04 | 9144.2ms/step | 2.2 samples/s
|
| 752 |
+
Step 7150/11700 | Loss: 2.5114 | LR: 1.84e-04 | 9143.2ms/step | 2.2 samples/s
|
| 753 |
+
Step 7160/11700 | Loss: 2.4243 | LR: 1.84e-04 | 9146.2ms/step | 2.2 samples/s
|
| 754 |
+
Step 7170/11700 | Loss: 2.4631 | LR: 1.83e-04 | 9135.0ms/step | 2.2 samples/s
|
| 755 |
+
Step 7180/11700 | Loss: 2.3695 | LR: 1.82e-04 | 9140.4ms/step | 2.2 samples/s
|
| 756 |
+
Step 7190/11700 | Loss: 2.3734 | LR: 1.82e-04 | 9140.6ms/step | 2.2 samples/s
|
| 757 |
+
Step 7200/11700 | Loss: 2.3384 | LR: 1.81e-04 | 9139.8ms/step | 2.2 samples/s
|
| 758 |
+
Step 7210/11700 | Loss: 2.2830 | LR: 1.80e-04 | 9138.4ms/step | 2.2 samples/s
|
| 759 |
+
Step 7220/11700 | Loss: 2.4199 | LR: 1.80e-04 | 9143.8ms/step | 2.2 samples/s
|
| 760 |
+
Step 7230/11700 | Loss: 2.4641 | LR: 1.79e-04 | 9141.6ms/step | 2.2 samples/s
|
| 761 |
+
Step 7240/11700 | Loss: 2.4233 | LR: 1.78e-04 | 9142.7ms/step | 2.2 samples/s
|
| 762 |
+
Step 7250/11700 | Loss: 2.2937 | LR: 1.78e-04 | 9145.0ms/step | 2.2 samples/s
|
| 763 |
+
Validation at step 7250: Loss = 2.4528, Perplexity = 11.62
|
| 764 |
+
Step 7260/11700 | Loss: 2.4109 | LR: 1.77e-04 | 22333.6ms/step | 0.9 samples/s
|
| 765 |
+
Step 7270/11700 | Loss: 2.4652 | LR: 1.76e-04 | 9145.3ms/step | 2.2 samples/s
|
| 766 |
+
Step 7280/11700 | Loss: 2.4718 | LR: 1.75e-04 | 9142.2ms/step | 2.2 samples/s
|
| 767 |
+
Step 7290/11700 | Loss: 2.4664 | LR: 1.75e-04 | 9140.0ms/step | 2.2 samples/s
|
| 768 |
+
Step 7300/11700 | Loss: 2.3871 | LR: 1.74e-04 | 9141.7ms/step | 2.2 samples/s
|
| 769 |
+
Step 7310/11700 | Loss: 2.5113 | LR: 1.73e-04 | 9137.1ms/step | 2.2 samples/s
|
| 770 |
+
Step 7320/11700 | Loss: 2.4106 | LR: 1.73e-04 | 9143.0ms/step | 2.2 samples/s
|
| 771 |
+
Step 7330/11700 | Loss: 2.4799 | LR: 1.72e-04 | 9146.5ms/step | 2.2 samples/s
|
| 772 |
+
Step 7340/11700 | Loss: 2.4191 | LR: 1.71e-04 | 9143.0ms/step | 2.2 samples/s
|
| 773 |
+
Step 7350/11700 | Loss: 2.4018 | LR: 1.71e-04 | 9146.7ms/step | 2.2 samples/s
|
| 774 |
+
Step 7360/11700 | Loss: 2.3563 | LR: 1.70e-04 | 9144.3ms/step | 2.2 samples/s
|
| 775 |
+
Step 7370/11700 | Loss: 2.4135 | LR: 1.69e-04 | 9142.6ms/step | 2.2 samples/s
|
| 776 |
+
Step 7380/11700 | Loss: 2.4531 | LR: 1.69e-04 | 9144.6ms/step | 2.2 samples/s
|
| 777 |
+
Step 7390/11700 | Loss: 2.4200 | LR: 1.68e-04 | 9147.0ms/step | 2.2 samples/s
|
| 778 |
+
Step 7400/11700 | Loss: 2.4536 | LR: 1.67e-04 | 9144.0ms/step | 2.2 samples/s
|
| 779 |
+
Step 7410/11700 | Loss: 2.3756 | LR: 1.67e-04 | 9149.2ms/step | 2.2 samples/s
|
| 780 |
+
Step 7420/11700 | Loss: 2.3599 | LR: 1.66e-04 | 9141.9ms/step | 2.2 samples/s
|
| 781 |
+
Step 7430/11700 | Loss: 2.3726 | LR: 1.65e-04 | 9147.2ms/step | 2.2 samples/s
|
| 782 |
+
Step 7440/11700 | Loss: 2.4754 | LR: 1.65e-04 | 9143.8ms/step | 2.2 samples/s
|
| 783 |
+
Step 7450/11700 | Loss: 2.4553 | LR: 1.64e-04 | 9147.7ms/step | 2.2 samples/s
|
| 784 |
+
Step 7460/11700 | Loss: 2.4626 | LR: 1.63e-04 | 9143.4ms/step | 2.2 samples/s
|
| 785 |
+
Step 7470/11700 | Loss: 2.4152 | LR: 1.63e-04 | 9140.5ms/step | 2.2 samples/s
|
| 786 |
+
Step 7480/11700 | Loss: 2.5465 | LR: 1.62e-04 | 9142.6ms/step | 2.2 samples/s
|
| 787 |
+
Step 7490/11700 | Loss: 2.3791 | LR: 1.61e-04 | 9146.4ms/step | 2.2 samples/s
|
| 788 |
+
Step 7500/11700 | Loss: 2.3715 | LR: 1.61e-04 | 9145.2ms/step | 2.2 samples/s
|
| 789 |
+
Validation at step 7500: Loss = 2.4463, Perplexity = 11.55
|
| 790 |
+
Step 7510/11700 | Loss: 2.4556 | LR: 1.60e-04 | 22336.7ms/step | 0.9 samples/s
|
| 791 |
+
Step 7520/11700 | Loss: 2.4793 | LR: 1.59e-04 | 9139.9ms/step | 2.2 samples/s
|
| 792 |
+
Step 7530/11700 | Loss: 2.3835 | LR: 1.59e-04 | 9139.9ms/step | 2.2 samples/s
|
| 793 |
+
Step 7540/11700 | Loss: 2.4624 | LR: 1.58e-04 | 9129.6ms/step | 2.2 samples/s
|
| 794 |
+
Step 7550/11700 | Loss: 2.4081 | LR: 1.57e-04 | 9143.2ms/step | 2.2 samples/s
|
| 795 |
+
Step 7560/11700 | Loss: 2.4680 | LR: 1.57e-04 | 9143.4ms/step | 2.2 samples/s
|
| 796 |
+
Step 7570/11700 | Loss: 2.4275 | LR: 1.56e-04 | 9142.9ms/step | 2.2 samples/s
|
| 797 |
+
Step 7580/11700 | Loss: 2.4140 | LR: 1.55e-04 | 9146.5ms/step | 2.2 samples/s
|
| 798 |
+
Step 7590/11700 | Loss: 2.3957 | LR: 1.55e-04 | 9144.1ms/step | 2.2 samples/s
|
| 799 |
+
Step 7600/11700 | Loss: 2.4526 | LR: 1.54e-04 | 9143.8ms/step | 2.2 samples/s
|
| 800 |
+
Step 7610/11700 | Loss: 2.5479 | LR: 1.53e-04 | 9148.2ms/step | 2.2 samples/s
|
| 801 |
+
Step 7620/11700 | Loss: 2.5143 | LR: 1.53e-04 | 9145.1ms/step | 2.2 samples/s
|
| 802 |
+
Step 7630/11700 | Loss: 2.3889 | LR: 1.52e-04 | 9143.9ms/step | 2.2 samples/s
|
| 803 |
+
Step 7640/11700 | Loss: 2.4230 | LR: 1.51e-04 | 9144.7ms/step | 2.2 samples/s
|
| 804 |
+
Step 7650/11700 | Loss: 2.4599 | LR: 1.51e-04 | 9138.4ms/step | 2.2 samples/s
|
| 805 |
+
Step 7660/11700 | Loss: 2.4075 | LR: 1.50e-04 | 9142.3ms/step | 2.2 samples/s
|
| 806 |
+
Step 7670/11700 | Loss: 2.3434 | LR: 1.49e-04 | 9145.0ms/step | 2.2 samples/s
|
| 807 |
+
Step 7680/11700 | Loss: 2.4047 | LR: 1.49e-04 | 9139.5ms/step | 2.2 samples/s
|
| 808 |
+
Step 7690/11700 | Loss: 2.4854 | LR: 1.48e-04 | 9107.0ms/step | 2.2 samples/s
|
| 809 |
+
Step 7700/11700 | Loss: 2.5507 | LR: 1.47e-04 | 9125.5ms/step | 2.2 samples/s
|
| 810 |
+
Step 7710/11700 | Loss: 2.3507 | LR: 1.47e-04 | 9109.7ms/step | 2.2 samples/s
|
| 811 |
+
Step 7720/11700 | Loss: 2.4551 | LR: 1.46e-04 | 9118.9ms/step | 2.2 samples/s
|
| 812 |
+
Step 7730/11700 | Loss: 2.3479 | LR: 1.45e-04 | 9115.3ms/step | 2.2 samples/s
|
| 813 |
+
Step 7740/11700 | Loss: 2.5090 | LR: 1.45e-04 | 9105.5ms/step | 2.2 samples/s
|
| 814 |
+
Step 7750/11700 | Loss: 2.4241 | LR: 1.44e-04 | 9106.0ms/step | 2.2 samples/s
|
| 815 |
+
Validation at step 7750: Loss = 2.4400, Perplexity = 11.47
|
| 816 |
+
Step 7760/11700 | Loss: 2.5415 | LR: 1.43e-04 | 22356.0ms/step | 0.9 samples/s
|
| 817 |
+
Step 7770/11700 | Loss: 2.4044 | LR: 1.43e-04 | 9160.6ms/step | 2.2 samples/s
|
| 818 |
+
Step 7780/11700 | Loss: 2.4987 | LR: 1.42e-04 | 9138.5ms/step | 2.2 samples/s
|
| 819 |
+
Step 7790/11700 | Loss: 2.4003 | LR: 1.41e-04 | 9124.7ms/step | 2.2 samples/s
|
| 820 |
+
Step 7800/11700 | Loss: 2.4857 | LR: 1.41e-04 | 9172.1ms/step | 2.2 samples/s
|
| 821 |
+
Step 7810/11700 | Loss: 2.3267 | LR: 1.40e-04 | 9160.9ms/step | 2.2 samples/s
|
| 822 |
+
Step 7820/11700 | Loss: 2.3706 | LR: 1.40e-04 | 9172.9ms/step | 2.2 samples/s
|
| 823 |
+
Step 7830/11700 | Loss: 2.3749 | LR: 1.39e-04 | 9172.6ms/step | 2.2 samples/s
|
| 824 |
+
Step 7840/11700 | Loss: 2.4368 | LR: 1.38e-04 | 9177.7ms/step | 2.2 samples/s
|
| 825 |
+
Step 7850/11700 | Loss: 2.4554 | LR: 1.38e-04 | 9170.9ms/step | 2.2 samples/s
|
| 826 |
+
Step 7860/11700 | Loss: 2.4551 | LR: 1.37e-04 | 9152.4ms/step | 2.2 samples/s
|
| 827 |
+
Step 7870/11700 | Loss: 2.4937 | LR: 1.36e-04 | 9161.1ms/step | 2.2 samples/s
|
| 828 |
+
Step 7880/11700 | Loss: 2.4503 | LR: 1.36e-04 | 9175.5ms/step | 2.2 samples/s
|
| 829 |
+
Step 7890/11700 | Loss: 2.4065 | LR: 1.35e-04 | 9181.0ms/step | 2.2 samples/s
|
| 830 |
+
Step 7900/11700 | Loss: 2.5095 | LR: 1.34e-04 | 9174.3ms/step | 2.2 samples/s
|
| 831 |
+
Step 7910/11700 | Loss: 2.2876 | LR: 1.34e-04 | 9173.1ms/step | 2.2 samples/s
|
| 832 |
+
Step 7920/11700 | Loss: 2.3911 | LR: 1.33e-04 | 9172.1ms/step | 2.2 samples/s
|
| 833 |
+
Step 7930/11700 | Loss: 2.3992 | LR: 1.33e-04 | 9181.6ms/step | 2.2 samples/s
|
| 834 |
+
Step 7940/11700 | Loss: 2.4527 | LR: 1.32e-04 | 9180.0ms/step | 2.2 samples/s
|
| 835 |
+
Step 7950/11700 | Loss: 2.4753 | LR: 1.31e-04 | 9176.4ms/step | 2.2 samples/s
|
| 836 |
+
Step 7960/11700 | Loss: 2.3989 | LR: 1.31e-04 | 9182.5ms/step | 2.2 samples/s
|
| 837 |
+
Step 7970/11700 | Loss: 2.3474 | LR: 1.30e-04 | 9164.9ms/step | 2.2 samples/s
|
| 838 |
+
Step 7980/11700 | Loss: 2.5024 | LR: 1.29e-04 | 9163.4ms/step | 2.2 samples/s
|
| 839 |
+
Step 7990/11700 | Loss: 2.4774 | LR: 1.29e-04 | 9167.9ms/step | 2.2 samples/s
|
| 840 |
+
Step 8000/11700 | Loss: 2.3614 | LR: 1.28e-04 | 9170.9ms/step | 2.2 samples/s
|
| 841 |
+
Validation at step 8000: Loss = 2.4351, Perplexity = 11.42
|
| 842 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-8000
|
| 843 |
+
Step 8010/11700 | Loss: 2.4495 | LR: 1.27e-04 | 22663.5ms/step | 0.9 samples/s
|
| 844 |
+
Step 8020/11700 | Loss: 2.3236 | LR: 1.27e-04 | 9170.4ms/step | 2.2 samples/s
|
| 845 |
+
Step 8030/11700 | Loss: 2.4147 | LR: 1.26e-04 | 9171.3ms/step | 2.2 samples/s
|
| 846 |
+
Step 8040/11700 | Loss: 2.4230 | LR: 1.26e-04 | 9167.4ms/step | 2.2 samples/s
|
| 847 |
+
Step 8050/11700 | Loss: 2.3836 | LR: 1.25e-04 | 9180.0ms/step | 2.2 samples/s
|
| 848 |
+
Step 8060/11700 | Loss: 2.4107 | LR: 1.24e-04 | 9176.3ms/step | 2.2 samples/s
|
| 849 |
+
Step 8070/11700 | Loss: 2.4668 | LR: 1.24e-04 | 9162.6ms/step | 2.2 samples/s
|
| 850 |
+
Step 8080/11700 | Loss: 2.4645 | LR: 1.23e-04 | 9166.3ms/step | 2.2 samples/s
|
| 851 |
+
Step 8090/11700 | Loss: 2.3952 | LR: 1.23e-04 | 9166.6ms/step | 2.2 samples/s
|
| 852 |
+
Step 8100/11700 | Loss: 2.4315 | LR: 1.22e-04 | 9136.1ms/step | 2.2 samples/s
|
| 853 |
+
Step 8110/11700 | Loss: 2.3796 | LR: 1.21e-04 | 9153.0ms/step | 2.2 samples/s
|
| 854 |
+
Step 8120/11700 | Loss: 2.3576 | LR: 1.21e-04 | 9156.1ms/step | 2.2 samples/s
|
| 855 |
+
Step 8130/11700 | Loss: 2.3386 | LR: 1.20e-04 | 9161.0ms/step | 2.2 samples/s
|
| 856 |
+
Step 8140/11700 | Loss: 2.3917 | LR: 1.19e-04 | 9164.5ms/step | 2.2 samples/s
|
| 857 |
+
Step 8150/11700 | Loss: 2.2752 | LR: 1.19e-04 | 9172.2ms/step | 2.2 samples/s
|
| 858 |
+
Step 8160/11700 | Loss: 2.5058 | LR: 1.18e-04 | 9167.1ms/step | 2.2 samples/s
|
| 859 |
+
Step 8170/11700 | Loss: 2.4014 | LR: 1.18e-04 | 9172.2ms/step | 2.2 samples/s
|
| 860 |
+
Step 8180/11700 | Loss: 2.4522 | LR: 1.17e-04 | 9165.4ms/step | 2.2 samples/s
|
| 861 |
+
Step 8190/11700 | Loss: 2.3676 | LR: 1.16e-04 | 9131.8ms/step | 2.2 samples/s
|
| 862 |
+
Step 8200/11700 | Loss: 2.4183 | LR: 1.16e-04 | 9163.8ms/step | 2.2 samples/s
|
| 863 |
+
Step 8210/11700 | Loss: 2.4511 | LR: 1.15e-04 | 9170.2ms/step | 2.2 samples/s
|
| 864 |
+
Step 8220/11700 | Loss: 2.4512 | LR: 1.15e-04 | 9172.0ms/step | 2.2 samples/s
|
| 865 |
+
Step 8230/11700 | Loss: 2.4338 | LR: 1.14e-04 | 9117.3ms/step | 2.2 samples/s
|
| 866 |
+
Step 8240/11700 | Loss: 2.4048 | LR: 1.13e-04 | 9151.1ms/step | 2.2 samples/s
|
| 867 |
+
Step 8250/11700 | Loss: 2.4346 | LR: 1.13e-04 | 9141.2ms/step | 2.2 samples/s
|
| 868 |
+
Validation at step 8250: Loss = 2.4308, Perplexity = 11.37
|
| 869 |
+
Step 8260/11700 | Loss: 2.3707 | LR: 1.12e-04 | 22385.4ms/step | 0.9 samples/s
|
| 870 |
+
Step 8270/11700 | Loss: 2.3795 | LR: 1.12e-04 | 9169.9ms/step | 2.2 samples/s
|
| 871 |
+
Step 8280/11700 | Loss: 2.5287 | LR: 1.11e-04 | 9174.3ms/step | 2.2 samples/s
|
| 872 |
+
Step 8290/11700 | Loss: 2.3900 | LR: 1.10e-04 | 9175.0ms/step | 2.2 samples/s
|
| 873 |
+
Step 8300/11700 | Loss: 2.4435 | LR: 1.10e-04 | 9165.9ms/step | 2.2 samples/s
|
| 874 |
+
Step 8310/11700 | Loss: 2.3756 | LR: 1.09e-04 | 9161.2ms/step | 2.2 samples/s
|
| 875 |
+
Step 8320/11700 | Loss: 2.4607 | LR: 1.09e-04 | 9165.4ms/step | 2.2 samples/s
|
| 876 |
+
Step 8330/11700 | Loss: 2.4689 | LR: 1.08e-04 | 9158.3ms/step | 2.2 samples/s
|
| 877 |
+
Step 8340/11700 | Loss: 2.4337 | LR: 1.07e-04 | 9161.7ms/step | 2.2 samples/s
|
| 878 |
+
Step 8350/11700 | Loss: 2.4452 | LR: 1.07e-04 | 9169.4ms/step | 2.2 samples/s
|
| 879 |
+
Step 8360/11700 | Loss: 2.3883 | LR: 1.06e-04 | 9162.0ms/step | 2.2 samples/s
|
| 880 |
+
Step 8370/11700 | Loss: 2.3772 | LR: 1.06e-04 | 9147.3ms/step | 2.2 samples/s
|
| 881 |
+
Step 8380/11700 | Loss: 2.4115 | LR: 1.05e-04 | 9101.6ms/step | 2.2 samples/s
|
| 882 |
+
Step 8390/11700 | Loss: 2.4160 | LR: 1.05e-04 | 9149.6ms/step | 2.2 samples/s
|
| 883 |
+
Step 8400/11700 | Loss: 2.4773 | LR: 1.04e-04 | 9163.5ms/step | 2.2 samples/s
|
| 884 |
+
Step 8410/11700 | Loss: 2.6004 | LR: 1.03e-04 | 9140.4ms/step | 2.2 samples/s
|
| 885 |
+
Step 8420/11700 | Loss: 2.3565 | LR: 1.03e-04 | 9154.5ms/step | 2.2 samples/s
|
| 886 |
+
Step 8430/11700 | Loss: 2.4777 | LR: 1.02e-04 | 9162.9ms/step | 2.2 samples/s
|
| 887 |
+
Step 8440/11700 | Loss: 2.4763 | LR: 1.02e-04 | 9172.2ms/step | 2.2 samples/s
|
| 888 |
+
Step 8450/11700 | Loss: 2.4180 | LR: 1.01e-04 | 9155.9ms/step | 2.2 samples/s
|
| 889 |
+
Step 8460/11700 | Loss: 2.4077 | LR: 1.00e-04 | 9165.0ms/step | 2.2 samples/s
|
| 890 |
+
Step 8470/11700 | Loss: 2.4033 | LR: 9.99e-05 | 9169.0ms/step | 2.2 samples/s
|
| 891 |
+
Step 8480/11700 | Loss: 2.4570 | LR: 9.93e-05 | 9164.0ms/step | 2.2 samples/s
|
| 892 |
+
Step 8490/11700 | Loss: 2.4096 | LR: 9.87e-05 | 9155.6ms/step | 2.2 samples/s
|
| 893 |
+
Step 8500/11700 | Loss: 2.2719 | LR: 9.82e-05 | 9153.6ms/step | 2.2 samples/s
|
| 894 |
+
Validation at step 8500: Loss = 2.4269, Perplexity = 11.32
|
| 895 |
+
Step 8510/11700 | Loss: 2.3755 | LR: 9.76e-05 | 22344.3ms/step | 0.9 samples/s
|
| 896 |
+
Step 8520/11700 | Loss: 2.4218 | LR: 9.70e-05 | 9159.8ms/step | 2.2 samples/s
|
| 897 |
+
Step 8530/11700 | Loss: 2.4472 | LR: 9.65e-05 | 9162.9ms/step | 2.2 samples/s
|
| 898 |
+
Step 8540/11700 | Loss: 2.4183 | LR: 9.59e-05 | 9156.1ms/step | 2.2 samples/s
|
| 899 |
+
Step 8550/11700 | Loss: 2.3086 | LR: 9.53e-05 | 9129.7ms/step | 2.2 samples/s
|
| 900 |
+
Step 8560/11700 | Loss: 2.4185 | LR: 9.48e-05 | 9157.5ms/step | 2.2 samples/s
|
| 901 |
+
Step 8570/11700 | Loss: 2.4111 | LR: 9.42e-05 | 9164.6ms/step | 2.2 samples/s
|
| 902 |
+
Step 8580/11700 | Loss: 2.3969 | LR: 9.36e-05 | 9161.1ms/step | 2.2 samples/s
|
| 903 |
+
Step 8590/11700 | Loss: 2.3657 | LR: 9.31e-05 | 9170.4ms/step | 2.2 samples/s
|
| 904 |
+
Step 8600/11700 | Loss: 2.4370 | LR: 9.25e-05 | 9162.9ms/step | 2.2 samples/s
|
| 905 |
+
Step 8610/11700 | Loss: 2.4129 | LR: 9.20e-05 | 9166.8ms/step | 2.2 samples/s
|
| 906 |
+
Step 8620/11700 | Loss: 2.4666 | LR: 9.14e-05 | 9162.5ms/step | 2.2 samples/s
|
| 907 |
+
Step 8630/11700 | Loss: 2.4403 | LR: 9.09e-05 | 9173.1ms/step | 2.2 samples/s
|
| 908 |
+
Step 8640/11700 | Loss: 2.3765 | LR: 9.03e-05 | 9163.9ms/step | 2.2 samples/s
|
| 909 |
+
Step 8650/11700 | Loss: 2.3332 | LR: 8.98e-05 | 9159.5ms/step | 2.2 samples/s
|
| 910 |
+
Step 8660/11700 | Loss: 2.3980 | LR: 8.92e-05 | 9172.3ms/step | 2.2 samples/s
|
| 911 |
+
Step 8670/11700 | Loss: 2.4891 | LR: 8.87e-05 | 9160.7ms/step | 2.2 samples/s
|
| 912 |
+
Step 8680/11700 | Loss: 2.3668 | LR: 8.81e-05 | 9165.0ms/step | 2.2 samples/s
|
| 913 |
+
Step 8690/11700 | Loss: 2.4908 | LR: 8.76e-05 | 9165.1ms/step | 2.2 samples/s
|
| 914 |
+
Step 8700/11700 | Loss: 2.4151 | LR: 8.70e-05 | 9170.5ms/step | 2.2 samples/s
|
| 915 |
+
Step 8710/11700 | Loss: 2.2804 | LR: 8.65e-05 | 9159.9ms/step | 2.2 samples/s
|
| 916 |
+
Step 8720/11700 | Loss: 2.3564 | LR: 8.59e-05 | 9171.8ms/step | 2.2 samples/s
|
| 917 |
+
Step 8730/11700 | Loss: 2.4057 | LR: 8.54e-05 | 9165.4ms/step | 2.2 samples/s
|
| 918 |
+
Step 8740/11700 | Loss: 2.4338 | LR: 8.49e-05 | 9168.4ms/step | 2.2 samples/s
|
| 919 |
+
Step 8750/11700 | Loss: 2.4397 | LR: 8.43e-05 | 9172.2ms/step | 2.2 samples/s
|
| 920 |
+
Validation at step 8750: Loss = 2.4242, Perplexity = 11.29
|
| 921 |
+
Step 8760/11700 | Loss: 2.3489 | LR: 8.38e-05 | 22404.7ms/step | 0.9 samples/s
|
| 922 |
+
Step 8770/11700 | Loss: 2.4060 | LR: 8.33e-05 | 9175.1ms/step | 2.2 samples/s
|
| 923 |
+
Step 8780/11700 | Loss: 2.4045 | LR: 8.27e-05 | 9181.0ms/step | 2.2 samples/s
|
| 924 |
+
Step 8790/11700 | Loss: 2.4425 | LR: 8.22e-05 | 9164.4ms/step | 2.2 samples/s
|
| 925 |
+
Step 8800/11700 | Loss: 2.4719 | LR: 8.17e-05 | 9135.8ms/step | 2.2 samples/s
|
| 926 |
+
Step 8810/11700 | Loss: 2.3267 | LR: 8.11e-05 | 9139.0ms/step | 2.2 samples/s
|
| 927 |
+
Step 8820/11700 | Loss: 2.3196 | LR: 8.06e-05 | 9139.1ms/step | 2.2 samples/s
|
| 928 |
+
Step 8830/11700 | Loss: 2.3807 | LR: 8.01e-05 | 9151.4ms/step | 2.2 samples/s
|
| 929 |
+
Step 8840/11700 | Loss: 2.4610 | LR: 7.95e-05 | 9164.5ms/step | 2.2 samples/s
|
| 930 |
+
Step 8850/11700 | Loss: 2.4271 | LR: 7.90e-05 | 9162.8ms/step | 2.2 samples/s
|
| 931 |
+
Step 8860/11700 | Loss: 2.3090 | LR: 7.85e-05 | 9172.6ms/step | 2.2 samples/s
|
| 932 |
+
Step 8870/11700 | Loss: 2.3629 | LR: 7.80e-05 | 9172.7ms/step | 2.2 samples/s
|
| 933 |
+
Step 8880/11700 | Loss: 2.4479 | LR: 7.75e-05 | 9170.8ms/step | 2.2 samples/s
|
| 934 |
+
Step 8890/11700 | Loss: 2.4114 | LR: 7.69e-05 | 9169.6ms/step | 2.2 samples/s
|
| 935 |
+
Step 8900/11700 | Loss: 2.3883 | LR: 7.64e-05 | 9167.5ms/step | 2.2 samples/s
|
| 936 |
+
Step 8910/11700 | Loss: 2.4879 | LR: 7.59e-05 | 9171.3ms/step | 2.2 samples/s
|
| 937 |
+
Step 8920/11700 | Loss: 2.4656 | LR: 7.54e-05 | 9169.8ms/step | 2.2 samples/s
|
| 938 |
+
Step 8930/11700 | Loss: 2.3480 | LR: 7.49e-05 | 9165.0ms/step | 2.2 samples/s
|
| 939 |
+
Step 8940/11700 | Loss: 2.4487 | LR: 7.44e-05 | 9157.1ms/step | 2.2 samples/s
|
| 940 |
+
Step 8950/11700 | Loss: 2.3798 | LR: 7.39e-05 | 9159.3ms/step | 2.2 samples/s
|
| 941 |
+
Step 8960/11700 | Loss: 2.3885 | LR: 7.34e-05 | 9160.4ms/step | 2.2 samples/s
|
| 942 |
+
Step 8970/11700 | Loss: 2.3963 | LR: 7.28e-05 | 9151.3ms/step | 2.2 samples/s
|
| 943 |
+
Step 8980/11700 | Loss: 2.3837 | LR: 7.23e-05 | 9131.0ms/step | 2.2 samples/s
|
| 944 |
+
Step 8990/11700 | Loss: 2.4221 | LR: 7.18e-05 | 9162.3ms/step | 2.2 samples/s
|
| 945 |
+
Step 9000/11700 | Loss: 2.4007 | LR: 7.13e-05 | 9160.8ms/step | 2.2 samples/s
|
| 946 |
+
Validation at step 9000: Loss = 2.4219, Perplexity = 11.27
|
| 947 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-9000
|
| 948 |
+
Step 9010/11700 | Loss: 2.4891 | LR: 7.08e-05 | 22653.2ms/step | 0.9 samples/s
|
| 949 |
+
Step 9020/11700 | Loss: 2.4024 | LR: 7.03e-05 | 9162.6ms/step | 2.2 samples/s
|
| 950 |
+
Step 9030/11700 | Loss: 2.3673 | LR: 6.98e-05 | 9109.2ms/step | 2.2 samples/s
|
| 951 |
+
Step 9040/11700 | Loss: 2.3385 | LR: 6.93e-05 | 9152.2ms/step | 2.2 samples/s
|
| 952 |
+
Step 9050/11700 | Loss: 2.4325 | LR: 6.88e-05 | 9169.0ms/step | 2.2 samples/s
|
| 953 |
+
Step 9060/11700 | Loss: 2.4489 | LR: 6.83e-05 | 9169.6ms/step | 2.2 samples/s
|
| 954 |
+
Step 9070/11700 | Loss: 2.4624 | LR: 6.79e-05 | 9154.7ms/step | 2.2 samples/s
|
| 955 |
+
Step 9080/11700 | Loss: 2.3576 | LR: 6.74e-05 | 9161.5ms/step | 2.2 samples/s
|
| 956 |
+
Step 9090/11700 | Loss: 2.3443 | LR: 6.69e-05 | 9164.2ms/step | 2.2 samples/s
|
| 957 |
+
Step 9100/11700 | Loss: 2.4335 | LR: 6.64e-05 | 9164.0ms/step | 2.2 samples/s
|
| 958 |
+
Step 9110/11700 | Loss: 2.4318 | LR: 6.59e-05 | 9128.5ms/step | 2.2 samples/s
|
| 959 |
+
Step 9120/11700 | Loss: 2.4627 | LR: 6.54e-05 | 9101.7ms/step | 2.2 samples/s
|
| 960 |
+
Step 9130/11700 | Loss: 2.3290 | LR: 6.49e-05 | 9144.0ms/step | 2.2 samples/s
|
| 961 |
+
Step 9140/11700 | Loss: 2.3076 | LR: 6.45e-05 | 9167.6ms/step | 2.2 samples/s
|
| 962 |
+
Step 9150/11700 | Loss: 2.4196 | LR: 6.40e-05 | 9174.0ms/step | 2.2 samples/s
|
| 963 |
+
Step 9160/11700 | Loss: 2.5077 | LR: 6.35e-05 | 9161.6ms/step | 2.2 samples/s
|
| 964 |
+
Step 9170/11700 | Loss: 2.4575 | LR: 6.30e-05 | 9132.4ms/step | 2.2 samples/s
|
| 965 |
+
Step 9180/11700 | Loss: 2.3932 | LR: 6.25e-05 | 9134.0ms/step | 2.2 samples/s
|
| 966 |
+
Step 9190/11700 | Loss: 2.4162 | LR: 6.21e-05 | 9152.2ms/step | 2.2 samples/s
|
| 967 |
+
Step 9200/11700 | Loss: 2.2983 | LR: 6.16e-05 | 9161.5ms/step | 2.2 samples/s
|
| 968 |
+
Step 9210/11700 | Loss: 2.4039 | LR: 6.11e-05 | 9155.3ms/step | 2.2 samples/s
|
| 969 |
+
Step 9220/11700 | Loss: 2.4864 | LR: 6.07e-05 | 9158.9ms/step | 2.2 samples/s
|
| 970 |
+
Step 9230/11700 | Loss: 2.3932 | LR: 6.02e-05 | 9157.0ms/step | 2.2 samples/s
|
| 971 |
+
Step 9240/11700 | Loss: 2.3804 | LR: 5.97e-05 | 9154.8ms/step | 2.2 samples/s
|
| 972 |
+
Step 9250/11700 | Loss: 2.4109 | LR: 5.93e-05 | 9155.3ms/step | 2.2 samples/s
|
| 973 |
+
Validation at step 9250: Loss = 2.4202, Perplexity = 11.25
|
| 974 |
+
Step 9260/11700 | Loss: 2.3340 | LR: 5.88e-05 | 22387.4ms/step | 0.9 samples/s
|
| 975 |
+
Step 9270/11700 | Loss: 2.4286 | LR: 5.83e-05 | 9163.9ms/step | 2.2 samples/s
|
| 976 |
+
Step 9280/11700 | Loss: 2.2828 | LR: 5.79e-05 | 9165.9ms/step | 2.2 samples/s
|
| 977 |
+
Step 9290/11700 | Loss: 2.3716 | LR: 5.74e-05 | 9163.2ms/step | 2.2 samples/s
|
| 978 |
+
Step 9300/11700 | Loss: 2.4253 | LR: 5.70e-05 | 9170.2ms/step | 2.2 samples/s
|
| 979 |
+
Step 9310/11700 | Loss: 2.3197 | LR: 5.65e-05 | 9154.8ms/step | 2.2 samples/s
|
| 980 |
+
Step 9320/11700 | Loss: 2.3636 | LR: 5.61e-05 | 9174.8ms/step | 2.2 samples/s
|
| 981 |
+
Step 9330/11700 | Loss: 2.3599 | LR: 5.56e-05 | 9161.4ms/step | 2.2 samples/s
|
| 982 |
+
Step 9340/11700 | Loss: 2.4703 | LR: 5.52e-05 | 9163.2ms/step | 2.2 samples/s
|
| 983 |
+
Step 9350/11700 | Loss: 2.3582 | LR: 5.47e-05 | 9165.9ms/step | 2.2 samples/s
|
| 984 |
+
Step 9360/11700 | Loss: 2.3917 | LR: 5.43e-05 | 9134.7ms/step | 2.2 samples/s
|
| 985 |
+
Step 9370/11700 | Loss: 2.4085 | LR: 5.38e-05 | 9151.9ms/step | 2.2 samples/s
|
| 986 |
+
Step 9380/11700 | Loss: 2.5106 | LR: 5.34e-05 | 9160.6ms/step | 2.2 samples/s
|
| 987 |
+
Step 9390/11700 | Loss: 2.4178 | LR: 5.29e-05 | 9159.9ms/step | 2.2 samples/s
|
| 988 |
+
Step 9400/11700 | Loss: 2.3949 | LR: 5.25e-05 | 9163.4ms/step | 2.2 samples/s
|
| 989 |
+
Step 9410/11700 | Loss: 2.4115 | LR: 5.20e-05 | 9163.4ms/step | 2.2 samples/s
|
| 990 |
+
Step 9420/11700 | Loss: 2.4736 | LR: 5.16e-05 | 9167.9ms/step | 2.2 samples/s
|
| 991 |
+
Step 9430/11700 | Loss: 2.2829 | LR: 5.12e-05 | 9157.2ms/step | 2.2 samples/s
|
| 992 |
+
Step 9440/11700 | Loss: 2.5008 | LR: 5.07e-05 | 9169.4ms/step | 2.2 samples/s
|
| 993 |
+
Step 9450/11700 | Loss: 2.4437 | LR: 5.03e-05 | 9171.1ms/step | 2.2 samples/s
|
| 994 |
+
Step 9460/11700 | Loss: 2.4427 | LR: 4.99e-05 | 9164.0ms/step | 2.2 samples/s
|
| 995 |
+
Step 9470/11700 | Loss: 2.4177 | LR: 4.94e-05 | 9160.0ms/step | 2.2 samples/s
|
| 996 |
+
Step 9480/11700 | Loss: 2.3766 | LR: 4.90e-05 | 9124.3ms/step | 2.2 samples/s
|
| 997 |
+
Step 9490/11700 | Loss: 2.5288 | LR: 4.86e-05 | 9091.2ms/step | 2.2 samples/s
|
| 998 |
+
Step 9500/11700 | Loss: 2.3329 | LR: 4.82e-05 | 9083.6ms/step | 2.2 samples/s
|
| 999 |
+
Validation at step 9500: Loss = 2.4189, Perplexity = 11.23
|
| 1000 |
+
Step 9510/11700 | Loss: 2.4208 | LR: 4.77e-05 | 22171.4ms/step | 0.9 samples/s
|
| 1001 |
+
Step 9520/11700 | Loss: 2.3744 | LR: 4.73e-05 | 9083.4ms/step | 2.2 samples/s
|
| 1002 |
+
Step 9530/11700 | Loss: 2.3762 | LR: 4.69e-05 | 9081.0ms/step | 2.2 samples/s
|
| 1003 |
+
Step 9540/11700 | Loss: 2.3385 | LR: 4.65e-05 | 9079.3ms/step | 2.2 samples/s
|
| 1004 |
+
Step 9550/11700 | Loss: 2.5458 | LR: 4.61e-05 | 9077.4ms/step | 2.2 samples/s
|
| 1005 |
+
Step 9560/11700 | Loss: 2.3236 | LR: 4.57e-05 | 9077.5ms/step | 2.2 samples/s
|
| 1006 |
+
Step 9570/11700 | Loss: 2.3622 | LR: 4.52e-05 | 9096.5ms/step | 2.2 samples/s
|
| 1007 |
+
Step 9580/11700 | Loss: 2.4830 | LR: 4.48e-05 | 9101.4ms/step | 2.2 samples/s
|
| 1008 |
+
Step 9590/11700 | Loss: 2.4136 | LR: 4.44e-05 | 9096.7ms/step | 2.2 samples/s
|
| 1009 |
+
Step 9600/11700 | Loss: 2.4221 | LR: 4.40e-05 | 9100.0ms/step | 2.2 samples/s
|
| 1010 |
+
Step 9610/11700 | Loss: 2.4217 | LR: 4.36e-05 | 9098.0ms/step | 2.2 samples/s
|
| 1011 |
+
Step 9620/11700 | Loss: 2.4346 | LR: 4.32e-05 | 9096.1ms/step | 2.2 samples/s
|
| 1012 |
+
Step 9630/11700 | Loss: 2.3820 | LR: 4.28e-05 | 9099.6ms/step | 2.2 samples/s
|
| 1013 |
+
Step 9640/11700 | Loss: 2.4332 | LR: 4.24e-05 | 9100.8ms/step | 2.2 samples/s
|
| 1014 |
+
Step 9650/11700 | Loss: 2.5114 | LR: 4.20e-05 | 9093.8ms/step | 2.2 samples/s
|
| 1015 |
+
Step 9660/11700 | Loss: 2.5194 | LR: 4.16e-05 | 9091.2ms/step | 2.2 samples/s
|
| 1016 |
+
Step 9670/11700 | Loss: 2.4134 | LR: 4.12e-05 | 9089.9ms/step | 2.2 samples/s
|
| 1017 |
+
Step 9680/11700 | Loss: 2.2336 | LR: 4.08e-05 | 9093.9ms/step | 2.2 samples/s
|
| 1018 |
+
Step 9690/11700 | Loss: 2.4744 | LR: 4.04e-05 | 9099.4ms/step | 2.2 samples/s
|
| 1019 |
+
Step 9700/11700 | Loss: 2.3434 | LR: 4.00e-05 | 9099.8ms/step | 2.2 samples/s
|
| 1020 |
+
Step 9710/11700 | Loss: 2.3949 | LR: 3.97e-05 | 9098.3ms/step | 2.2 samples/s
|
| 1021 |
+
Step 9720/11700 | Loss: 2.4678 | LR: 3.93e-05 | 9099.1ms/step | 2.2 samples/s
|
| 1022 |
+
Step 9730/11700 | Loss: 2.3112 | LR: 3.89e-05 | 9103.3ms/step | 2.2 samples/s
|
| 1023 |
+
Step 9740/11700 | Loss: 2.4822 | LR: 3.85e-05 | 9099.7ms/step | 2.2 samples/s
|
| 1024 |
+
Step 9750/11700 | Loss: 2.4073 | LR: 3.81e-05 | 9098.1ms/step | 2.2 samples/s
|
| 1025 |
+
Validation at step 9750: Loss = 2.4181, Perplexity = 11.22
|
| 1026 |
+
Step 9760/11700 | Loss: 2.3913 | LR: 3.77e-05 | 22216.4ms/step | 0.9 samples/s
|
| 1027 |
+
Step 9770/11700 | Loss: 2.3342 | LR: 3.74e-05 | 9100.0ms/step | 2.2 samples/s
|
| 1028 |
+
Step 9780/11700 | Loss: 2.5310 | LR: 3.70e-05 | 9106.5ms/step | 2.2 samples/s
|
| 1029 |
+
Step 9790/11700 | Loss: 2.3876 | LR: 3.66e-05 | 9098.7ms/step | 2.2 samples/s
|
| 1030 |
+
Step 9800/11700 | Loss: 2.3358 | LR: 3.62e-05 | 9102.9ms/step | 2.2 samples/s
|
| 1031 |
+
Step 9810/11700 | Loss: 2.4528 | LR: 3.59e-05 | 9103.0ms/step | 2.2 samples/s
|
| 1032 |
+
Step 9820/11700 | Loss: 2.3030 | LR: 3.55e-05 | 9102.3ms/step | 2.2 samples/s
|
| 1033 |
+
Step 9830/11700 | Loss: 2.3220 | LR: 3.51e-05 | 9098.1ms/step | 2.2 samples/s
|
| 1034 |
+
Step 9840/11700 | Loss: 2.3566 | LR: 3.48e-05 | 9097.1ms/step | 2.2 samples/s
|
| 1035 |
+
Step 9850/11700 | Loss: 2.4318 | LR: 3.44e-05 | 9095.0ms/step | 2.2 samples/s
|
| 1036 |
+
Step 9860/11700 | Loss: 2.4119 | LR: 3.40e-05 | 9098.3ms/step | 2.2 samples/s
|
| 1037 |
+
Step 9870/11700 | Loss: 2.3591 | LR: 3.37e-05 | 9093.9ms/step | 2.2 samples/s
|
| 1038 |
+
Step 9880/11700 | Loss: 2.4002 | LR: 3.33e-05 | 9091.3ms/step | 2.2 samples/s
|
| 1039 |
+
Step 9890/11700 | Loss: 2.3055 | LR: 3.30e-05 | 9094.7ms/step | 2.2 samples/s
|
| 1040 |
+
Step 9900/11700 | Loss: 2.4387 | LR: 3.26e-05 | 9094.9ms/step | 2.2 samples/s
|
| 1041 |
+
Step 9910/11700 | Loss: 2.4232 | LR: 3.22e-05 | 9093.2ms/step | 2.2 samples/s
|
| 1042 |
+
Step 9920/11700 | Loss: 2.3867 | LR: 3.19e-05 | 9092.9ms/step | 2.2 samples/s
|
| 1043 |
+
Step 9930/11700 | Loss: 2.4695 | LR: 3.15e-05 | 9083.8ms/step | 2.2 samples/s
|
| 1044 |
+
Step 9940/11700 | Loss: 2.2801 | LR: 3.12e-05 | 9077.9ms/step | 2.2 samples/s
|
| 1045 |
+
Step 9950/11700 | Loss: 2.3903 | LR: 3.09e-05 | 9083.8ms/step | 2.2 samples/s
|
| 1046 |
+
Step 9960/11700 | Loss: 2.3374 | LR: 3.05e-05 | 9088.2ms/step | 2.2 samples/s
|
| 1047 |
+
Step 9970/11700 | Loss: 2.4123 | LR: 3.02e-05 | 9099.7ms/step | 2.2 samples/s
|
| 1048 |
+
Step 9980/11700 | Loss: 2.3435 | LR: 2.98e-05 | 9094.8ms/step | 2.2 samples/s
|
| 1049 |
+
Step 9990/11700 | Loss: 2.2962 | LR: 2.95e-05 | 9103.3ms/step | 2.2 samples/s
|
| 1050 |
+
Step 10000/11700 | Loss: 2.4109 | LR: 2.92e-05 | 9106.0ms/step | 2.2 samples/s
|
| 1051 |
+
Validation at step 10000: Loss = 2.4175, Perplexity = 11.22
|
| 1052 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-10000
|
| 1053 |
+
Step 10010/11700 | Loss: 2.5097 | LR: 2.88e-05 | 22548.5ms/step | 0.9 samples/s
|
| 1054 |
+
Step 10020/11700 | Loss: 2.4932 | LR: 2.85e-05 | 9106.1ms/step | 2.2 samples/s
|
| 1055 |
+
Step 10030/11700 | Loss: 2.3928 | LR: 2.82e-05 | 9105.3ms/step | 2.2 samples/s
|
| 1056 |
+
Step 10040/11700 | Loss: 2.4463 | LR: 2.78e-05 | 9099.3ms/step | 2.2 samples/s
|
| 1057 |
+
Step 10050/11700 | Loss: 2.4821 | LR: 2.75e-05 | 9103.3ms/step | 2.2 samples/s
|
| 1058 |
+
Step 10060/11700 | Loss: 2.3571 | LR: 2.72e-05 | 9102.9ms/step | 2.2 samples/s
|
| 1059 |
+
Step 10070/11700 | Loss: 2.4759 | LR: 2.68e-05 | 9102.8ms/step | 2.2 samples/s
|
| 1060 |
+
Step 10080/11700 | Loss: 2.3518 | LR: 2.65e-05 | 9100.5ms/step | 2.2 samples/s
|
| 1061 |
+
Step 10090/11700 | Loss: 2.4206 | LR: 2.62e-05 | 9102.1ms/step | 2.2 samples/s
|
| 1062 |
+
Step 10100/11700 | Loss: 2.3821 | LR: 2.59e-05 | 9102.8ms/step | 2.2 samples/s
|
| 1063 |
+
Step 10110/11700 | Loss: 2.4557 | LR: 2.56e-05 | 9097.5ms/step | 2.2 samples/s
|
| 1064 |
+
Step 10120/11700 | Loss: 2.3226 | LR: 2.52e-05 | 9100.8ms/step | 2.2 samples/s
|
| 1065 |
+
Step 10130/11700 | Loss: 2.4919 | LR: 2.49e-05 | 9102.8ms/step | 2.2 samples/s
|
| 1066 |
+
Step 10140/11700 | Loss: 2.5247 | LR: 2.46e-05 | 9104.9ms/step | 2.2 samples/s
|
| 1067 |
+
Step 10150/11700 | Loss: 2.4309 | LR: 2.43e-05 | 9100.1ms/step | 2.2 samples/s
|
| 1068 |
+
Step 10160/11700 | Loss: 2.4178 | LR: 2.40e-05 | 9100.3ms/step | 2.2 samples/s
|
| 1069 |
+
Step 10170/11700 | Loss: 2.4020 | LR: 2.37e-05 | 9104.2ms/step | 2.2 samples/s
|
| 1070 |
+
Step 10180/11700 | Loss: 2.4233 | LR: 2.34e-05 | 9107.6ms/step | 2.2 samples/s
|
| 1071 |
+
Step 10190/11700 | Loss: 2.3305 | LR: 2.31e-05 | 9101.2ms/step | 2.2 samples/s
|
| 1072 |
+
Step 10200/11700 | Loss: 2.3575 | LR: 2.28e-05 | 9103.3ms/step | 2.2 samples/s
|
| 1073 |
+
Step 10210/11700 | Loss: 2.4528 | LR: 2.25e-05 | 9101.5ms/step | 2.2 samples/s
|
| 1074 |
+
Step 10220/11700 | Loss: 2.4093 | LR: 2.22e-05 | 9099.6ms/step | 2.2 samples/s
|
| 1075 |
+
Step 10230/11700 | Loss: 2.5148 | LR: 2.19e-05 | 9105.7ms/step | 2.2 samples/s
|
| 1076 |
+
Step 10240/11700 | Loss: 2.5010 | LR: 2.16e-05 | 9106.7ms/step | 2.2 samples/s
|
| 1077 |
+
Step 10250/11700 | Loss: 2.4495 | LR: 2.13e-05 | 9101.9ms/step | 2.2 samples/s
|
| 1078 |
+
Validation at step 10250: Loss = 2.4171, Perplexity = 11.21
|
| 1079 |
+
Step 10260/11700 | Loss: 2.4596 | LR: 2.10e-05 | 22198.9ms/step | 0.9 samples/s
|
| 1080 |
+
Step 10270/11700 | Loss: 2.3600 | LR: 2.07e-05 | 9086.5ms/step | 2.2 samples/s
|
| 1081 |
+
Step 10280/11700 | Loss: 2.4585 | LR: 2.05e-05 | 9123.9ms/step | 2.2 samples/s
|
| 1082 |
+
Step 10290/11700 | Loss: 2.4423 | LR: 2.02e-05 | 9139.1ms/step | 2.2 samples/s
|
| 1083 |
+
Step 10300/11700 | Loss: 2.2947 | LR: 1.99e-05 | 9116.9ms/step | 2.2 samples/s
|
| 1084 |
+
Step 10310/11700 | Loss: 2.4039 | LR: 1.96e-05 | 9134.9ms/step | 2.2 samples/s
|
| 1085 |
+
Step 10320/11700 | Loss: 2.3497 | LR: 1.93e-05 | 9112.7ms/step | 2.2 samples/s
|
| 1086 |
+
Step 10330/11700 | Loss: 2.3420 | LR: 1.91e-05 | 9080.9ms/step | 2.2 samples/s
|
| 1087 |
+
Step 10340/11700 | Loss: 2.5107 | LR: 1.88e-05 | 9092.8ms/step | 2.2 samples/s
|
| 1088 |
+
Step 10350/11700 | Loss: 2.4409 | LR: 1.85e-05 | 9096.1ms/step | 2.2 samples/s
|
| 1089 |
+
Step 10360/11700 | Loss: 2.4368 | LR: 1.82e-05 | 9141.3ms/step | 2.2 samples/s
|
| 1090 |
+
Step 10370/11700 | Loss: 2.4174 | LR: 1.80e-05 | 9124.1ms/step | 2.2 samples/s
|
| 1091 |
+
Step 10380/11700 | Loss: 2.2527 | LR: 1.77e-05 | 9139.7ms/step | 2.2 samples/s
|
| 1092 |
+
Step 10390/11700 | Loss: 2.5497 | LR: 1.75e-05 | 9136.6ms/step | 2.2 samples/s
|
| 1093 |
+
Step 10400/11700 | Loss: 2.4588 | LR: 1.72e-05 | 9118.6ms/step | 2.2 samples/s
|
| 1094 |
+
Step 10410/11700 | Loss: 2.4077 | LR: 1.69e-05 | 9116.5ms/step | 2.2 samples/s
|
| 1095 |
+
Step 10420/11700 | Loss: 2.3855 | LR: 1.67e-05 | 9127.3ms/step | 2.2 samples/s
|
| 1096 |
+
Step 10430/11700 | Loss: 2.3808 | LR: 1.64e-05 | 9119.5ms/step | 2.2 samples/s
|
| 1097 |
+
Step 10440/11700 | Loss: 2.3537 | LR: 1.62e-05 | 9131.4ms/step | 2.2 samples/s
|
| 1098 |
+
Step 10450/11700 | Loss: 2.4298 | LR: 1.59e-05 | 9135.3ms/step | 2.2 samples/s
|
| 1099 |
+
Step 10460/11700 | Loss: 2.4546 | LR: 1.57e-05 | 9150.5ms/step | 2.2 samples/s
|
| 1100 |
+
Step 10470/11700 | Loss: 2.3395 | LR: 1.54e-05 | 9126.9ms/step | 2.2 samples/s
|
| 1101 |
+
Step 10480/11700 | Loss: 2.3809 | LR: 1.52e-05 | 9132.6ms/step | 2.2 samples/s
|
| 1102 |
+
Step 10490/11700 | Loss: 2.3225 | LR: 1.49e-05 | 9135.3ms/step | 2.2 samples/s
|
| 1103 |
+
Step 10500/11700 | Loss: 2.4105 | LR: 1.47e-05 | 9121.3ms/step | 2.2 samples/s
|
| 1104 |
+
Validation at step 10500: Loss = 2.4170, Perplexity = 11.21
|
| 1105 |
+
Step 10510/11700 | Loss: 2.5349 | LR: 1.44e-05 | 22222.6ms/step | 0.9 samples/s
|
| 1106 |
+
Step 10520/11700 | Loss: 2.4357 | LR: 1.42e-05 | 9102.4ms/step | 2.2 samples/s
|
| 1107 |
+
Step 10530/11700 | Loss: 2.4227 | LR: 1.40e-05 | 9107.3ms/step | 2.2 samples/s
|
| 1108 |
+
Step 10540/11700 | Loss: 2.2726 | LR: 1.37e-05 | 9111.3ms/step | 2.2 samples/s
|
| 1109 |
+
Step 10550/11700 | Loss: 2.4487 | LR: 1.35e-05 | 9109.7ms/step | 2.2 samples/s
|
| 1110 |
+
Step 10560/11700 | Loss: 2.5486 | LR: 1.33e-05 | 9103.0ms/step | 2.2 samples/s
|
| 1111 |
+
Step 10570/11700 | Loss: 2.3045 | LR: 1.30e-05 | 9108.7ms/step | 2.2 samples/s
|
| 1112 |
+
Step 10580/11700 | Loss: 2.4801 | LR: 1.28e-05 | 9117.5ms/step | 2.2 samples/s
|
| 1113 |
+
Step 10590/11700 | Loss: 2.3417 | LR: 1.26e-05 | 9126.4ms/step | 2.2 samples/s
|
| 1114 |
+
Step 10600/11700 | Loss: 2.4317 | LR: 1.23e-05 | 9124.0ms/step | 2.2 samples/s
|
| 1115 |
+
Step 10610/11700 | Loss: 2.4008 | LR: 1.21e-05 | 9121.9ms/step | 2.2 samples/s
|
| 1116 |
+
Step 10620/11700 | Loss: 2.3191 | LR: 1.19e-05 | 9113.9ms/step | 2.2 samples/s
|
| 1117 |
+
Step 10630/11700 | Loss: 2.3881 | LR: 1.17e-05 | 9120.8ms/step | 2.2 samples/s
|
| 1118 |
+
Step 10640/11700 | Loss: 2.3732 | LR: 1.15e-05 | 9137.7ms/step | 2.2 samples/s
|
| 1119 |
+
Step 10650/11700 | Loss: 2.4272 | LR: 1.13e-05 | 9141.0ms/step | 2.2 samples/s
|
| 1120 |
+
Step 10660/11700 | Loss: 2.3779 | LR: 1.10e-05 | 9106.6ms/step | 2.2 samples/s
|
| 1121 |
+
Step 10670/11700 | Loss: 2.3638 | LR: 1.08e-05 | 9103.7ms/step | 2.2 samples/s
|
| 1122 |
+
Step 10680/11700 | Loss: 2.4410 | LR: 1.06e-05 | 9121.0ms/step | 2.2 samples/s
|
| 1123 |
+
Step 10690/11700 | Loss: 2.4300 | LR: 1.04e-05 | 9104.0ms/step | 2.2 samples/s
|
| 1124 |
+
Step 10700/11700 | Loss: 2.3986 | LR: 1.02e-05 | 9121.9ms/step | 2.2 samples/s
|
| 1125 |
+
Step 10710/11700 | Loss: 2.4302 | LR: 1.00e-05 | 9112.8ms/step | 2.2 samples/s
|
| 1126 |
+
Step 10720/11700 | Loss: 2.3663 | LR: 9.82e-06 | 9108.8ms/step | 2.2 samples/s
|
| 1127 |
+
Step 10730/11700 | Loss: 2.4699 | LR: 9.62e-06 | 9133.5ms/step | 2.2 samples/s
|
| 1128 |
+
Step 10740/11700 | Loss: 2.2946 | LR: 9.42e-06 | 9148.5ms/step | 2.2 samples/s
|
| 1129 |
+
Step 10750/11700 | Loss: 2.3235 | LR: 9.23e-06 | 9152.1ms/step | 2.2 samples/s
|
| 1130 |
+
Validation at step 10750: Loss = 2.4169, Perplexity = 11.21
|
| 1131 |
+
Step 10760/11700 | Loss: 2.4929 | LR: 9.04e-06 | 22361.3ms/step | 0.9 samples/s
|
| 1132 |
+
Step 10770/11700 | Loss: 2.4670 | LR: 8.85e-06 | 9158.1ms/step | 2.2 samples/s
|
| 1133 |
+
Step 10780/11700 | Loss: 2.4953 | LR: 8.66e-06 | 9165.7ms/step | 2.2 samples/s
|
| 1134 |
+
Step 10790/11700 | Loss: 2.4054 | LR: 8.47e-06 | 9159.9ms/step | 2.2 samples/s
|
| 1135 |
+
Step 10800/11700 | Loss: 2.5014 | LR: 8.29e-06 | 9151.5ms/step | 2.2 samples/s
|
| 1136 |
+
Step 10810/11700 | Loss: 2.3980 | LR: 8.11e-06 | 9149.6ms/step | 2.2 samples/s
|
| 1137 |
+
Step 10820/11700 | Loss: 2.4854 | LR: 7.93e-06 | 9148.7ms/step | 2.2 samples/s
|
| 1138 |
+
Step 10830/11700 | Loss: 2.3843 | LR: 7.75e-06 | 9151.3ms/step | 2.2 samples/s
|
| 1139 |
+
Step 10840/11700 | Loss: 2.3658 | LR: 7.57e-06 | 9150.1ms/step | 2.2 samples/s
|
| 1140 |
+
Step 10850/11700 | Loss: 2.3163 | LR: 7.40e-06 | 9134.2ms/step | 2.2 samples/s
|
| 1141 |
+
Step 10860/11700 | Loss: 2.3183 | LR: 7.22e-06 | 9140.0ms/step | 2.2 samples/s
|
| 1142 |
+
Step 10870/11700 | Loss: 2.3455 | LR: 7.05e-06 | 9145.6ms/step | 2.2 samples/s
|
| 1143 |
+
Step 10880/11700 | Loss: 2.3252 | LR: 6.89e-06 | 9148.4ms/step | 2.2 samples/s
|
| 1144 |
+
Step 10890/11700 | Loss: 2.3133 | LR: 6.72e-06 | 9144.5ms/step | 2.2 samples/s
|
| 1145 |
+
Step 10900/11700 | Loss: 2.4186 | LR: 6.56e-06 | 9151.7ms/step | 2.2 samples/s
|
| 1146 |
+
Step 10910/11700 | Loss: 2.4580 | LR: 6.39e-06 | 9152.0ms/step | 2.2 samples/s
|
| 1147 |
+
Step 10920/11700 | Loss: 2.3813 | LR: 6.23e-06 | 9152.0ms/step | 2.2 samples/s
|
| 1148 |
+
Step 10930/11700 | Loss: 2.3034 | LR: 6.08e-06 | 9139.7ms/step | 2.2 samples/s
|
| 1149 |
+
Step 10940/11700 | Loss: 2.3248 | LR: 5.92e-06 | 9138.4ms/step | 2.2 samples/s
|
| 1150 |
+
Step 10950/11700 | Loss: 2.3327 | LR: 5.77e-06 | 9149.5ms/step | 2.2 samples/s
|
| 1151 |
+
Step 10960/11700 | Loss: 2.4807 | LR: 5.61e-06 | 9130.0ms/step | 2.2 samples/s
|
| 1152 |
+
Step 10970/11700 | Loss: 2.4386 | LR: 5.46e-06 | 9108.1ms/step | 2.2 samples/s
|
| 1153 |
+
Step 10980/11700 | Loss: 2.4359 | LR: 5.31e-06 | 9124.5ms/step | 2.2 samples/s
|
| 1154 |
+
Step 10990/11700 | Loss: 2.3741 | LR: 5.17e-06 | 9128.7ms/step | 2.2 samples/s
|
| 1155 |
+
Step 11000/11700 | Loss: 2.3525 | LR: 5.02e-06 | 9133.9ms/step | 2.2 samples/s
|
| 1156 |
+
Validation at step 11000: Loss = 2.4168, Perplexity = 11.21
|
| 1157 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-11000
|
| 1158 |
+
Step 11010/11700 | Loss: 2.3427 | LR: 4.88e-06 | 22607.8ms/step | 0.9 samples/s
|
| 1159 |
+
Step 11020/11700 | Loss: 2.4104 | LR: 4.74e-06 | 9131.2ms/step | 2.2 samples/s
|
| 1160 |
+
Step 11030/11700 | Loss: 2.4457 | LR: 4.60e-06 | 9135.8ms/step | 2.2 samples/s
|
| 1161 |
+
Step 11040/11700 | Loss: 2.4375 | LR: 4.47e-06 | 9135.8ms/step | 2.2 samples/s
|
| 1162 |
+
Step 11050/11700 | Loss: 2.4598 | LR: 4.33e-06 | 9131.4ms/step | 2.2 samples/s
|
| 1163 |
+
Step 11060/11700 | Loss: 2.3929 | LR: 4.20e-06 | 9131.5ms/step | 2.2 samples/s
|
| 1164 |
+
Step 11070/11700 | Loss: 2.3549 | LR: 4.07e-06 | 9129.3ms/step | 2.2 samples/s
|
| 1165 |
+
Step 11080/11700 | Loss: 2.4145 | LR: 3.94e-06 | 9122.4ms/step | 2.2 samples/s
|
| 1166 |
+
Step 11090/11700 | Loss: 2.3997 | LR: 3.82e-06 | 9150.4ms/step | 2.2 samples/s
|
| 1167 |
+
Step 11100/11700 | Loss: 2.4456 | LR: 3.69e-06 | 9149.3ms/step | 2.2 samples/s
|
| 1168 |
+
Step 11110/11700 | Loss: 2.3905 | LR: 3.57e-06 | 9144.8ms/step | 2.2 samples/s
|
| 1169 |
+
Step 11120/11700 | Loss: 2.4358 | LR: 3.45e-06 | 9146.0ms/step | 2.2 samples/s
|
| 1170 |
+
Step 11130/11700 | Loss: 2.4798 | LR: 3.34e-06 | 9148.4ms/step | 2.2 samples/s
|
| 1171 |
+
Step 11140/11700 | Loss: 2.3864 | LR: 3.22e-06 | 9144.8ms/step | 2.2 samples/s
|
| 1172 |
+
Step 11150/11700 | Loss: 2.4082 | LR: 3.11e-06 | 9144.2ms/step | 2.2 samples/s
|
| 1173 |
+
Step 11160/11700 | Loss: 2.3546 | LR: 2.99e-06 | 9146.1ms/step | 2.2 samples/s
|
| 1174 |
+
Step 11170/11700 | Loss: 2.4663 | LR: 2.88e-06 | 9143.8ms/step | 2.2 samples/s
|
| 1175 |
+
Step 11180/11700 | Loss: 2.3766 | LR: 2.78e-06 | 9140.5ms/step | 2.2 samples/s
|
| 1176 |
+
Step 11190/11700 | Loss: 2.4169 | LR: 2.67e-06 | 9141.8ms/step | 2.2 samples/s
|
| 1177 |
+
Step 11200/11700 | Loss: 2.4129 | LR: 2.57e-06 | 9143.4ms/step | 2.2 samples/s
|
| 1178 |
+
Step 11210/11700 | Loss: 2.4614 | LR: 2.47e-06 | 9144.1ms/step | 2.2 samples/s
|
| 1179 |
+
Step 11220/11700 | Loss: 2.3528 | LR: 2.37e-06 | 9143.7ms/step | 2.2 samples/s
|
| 1180 |
+
Step 11230/11700 | Loss: 2.2803 | LR: 2.27e-06 | 9141.3ms/step | 2.2 samples/s
|
| 1181 |
+
Step 11240/11700 | Loss: 2.3476 | LR: 2.17e-06 | 9144.4ms/step | 2.2 samples/s
|
| 1182 |
+
Step 11250/11700 | Loss: 2.3648 | LR: 2.08e-06 | 9144.3ms/step | 2.2 samples/s
|
| 1183 |
+
Validation at step 11250: Loss = 2.4168, Perplexity = 11.21
|
| 1184 |
+
Step 11260/11700 | Loss: 2.3975 | LR: 1.99e-06 | 22340.4ms/step | 0.9 samples/s
|
| 1185 |
+
Step 11270/11700 | Loss: 2.4421 | LR: 1.90e-06 | 9141.8ms/step | 2.2 samples/s
|
| 1186 |
+
Step 11280/11700 | Loss: 2.3819 | LR: 1.81e-06 | 9143.4ms/step | 2.2 samples/s
|
| 1187 |
+
Step 11290/11700 | Loss: 2.4519 | LR: 1.73e-06 | 9145.6ms/step | 2.2 samples/s
|
| 1188 |
+
Step 11300/11700 | Loss: 2.4193 | LR: 1.64e-06 | 9146.9ms/step | 2.2 samples/s
|
| 1189 |
+
Step 11310/11700 | Loss: 2.3875 | LR: 1.56e-06 | 9145.4ms/step | 2.2 samples/s
|
| 1190 |
+
Step 11320/11700 | Loss: 2.4187 | LR: 1.48e-06 | 9143.3ms/step | 2.2 samples/s
|
| 1191 |
+
Step 11330/11700 | Loss: 2.3440 | LR: 1.41e-06 | 9143.8ms/step | 2.2 samples/s
|
| 1192 |
+
Step 11340/11700 | Loss: 2.3200 | LR: 1.33e-06 | 9143.5ms/step | 2.2 samples/s
|
| 1193 |
+
Step 11350/11700 | Loss: 2.4451 | LR: 1.26e-06 | 9141.5ms/step | 2.2 samples/s
|
| 1194 |
+
Step 11360/11700 | Loss: 2.4379 | LR: 1.19e-06 | 9145.6ms/step | 2.2 samples/s
|
| 1195 |
+
Step 11370/11700 | Loss: 2.3974 | LR: 1.12e-06 | 9143.7ms/step | 2.2 samples/s
|
| 1196 |
+
Step 11380/11700 | Loss: 2.3833 | LR: 1.05e-06 | 9141.1ms/step | 2.2 samples/s
|
| 1197 |
+
Step 11390/11700 | Loss: 2.4030 | LR: 9.88e-07 | 9144.8ms/step | 2.2 samples/s
|
| 1198 |
+
Step 11400/11700 | Loss: 2.4292 | LR: 9.25e-07 | 9144.1ms/step | 2.2 samples/s
|
| 1199 |
+
Step 11410/11700 | Loss: 2.4034 | LR: 8.65e-07 | 9146.5ms/step | 2.2 samples/s
|
| 1200 |
+
Step 11420/11700 | Loss: 2.2530 | LR: 8.06e-07 | 9145.4ms/step | 2.2 samples/s
|
| 1201 |
+
Step 11430/11700 | Loss: 2.3557 | LR: 7.50e-07 | 9142.0ms/step | 2.2 samples/s
|
| 1202 |
+
Step 11440/11700 | Loss: 2.4461 | LR: 6.95e-07 | 9145.1ms/step | 2.2 samples/s
|
| 1203 |
+
Step 11450/11700 | Loss: 2.2934 | LR: 6.43e-07 | 9147.1ms/step | 2.2 samples/s
|
| 1204 |
+
Step 11460/11700 | Loss: 2.4179 | LR: 5.92e-07 | 9143.5ms/step | 2.2 samples/s
|
| 1205 |
+
Step 11470/11700 | Loss: 2.4641 | LR: 5.44e-07 | 9144.8ms/step | 2.2 samples/s
|
| 1206 |
+
Step 11480/11700 | Loss: 2.3326 | LR: 4.98e-07 | 9142.3ms/step | 2.2 samples/s
|
| 1207 |
+
Step 11490/11700 | Loss: 2.4337 | LR: 4.54e-07 | 9144.3ms/step | 2.2 samples/s
|
| 1208 |
+
Step 11500/11700 | Loss: 2.3676 | LR: 4.11e-07 | 9149.6ms/step | 2.2 samples/s
|
| 1209 |
+
Validation at step 11500: Loss = 2.4168, Perplexity = 11.21
|
| 1210 |
+
Step 11510/11700 | Loss: 2.3525 | LR: 3.71e-07 | 22351.6ms/step | 0.9 samples/s
|
| 1211 |
+
Step 11520/11700 | Loss: 2.4273 | LR: 3.33e-07 | 9148.2ms/step | 2.2 samples/s
|
| 1212 |
+
Step 11530/11700 | Loss: 2.3713 | LR: 2.97e-07 | 9147.8ms/step | 2.2 samples/s
|
| 1213 |
+
Step 11540/11700 | Loss: 2.3495 | LR: 2.63e-07 | 9147.0ms/step | 2.2 samples/s
|
| 1214 |
+
Step 11550/11700 | Loss: 2.4695 | LR: 2.31e-07 | 9145.7ms/step | 2.2 samples/s
|
| 1215 |
+
Step 11560/11700 | Loss: 2.3688 | LR: 2.02e-07 | 9145.8ms/step | 2.2 samples/s
|
| 1216 |
+
Step 11570/11700 | Loss: 2.4825 | LR: 1.74e-07 | 9144.8ms/step | 2.2 samples/s
|
| 1217 |
+
Step 11580/11700 | Loss: 2.3171 | LR: 1.48e-07 | 9145.7ms/step | 2.2 samples/s
|
| 1218 |
+
Step 11590/11700 | Loss: 2.4345 | LR: 1.24e-07 | 9147.6ms/step | 2.2 samples/s
|
| 1219 |
+
Step 11600/11700 | Loss: 2.4364 | LR: 1.03e-07 | 9145.1ms/step | 2.2 samples/s
|
| 1220 |
+
Step 11610/11700 | Loss: 2.4370 | LR: 8.33e-08 | 9146.4ms/step | 2.2 samples/s
|
| 1221 |
+
Step 11620/11700 | Loss: 2.3535 | LR: 6.58e-08 | 9146.2ms/step | 2.2 samples/s
|
| 1222 |
+
Step 11630/11700 | Loss: 2.3021 | LR: 5.04e-08 | 9143.1ms/step | 2.2 samples/s
|
| 1223 |
+
Step 11640/11700 | Loss: 2.4361 | LR: 3.70e-08 | 9145.8ms/step | 2.2 samples/s
|
| 1224 |
+
Step 11650/11700 | Loss: 2.4875 | LR: 2.57e-08 | 9145.5ms/step | 2.2 samples/s
|
| 1225 |
+
Step 11660/11700 | Loss: 2.2632 | LR: 1.65e-08 | 9147.1ms/step | 2.2 samples/s
|
| 1226 |
+
Step 11670/11700 | Loss: 2.4166 | LR: 9.26e-09 | 9149.2ms/step | 2.2 samples/s
|
| 1227 |
+
Step 11680/11700 | Loss: 2.4306 | LR: 4.12e-09 | 9147.5ms/step | 2.2 samples/s
|
| 1228 |
+
Step 11690/11700 | Loss: 2.3321 | LR: 1.03e-09 | 9147.6ms/step | 2.2 samples/s
|
| 1229 |
+
Step 11700/11700 | Loss: 2.3975 | LR: 0.00e+00 | 9147.1ms/step | 2.2 samples/s
|
| 1230 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752/checkpoint-11700
|
epfl_final/SparseAtt/wandb/run-20260707_035828-jmi9928p/files/requirements.txt
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
epfl_final/SparseAtt/wandb/run-20260707_035828-jmi9928p/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-07-07T03:58:28.689957Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"configs/ladder30k/dense.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/TokenPrune.git",
|
| 14 |
+
"commit": "88298c7a418622530ca5426b22b1e5ebd10ca0cb"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "h100-train-4-gpu-0-0",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 23 |
+
"gpu_count": 4,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "9374931091456",
|
| 27 |
+
"used": "4873970147328"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1623005732864"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-85844f64-d144-1c84-306d-f88b5f232a77"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 43 |
+
"memoryTotal": "85520809984",
|
| 44 |
+
"cudaCores": 16896,
|
| 45 |
+
"architecture": "Hopper",
|
| 46 |
+
"uuid": "GPU-4e4c2343-4f5b-a2ff-312e-03d402a27803"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 50 |
+
"memoryTotal": "85520809984",
|
| 51 |
+
"cudaCores": 16896,
|
| 52 |
+
"architecture": "Hopper",
|
| 53 |
+
"uuid": "GPU-fbf0ccd1-8bd4-5f57-6e74-6c0cf7eb70c9"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 57 |
+
"memoryTotal": "85520809984",
|
| 58 |
+
"cudaCores": 16896,
|
| 59 |
+
"architecture": "Hopper",
|
| 60 |
+
"uuid": "GPU-410cba62-f098-3454-c960-ebc55bd081af"
|
| 61 |
+
}
|
| 62 |
+
],
|
| 63 |
+
"cudaVersion": "13.0",
|
| 64 |
+
"writerId": "wt58lqx6cy40hdn2zs4v7es7obp06md4"
|
| 65 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260707_035828-jmi9928p/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"eval/perplexity":11.210252120048505,"speed/samples_per_sec":2.1864967048822845,"train/learning_rate":0,"eval/dense_interval_start":-1,"eval/uses_dense_attention":1,"speed/steps_per_sec":0.10932483524411422,"eval/loss":2.416828727468848,"_wandb":{"runtime":112918},"train/global_step":11700,"speed/ms_per_step":9147.052431106567,"_runtime":112918.236246079,"_timestamp":1.7835096251027942e+09,"eval/dense_interval_active":0,"train/epoch":0.3122998078155029,"train/loss":2.3975004887208344,"_step":11700,"eval/uses_sparse_attention":0,"eval/dense_interval_end":-1}
|
epfl_final/SparseAtt/wandb/run-20260707_035828-jmi9928p/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260707_035828-jmi9928p/logs/debug.log
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-07-07 03:58:28,711 INFO MainThread:2484 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-07-07 03:58:28,711 INFO MainThread:2484 [wandb_setup.py:_flush():81] Configure stats pid to 2484
|
| 3 |
+
2026-07-07 03:58:28,711 INFO MainThread:2484 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-07-07 03:58:28,711 INFO MainThread:2484 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260707_035828-jmi9928p/logs/debug.log
|
| 5 |
+
2026-07-07 03:58:28,712 INFO MainThread:2484 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260707_035828-jmi9928p/logs/debug-internal.log
|
| 6 |
+
2026-07-07 03:58:28,712 INFO MainThread:2484 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-07-07 03:58:28,712 INFO MainThread:2484 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/longctx30k/tokenized', 'val_path': None, 'max_seq_length': 30720, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l30k365m_dense/run_20260707_035752', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 11700, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 20, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m dense 30k', 'wandb_tags': ['qwen3', '365m', '30k', 'scaleladder', 'dense'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 120, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-07-07 03:58:28,712 INFO MainThread:2484 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-07-07 03:58:28,965 INFO MainThread:2484 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-07-07 03:58:29,292 INFO MainThread:2484 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-07-07 03:58:29,294 INFO MainThread:2484 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-07-07 03:58:29,309 INFO MainThread:2484 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-07-07 03:58:29,899 INFO MainThread:2484 [wandb_init.py:init():1056] starting run threads in backend
|
| 15 |
+
2026-07-07 03:58:30,143 INFO MainThread:2484 [wandb_run.py:_console_start():2554] atexit reg
|
| 16 |
+
2026-07-07 03:58:30,143 INFO MainThread:2484 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 17 |
+
2026-07-07 03:58:30,143 INFO MainThread:2484 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 18 |
+
2026-07-07 03:58:30,143 INFO MainThread:2484 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 19 |
+
2026-07-07 03:58:30,147 INFO MainThread:2484 [wandb_init.py:init():1094] run started, returning control to user process
|
| 20 |
+
2026-07-08 11:20:28,135 INFO MainThread:2484 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/jmi9928p
|
| 21 |
+
2026-07-08 11:20:28,135 INFO MainThread:2484 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
|
| 22 |
+
2026-07-08 11:20:28,136 INFO MainThread:2484 [wandb_run.py:_restore():2502] restore
|
| 23 |
+
2026-07-08 11:20:28,136 INFO MainThread:2484 [wandb_run.py:_restore():2508] restore done
|
| 24 |
+
2026-07-08 11:20:34,193 INFO MainThread:2484 [wandb_run.py:_footer_sync_info():3889] logging synced files
|
epfl_final/SparseAtt/wandb/run-20260707_035828-jmi9928p/run-jmi9928p.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:266f6ce525ad9f31a8ece5857d5b98993edc2405a8161e76d8cedc5c3070861e
|
| 3 |
+
size 35421241
|
epfl_final/SparseAtt/wandb/run-20260707_035835-sh1m3lmh/files/config.yaml
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.26.1
|
| 4 |
+
e:
|
| 5 |
+
pkavbxq1hkh38trzbvq50jk6zamishzu:
|
| 6 |
+
args:
|
| 7 |
+
- --config
|
| 8 |
+
- configs/ladder30k/resampled.json
|
| 9 |
+
codePath: train.py
|
| 10 |
+
codePathLocal: train.py
|
| 11 |
+
cpu_count: 96
|
| 12 |
+
cpu_count_logical: 192
|
| 13 |
+
cudaVersion: "13.0"
|
| 14 |
+
disk:
|
| 15 |
+
/:
|
| 16 |
+
total: "9374931091456"
|
| 17 |
+
used: "4873970151424"
|
| 18 |
+
email: ruben.navasardyan@mail.utoronto.ca
|
| 19 |
+
executable: /mloscratch/homes/navasard/conda/bin/python
|
| 20 |
+
git:
|
| 21 |
+
commit: 88298c7a418622530ca5426b22b1e5ebd10ca0cb
|
| 22 |
+
remote: https://github.com/rubentium/TokenPrune.git
|
| 23 |
+
gpu: NVIDIA H100 80GB HBM3
|
| 24 |
+
gpu_count: 4
|
| 25 |
+
gpu_nvidia:
|
| 26 |
+
- architecture: Hopper
|
| 27 |
+
cudaCores: 16896
|
| 28 |
+
memoryTotal: "85520809984"
|
| 29 |
+
name: NVIDIA H100 80GB HBM3
|
| 30 |
+
uuid: GPU-85844f64-d144-1c84-306d-f88b5f232a77
|
| 31 |
+
- architecture: Hopper
|
| 32 |
+
cudaCores: 16896
|
| 33 |
+
memoryTotal: "85520809984"
|
| 34 |
+
name: NVIDIA H100 80GB HBM3
|
| 35 |
+
uuid: GPU-4e4c2343-4f5b-a2ff-312e-03d402a27803
|
| 36 |
+
- architecture: Hopper
|
| 37 |
+
cudaCores: 16896
|
| 38 |
+
memoryTotal: "85520809984"
|
| 39 |
+
name: NVIDIA H100 80GB HBM3
|
| 40 |
+
uuid: GPU-fbf0ccd1-8bd4-5f57-6e74-6c0cf7eb70c9
|
| 41 |
+
- architecture: Hopper
|
| 42 |
+
cudaCores: 16896
|
| 43 |
+
memoryTotal: "85520809984"
|
| 44 |
+
name: NVIDIA H100 80GB HBM3
|
| 45 |
+
uuid: GPU-410cba62-f098-3454-c960-ebc55bd081af
|
| 46 |
+
host: h100-train-4-gpu-0-0
|
| 47 |
+
memory:
|
| 48 |
+
total: "1623005732864"
|
| 49 |
+
os: Linux-6.8.0-124-generic-x86_64-with-glibc2.35
|
| 50 |
+
program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
|
| 51 |
+
python: CPython 3.13.13
|
| 52 |
+
root: /mloscratch/homes/navasard/sparse/SparseAtt
|
| 53 |
+
startedAt: "2026-07-07T03:58:35.279877Z"
|
| 54 |
+
writerId: pkavbxq1hkh38trzbvq50jk6zamishzu
|
| 55 |
+
m: []
|
| 56 |
+
python_version: 3.13.13
|
| 57 |
+
t:
|
| 58 |
+
"1":
|
| 59 |
+
- 1
|
| 60 |
+
- 5
|
| 61 |
+
- 11
|
| 62 |
+
- 49
|
| 63 |
+
- 51
|
| 64 |
+
- 53
|
| 65 |
+
- 71
|
| 66 |
+
"2":
|
| 67 |
+
- 1
|
| 68 |
+
- 5
|
| 69 |
+
- 11
|
| 70 |
+
- 49
|
| 71 |
+
- 51
|
| 72 |
+
- 53
|
| 73 |
+
- 71
|
| 74 |
+
"3":
|
| 75 |
+
- 2
|
| 76 |
+
- 13
|
| 77 |
+
- 15
|
| 78 |
+
- 16
|
| 79 |
+
- 61
|
| 80 |
+
"4": 3.13.13
|
| 81 |
+
"5": 0.26.1
|
| 82 |
+
"6": 5.7.0
|
| 83 |
+
"12": 0.26.1
|
| 84 |
+
"13": linux-x86_64
|
| 85 |
+
attention_type:
|
| 86 |
+
value: sparse_stochastic
|
| 87 |
+
checkpoint_activations:
|
| 88 |
+
value: false
|
| 89 |
+
dense_mode:
|
| 90 |
+
value: false
|
| 91 |
+
dense_step_intervals:
|
| 92 |
+
value: null
|
| 93 |
+
eval_batches:
|
| 94 |
+
value: 50
|
| 95 |
+
eval_steps:
|
| 96 |
+
value: 250
|
| 97 |
+
force_new_run:
|
| 98 |
+
value: true
|
| 99 |
+
gradient_accumulation_steps:
|
| 100 |
+
value: 20
|
| 101 |
+
head_dim:
|
| 102 |
+
value: null
|
| 103 |
+
hidden_size:
|
| 104 |
+
value: 768
|
| 105 |
+
init_from_scratch:
|
| 106 |
+
value: true
|
| 107 |
+
intermediate_size:
|
| 108 |
+
value: 3072
|
| 109 |
+
learning_rate:
|
| 110 |
+
value: 0.0005
|
| 111 |
+
logging_steps:
|
| 112 |
+
value: 10
|
| 113 |
+
lr_scheduler:
|
| 114 |
+
value: cosine
|
| 115 |
+
max_grad_norm:
|
| 116 |
+
value: 1
|
| 117 |
+
max_seq_length:
|
| 118 |
+
value: 30720
|
| 119 |
+
max_steps:
|
| 120 |
+
value: 11700
|
| 121 |
+
mixed_precision:
|
| 122 |
+
value: bf16
|
| 123 |
+
model_dtype:
|
| 124 |
+
value: bfloat16
|
| 125 |
+
model_path:
|
| 126 |
+
value: /mloscratch/homes/navasard/qwen3_06b_base
|
| 127 |
+
num_attention_heads:
|
| 128 |
+
value: 6
|
| 129 |
+
num_epochs:
|
| 130 |
+
value: 1
|
| 131 |
+
num_hidden_layers:
|
| 132 |
+
value: 28
|
| 133 |
+
num_key_value_heads:
|
| 134 |
+
value: 2
|
| 135 |
+
num_workers:
|
| 136 |
+
value: 8
|
| 137 |
+
output_dir:
|
| 138 |
+
value: /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753
|
| 139 |
+
per_device_batch_size:
|
| 140 |
+
value: 1
|
| 141 |
+
resume_from_checkpoint:
|
| 142 |
+
value: null
|
| 143 |
+
save_steps:
|
| 144 |
+
value: 1000
|
| 145 |
+
seed:
|
| 146 |
+
value: 42
|
| 147 |
+
sparse_block_size:
|
| 148 |
+
value: 128
|
| 149 |
+
sparse_layer_indices:
|
| 150 |
+
value: null
|
| 151 |
+
sparse_mask_cache_dir:
|
| 152 |
+
value: /mloscratch/homes/navasard/sparse/mask_cache_30k_b128_nrb120
|
| 153 |
+
sparse_mask_pool_size:
|
| 154 |
+
value: 1000
|
| 155 |
+
sparse_mask_seed:
|
| 156 |
+
value: 0
|
| 157 |
+
sparse_num_random_blocks:
|
| 158 |
+
value: 120
|
| 159 |
+
sparse_val_dense_compare:
|
| 160 |
+
value: false
|
| 161 |
+
strategy:
|
| 162 |
+
value: ddp
|
| 163 |
+
train_path:
|
| 164 |
+
value: /mloscratch/homes/navasard/data/longctx30k/tokenized
|
| 165 |
+
val_path:
|
| 166 |
+
value: null
|
| 167 |
+
verbose:
|
| 168 |
+
value: true
|
| 169 |
+
wandb_enabled:
|
| 170 |
+
value: true
|
| 171 |
+
wandb_name:
|
| 172 |
+
value: 365m resampled 30k nrb120
|
| 173 |
+
wandb_project:
|
| 174 |
+
value: stochastic_sparsity
|
| 175 |
+
wandb_tags:
|
| 176 |
+
value:
|
| 177 |
+
- qwen3
|
| 178 |
+
- 365m
|
| 179 |
+
- 30k
|
| 180 |
+
- scaleladder
|
| 181 |
+
- resampled
|
| 182 |
+
warmup_steps:
|
| 183 |
+
value: 750
|
| 184 |
+
weight_decay:
|
| 185 |
+
value: 0.01
|
epfl_final/SparseAtt/wandb/run-20260707_035835-sh1m3lmh/files/output.log
ADDED
|
@@ -0,0 +1,1230 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Starting training...
|
| 2 |
+
Switching attention mode for step 1: sparse_stochastic (configured sparse attention)
|
| 3 |
+
Step 10/11700 | Loss: 12.0479 | LR: 6.67e-06 | 11329.2ms/step | 1.8 samples/s
|
| 4 |
+
Step 20/11700 | Loss: 11.9647 | LR: 1.33e-05 | 7600.0ms/step | 2.6 samples/s
|
| 5 |
+
Step 30/11700 | Loss: 11.6988 | LR: 2.00e-05 | 7599.8ms/step | 2.6 samples/s
|
| 6 |
+
Step 40/11700 | Loss: 11.1859 | LR: 2.67e-05 | 7591.9ms/step | 2.6 samples/s
|
| 7 |
+
Step 50/11700 | Loss: 10.8574 | LR: 3.33e-05 | 7571.2ms/step | 2.6 samples/s
|
| 8 |
+
Step 60/11700 | Loss: 10.5733 | LR: 4.00e-05 | 7557.1ms/step | 2.6 samples/s
|
| 9 |
+
Step 70/11700 | Loss: 10.2937 | LR: 4.67e-05 | 7547.9ms/step | 2.6 samples/s
|
| 10 |
+
Step 80/11700 | Loss: 10.0250 | LR: 5.33e-05 | 7550.5ms/step | 2.6 samples/s
|
| 11 |
+
Step 90/11700 | Loss: 9.7320 | LR: 6.00e-05 | 7539.6ms/step | 2.7 samples/s
|
| 12 |
+
Step 100/11700 | Loss: 9.3457 | LR: 6.67e-05 | 7543.7ms/step | 2.7 samples/s
|
| 13 |
+
Step 110/11700 | Loss: 8.8785 | LR: 7.33e-05 | 7540.9ms/step | 2.7 samples/s
|
| 14 |
+
Step 120/11700 | Loss: 8.3278 | LR: 8.00e-05 | 7543.5ms/step | 2.7 samples/s
|
| 15 |
+
Step 130/11700 | Loss: 7.8522 | LR: 8.67e-05 | 7546.4ms/step | 2.7 samples/s
|
| 16 |
+
Step 140/11700 | Loss: 7.5728 | LR: 9.33e-05 | 7549.7ms/step | 2.6 samples/s
|
| 17 |
+
Step 150/11700 | Loss: 7.3827 | LR: 1.00e-04 | 7551.9ms/step | 2.6 samples/s
|
| 18 |
+
Step 160/11700 | Loss: 7.1803 | LR: 1.07e-04 | 7559.6ms/step | 2.6 samples/s
|
| 19 |
+
Step 170/11700 | Loss: 7.0660 | LR: 1.13e-04 | 7565.9ms/step | 2.6 samples/s
|
| 20 |
+
Step 180/11700 | Loss: 6.9225 | LR: 1.20e-04 | 7574.8ms/step | 2.6 samples/s
|
| 21 |
+
Step 190/11700 | Loss: 6.8360 | LR: 1.27e-04 | 7583.7ms/step | 2.6 samples/s
|
| 22 |
+
Step 200/11700 | Loss: 6.6722 | LR: 1.33e-04 | 7582.5ms/step | 2.6 samples/s
|
| 23 |
+
Step 210/11700 | Loss: 6.5912 | LR: 1.40e-04 | 7592.1ms/step | 2.6 samples/s
|
| 24 |
+
Step 220/11700 | Loss: 6.4450 | LR: 1.47e-04 | 7599.8ms/step | 2.6 samples/s
|
| 25 |
+
Step 230/11700 | Loss: 6.3792 | LR: 1.53e-04 | 7594.3ms/step | 2.6 samples/s
|
| 26 |
+
Step 240/11700 | Loss: 6.2398 | LR: 1.60e-04 | 7597.7ms/step | 2.6 samples/s
|
| 27 |
+
Step 250/11700 | Loss: 6.2632 | LR: 1.67e-04 | 7600.1ms/step | 2.6 samples/s
|
| 28 |
+
Validation at step 250: Loss = 6.1315, Perplexity = 460.10
|
| 29 |
+
Step 260/11700 | Loss: 6.1782 | LR: 1.73e-04 | 21230.3ms/step | 0.9 samples/s
|
| 30 |
+
Step 270/11700 | Loss: 6.0482 | LR: 1.80e-04 | 7604.0ms/step | 2.6 samples/s
|
| 31 |
+
Step 280/11700 | Loss: 5.9825 | LR: 1.87e-04 | 7600.6ms/step | 2.6 samples/s
|
| 32 |
+
Step 290/11700 | Loss: 5.8524 | LR: 1.93e-04 | 7611.6ms/step | 2.6 samples/s
|
| 33 |
+
Step 300/11700 | Loss: 5.8357 | LR: 2.00e-04 | 7606.3ms/step | 2.6 samples/s
|
| 34 |
+
Step 310/11700 | Loss: 5.8035 | LR: 2.07e-04 | 7609.8ms/step | 2.6 samples/s
|
| 35 |
+
Step 320/11700 | Loss: 5.6966 | LR: 2.13e-04 | 7605.5ms/step | 2.6 samples/s
|
| 36 |
+
Step 330/11700 | Loss: 5.5916 | LR: 2.20e-04 | 7600.5ms/step | 2.6 samples/s
|
| 37 |
+
Step 340/11700 | Loss: 5.5418 | LR: 2.27e-04 | 7600.1ms/step | 2.6 samples/s
|
| 38 |
+
Step 350/11700 | Loss: 5.3863 | LR: 2.33e-04 | 7604.4ms/step | 2.6 samples/s
|
| 39 |
+
Step 360/11700 | Loss: 5.4568 | LR: 2.40e-04 | 7605.7ms/step | 2.6 samples/s
|
| 40 |
+
Step 370/11700 | Loss: 5.4807 | LR: 2.47e-04 | 7601.9ms/step | 2.6 samples/s
|
| 41 |
+
Step 380/11700 | Loss: 5.4711 | LR: 2.53e-04 | 7608.6ms/step | 2.6 samples/s
|
| 42 |
+
Step 390/11700 | Loss: 5.4323 | LR: 2.60e-04 | 7601.6ms/step | 2.6 samples/s
|
| 43 |
+
Step 400/11700 | Loss: 5.2830 | LR: 2.67e-04 | 7600.1ms/step | 2.6 samples/s
|
| 44 |
+
Step 410/11700 | Loss: 5.2953 | LR: 2.73e-04 | 7598.0ms/step | 2.6 samples/s
|
| 45 |
+
Step 420/11700 | Loss: 5.2572 | LR: 2.80e-04 | 7600.0ms/step | 2.6 samples/s
|
| 46 |
+
Step 430/11700 | Loss: 5.2149 | LR: 2.87e-04 | 7604.0ms/step | 2.6 samples/s
|
| 47 |
+
Step 440/11700 | Loss: 5.2148 | LR: 2.93e-04 | 7600.2ms/step | 2.6 samples/s
|
| 48 |
+
Step 450/11700 | Loss: 5.1298 | LR: 3.00e-04 | 7600.0ms/step | 2.6 samples/s
|
| 49 |
+
Step 460/11700 | Loss: 5.1517 | LR: 3.07e-04 | 7600.1ms/step | 2.6 samples/s
|
| 50 |
+
Step 470/11700 | Loss: 5.1186 | LR: 3.13e-04 | 7600.0ms/step | 2.6 samples/s
|
| 51 |
+
Step 480/11700 | Loss: 5.1114 | LR: 3.20e-04 | 7599.9ms/step | 2.6 samples/s
|
| 52 |
+
Step 490/11700 | Loss: 5.0286 | LR: 3.27e-04 | 7602.3ms/step | 2.6 samples/s
|
| 53 |
+
Step 500/11700 | Loss: 4.9817 | LR: 3.33e-04 | 7606.0ms/step | 2.6 samples/s
|
| 54 |
+
Validation at step 500: Loss = 4.9360, Perplexity = 139.22
|
| 55 |
+
Step 510/11700 | Loss: 4.9004 | LR: 3.40e-04 | 18995.0ms/step | 1.1 samples/s
|
| 56 |
+
Step 520/11700 | Loss: 4.9012 | LR: 3.47e-04 | 7627.4ms/step | 2.6 samples/s
|
| 57 |
+
Step 530/11700 | Loss: 4.8322 | LR: 3.53e-04 | 7626.6ms/step | 2.6 samples/s
|
| 58 |
+
Step 540/11700 | Loss: 4.7308 | LR: 3.60e-04 | 7633.1ms/step | 2.6 samples/s
|
| 59 |
+
Step 550/11700 | Loss: 4.7559 | LR: 3.67e-04 | 7629.6ms/step | 2.6 samples/s
|
| 60 |
+
Step 560/11700 | Loss: 4.7494 | LR: 3.73e-04 | 7632.7ms/step | 2.6 samples/s
|
| 61 |
+
Step 570/11700 | Loss: 4.5046 | LR: 3.80e-04 | 7639.8ms/step | 2.6 samples/s
|
| 62 |
+
Step 580/11700 | Loss: 4.5455 | LR: 3.87e-04 | 7633.7ms/step | 2.6 samples/s
|
| 63 |
+
Step 590/11700 | Loss: 4.5063 | LR: 3.93e-04 | 7628.7ms/step | 2.6 samples/s
|
| 64 |
+
Step 600/11700 | Loss: 4.5620 | LR: 4.00e-04 | 7628.2ms/step | 2.6 samples/s
|
| 65 |
+
Step 610/11700 | Loss: 4.5280 | LR: 4.07e-04 | 7630.2ms/step | 2.6 samples/s
|
| 66 |
+
Step 620/11700 | Loss: 4.5137 | LR: 4.13e-04 | 7632.8ms/step | 2.6 samples/s
|
| 67 |
+
Step 630/11700 | Loss: 4.3595 | LR: 4.20e-04 | 7637.1ms/step | 2.6 samples/s
|
| 68 |
+
Step 640/11700 | Loss: 4.4466 | LR: 4.27e-04 | 7638.8ms/step | 2.6 samples/s
|
| 69 |
+
Step 650/11700 | Loss: 4.3167 | LR: 4.33e-04 | 7632.1ms/step | 2.6 samples/s
|
| 70 |
+
Step 660/11700 | Loss: 4.4288 | LR: 4.40e-04 | 7630.8ms/step | 2.6 samples/s
|
| 71 |
+
Step 670/11700 | Loss: 4.3230 | LR: 4.47e-04 | 7634.9ms/step | 2.6 samples/s
|
| 72 |
+
Step 680/11700 | Loss: 4.2359 | LR: 4.53e-04 | 7629.6ms/step | 2.6 samples/s
|
| 73 |
+
Step 690/11700 | Loss: 4.0769 | LR: 4.60e-04 | 7633.8ms/step | 2.6 samples/s
|
| 74 |
+
Step 700/11700 | Loss: 4.2452 | LR: 4.67e-04 | 7636.3ms/step | 2.6 samples/s
|
| 75 |
+
Step 710/11700 | Loss: 4.2274 | LR: 4.73e-04 | 7637.9ms/step | 2.6 samples/s
|
| 76 |
+
Step 720/11700 | Loss: 4.1034 | LR: 4.80e-04 | 7636.5ms/step | 2.6 samples/s
|
| 77 |
+
Step 730/11700 | Loss: 3.9870 | LR: 4.87e-04 | 7636.2ms/step | 2.6 samples/s
|
| 78 |
+
Step 740/11700 | Loss: 4.1412 | LR: 4.93e-04 | 7641.2ms/step | 2.6 samples/s
|
| 79 |
+
Step 750/11700 | Loss: 4.0230 | LR: 5.00e-04 | 7638.8ms/step | 2.6 samples/s
|
| 80 |
+
Validation at step 750: Loss = 3.9706, Perplexity = 53.02
|
| 81 |
+
Step 760/11700 | Loss: 4.0455 | LR: 5.00e-04 | 19000.0ms/step | 1.1 samples/s
|
| 82 |
+
Step 770/11700 | Loss: 3.8142 | LR: 5.00e-04 | 7634.6ms/step | 2.6 samples/s
|
| 83 |
+
Step 780/11700 | Loss: 4.0223 | LR: 5.00e-04 | 7632.9ms/step | 2.6 samples/s
|
| 84 |
+
Step 790/11700 | Loss: 3.7921 | LR: 5.00e-04 | 7633.7ms/step | 2.6 samples/s
|
| 85 |
+
Step 800/11700 | Loss: 3.7885 | LR: 5.00e-04 | 7629.8ms/step | 2.6 samples/s
|
| 86 |
+
Step 810/11700 | Loss: 3.8295 | LR: 5.00e-04 | 7632.8ms/step | 2.6 samples/s
|
| 87 |
+
Step 820/11700 | Loss: 3.8057 | LR: 5.00e-04 | 7634.0ms/step | 2.6 samples/s
|
| 88 |
+
Step 830/11700 | Loss: 3.8012 | LR: 5.00e-04 | 7624.5ms/step | 2.6 samples/s
|
| 89 |
+
Step 840/11700 | Loss: 3.8010 | LR: 5.00e-04 | 7634.0ms/step | 2.6 samples/s
|
| 90 |
+
Step 850/11700 | Loss: 3.8501 | LR: 5.00e-04 | 7629.5ms/step | 2.6 samples/s
|
| 91 |
+
Step 860/11700 | Loss: 3.6847 | LR: 5.00e-04 | 7634.0ms/step | 2.6 samples/s
|
| 92 |
+
Step 870/11700 | Loss: 3.7236 | LR: 5.00e-04 | 7638.1ms/step | 2.6 samples/s
|
| 93 |
+
Step 880/11700 | Loss: 3.7636 | LR: 5.00e-04 | 7632.9ms/step | 2.6 samples/s
|
| 94 |
+
Step 890/11700 | Loss: 3.6095 | LR: 5.00e-04 | 7636.6ms/step | 2.6 samples/s
|
| 95 |
+
Step 900/11700 | Loss: 3.5485 | LR: 5.00e-04 | 7629.0ms/step | 2.6 samples/s
|
| 96 |
+
Step 910/11700 | Loss: 3.7111 | LR: 5.00e-04 | 7636.1ms/step | 2.6 samples/s
|
| 97 |
+
Step 920/11700 | Loss: 3.5776 | LR: 5.00e-04 | 7617.8ms/step | 2.6 samples/s
|
| 98 |
+
Step 930/11700 | Loss: 3.5592 | LR: 5.00e-04 | 7633.3ms/step | 2.6 samples/s
|
| 99 |
+
Step 940/11700 | Loss: 3.6484 | LR: 5.00e-04 | 7628.7ms/step | 2.6 samples/s
|
| 100 |
+
Step 950/11700 | Loss: 3.4823 | LR: 5.00e-04 | 7630.0ms/step | 2.6 samples/s
|
| 101 |
+
Step 960/11700 | Loss: 3.5535 | LR: 5.00e-04 | 7622.5ms/step | 2.6 samples/s
|
| 102 |
+
Step 970/11700 | Loss: 3.6103 | LR: 5.00e-04 | 7631.1ms/step | 2.6 samples/s
|
| 103 |
+
Step 980/11700 | Loss: 3.3337 | LR: 4.99e-04 | 7633.1ms/step | 2.6 samples/s
|
| 104 |
+
Step 990/11700 | Loss: 3.5460 | LR: 4.99e-04 | 7635.6ms/step | 2.6 samples/s
|
| 105 |
+
Step 1000/11700 | Loss: 3.4406 | LR: 4.99e-04 | 7624.1ms/step | 2.6 samples/s
|
| 106 |
+
Validation at step 1000: Loss = 3.4950, Perplexity = 32.95
|
| 107 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-1000
|
| 108 |
+
Step 1010/11700 | Loss: 3.5669 | LR: 4.99e-04 | 19271.6ms/step | 1.0 samples/s
|
| 109 |
+
Step 1020/11700 | Loss: 3.4663 | LR: 4.99e-04 | 7600.2ms/step | 2.6 samples/s
|
| 110 |
+
Step 1030/11700 | Loss: 3.4848 | LR: 4.99e-04 | 7600.0ms/step | 2.6 samples/s
|
| 111 |
+
Step 1040/11700 | Loss: 3.5081 | LR: 4.99e-04 | 7600.0ms/step | 2.6 samples/s
|
| 112 |
+
Step 1050/11700 | Loss: 3.4249 | LR: 4.99e-04 | 7604.0ms/step | 2.6 samples/s
|
| 113 |
+
Step 1060/11700 | Loss: 3.4844 | LR: 4.99e-04 | 7600.0ms/step | 2.6 samples/s
|
| 114 |
+
Step 1070/11700 | Loss: 3.4368 | LR: 4.99e-04 | 7606.0ms/step | 2.6 samples/s
|
| 115 |
+
Step 1080/11700 | Loss: 3.3002 | LR: 4.99e-04 | 7600.1ms/step | 2.6 samples/s
|
| 116 |
+
Step 1090/11700 | Loss: 3.3574 | LR: 4.99e-04 | 7600.0ms/step | 2.6 samples/s
|
| 117 |
+
Step 1100/11700 | Loss: 3.5562 | LR: 4.99e-04 | 7600.0ms/step | 2.6 samples/s
|
| 118 |
+
Step 1110/11700 | Loss: 3.5005 | LR: 4.99e-04 | 7600.1ms/step | 2.6 samples/s
|
| 119 |
+
Step 1120/11700 | Loss: 3.3888 | LR: 4.99e-04 | 7599.9ms/step | 2.6 samples/s
|
| 120 |
+
Step 1130/11700 | Loss: 3.3930 | LR: 4.99e-04 | 7600.0ms/step | 2.6 samples/s
|
| 121 |
+
Step 1140/11700 | Loss: 3.1697 | LR: 4.98e-04 | 7600.2ms/step | 2.6 samples/s
|
| 122 |
+
Step 1150/11700 | Loss: 3.3267 | LR: 4.98e-04 | 7600.1ms/step | 2.6 samples/s
|
| 123 |
+
Step 1160/11700 | Loss: 3.3372 | LR: 4.98e-04 | 7610.0ms/step | 2.6 samples/s
|
| 124 |
+
Step 1170/11700 | Loss: 3.3013 | LR: 4.98e-04 | 7599.9ms/step | 2.6 samples/s
|
| 125 |
+
Step 1180/11700 | Loss: 3.2397 | LR: 4.98e-04 | 7626.6ms/step | 2.6 samples/s
|
| 126 |
+
Step 1190/11700 | Loss: 3.2126 | LR: 4.98e-04 | 7629.5ms/step | 2.6 samples/s
|
| 127 |
+
Step 1200/11700 | Loss: 3.1437 | LR: 4.98e-04 | 7634.4ms/step | 2.6 samples/s
|
| 128 |
+
Step 1210/11700 | Loss: 3.4685 | LR: 4.98e-04 | 7628.3ms/step | 2.6 samples/s
|
| 129 |
+
Step 1220/11700 | Loss: 3.2654 | LR: 4.98e-04 | 7638.9ms/step | 2.6 samples/s
|
| 130 |
+
Step 1230/11700 | Loss: 3.3039 | LR: 4.98e-04 | 7636.8ms/step | 2.6 samples/s
|
| 131 |
+
Step 1240/11700 | Loss: 3.2457 | LR: 4.98e-04 | 7630.8ms/step | 2.6 samples/s
|
| 132 |
+
Step 1250/11700 | Loss: 3.2235 | LR: 4.97e-04 | 7635.2ms/step | 2.6 samples/s
|
| 133 |
+
Validation at step 1250: Loss = 3.2701, Perplexity = 26.31
|
| 134 |
+
Step 1260/11700 | Loss: 3.3010 | LR: 4.97e-04 | 18991.9ms/step | 1.1 samples/s
|
| 135 |
+
Step 1270/11700 | Loss: 3.1347 | LR: 4.97e-04 | 7634.7ms/step | 2.6 samples/s
|
| 136 |
+
Step 1280/11700 | Loss: 3.2432 | LR: 4.97e-04 | 7638.1ms/step | 2.6 samples/s
|
| 137 |
+
Step 1290/11700 | Loss: 3.2414 | LR: 4.97e-04 | 7639.6ms/step | 2.6 samples/s
|
| 138 |
+
Step 1300/11700 | Loss: 3.2508 | LR: 4.97e-04 | 7631.5ms/step | 2.6 samples/s
|
| 139 |
+
Step 1310/11700 | Loss: 3.3061 | LR: 4.97e-04 | 7640.9ms/step | 2.6 samples/s
|
| 140 |
+
Step 1320/11700 | Loss: 3.2569 | LR: 4.97e-04 | 7638.5ms/step | 2.6 samples/s
|
| 141 |
+
Step 1330/11700 | Loss: 3.1805 | LR: 4.97e-04 | 7635.2ms/step | 2.6 samples/s
|
| 142 |
+
Step 1340/11700 | Loss: 3.2266 | LR: 4.96e-04 | 7632.4ms/step | 2.6 samples/s
|
| 143 |
+
Step 1350/11700 | Loss: 3.2419 | LR: 4.96e-04 | 7630.5ms/step | 2.6 samples/s
|
| 144 |
+
Step 1360/11700 | Loss: 3.2054 | LR: 4.96e-04 | 7633.6ms/step | 2.6 samples/s
|
| 145 |
+
Step 1370/11700 | Loss: 3.1435 | LR: 4.96e-04 | 7636.3ms/step | 2.6 samples/s
|
| 146 |
+
Step 1380/11700 | Loss: 3.0078 | LR: 4.96e-04 | 7637.6ms/step | 2.6 samples/s
|
| 147 |
+
Step 1390/11700 | Loss: 3.1693 | LR: 4.96e-04 | 7631.2ms/step | 2.6 samples/s
|
| 148 |
+
Step 1400/11700 | Loss: 3.2617 | LR: 4.96e-04 | 7636.3ms/step | 2.6 samples/s
|
| 149 |
+
Step 1410/11700 | Loss: 3.1966 | LR: 4.96e-04 | 7638.8ms/step | 2.6 samples/s
|
| 150 |
+
Step 1420/11700 | Loss: 3.0909 | LR: 4.95e-04 | 7638.4ms/step | 2.6 samples/s
|
| 151 |
+
Step 1430/11700 | Loss: 3.1389 | LR: 4.95e-04 | 7637.6ms/step | 2.6 samples/s
|
| 152 |
+
Step 1440/11700 | Loss: 3.1319 | LR: 4.95e-04 | 7618.6ms/step | 2.6 samples/s
|
| 153 |
+
Step 1450/11700 | Loss: 3.0518 | LR: 4.95e-04 | 7611.5ms/step | 2.6 samples/s
|
| 154 |
+
Step 1460/11700 | Loss: 3.1102 | LR: 4.95e-04 | 7610.3ms/step | 2.6 samples/s
|
| 155 |
+
Step 1470/11700 | Loss: 3.1211 | LR: 4.95e-04 | 7611.9ms/step | 2.6 samples/s
|
| 156 |
+
Step 1480/11700 | Loss: 3.0520 | LR: 4.95e-04 | 7616.5ms/step | 2.6 samples/s
|
| 157 |
+
Step 1490/11700 | Loss: 3.2031 | LR: 4.94e-04 | 7615.6ms/step | 2.6 samples/s
|
| 158 |
+
Step 1500/11700 | Loss: 3.0442 | LR: 4.94e-04 | 7616.7ms/step | 2.6 samples/s
|
| 159 |
+
Validation at step 1500: Loss = 3.1245, Perplexity = 22.75
|
| 160 |
+
Step 1510/11700 | Loss: 3.1192 | LR: 4.94e-04 | 18947.7ms/step | 1.1 samples/s
|
| 161 |
+
Step 1520/11700 | Loss: 3.1062 | LR: 4.94e-04 | 7608.0ms/step | 2.6 samples/s
|
| 162 |
+
Step 1530/11700 | Loss: 3.1092 | LR: 4.94e-04 | 7605.8ms/step | 2.6 samples/s
|
| 163 |
+
Step 1540/11700 | Loss: 3.2591 | LR: 4.94e-04 | 7606.1ms/step | 2.6 samples/s
|
| 164 |
+
Step 1550/11700 | Loss: 3.1102 | LR: 4.93e-04 | 7604.1ms/step | 2.6 samples/s
|
| 165 |
+
Step 1560/11700 | Loss: 3.1356 | LR: 4.93e-04 | 7606.2ms/step | 2.6 samples/s
|
| 166 |
+
Step 1570/11700 | Loss: 3.1269 | LR: 4.93e-04 | 7606.5ms/step | 2.6 samples/s
|
| 167 |
+
Step 1580/11700 | Loss: 3.0856 | LR: 4.93e-04 | 7601.3ms/step | 2.6 samples/s
|
| 168 |
+
Step 1590/11700 | Loss: 3.2140 | LR: 4.93e-04 | 7600.5ms/step | 2.6 samples/s
|
| 169 |
+
Step 1600/11700 | Loss: 3.1014 | LR: 4.93e-04 | 7610.0ms/step | 2.6 samples/s
|
| 170 |
+
Step 1610/11700 | Loss: 3.1492 | LR: 4.92e-04 | 7605.6ms/step | 2.6 samples/s
|
| 171 |
+
Step 1620/11700 | Loss: 3.0294 | LR: 4.92e-04 | 7604.3ms/step | 2.6 samples/s
|
| 172 |
+
Step 1630/11700 | Loss: 2.9351 | LR: 4.92e-04 | 7607.8ms/step | 2.6 samples/s
|
| 173 |
+
Step 1640/11700 | Loss: 3.1176 | LR: 4.92e-04 | 7607.5ms/step | 2.6 samples/s
|
| 174 |
+
Step 1650/11700 | Loss: 3.0504 | LR: 4.92e-04 | 7604.8ms/step | 2.6 samples/s
|
| 175 |
+
Step 1660/11700 | Loss: 3.0839 | LR: 4.92e-04 | 7601.3ms/step | 2.6 samples/s
|
| 176 |
+
Step 1670/11700 | Loss: 3.2062 | LR: 4.91e-04 | 7604.7ms/step | 2.6 samples/s
|
| 177 |
+
Step 1680/11700 | Loss: 3.0530 | LR: 4.91e-04 | 7603.2ms/step | 2.6 samples/s
|
| 178 |
+
Step 1690/11700 | Loss: 2.9155 | LR: 4.91e-04 | 7602.9ms/step | 2.6 samples/s
|
| 179 |
+
Step 1700/11700 | Loss: 2.9772 | LR: 4.91e-04 | 7604.0ms/step | 2.6 samples/s
|
| 180 |
+
Step 1710/11700 | Loss: 3.1062 | LR: 4.91e-04 | 7605.0ms/step | 2.6 samples/s
|
| 181 |
+
Step 1720/11700 | Loss: 3.0288 | LR: 4.90e-04 | 7603.2ms/step | 2.6 samples/s
|
| 182 |
+
Step 1730/11700 | Loss: 3.0550 | LR: 4.90e-04 | 7600.3ms/step | 2.6 samples/s
|
| 183 |
+
Step 1740/11700 | Loss: 2.9532 | LR: 4.90e-04 | 7603.7ms/step | 2.6 samples/s
|
| 184 |
+
Step 1750/11700 | Loss: 2.9292 | LR: 4.90e-04 | 7600.0ms/step | 2.6 samples/s
|
| 185 |
+
Validation at step 1750: Loss = 3.0491, Perplexity = 21.10
|
| 186 |
+
Step 1760/11700 | Loss: 3.0268 | LR: 4.90e-04 | 18936.3ms/step | 1.1 samples/s
|
| 187 |
+
Step 1770/11700 | Loss: 3.0251 | LR: 4.89e-04 | 7602.7ms/step | 2.6 samples/s
|
| 188 |
+
Step 1780/11700 | Loss: 3.0488 | LR: 4.89e-04 | 7603.3ms/step | 2.6 samples/s
|
| 189 |
+
Step 1790/11700 | Loss: 3.0190 | LR: 4.89e-04 | 7602.2ms/step | 2.6 samples/s
|
| 190 |
+
Step 1800/11700 | Loss: 3.0347 | LR: 4.89e-04 | 7606.0ms/step | 2.6 samples/s
|
| 191 |
+
Step 1810/11700 | Loss: 3.0920 | LR: 4.89e-04 | 7606.3ms/step | 2.6 samples/s
|
| 192 |
+
Step 1820/11700 | Loss: 3.0454 | LR: 4.88e-04 | 7612.4ms/step | 2.6 samples/s
|
| 193 |
+
Step 1830/11700 | Loss: 3.0013 | LR: 4.88e-04 | 7606.7ms/step | 2.6 samples/s
|
| 194 |
+
Step 1840/11700 | Loss: 2.9907 | LR: 4.88e-04 | 7604.7ms/step | 2.6 samples/s
|
| 195 |
+
Step 1850/11700 | Loss: 3.0100 | LR: 4.88e-04 | 7599.9ms/step | 2.6 samples/s
|
| 196 |
+
Step 1860/11700 | Loss: 2.9925 | LR: 4.87e-04 | 7600.2ms/step | 2.6 samples/s
|
| 197 |
+
Step 1870/11700 | Loss: 2.9190 | LR: 4.87e-04 | 7600.1ms/step | 2.6 samples/s
|
| 198 |
+
Step 1880/11700 | Loss: 2.9439 | LR: 4.87e-04 | 7602.0ms/step | 2.6 samples/s
|
| 199 |
+
Step 1890/11700 | Loss: 3.0621 | LR: 4.87e-04 | 7600.0ms/step | 2.6 samples/s
|
| 200 |
+
Step 1900/11700 | Loss: 3.0059 | LR: 4.87e-04 | 7602.3ms/step | 2.6 samples/s
|
| 201 |
+
Step 1910/11700 | Loss: 3.0302 | LR: 4.86e-04 | 7600.1ms/step | 2.6 samples/s
|
| 202 |
+
Step 1920/11700 | Loss: 2.8230 | LR: 4.86e-04 | 7603.9ms/step | 2.6 samples/s
|
| 203 |
+
Step 1930/11700 | Loss: 3.0104 | LR: 4.86e-04 | 7602.1ms/step | 2.6 samples/s
|
| 204 |
+
Step 1940/11700 | Loss: 2.9483 | LR: 4.86e-04 | 7599.8ms/step | 2.6 samples/s
|
| 205 |
+
Step 1950/11700 | Loss: 2.8686 | LR: 4.85e-04 | 7600.3ms/step | 2.6 samples/s
|
| 206 |
+
Step 1960/11700 | Loss: 2.9357 | LR: 4.85e-04 | 7599.9ms/step | 2.6 samples/s
|
| 207 |
+
Step 1970/11700 | Loss: 2.9603 | LR: 4.85e-04 | 7599.8ms/step | 2.6 samples/s
|
| 208 |
+
Step 1980/11700 | Loss: 2.9403 | LR: 4.85e-04 | 7600.1ms/step | 2.6 samples/s
|
| 209 |
+
Step 1990/11700 | Loss: 2.8852 | LR: 4.84e-04 | 7602.6ms/step | 2.6 samples/s
|
| 210 |
+
Step 2000/11700 | Loss: 3.0944 | LR: 4.84e-04 | 7607.7ms/step | 2.6 samples/s
|
| 211 |
+
Validation at step 2000: Loss = 2.9636, Perplexity = 19.37
|
| 212 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-2000
|
| 213 |
+
Step 2010/11700 | Loss: 2.9827 | LR: 4.84e-04 | 19260.5ms/step | 1.0 samples/s
|
| 214 |
+
Step 2020/11700 | Loss: 3.0175 | LR: 4.84e-04 | 7613.7ms/step | 2.6 samples/s
|
| 215 |
+
Step 2030/11700 | Loss: 2.9211 | LR: 4.83e-04 | 7616.1ms/step | 2.6 samples/s
|
| 216 |
+
Step 2040/11700 | Loss: 2.9654 | LR: 4.83e-04 | 7616.1ms/step | 2.6 samples/s
|
| 217 |
+
Step 2050/11700 | Loss: 2.8388 | LR: 4.83e-04 | 7621.4ms/step | 2.6 samples/s
|
| 218 |
+
Step 2060/11700 | Loss: 2.9127 | LR: 4.83e-04 | 7608.7ms/step | 2.6 samples/s
|
| 219 |
+
Step 2070/11700 | Loss: 2.9145 | LR: 4.82e-04 | 7608.3ms/step | 2.6 samples/s
|
| 220 |
+
Step 2080/11700 | Loss: 2.8878 | LR: 4.82e-04 | 7611.9ms/step | 2.6 samples/s
|
| 221 |
+
Step 2090/11700 | Loss: 2.9154 | LR: 4.82e-04 | 7611.8ms/step | 2.6 samples/s
|
| 222 |
+
Step 2100/11700 | Loss: 3.0062 | LR: 4.81e-04 | 7614.5ms/step | 2.6 samples/s
|
| 223 |
+
Step 2110/11700 | Loss: 3.0052 | LR: 4.81e-04 | 7609.7ms/step | 2.6 samples/s
|
| 224 |
+
Step 2120/11700 | Loss: 2.9525 | LR: 4.81e-04 | 7638.7ms/step | 2.6 samples/s
|
| 225 |
+
Step 2130/11700 | Loss: 2.9403 | LR: 4.81e-04 | 7627.2ms/step | 2.6 samples/s
|
| 226 |
+
Step 2140/11700 | Loss: 3.0496 | LR: 4.80e-04 | 7618.3ms/step | 2.6 samples/s
|
| 227 |
+
Step 2150/11700 | Loss: 2.9641 | LR: 4.80e-04 | 7616.1ms/step | 2.6 samples/s
|
| 228 |
+
Step 2160/11700 | Loss: 2.9237 | LR: 4.80e-04 | 7615.9ms/step | 2.6 samples/s
|
| 229 |
+
Step 2170/11700 | Loss: 2.8569 | LR: 4.80e-04 | 7610.7ms/step | 2.6 samples/s
|
| 230 |
+
Step 2180/11700 | Loss: 2.8931 | LR: 4.79e-04 | 7613.6ms/step | 2.6 samples/s
|
| 231 |
+
Step 2190/11700 | Loss: 2.8802 | LR: 4.79e-04 | 7608.1ms/step | 2.6 samples/s
|
| 232 |
+
Step 2200/11700 | Loss: 2.9422 | LR: 4.79e-04 | 7611.9ms/step | 2.6 samples/s
|
| 233 |
+
Step 2210/11700 | Loss: 2.9791 | LR: 4.78e-04 | 7604.0ms/step | 2.6 samples/s
|
| 234 |
+
Step 2220/11700 | Loss: 2.7883 | LR: 4.78e-04 | 7604.0ms/step | 2.6 samples/s
|
| 235 |
+
Step 2230/11700 | Loss: 2.8241 | LR: 4.78e-04 | 7602.0ms/step | 2.6 samples/s
|
| 236 |
+
Step 2240/11700 | Loss: 2.8108 | LR: 4.78e-04 | 7612.1ms/step | 2.6 samples/s
|
| 237 |
+
Step 2250/11700 | Loss: 3.0036 | LR: 4.77e-04 | 7610.2ms/step | 2.6 samples/s
|
| 238 |
+
Validation at step 2250: Loss = 2.9141, Perplexity = 18.43
|
| 239 |
+
Step 2260/11700 | Loss: 2.7842 | LR: 4.77e-04 | 18947.9ms/step | 1.1 samples/s
|
| 240 |
+
Step 2270/11700 | Loss: 2.9975 | LR: 4.77e-04 | 7610.2ms/step | 2.6 samples/s
|
| 241 |
+
Step 2280/11700 | Loss: 2.9094 | LR: 4.76e-04 | 7614.0ms/step | 2.6 samples/s
|
| 242 |
+
Step 2290/11700 | Loss: 2.9258 | LR: 4.76e-04 | 7611.9ms/step | 2.6 samples/s
|
| 243 |
+
Step 2300/11700 | Loss: 2.8657 | LR: 4.76e-04 | 7608.3ms/step | 2.6 samples/s
|
| 244 |
+
Step 2310/11700 | Loss: 2.8510 | LR: 4.75e-04 | 7611.6ms/step | 2.6 samples/s
|
| 245 |
+
Step 2320/11700 | Loss: 3.0006 | LR: 4.75e-04 | 7612.7ms/step | 2.6 samples/s
|
| 246 |
+
Step 2330/11700 | Loss: 2.8261 | LR: 4.75e-04 | 7613.2ms/step | 2.6 samples/s
|
| 247 |
+
Step 2340/11700 | Loss: 2.8632 | LR: 4.74e-04 | 7612.4ms/step | 2.6 samples/s
|
| 248 |
+
Step 2350/11700 | Loss: 2.7935 | LR: 4.74e-04 | 7608.2ms/step | 2.6 samples/s
|
| 249 |
+
Step 2360/11700 | Loss: 2.8026 | LR: 4.74e-04 | 7608.0ms/step | 2.6 samples/s
|
| 250 |
+
Step 2370/11700 | Loss: 2.7833 | LR: 4.73e-04 | 7609.8ms/step | 2.6 samples/s
|
| 251 |
+
Step 2380/11700 | Loss: 2.7778 | LR: 4.73e-04 | 7616.0ms/step | 2.6 samples/s
|
| 252 |
+
Step 2390/11700 | Loss: 2.9199 | LR: 4.73e-04 | 7611.9ms/step | 2.6 samples/s
|
| 253 |
+
Step 2400/11700 | Loss: 2.8552 | LR: 4.73e-04 | 7610.3ms/step | 2.6 samples/s
|
| 254 |
+
Step 2410/11700 | Loss: 2.8733 | LR: 4.72e-04 | 7610.0ms/step | 2.6 samples/s
|
| 255 |
+
Step 2420/11700 | Loss: 2.7364 | LR: 4.72e-04 | 7611.3ms/step | 2.6 samples/s
|
| 256 |
+
Step 2430/11700 | Loss: 2.9123 | LR: 4.72e-04 | 7612.6ms/step | 2.6 samples/s
|
| 257 |
+
Step 2440/11700 | Loss: 2.9334 | LR: 4.71e-04 | 7612.2ms/step | 2.6 samples/s
|
| 258 |
+
Step 2450/11700 | Loss: 2.7649 | LR: 4.71e-04 | 7612.0ms/step | 2.6 samples/s
|
| 259 |
+
Step 2460/11700 | Loss: 2.8805 | LR: 4.71e-04 | 7608.6ms/step | 2.6 samples/s
|
| 260 |
+
Step 2470/11700 | Loss: 2.8871 | LR: 4.70e-04 | 7611.8ms/step | 2.6 samples/s
|
| 261 |
+
Step 2480/11700 | Loss: 2.8158 | LR: 4.70e-04 | 7608.0ms/step | 2.6 samples/s
|
| 262 |
+
Step 2490/11700 | Loss: 2.9086 | LR: 4.69e-04 | 7609.5ms/step | 2.6 samples/s
|
| 263 |
+
Step 2500/11700 | Loss: 2.7867 | LR: 4.69e-04 | 7610.2ms/step | 2.6 samples/s
|
| 264 |
+
Validation at step 2500: Loss = 2.8493, Perplexity = 17.28
|
| 265 |
+
Step 2510/11700 | Loss: 2.7591 | LR: 4.69e-04 | 18952.3ms/step | 1.1 samples/s
|
| 266 |
+
Step 2520/11700 | Loss: 2.9264 | LR: 4.68e-04 | 7608.4ms/step | 2.6 samples/s
|
| 267 |
+
Step 2530/11700 | Loss: 2.8196 | LR: 4.68e-04 | 7612.4ms/step | 2.6 samples/s
|
| 268 |
+
Step 2540/11700 | Loss: 2.7508 | LR: 4.68e-04 | 7611.4ms/step | 2.6 samples/s
|
| 269 |
+
Step 2550/11700 | Loss: 2.9211 | LR: 4.67e-04 | 7608.3ms/step | 2.6 samples/s
|
| 270 |
+
Step 2560/11700 | Loss: 2.8610 | LR: 4.67e-04 | 7609.7ms/step | 2.6 samples/s
|
| 271 |
+
Step 2570/11700 | Loss: 2.8221 | LR: 4.67e-04 | 7610.4ms/step | 2.6 samples/s
|
| 272 |
+
Step 2580/11700 | Loss: 2.8414 | LR: 4.66e-04 | 7611.9ms/step | 2.6 samples/s
|
| 273 |
+
Step 2590/11700 | Loss: 2.7814 | LR: 4.66e-04 | 7609.1ms/step | 2.6 samples/s
|
| 274 |
+
Step 2600/11700 | Loss: 2.8094 | LR: 4.66e-04 | 7611.1ms/step | 2.6 samples/s
|
| 275 |
+
Step 2610/11700 | Loss: 2.7369 | LR: 4.65e-04 | 7615.8ms/step | 2.6 samples/s
|
| 276 |
+
Step 2620/11700 | Loss: 2.7949 | LR: 4.65e-04 | 7612.1ms/step | 2.6 samples/s
|
| 277 |
+
Step 2630/11700 | Loss: 2.7418 | LR: 4.65e-04 | 7610.1ms/step | 2.6 samples/s
|
| 278 |
+
Step 2640/11700 | Loss: 2.8498 | LR: 4.64e-04 | 7610.5ms/step | 2.6 samples/s
|
| 279 |
+
Step 2650/11700 | Loss: 2.9133 | LR: 4.64e-04 | 7605.9ms/step | 2.6 samples/s
|
| 280 |
+
Step 2660/11700 | Loss: 2.8431 | LR: 4.63e-04 | 7608.1ms/step | 2.6 samples/s
|
| 281 |
+
Step 2670/11700 | Loss: 2.7867 | LR: 4.63e-04 | 7609.9ms/step | 2.6 samples/s
|
| 282 |
+
Step 2680/11700 | Loss: 2.8282 | LR: 4.63e-04 | 7611.7ms/step | 2.6 samples/s
|
| 283 |
+
Step 2690/11700 | Loss: 2.9309 | LR: 4.62e-04 | 7608.5ms/step | 2.6 samples/s
|
| 284 |
+
Step 2700/11700 | Loss: 2.8279 | LR: 4.62e-04 | 7610.2ms/step | 2.6 samples/s
|
| 285 |
+
Step 2710/11700 | Loss: 2.8247 | LR: 4.62e-04 | 7611.9ms/step | 2.6 samples/s
|
| 286 |
+
Step 2720/11700 | Loss: 2.8291 | LR: 4.61e-04 | 7607.9ms/step | 2.6 samples/s
|
| 287 |
+
Step 2730/11700 | Loss: 2.7304 | LR: 4.61e-04 | 7610.0ms/step | 2.6 samples/s
|
| 288 |
+
Step 2740/11700 | Loss: 2.8146 | LR: 4.60e-04 | 7608.0ms/step | 2.6 samples/s
|
| 289 |
+
Step 2750/11700 | Loss: 2.8669 | LR: 4.60e-04 | 7614.1ms/step | 2.6 samples/s
|
| 290 |
+
Validation at step 2750: Loss = 2.8076, Perplexity = 16.57
|
| 291 |
+
Step 2760/11700 | Loss: 2.6176 | LR: 4.60e-04 | 18948.5ms/step | 1.1 samples/s
|
| 292 |
+
Step 2770/11700 | Loss: 2.7700 | LR: 4.59e-04 | 7611.5ms/step | 2.6 samples/s
|
| 293 |
+
Step 2780/11700 | Loss: 2.7771 | LR: 4.59e-04 | 7606.3ms/step | 2.6 samples/s
|
| 294 |
+
Step 2790/11700 | Loss: 2.8256 | LR: 4.58e-04 | 7612.0ms/step | 2.6 samples/s
|
| 295 |
+
Step 2800/11700 | Loss: 2.7337 | LR: 4.58e-04 | 7607.8ms/step | 2.6 samples/s
|
| 296 |
+
Step 2810/11700 | Loss: 2.8341 | LR: 4.58e-04 | 7609.9ms/step | 2.6 samples/s
|
| 297 |
+
Step 2820/11700 | Loss: 2.8189 | LR: 4.57e-04 | 7606.6ms/step | 2.6 samples/s
|
| 298 |
+
Step 2830/11700 | Loss: 2.7889 | LR: 4.57e-04 | 7609.6ms/step | 2.6 samples/s
|
| 299 |
+
Step 2840/11700 | Loss: 2.8658 | LR: 4.56e-04 | 7610.8ms/step | 2.6 samples/s
|
| 300 |
+
Step 2850/11700 | Loss: 2.8974 | LR: 4.56e-04 | 7613.5ms/step | 2.6 samples/s
|
| 301 |
+
Step 2860/11700 | Loss: 2.7809 | LR: 4.56e-04 | 7610.2ms/step | 2.6 samples/s
|
| 302 |
+
Step 2870/11700 | Loss: 2.8514 | LR: 4.55e-04 | 7613.8ms/step | 2.6 samples/s
|
| 303 |
+
Step 2880/11700 | Loss: 2.5813 | LR: 4.55e-04 | 7610.0ms/step | 2.6 samples/s
|
| 304 |
+
Step 2890/11700 | Loss: 2.8308 | LR: 4.54e-04 | 7612.3ms/step | 2.6 samples/s
|
| 305 |
+
Step 2900/11700 | Loss: 2.7753 | LR: 4.54e-04 | 7608.1ms/step | 2.6 samples/s
|
| 306 |
+
Step 2910/11700 | Loss: 2.8491 | LR: 4.54e-04 | 7611.9ms/step | 2.6 samples/s
|
| 307 |
+
Step 2920/11700 | Loss: 2.7774 | LR: 4.53e-04 | 7611.0ms/step | 2.6 samples/s
|
| 308 |
+
Step 2930/11700 | Loss: 2.8587 | LR: 4.53e-04 | 7613.1ms/step | 2.6 samples/s
|
| 309 |
+
Step 2940/11700 | Loss: 2.7871 | LR: 4.52e-04 | 7610.5ms/step | 2.6 samples/s
|
| 310 |
+
Step 2950/11700 | Loss: 2.8360 | LR: 4.52e-04 | 7607.9ms/step | 2.6 samples/s
|
| 311 |
+
Step 2960/11700 | Loss: 2.7305 | LR: 4.51e-04 | 7608.6ms/step | 2.6 samples/s
|
| 312 |
+
Step 2970/11700 | Loss: 2.6809 | LR: 4.51e-04 | 7611.6ms/step | 2.6 samples/s
|
| 313 |
+
Step 2980/11700 | Loss: 2.7625 | LR: 4.51e-04 | 7611.5ms/step | 2.6 samples/s
|
| 314 |
+
Step 2990/11700 | Loss: 2.6966 | LR: 4.50e-04 | 7606.3ms/step | 2.6 samples/s
|
| 315 |
+
Step 3000/11700 | Loss: 2.7739 | LR: 4.50e-04 | 7612.8ms/step | 2.6 samples/s
|
| 316 |
+
Validation at step 3000: Loss = 2.7782, Perplexity = 16.09
|
| 317 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-3000
|
| 318 |
+
Step 3010/11700 | Loss: 2.7432 | LR: 4.49e-04 | 19268.7ms/step | 1.0 samples/s
|
| 319 |
+
Step 3020/11700 | Loss: 2.8894 | LR: 4.49e-04 | 7612.2ms/step | 2.6 samples/s
|
| 320 |
+
Step 3030/11700 | Loss: 2.7245 | LR: 4.48e-04 | 7614.5ms/step | 2.6 samples/s
|
| 321 |
+
Step 3040/11700 | Loss: 2.8307 | LR: 4.48e-04 | 7616.1ms/step | 2.6 samples/s
|
| 322 |
+
Step 3050/11700 | Loss: 2.7633 | LR: 4.48e-04 | 7614.0ms/step | 2.6 samples/s
|
| 323 |
+
Step 3060/11700 | Loss: 2.9458 | LR: 4.47e-04 | 7609.8ms/step | 2.6 samples/s
|
| 324 |
+
Step 3070/11700 | Loss: 2.7247 | LR: 4.47e-04 | 7614.3ms/step | 2.6 samples/s
|
| 325 |
+
Step 3080/11700 | Loss: 2.7233 | LR: 4.46e-04 | 7616.7ms/step | 2.6 samples/s
|
| 326 |
+
Step 3090/11700 | Loss: 2.7289 | LR: 4.46e-04 | 7615.1ms/step | 2.6 samples/s
|
| 327 |
+
Step 3100/11700 | Loss: 2.8609 | LR: 4.45e-04 | 7612.2ms/step | 2.6 samples/s
|
| 328 |
+
Step 3110/11700 | Loss: 2.6425 | LR: 4.45e-04 | 7610.6ms/step | 2.6 samples/s
|
| 329 |
+
Step 3120/11700 | Loss: 2.7153 | LR: 4.44e-04 | 7611.3ms/step | 2.6 samples/s
|
| 330 |
+
Step 3130/11700 | Loss: 2.7524 | LR: 4.44e-04 | 7610.3ms/step | 2.6 samples/s
|
| 331 |
+
Step 3140/11700 | Loss: 2.7032 | LR: 4.43e-04 | 7608.6ms/step | 2.6 samples/s
|
| 332 |
+
Step 3150/11700 | Loss: 2.7226 | LR: 4.43e-04 | 7609.2ms/step | 2.6 samples/s
|
| 333 |
+
Step 3160/11700 | Loss: 2.7232 | LR: 4.43e-04 | 7607.6ms/step | 2.6 samples/s
|
| 334 |
+
Step 3170/11700 | Loss: 2.8159 | LR: 4.42e-04 | 7611.0ms/step | 2.6 samples/s
|
| 335 |
+
Step 3180/11700 | Loss: 2.7809 | LR: 4.42e-04 | 7609.9ms/step | 2.6 samples/s
|
| 336 |
+
Step 3190/11700 | Loss: 2.6536 | LR: 4.41e-04 | 7610.7ms/step | 2.6 samples/s
|
| 337 |
+
Step 3200/11700 | Loss: 2.6841 | LR: 4.41e-04 | 7613.1ms/step | 2.6 samples/s
|
| 338 |
+
Step 3210/11700 | Loss: 2.7078 | LR: 4.40e-04 | 7608.4ms/step | 2.6 samples/s
|
| 339 |
+
Step 3220/11700 | Loss: 2.7307 | LR: 4.40e-04 | 7612.4ms/step | 2.6 samples/s
|
| 340 |
+
Step 3230/11700 | Loss: 2.6818 | LR: 4.39e-04 | 7613.6ms/step | 2.6 samples/s
|
| 341 |
+
Step 3240/11700 | Loss: 2.7486 | LR: 4.39e-04 | 7611.8ms/step | 2.6 samples/s
|
| 342 |
+
Step 3250/11700 | Loss: 2.8363 | LR: 4.38e-04 | 7612.4ms/step | 2.6 samples/s
|
| 343 |
+
Validation at step 3250: Loss = 2.7420, Perplexity = 15.52
|
| 344 |
+
Step 3260/11700 | Loss: 2.7500 | LR: 4.38e-04 | 18947.9ms/step | 1.1 samples/s
|
| 345 |
+
Step 3270/11700 | Loss: 2.6923 | LR: 4.37e-04 | 7610.1ms/step | 2.6 samples/s
|
| 346 |
+
Step 3280/11700 | Loss: 2.7506 | LR: 4.37e-04 | 7606.4ms/step | 2.6 samples/s
|
| 347 |
+
Step 3290/11700 | Loss: 2.6981 | LR: 4.37e-04 | 7607.9ms/step | 2.6 samples/s
|
| 348 |
+
Step 3300/11700 | Loss: 2.7601 | LR: 4.36e-04 | 7612.2ms/step | 2.6 samples/s
|
| 349 |
+
Step 3310/11700 | Loss: 2.7601 | LR: 4.36e-04 | 7609.8ms/step | 2.6 samples/s
|
| 350 |
+
Step 3320/11700 | Loss: 2.6774 | LR: 4.35e-04 | 7613.8ms/step | 2.6 samples/s
|
| 351 |
+
Step 3330/11700 | Loss: 2.6634 | LR: 4.35e-04 | 7610.2ms/step | 2.6 samples/s
|
| 352 |
+
Step 3340/11700 | Loss: 2.7992 | LR: 4.34e-04 | 7609.9ms/step | 2.6 samples/s
|
| 353 |
+
Step 3350/11700 | Loss: 2.7002 | LR: 4.34e-04 | 7606.3ms/step | 2.6 samples/s
|
| 354 |
+
Step 3360/11700 | Loss: 2.7788 | LR: 4.33e-04 | 7610.0ms/step | 2.6 samples/s
|
| 355 |
+
Step 3370/11700 | Loss: 2.6616 | LR: 4.33e-04 | 7610.6ms/step | 2.6 samples/s
|
| 356 |
+
Step 3380/11700 | Loss: 2.7292 | LR: 4.32e-04 | 7615.1ms/step | 2.6 samples/s
|
| 357 |
+
Step 3390/11700 | Loss: 2.6927 | LR: 4.32e-04 | 7610.3ms/step | 2.6 samples/s
|
| 358 |
+
Step 3400/11700 | Loss: 2.8020 | LR: 4.31e-04 | 7610.3ms/step | 2.6 samples/s
|
| 359 |
+
Step 3410/11700 | Loss: 2.7439 | LR: 4.31e-04 | 7613.8ms/step | 2.6 samples/s
|
| 360 |
+
Step 3420/11700 | Loss: 2.5693 | LR: 4.30e-04 | 7611.9ms/step | 2.6 samples/s
|
| 361 |
+
Step 3430/11700 | Loss: 2.7554 | LR: 4.30e-04 | 7611.2ms/step | 2.6 samples/s
|
| 362 |
+
Step 3440/11700 | Loss: 2.6121 | LR: 4.29e-04 | 7605.3ms/step | 2.6 samples/s
|
| 363 |
+
Step 3450/11700 | Loss: 2.7246 | LR: 4.29e-04 | 7611.5ms/step | 2.6 samples/s
|
| 364 |
+
Step 3460/11700 | Loss: 2.7085 | LR: 4.28e-04 | 7610.2ms/step | 2.6 samples/s
|
| 365 |
+
Step 3470/11700 | Loss: 2.5974 | LR: 4.28e-04 | 7612.2ms/step | 2.6 samples/s
|
| 366 |
+
Step 3480/11700 | Loss: 2.5218 | LR: 4.27e-04 | 7613.7ms/step | 2.6 samples/s
|
| 367 |
+
Step 3490/11700 | Loss: 2.8501 | LR: 4.27e-04 | 7612.0ms/step | 2.6 samples/s
|
| 368 |
+
Step 3500/11700 | Loss: 2.7622 | LR: 4.26e-04 | 7614.3ms/step | 2.6 samples/s
|
| 369 |
+
Validation at step 3500: Loss = 2.7115, Perplexity = 15.05
|
| 370 |
+
Step 3510/11700 | Loss: 2.7423 | LR: 4.26e-04 | 18958.3ms/step | 1.1 samples/s
|
| 371 |
+
Step 3520/11700 | Loss: 2.7299 | LR: 4.25e-04 | 7617.2ms/step | 2.6 samples/s
|
| 372 |
+
Step 3530/11700 | Loss: 2.7402 | LR: 4.25e-04 | 7609.4ms/step | 2.6 samples/s
|
| 373 |
+
Step 3540/11700 | Loss: 2.7489 | LR: 4.24e-04 | 7615.1ms/step | 2.6 samples/s
|
| 374 |
+
Step 3550/11700 | Loss: 2.6690 | LR: 4.24e-04 | 7608.3ms/step | 2.6 samples/s
|
| 375 |
+
Step 3560/11700 | Loss: 2.8603 | LR: 4.23e-04 | 7607.9ms/step | 2.6 samples/s
|
| 376 |
+
Step 3570/11700 | Loss: 2.7303 | LR: 4.23e-04 | 7605.9ms/step | 2.6 samples/s
|
| 377 |
+
Step 3580/11700 | Loss: 2.7424 | LR: 4.22e-04 | 7606.1ms/step | 2.6 samples/s
|
| 378 |
+
Step 3590/11700 | Loss: 2.6399 | LR: 4.22e-04 | 7622.5ms/step | 2.6 samples/s
|
| 379 |
+
Step 3600/11700 | Loss: 2.6372 | LR: 4.21e-04 | 7611.4ms/step | 2.6 samples/s
|
| 380 |
+
Step 3610/11700 | Loss: 2.7694 | LR: 4.20e-04 | 7612.6ms/step | 2.6 samples/s
|
| 381 |
+
Step 3620/11700 | Loss: 2.6361 | LR: 4.20e-04 | 7619.8ms/step | 2.6 samples/s
|
| 382 |
+
Step 3630/11700 | Loss: 2.6453 | LR: 4.19e-04 | 7608.4ms/step | 2.6 samples/s
|
| 383 |
+
Step 3640/11700 | Loss: 2.5932 | LR: 4.19e-04 | 7609.8ms/step | 2.6 samples/s
|
| 384 |
+
Step 3650/11700 | Loss: 2.7232 | LR: 4.18e-04 | 7608.1ms/step | 2.6 samples/s
|
| 385 |
+
Step 3660/11700 | Loss: 2.6810 | LR: 4.18e-04 | 7611.7ms/step | 2.6 samples/s
|
| 386 |
+
Step 3670/11700 | Loss: 2.6434 | LR: 4.17e-04 | 7610.3ms/step | 2.6 samples/s
|
| 387 |
+
Step 3680/11700 | Loss: 2.7385 | LR: 4.17e-04 | 7610.6ms/step | 2.6 samples/s
|
| 388 |
+
Step 3690/11700 | Loss: 2.6741 | LR: 4.16e-04 | 7615.1ms/step | 2.6 samples/s
|
| 389 |
+
Step 3700/11700 | Loss: 2.7155 | LR: 4.16e-04 | 7611.5ms/step | 2.6 samples/s
|
| 390 |
+
Step 3710/11700 | Loss: 2.6129 | LR: 4.15e-04 | 7607.6ms/step | 2.6 samples/s
|
| 391 |
+
Step 3720/11700 | Loss: 2.6344 | LR: 4.15e-04 | 7605.3ms/step | 2.6 samples/s
|
| 392 |
+
Step 3730/11700 | Loss: 2.6931 | LR: 4.14e-04 | 7606.2ms/step | 2.6 samples/s
|
| 393 |
+
Step 3740/11700 | Loss: 2.6414 | LR: 4.14e-04 | 7608.1ms/step | 2.6 samples/s
|
| 394 |
+
Step 3750/11700 | Loss: 2.7151 | LR: 4.13e-04 | 7611.8ms/step | 2.6 samples/s
|
| 395 |
+
Validation at step 3750: Loss = 2.6834, Perplexity = 14.64
|
| 396 |
+
Step 3760/11700 | Loss: 2.7246 | LR: 4.12e-04 | 18946.0ms/step | 1.1 samples/s
|
| 397 |
+
Step 3770/11700 | Loss: 2.7152 | LR: 4.12e-04 | 7602.4ms/step | 2.6 samples/s
|
| 398 |
+
Step 3780/11700 | Loss: 2.6642 | LR: 4.11e-04 | 7599.8ms/step | 2.6 samples/s
|
| 399 |
+
Step 3790/11700 | Loss: 2.8064 | LR: 4.11e-04 | 7599.9ms/step | 2.6 samples/s
|
| 400 |
+
Step 3800/11700 | Loss: 2.6964 | LR: 4.10e-04 | 7600.2ms/step | 2.6 samples/s
|
| 401 |
+
Step 3810/11700 | Loss: 2.6567 | LR: 4.10e-04 | 7603.9ms/step | 2.6 samples/s
|
| 402 |
+
Step 3820/11700 | Loss: 2.6447 | LR: 4.09e-04 | 7606.6ms/step | 2.6 samples/s
|
| 403 |
+
Step 3830/11700 | Loss: 2.6665 | LR: 4.09e-04 | 7603.7ms/step | 2.6 samples/s
|
| 404 |
+
Step 3840/11700 | Loss: 2.5959 | LR: 4.08e-04 | 7602.2ms/step | 2.6 samples/s
|
| 405 |
+
Step 3850/11700 | Loss: 2.7130 | LR: 4.07e-04 | 7601.9ms/step | 2.6 samples/s
|
| 406 |
+
Step 3860/11700 | Loss: 2.6326 | LR: 4.07e-04 | 7601.7ms/step | 2.6 samples/s
|
| 407 |
+
Step 3870/11700 | Loss: 2.6648 | LR: 4.06e-04 | 7600.2ms/step | 2.6 samples/s
|
| 408 |
+
Step 3880/11700 | Loss: 2.5469 | LR: 4.06e-04 | 7600.3ms/step | 2.6 samples/s
|
| 409 |
+
Step 3890/11700 | Loss: 2.7198 | LR: 4.05e-04 | 7599.9ms/step | 2.6 samples/s
|
| 410 |
+
Step 3900/11700 | Loss: 2.5934 | LR: 4.05e-04 | 7599.8ms/step | 2.6 samples/s
|
| 411 |
+
Step 3910/11700 | Loss: 2.6807 | LR: 4.04e-04 | 7600.1ms/step | 2.6 samples/s
|
| 412 |
+
Step 3920/11700 | Loss: 2.5899 | LR: 4.04e-04 | 7602.2ms/step | 2.6 samples/s
|
| 413 |
+
Step 3930/11700 | Loss: 2.6513 | LR: 4.03e-04 | 7600.4ms/step | 2.6 samples/s
|
| 414 |
+
Step 3940/11700 | Loss: 2.7888 | LR: 4.02e-04 | 7601.5ms/step | 2.6 samples/s
|
| 415 |
+
Step 3950/11700 | Loss: 2.6122 | LR: 4.02e-04 | 7602.2ms/step | 2.6 samples/s
|
| 416 |
+
Step 3960/11700 | Loss: 2.6588 | LR: 4.01e-04 | 7600.3ms/step | 2.6 samples/s
|
| 417 |
+
Step 3970/11700 | Loss: 2.6418 | LR: 4.01e-04 | 7603.9ms/step | 2.6 samples/s
|
| 418 |
+
Step 3980/11700 | Loss: 2.7049 | LR: 4.00e-04 | 7604.7ms/step | 2.6 samples/s
|
| 419 |
+
Step 3990/11700 | Loss: 2.6874 | LR: 4.00e-04 | 7603.4ms/step | 2.6 samples/s
|
| 420 |
+
Step 4000/11700 | Loss: 2.7379 | LR: 3.99e-04 | 7603.9ms/step | 2.6 samples/s
|
| 421 |
+
Validation at step 4000: Loss = 2.6632, Perplexity = 14.34
|
| 422 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-4000
|
| 423 |
+
Step 4010/11700 | Loss: 2.5568 | LR: 3.98e-04 | 19278.4ms/step | 1.0 samples/s
|
| 424 |
+
Step 4020/11700 | Loss: 2.6559 | LR: 3.98e-04 | 7612.1ms/step | 2.6 samples/s
|
| 425 |
+
Step 4030/11700 | Loss: 2.5482 | LR: 3.97e-04 | 7613.7ms/step | 2.6 samples/s
|
| 426 |
+
Step 4040/11700 | Loss: 2.6874 | LR: 3.97e-04 | 7610.2ms/step | 2.6 samples/s
|
| 427 |
+
Step 4050/11700 | Loss: 2.6820 | LR: 3.96e-04 | 7610.8ms/step | 2.6 samples/s
|
| 428 |
+
Step 4060/11700 | Loss: 2.5577 | LR: 3.95e-04 | 7615.0ms/step | 2.6 samples/s
|
| 429 |
+
Step 4070/11700 | Loss: 2.7379 | LR: 3.95e-04 | 7616.5ms/step | 2.6 samples/s
|
| 430 |
+
Step 4080/11700 | Loss: 2.6271 | LR: 3.94e-04 | 7609.7ms/step | 2.6 samples/s
|
| 431 |
+
Step 4090/11700 | Loss: 2.6038 | LR: 3.94e-04 | 7612.3ms/step | 2.6 samples/s
|
| 432 |
+
Step 4100/11700 | Loss: 2.6451 | LR: 3.93e-04 | 7615.7ms/step | 2.6 samples/s
|
| 433 |
+
Step 4110/11700 | Loss: 2.6288 | LR: 3.93e-04 | 7610.4ms/step | 2.6 samples/s
|
| 434 |
+
Step 4120/11700 | Loss: 2.6967 | LR: 3.92e-04 | 7606.2ms/step | 2.6 samples/s
|
| 435 |
+
Step 4130/11700 | Loss: 2.5616 | LR: 3.91e-04 | 7619.8ms/step | 2.6 samples/s
|
| 436 |
+
Step 4140/11700 | Loss: 2.6132 | LR: 3.91e-04 | 7652.1ms/step | 2.6 samples/s
|
| 437 |
+
Step 4150/11700 | Loss: 2.5544 | LR: 3.90e-04 | 7618.1ms/step | 2.6 samples/s
|
| 438 |
+
Step 4160/11700 | Loss: 2.6283 | LR: 3.90e-04 | 7617.9ms/step | 2.6 samples/s
|
| 439 |
+
Step 4170/11700 | Loss: 2.6882 | LR: 3.89e-04 | 7616.1ms/step | 2.6 samples/s
|
| 440 |
+
Step 4180/11700 | Loss: 2.6284 | LR: 3.88e-04 | 7616.2ms/step | 2.6 samples/s
|
| 441 |
+
Step 4190/11700 | Loss: 2.6597 | LR: 3.88e-04 | 7626.3ms/step | 2.6 samples/s
|
| 442 |
+
Step 4200/11700 | Loss: 2.5740 | LR: 3.87e-04 | 7617.9ms/step | 2.6 samples/s
|
| 443 |
+
Step 4210/11700 | Loss: 2.6776 | LR: 3.87e-04 | 7615.7ms/step | 2.6 samples/s
|
| 444 |
+
Step 4220/11700 | Loss: 2.6415 | LR: 3.86e-04 | 7614.1ms/step | 2.6 samples/s
|
| 445 |
+
Step 4230/11700 | Loss: 2.5606 | LR: 3.85e-04 | 7615.4ms/step | 2.6 samples/s
|
| 446 |
+
Step 4240/11700 | Loss: 2.6703 | LR: 3.85e-04 | 7612.7ms/step | 2.6 samples/s
|
| 447 |
+
Step 4250/11700 | Loss: 2.7707 | LR: 3.84e-04 | 7615.7ms/step | 2.6 samples/s
|
| 448 |
+
Validation at step 4250: Loss = 2.6421, Perplexity = 14.04
|
| 449 |
+
Step 4260/11700 | Loss: 2.7367 | LR: 3.84e-04 | 18952.5ms/step | 1.1 samples/s
|
| 450 |
+
Step 4270/11700 | Loss: 2.5886 | LR: 3.83e-04 | 7614.1ms/step | 2.6 samples/s
|
| 451 |
+
Step 4280/11700 | Loss: 2.6220 | LR: 3.82e-04 | 7612.4ms/step | 2.6 samples/s
|
| 452 |
+
Step 4290/11700 | Loss: 2.6274 | LR: 3.82e-04 | 7611.8ms/step | 2.6 samples/s
|
| 453 |
+
Step 4300/11700 | Loss: 2.6818 | LR: 3.81e-04 | 7613.6ms/step | 2.6 samples/s
|
| 454 |
+
Step 4310/11700 | Loss: 2.6037 | LR: 3.81e-04 | 7612.2ms/step | 2.6 samples/s
|
| 455 |
+
Step 4320/11700 | Loss: 2.5811 | LR: 3.80e-04 | 7612.1ms/step | 2.6 samples/s
|
| 456 |
+
Step 4330/11700 | Loss: 2.5921 | LR: 3.79e-04 | 7610.7ms/step | 2.6 samples/s
|
| 457 |
+
Step 4340/11700 | Loss: 2.6630 | LR: 3.79e-04 | 7610.2ms/step | 2.6 samples/s
|
| 458 |
+
Step 4350/11700 | Loss: 2.7361 | LR: 3.78e-04 | 7611.3ms/step | 2.6 samples/s
|
| 459 |
+
Step 4360/11700 | Loss: 2.5722 | LR: 3.77e-04 | 7610.0ms/step | 2.6 samples/s
|
| 460 |
+
Step 4370/11700 | Loss: 2.6058 | LR: 3.77e-04 | 7612.2ms/step | 2.6 samples/s
|
| 461 |
+
Step 4380/11700 | Loss: 2.5664 | LR: 3.76e-04 | 7614.0ms/step | 2.6 samples/s
|
| 462 |
+
Step 4390/11700 | Loss: 2.6413 | LR: 3.76e-04 | 7616.1ms/step | 2.6 samples/s
|
| 463 |
+
Step 4400/11700 | Loss: 2.6297 | LR: 3.75e-04 | 7613.6ms/step | 2.6 samples/s
|
| 464 |
+
Step 4410/11700 | Loss: 2.6474 | LR: 3.74e-04 | 7612.5ms/step | 2.6 samples/s
|
| 465 |
+
Step 4420/11700 | Loss: 2.6363 | LR: 3.74e-04 | 7614.0ms/step | 2.6 samples/s
|
| 466 |
+
Step 4430/11700 | Loss: 2.6625 | LR: 3.73e-04 | 7620.2ms/step | 2.6 samples/s
|
| 467 |
+
Step 4440/11700 | Loss: 2.6639 | LR: 3.73e-04 | 7618.3ms/step | 2.6 samples/s
|
| 468 |
+
Step 4450/11700 | Loss: 2.6055 | LR: 3.72e-04 | 7614.1ms/step | 2.6 samples/s
|
| 469 |
+
Step 4460/11700 | Loss: 2.7131 | LR: 3.71e-04 | 7615.8ms/step | 2.6 samples/s
|
| 470 |
+
Step 4470/11700 | Loss: 2.5277 | LR: 3.71e-04 | 7616.2ms/step | 2.6 samples/s
|
| 471 |
+
Step 4480/11700 | Loss: 2.5976 | LR: 3.70e-04 | 7613.9ms/step | 2.6 samples/s
|
| 472 |
+
Step 4490/11700 | Loss: 2.6095 | LR: 3.69e-04 | 7610.0ms/step | 2.6 samples/s
|
| 473 |
+
Step 4500/11700 | Loss: 2.6247 | LR: 3.69e-04 | 7611.8ms/step | 2.6 samples/s
|
| 474 |
+
Validation at step 4500: Loss = 2.6261, Perplexity = 13.82
|
| 475 |
+
Step 4510/11700 | Loss: 2.5218 | LR: 3.68e-04 | 18949.1ms/step | 1.1 samples/s
|
| 476 |
+
Step 4520/11700 | Loss: 2.6445 | LR: 3.67e-04 | 7611.2ms/step | 2.6 samples/s
|
| 477 |
+
Step 4530/11700 | Loss: 2.7395 | LR: 3.67e-04 | 7610.3ms/step | 2.6 samples/s
|
| 478 |
+
Step 4540/11700 | Loss: 2.6643 | LR: 3.66e-04 | 7612.1ms/step | 2.6 samples/s
|
| 479 |
+
Step 4550/11700 | Loss: 2.5990 | LR: 3.66e-04 | 7611.9ms/step | 2.6 samples/s
|
| 480 |
+
Step 4560/11700 | Loss: 2.5714 | LR: 3.65e-04 | 7616.6ms/step | 2.6 samples/s
|
| 481 |
+
Step 4570/11700 | Loss: 2.5919 | LR: 3.64e-04 | 7613.8ms/step | 2.6 samples/s
|
| 482 |
+
Step 4580/11700 | Loss: 2.5569 | LR: 3.64e-04 | 7615.6ms/step | 2.6 samples/s
|
| 483 |
+
Step 4590/11700 | Loss: 2.7075 | LR: 3.63e-04 | 7613.6ms/step | 2.6 samples/s
|
| 484 |
+
Step 4600/11700 | Loss: 2.4525 | LR: 3.62e-04 | 7616.7ms/step | 2.6 samples/s
|
| 485 |
+
Step 4610/11700 | Loss: 2.5079 | LR: 3.62e-04 | 7614.5ms/step | 2.6 samples/s
|
| 486 |
+
Step 4620/11700 | Loss: 2.6193 | LR: 3.61e-04 | 7611.0ms/step | 2.6 samples/s
|
| 487 |
+
Step 4630/11700 | Loss: 2.6276 | LR: 3.60e-04 | 7612.4ms/step | 2.6 samples/s
|
| 488 |
+
Step 4640/11700 | Loss: 2.5736 | LR: 3.60e-04 | 7608.3ms/step | 2.6 samples/s
|
| 489 |
+
Step 4650/11700 | Loss: 2.5727 | LR: 3.59e-04 | 7613.9ms/step | 2.6 samples/s
|
| 490 |
+
Step 4660/11700 | Loss: 2.6427 | LR: 3.59e-04 | 7616.4ms/step | 2.6 samples/s
|
| 491 |
+
Step 4670/11700 | Loss: 2.5595 | LR: 3.58e-04 | 7618.0ms/step | 2.6 samples/s
|
| 492 |
+
Step 4680/11700 | Loss: 2.6462 | LR: 3.57e-04 | 7616.5ms/step | 2.6 samples/s
|
| 493 |
+
Step 4690/11700 | Loss: 2.5448 | LR: 3.57e-04 | 7613.3ms/step | 2.6 samples/s
|
| 494 |
+
Step 4700/11700 | Loss: 2.5456 | LR: 3.56e-04 | 7628.1ms/step | 2.6 samples/s
|
| 495 |
+
Step 4710/11700 | Loss: 2.7032 | LR: 3.55e-04 | 7620.7ms/step | 2.6 samples/s
|
| 496 |
+
Step 4720/11700 | Loss: 2.5688 | LR: 3.55e-04 | 7615.6ms/step | 2.6 samples/s
|
| 497 |
+
Step 4730/11700 | Loss: 2.6316 | LR: 3.54e-04 | 7613.7ms/step | 2.6 samples/s
|
| 498 |
+
Step 4740/11700 | Loss: 2.6416 | LR: 3.53e-04 | 7612.0ms/step | 2.6 samples/s
|
| 499 |
+
Step 4750/11700 | Loss: 2.5873 | LR: 3.53e-04 | 7614.5ms/step | 2.6 samples/s
|
| 500 |
+
Validation at step 4750: Loss = 2.6044, Perplexity = 13.52
|
| 501 |
+
Step 4760/11700 | Loss: 2.6811 | LR: 3.52e-04 | 18960.5ms/step | 1.1 samples/s
|
| 502 |
+
Step 4770/11700 | Loss: 2.6368 | LR: 3.51e-04 | 7615.7ms/step | 2.6 samples/s
|
| 503 |
+
Step 4780/11700 | Loss: 2.6034 | LR: 3.51e-04 | 7613.9ms/step | 2.6 samples/s
|
| 504 |
+
Step 4790/11700 | Loss: 2.5793 | LR: 3.50e-04 | 7613.7ms/step | 2.6 samples/s
|
| 505 |
+
Step 4800/11700 | Loss: 2.5218 | LR: 3.49e-04 | 7614.6ms/step | 2.6 samples/s
|
| 506 |
+
Step 4810/11700 | Loss: 2.4933 | LR: 3.49e-04 | 7620.1ms/step | 2.6 samples/s
|
| 507 |
+
Step 4820/11700 | Loss: 2.5955 | LR: 3.48e-04 | 7618.0ms/step | 2.6 samples/s
|
| 508 |
+
Step 4830/11700 | Loss: 2.6151 | LR: 3.47e-04 | 7616.0ms/step | 2.6 samples/s
|
| 509 |
+
Step 4840/11700 | Loss: 2.6124 | LR: 3.47e-04 | 7612.1ms/step | 2.6 samples/s
|
| 510 |
+
Step 4850/11700 | Loss: 2.6185 | LR: 3.46e-04 | 7609.7ms/step | 2.6 samples/s
|
| 511 |
+
Step 4860/11700 | Loss: 2.5633 | LR: 3.45e-04 | 7610.9ms/step | 2.6 samples/s
|
| 512 |
+
Step 4870/11700 | Loss: 2.6896 | LR: 3.45e-04 | 7613.6ms/step | 2.6 samples/s
|
| 513 |
+
Step 4880/11700 | Loss: 2.6185 | LR: 3.44e-04 | 7616.0ms/step | 2.6 samples/s
|
| 514 |
+
Step 4890/11700 | Loss: 2.6802 | LR: 3.43e-04 | 7615.8ms/step | 2.6 samples/s
|
| 515 |
+
Step 4900/11700 | Loss: 2.6200 | LR: 3.43e-04 | 7613.3ms/step | 2.6 samples/s
|
| 516 |
+
Step 4910/11700 | Loss: 2.5694 | LR: 3.42e-04 | 7612.7ms/step | 2.6 samples/s
|
| 517 |
+
Step 4920/11700 | Loss: 2.5422 | LR: 3.41e-04 | 7614.5ms/step | 2.6 samples/s
|
| 518 |
+
Step 4930/11700 | Loss: 2.5080 | LR: 3.41e-04 | 7613.6ms/step | 2.6 samples/s
|
| 519 |
+
Step 4940/11700 | Loss: 2.6328 | LR: 3.40e-04 | 7607.2ms/step | 2.6 samples/s
|
| 520 |
+
Step 4950/11700 | Loss: 2.4516 | LR: 3.39e-04 | 7615.2ms/step | 2.6 samples/s
|
| 521 |
+
Step 4960/11700 | Loss: 2.6128 | LR: 3.39e-04 | 7614.4ms/step | 2.6 samples/s
|
| 522 |
+
Step 4970/11700 | Loss: 2.5995 | LR: 3.38e-04 | 7615.5ms/step | 2.6 samples/s
|
| 523 |
+
Step 4980/11700 | Loss: 2.7101 | LR: 3.37e-04 | 7612.4ms/step | 2.6 samples/s
|
| 524 |
+
Step 4990/11700 | Loss: 2.5288 | LR: 3.37e-04 | 7613.8ms/step | 2.6 samples/s
|
| 525 |
+
Step 5000/11700 | Loss: 2.6903 | LR: 3.36e-04 | 7614.7ms/step | 2.6 samples/s
|
| 526 |
+
Validation at step 5000: Loss = 2.5881, Perplexity = 13.30
|
| 527 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-5000
|
| 528 |
+
Step 5010/11700 | Loss: 2.6881 | LR: 3.35e-04 | 19253.6ms/step | 1.0 samples/s
|
| 529 |
+
Step 5020/11700 | Loss: 2.6719 | LR: 3.35e-04 | 7608.5ms/step | 2.6 samples/s
|
| 530 |
+
Step 5030/11700 | Loss: 2.6653 | LR: 3.34e-04 | 7611.7ms/step | 2.6 samples/s
|
| 531 |
+
Step 5040/11700 | Loss: 2.4995 | LR: 3.33e-04 | 7611.9ms/step | 2.6 samples/s
|
| 532 |
+
Step 5050/11700 | Loss: 2.4390 | LR: 3.33e-04 | 7613.9ms/step | 2.6 samples/s
|
| 533 |
+
Step 5060/11700 | Loss: 2.6079 | LR: 3.32e-04 | 7617.5ms/step | 2.6 samples/s
|
| 534 |
+
Step 5070/11700 | Loss: 2.5679 | LR: 3.31e-04 | 7618.9ms/step | 2.6 samples/s
|
| 535 |
+
Step 5080/11700 | Loss: 2.7003 | LR: 3.31e-04 | 7621.9ms/step | 2.6 samples/s
|
| 536 |
+
Step 5090/11700 | Loss: 2.6001 | LR: 3.30e-04 | 7609.9ms/step | 2.6 samples/s
|
| 537 |
+
Step 5100/11700 | Loss: 2.6997 | LR: 3.29e-04 | 7610.4ms/step | 2.6 samples/s
|
| 538 |
+
Step 5110/11700 | Loss: 2.5994 | LR: 3.29e-04 | 7613.7ms/step | 2.6 samples/s
|
| 539 |
+
Step 5120/11700 | Loss: 2.5507 | LR: 3.28e-04 | 7613.9ms/step | 2.6 samples/s
|
| 540 |
+
Step 5130/11700 | Loss: 2.4661 | LR: 3.27e-04 | 7620.5ms/step | 2.6 samples/s
|
| 541 |
+
Step 5140/11700 | Loss: 2.5179 | LR: 3.27e-04 | 7614.0ms/step | 2.6 samples/s
|
| 542 |
+
Step 5150/11700 | Loss: 2.5856 | LR: 3.26e-04 | 7608.4ms/step | 2.6 samples/s
|
| 543 |
+
Step 5160/11700 | Loss: 2.5308 | LR: 3.25e-04 | 7613.7ms/step | 2.6 samples/s
|
| 544 |
+
Step 5170/11700 | Loss: 2.5376 | LR: 3.25e-04 | 7608.9ms/step | 2.6 samples/s
|
| 545 |
+
Step 5180/11700 | Loss: 2.5178 | LR: 3.24e-04 | 7613.8ms/step | 2.6 samples/s
|
| 546 |
+
Step 5190/11700 | Loss: 2.5916 | LR: 3.23e-04 | 7611.6ms/step | 2.6 samples/s
|
| 547 |
+
Step 5200/11700 | Loss: 2.4277 | LR: 3.22e-04 | 7609.6ms/step | 2.6 samples/s
|
| 548 |
+
Step 5210/11700 | Loss: 2.5445 | LR: 3.22e-04 | 7608.2ms/step | 2.6 samples/s
|
| 549 |
+
Step 5220/11700 | Loss: 2.5988 | LR: 3.21e-04 | 7606.7ms/step | 2.6 samples/s
|
| 550 |
+
Step 5230/11700 | Loss: 2.6751 | LR: 3.20e-04 | 7611.4ms/step | 2.6 samples/s
|
| 551 |
+
Step 5240/11700 | Loss: 2.5496 | LR: 3.20e-04 | 7612.2ms/step | 2.6 samples/s
|
| 552 |
+
Step 5250/11700 | Loss: 2.5894 | LR: 3.19e-04 | 7615.8ms/step | 2.6 samples/s
|
| 553 |
+
Validation at step 5250: Loss = 2.5715, Perplexity = 13.09
|
| 554 |
+
Step 5260/11700 | Loss: 2.5818 | LR: 3.18e-04 | 18951.6ms/step | 1.1 samples/s
|
| 555 |
+
Step 5270/11700 | Loss: 2.5617 | LR: 3.18e-04 | 7616.6ms/step | 2.6 samples/s
|
| 556 |
+
Step 5280/11700 | Loss: 2.5183 | LR: 3.17e-04 | 7615.1ms/step | 2.6 samples/s
|
| 557 |
+
Step 5290/11700 | Loss: 2.5248 | LR: 3.16e-04 | 7613.1ms/step | 2.6 samples/s
|
| 558 |
+
Step 5300/11700 | Loss: 2.5409 | LR: 3.16e-04 | 7610.3ms/step | 2.6 samples/s
|
| 559 |
+
Step 5310/11700 | Loss: 2.5336 | LR: 3.15e-04 | 7612.8ms/step | 2.6 samples/s
|
| 560 |
+
Step 5320/11700 | Loss: 2.5087 | LR: 3.14e-04 | 7611.1ms/step | 2.6 samples/s
|
| 561 |
+
Step 5330/11700 | Loss: 2.4975 | LR: 3.13e-04 | 7611.6ms/step | 2.6 samples/s
|
| 562 |
+
Step 5340/11700 | Loss: 2.5320 | LR: 3.13e-04 | 7610.2ms/step | 2.6 samples/s
|
| 563 |
+
Step 5350/11700 | Loss: 2.5301 | LR: 3.12e-04 | 7613.9ms/step | 2.6 samples/s
|
| 564 |
+
Step 5360/11700 | Loss: 2.4581 | LR: 3.11e-04 | 7610.6ms/step | 2.6 samples/s
|
| 565 |
+
Step 5370/11700 | Loss: 2.5634 | LR: 3.11e-04 | 7610.5ms/step | 2.6 samples/s
|
| 566 |
+
Step 5380/11700 | Loss: 2.4609 | LR: 3.10e-04 | 7611.5ms/step | 2.6 samples/s
|
| 567 |
+
Step 5390/11700 | Loss: 2.6007 | LR: 3.09e-04 | 7611.8ms/step | 2.6 samples/s
|
| 568 |
+
Step 5400/11700 | Loss: 2.5092 | LR: 3.09e-04 | 7610.5ms/step | 2.6 samples/s
|
| 569 |
+
Step 5410/11700 | Loss: 2.4945 | LR: 3.08e-04 | 7607.9ms/step | 2.6 samples/s
|
| 570 |
+
Step 5420/11700 | Loss: 2.5610 | LR: 3.07e-04 | 7612.0ms/step | 2.6 samples/s
|
| 571 |
+
Step 5430/11700 | Loss: 2.6332 | LR: 3.07e-04 | 7612.4ms/step | 2.6 samples/s
|
| 572 |
+
Step 5440/11700 | Loss: 2.6385 | LR: 3.06e-04 | 7613.6ms/step | 2.6 samples/s
|
| 573 |
+
Step 5450/11700 | Loss: 2.5227 | LR: 3.05e-04 | 7613.9ms/step | 2.6 samples/s
|
| 574 |
+
Step 5460/11700 | Loss: 2.5622 | LR: 3.04e-04 | 7610.8ms/step | 2.6 samples/s
|
| 575 |
+
Step 5470/11700 | Loss: 2.4257 | LR: 3.04e-04 | 7613.6ms/step | 2.6 samples/s
|
| 576 |
+
Step 5480/11700 | Loss: 2.5297 | LR: 3.03e-04 | 7610.9ms/step | 2.6 samples/s
|
| 577 |
+
Step 5490/11700 | Loss: 2.6934 | LR: 3.02e-04 | 7613.1ms/step | 2.6 samples/s
|
| 578 |
+
Step 5500/11700 | Loss: 2.5976 | LR: 3.02e-04 | 7610.0ms/step | 2.6 samples/s
|
| 579 |
+
Validation at step 5500: Loss = 2.5588, Perplexity = 12.92
|
| 580 |
+
Step 5510/11700 | Loss: 2.4423 | LR: 3.01e-04 | 18947.6ms/step | 1.1 samples/s
|
| 581 |
+
Step 5520/11700 | Loss: 2.4193 | LR: 3.00e-04 | 7610.8ms/step | 2.6 samples/s
|
| 582 |
+
Step 5530/11700 | Loss: 2.5742 | LR: 3.00e-04 | 7608.0ms/step | 2.6 samples/s
|
| 583 |
+
Step 5540/11700 | Loss: 2.5540 | LR: 2.99e-04 | 7610.2ms/step | 2.6 samples/s
|
| 584 |
+
Step 5550/11700 | Loss: 2.5597 | LR: 2.98e-04 | 7611.7ms/step | 2.6 samples/s
|
| 585 |
+
Step 5560/11700 | Loss: 2.6001 | LR: 2.97e-04 | 7610.3ms/step | 2.6 samples/s
|
| 586 |
+
Step 5570/11700 | Loss: 2.4737 | LR: 2.97e-04 | 7611.8ms/step | 2.6 samples/s
|
| 587 |
+
Step 5580/11700 | Loss: 2.5655 | LR: 2.96e-04 | 7613.4ms/step | 2.6 samples/s
|
| 588 |
+
Step 5590/11700 | Loss: 2.5908 | LR: 2.95e-04 | 7611.0ms/step | 2.6 samples/s
|
| 589 |
+
Step 5600/11700 | Loss: 2.5365 | LR: 2.95e-04 | 7609.8ms/step | 2.6 samples/s
|
| 590 |
+
Step 5610/11700 | Loss: 2.3945 | LR: 2.94e-04 | 7613.4ms/step | 2.6 samples/s
|
| 591 |
+
Step 5620/11700 | Loss: 2.5542 | LR: 2.93e-04 | 7608.6ms/step | 2.6 samples/s
|
| 592 |
+
Step 5630/11700 | Loss: 2.4460 | LR: 2.92e-04 | 7607.6ms/step | 2.6 samples/s
|
| 593 |
+
Step 5640/11700 | Loss: 2.5780 | LR: 2.92e-04 | 7606.5ms/step | 2.6 samples/s
|
| 594 |
+
Step 5650/11700 | Loss: 2.6303 | LR: 2.91e-04 | 7608.5ms/step | 2.6 samples/s
|
| 595 |
+
Step 5660/11700 | Loss: 2.5373 | LR: 2.90e-04 | 7614.2ms/step | 2.6 samples/s
|
| 596 |
+
Step 5670/11700 | Loss: 2.5207 | LR: 2.90e-04 | 7613.6ms/step | 2.6 samples/s
|
| 597 |
+
Step 5680/11700 | Loss: 2.5938 | LR: 2.89e-04 | 7612.2ms/step | 2.6 samples/s
|
| 598 |
+
Step 5690/11700 | Loss: 2.5069 | LR: 2.88e-04 | 7617.7ms/step | 2.6 samples/s
|
| 599 |
+
Step 5700/11700 | Loss: 2.5292 | LR: 2.88e-04 | 7614.3ms/step | 2.6 samples/s
|
| 600 |
+
Step 5710/11700 | Loss: 2.5033 | LR: 2.87e-04 | 7613.4ms/step | 2.6 samples/s
|
| 601 |
+
Step 5720/11700 | Loss: 2.4810 | LR: 2.86e-04 | 7615.3ms/step | 2.6 samples/s
|
| 602 |
+
Step 5730/11700 | Loss: 2.5955 | LR: 2.85e-04 | 7614.3ms/step | 2.6 samples/s
|
| 603 |
+
Step 5740/11700 | Loss: 2.4630 | LR: 2.85e-04 | 7611.3ms/step | 2.6 samples/s
|
| 604 |
+
Step 5750/11700 | Loss: 2.4985 | LR: 2.84e-04 | 7614.4ms/step | 2.6 samples/s
|
| 605 |
+
Validation at step 5750: Loss = 2.5464, Perplexity = 12.76
|
| 606 |
+
Step 5760/11700 | Loss: 2.5372 | LR: 2.83e-04 | 18956.0ms/step | 1.1 samples/s
|
| 607 |
+
Step 5770/11700 | Loss: 2.6660 | LR: 2.83e-04 | 7612.8ms/step | 2.6 samples/s
|
| 608 |
+
Step 5780/11700 | Loss: 2.5048 | LR: 2.82e-04 | 7611.0ms/step | 2.6 samples/s
|
| 609 |
+
Step 5790/11700 | Loss: 2.6276 | LR: 2.81e-04 | 7614.8ms/step | 2.6 samples/s
|
| 610 |
+
Step 5800/11700 | Loss: 2.5949 | LR: 2.80e-04 | 7614.8ms/step | 2.6 samples/s
|
| 611 |
+
Step 5810/11700 | Loss: 2.5684 | LR: 2.80e-04 | 7610.6ms/step | 2.6 samples/s
|
| 612 |
+
Step 5820/11700 | Loss: 2.4634 | LR: 2.79e-04 | 7611.6ms/step | 2.6 samples/s
|
| 613 |
+
Step 5830/11700 | Loss: 2.5140 | LR: 2.78e-04 | 7618.2ms/step | 2.6 samples/s
|
| 614 |
+
Step 5840/11700 | Loss: 2.5053 | LR: 2.78e-04 | 7610.2ms/step | 2.6 samples/s
|
| 615 |
+
Step 5850/11700 | Loss: 2.5132 | LR: 2.77e-04 | 7608.2ms/step | 2.6 samples/s
|
| 616 |
+
Step 5860/11700 | Loss: 2.5853 | LR: 2.76e-04 | 7610.2ms/step | 2.6 samples/s
|
| 617 |
+
Step 5870/11700 | Loss: 2.5949 | LR: 2.75e-04 | 7612.2ms/step | 2.6 samples/s
|
| 618 |
+
Step 5880/11700 | Loss: 2.5736 | LR: 2.75e-04 | 7609.7ms/step | 2.6 samples/s
|
| 619 |
+
Step 5890/11700 | Loss: 2.5102 | LR: 2.74e-04 | 7610.0ms/step | 2.6 samples/s
|
| 620 |
+
Step 5900/11700 | Loss: 2.5212 | LR: 2.73e-04 | 7610.1ms/step | 2.6 samples/s
|
| 621 |
+
Step 5910/11700 | Loss: 2.5679 | LR: 2.73e-04 | 7605.6ms/step | 2.6 samples/s
|
| 622 |
+
Step 5920/11700 | Loss: 2.4775 | LR: 2.72e-04 | 7606.6ms/step | 2.6 samples/s
|
| 623 |
+
Step 5930/11700 | Loss: 2.5086 | LR: 2.71e-04 | 7605.9ms/step | 2.6 samples/s
|
| 624 |
+
Step 5940/11700 | Loss: 2.4383 | LR: 2.70e-04 | 7610.0ms/step | 2.6 samples/s
|
| 625 |
+
Step 5950/11700 | Loss: 2.6060 | LR: 2.70e-04 | 7606.2ms/step | 2.6 samples/s
|
| 626 |
+
Step 5960/11700 | Loss: 2.6309 | LR: 2.69e-04 | 7608.3ms/step | 2.6 samples/s
|
| 627 |
+
Step 5970/11700 | Loss: 2.5211 | LR: 2.68e-04 | 7606.2ms/step | 2.6 samples/s
|
| 628 |
+
Step 5980/11700 | Loss: 2.5405 | LR: 2.68e-04 | 7610.2ms/step | 2.6 samples/s
|
| 629 |
+
Step 5990/11700 | Loss: 2.5309 | LR: 2.67e-04 | 7611.4ms/step | 2.6 samples/s
|
| 630 |
+
Step 6000/11700 | Loss: 2.5256 | LR: 2.66e-04 | 7608.2ms/step | 2.6 samples/s
|
| 631 |
+
Validation at step 6000: Loss = 2.5346, Perplexity = 12.61
|
| 632 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-6000
|
| 633 |
+
Step 6010/11700 | Loss: 2.5801 | LR: 2.65e-04 | 19240.2ms/step | 1.0 samples/s
|
| 634 |
+
Step 6020/11700 | Loss: 2.4423 | LR: 2.65e-04 | 7604.7ms/step | 2.6 samples/s
|
| 635 |
+
Step 6030/11700 | Loss: 2.5027 | LR: 2.64e-04 | 7611.0ms/step | 2.6 samples/s
|
| 636 |
+
Step 6040/11700 | Loss: 2.4667 | LR: 2.63e-04 | 7609.2ms/step | 2.6 samples/s
|
| 637 |
+
Step 6050/11700 | Loss: 2.4700 | LR: 2.63e-04 | 7613.4ms/step | 2.6 samples/s
|
| 638 |
+
Step 6060/11700 | Loss: 2.5714 | LR: 2.62e-04 | 7616.1ms/step | 2.6 samples/s
|
| 639 |
+
Step 6070/11700 | Loss: 2.5628 | LR: 2.61e-04 | 7617.8ms/step | 2.6 samples/s
|
| 640 |
+
Step 6080/11700 | Loss: 2.5917 | LR: 2.60e-04 | 7606.3ms/step | 2.6 samples/s
|
| 641 |
+
Step 6090/11700 | Loss: 2.4602 | LR: 2.60e-04 | 7609.8ms/step | 2.6 samples/s
|
| 642 |
+
Step 6100/11700 | Loss: 2.5336 | LR: 2.59e-04 | 7607.7ms/step | 2.6 samples/s
|
| 643 |
+
Step 6110/11700 | Loss: 2.4669 | LR: 2.58e-04 | 7604.4ms/step | 2.6 samples/s
|
| 644 |
+
Step 6120/11700 | Loss: 2.5934 | LR: 2.58e-04 | 7604.2ms/step | 2.6 samples/s
|
| 645 |
+
Step 6130/11700 | Loss: 2.5225 | LR: 2.57e-04 | 7603.7ms/step | 2.6 samples/s
|
| 646 |
+
Step 6140/11700 | Loss: 2.5335 | LR: 2.56e-04 | 7614.6ms/step | 2.6 samples/s
|
| 647 |
+
Step 6150/11700 | Loss: 2.5068 | LR: 2.55e-04 | 7609.6ms/step | 2.6 samples/s
|
| 648 |
+
Step 6160/11700 | Loss: 2.4444 | LR: 2.55e-04 | 7606.8ms/step | 2.6 samples/s
|
| 649 |
+
Step 6170/11700 | Loss: 2.4552 | LR: 2.54e-04 | 7607.4ms/step | 2.6 samples/s
|
| 650 |
+
Step 6180/11700 | Loss: 2.4243 | LR: 2.53e-04 | 7603.7ms/step | 2.6 samples/s
|
| 651 |
+
Step 6190/11700 | Loss: 2.4845 | LR: 2.53e-04 | 7604.4ms/step | 2.6 samples/s
|
| 652 |
+
Step 6200/11700 | Loss: 2.6105 | LR: 2.52e-04 | 7606.3ms/step | 2.6 samples/s
|
| 653 |
+
Step 6210/11700 | Loss: 2.5940 | LR: 2.51e-04 | 7609.3ms/step | 2.6 samples/s
|
| 654 |
+
Step 6220/11700 | Loss: 2.5409 | LR: 2.50e-04 | 7608.2ms/step | 2.6 samples/s
|
| 655 |
+
Step 6230/11700 | Loss: 2.6486 | LR: 2.50e-04 | 7609.1ms/step | 2.6 samples/s
|
| 656 |
+
Step 6240/11700 | Loss: 2.5075 | LR: 2.49e-04 | 7610.4ms/step | 2.6 samples/s
|
| 657 |
+
Step 6250/11700 | Loss: 2.5440 | LR: 2.48e-04 | 7609.1ms/step | 2.6 samples/s
|
| 658 |
+
Validation at step 6250: Loss = 2.5223, Perplexity = 12.46
|
| 659 |
+
Step 6260/11700 | Loss: 2.4386 | LR: 2.47e-04 | 18944.2ms/step | 1.1 samples/s
|
| 660 |
+
Step 6270/11700 | Loss: 2.5351 | LR: 2.47e-04 | 7610.0ms/step | 2.6 samples/s
|
| 661 |
+
Step 6280/11700 | Loss: 2.5141 | LR: 2.46e-04 | 7605.9ms/step | 2.6 samples/s
|
| 662 |
+
Step 6290/11700 | Loss: 2.4930 | LR: 2.45e-04 | 7604.4ms/step | 2.6 samples/s
|
| 663 |
+
Step 6300/11700 | Loss: 2.5017 | LR: 2.45e-04 | 7605.2ms/step | 2.6 samples/s
|
| 664 |
+
Step 6310/11700 | Loss: 2.4942 | LR: 2.44e-04 | 7600.7ms/step | 2.6 samples/s
|
| 665 |
+
Step 6320/11700 | Loss: 2.5595 | LR: 2.43e-04 | 7603.8ms/step | 2.6 samples/s
|
| 666 |
+
Step 6330/11700 | Loss: 2.6230 | LR: 2.42e-04 | 7606.1ms/step | 2.6 samples/s
|
| 667 |
+
Step 6340/11700 | Loss: 2.5357 | LR: 2.42e-04 | 7607.9ms/step | 2.6 samples/s
|
| 668 |
+
Step 6350/11700 | Loss: 2.5446 | LR: 2.41e-04 | 7612.2ms/step | 2.6 samples/s
|
| 669 |
+
Step 6360/11700 | Loss: 2.4417 | LR: 2.40e-04 | 7607.9ms/step | 2.6 samples/s
|
| 670 |
+
Step 6370/11700 | Loss: 2.4258 | LR: 2.40e-04 | 7612.2ms/step | 2.6 samples/s
|
| 671 |
+
Step 6380/11700 | Loss: 2.3967 | LR: 2.39e-04 | 7617.9ms/step | 2.6 samples/s
|
| 672 |
+
Step 6390/11700 | Loss: 2.5286 | LR: 2.38e-04 | 7614.7ms/step | 2.6 samples/s
|
| 673 |
+
Step 6400/11700 | Loss: 2.5045 | LR: 2.37e-04 | 7609.5ms/step | 2.6 samples/s
|
| 674 |
+
Step 6410/11700 | Loss: 2.4336 | LR: 2.37e-04 | 7608.1ms/step | 2.6 samples/s
|
| 675 |
+
Step 6420/11700 | Loss: 2.4319 | LR: 2.36e-04 | 7610.4ms/step | 2.6 samples/s
|
| 676 |
+
Step 6430/11700 | Loss: 2.3275 | LR: 2.35e-04 | 7610.5ms/step | 2.6 samples/s
|
| 677 |
+
Step 6440/11700 | Loss: 2.4991 | LR: 2.35e-04 | 7609.7ms/step | 2.6 samples/s
|
| 678 |
+
Step 6450/11700 | Loss: 2.4399 | LR: 2.34e-04 | 7608.1ms/step | 2.6 samples/s
|
| 679 |
+
Step 6460/11700 | Loss: 2.5711 | LR: 2.33e-04 | 7609.5ms/step | 2.6 samples/s
|
| 680 |
+
Step 6470/11700 | Loss: 2.4356 | LR: 2.32e-04 | 7610.3ms/step | 2.6 samples/s
|
| 681 |
+
Step 6480/11700 | Loss: 2.6864 | LR: 2.32e-04 | 7605.7ms/step | 2.6 samples/s
|
| 682 |
+
Step 6490/11700 | Loss: 2.4923 | LR: 2.31e-04 | 7607.9ms/step | 2.6 samples/s
|
| 683 |
+
Step 6500/11700 | Loss: 2.5258 | LR: 2.30e-04 | 7600.5ms/step | 2.6 samples/s
|
| 684 |
+
Validation at step 6500: Loss = 2.5129, Perplexity = 12.34
|
| 685 |
+
Step 6510/11700 | Loss: 2.5046 | LR: 2.30e-04 | 18946.0ms/step | 1.1 samples/s
|
| 686 |
+
Step 6520/11700 | Loss: 2.5058 | LR: 2.29e-04 | 7606.2ms/step | 2.6 samples/s
|
| 687 |
+
Step 6530/11700 | Loss: 2.4228 | LR: 2.28e-04 | 7607.9ms/step | 2.6 samples/s
|
| 688 |
+
Step 6540/11700 | Loss: 2.5749 | LR: 2.27e-04 | 7608.2ms/step | 2.6 samples/s
|
| 689 |
+
Step 6550/11700 | Loss: 2.4751 | LR: 2.27e-04 | 7608.3ms/step | 2.6 samples/s
|
| 690 |
+
Step 6560/11700 | Loss: 2.4266 | LR: 2.26e-04 | 7607.6ms/step | 2.6 samples/s
|
| 691 |
+
Step 6570/11700 | Loss: 2.3825 | LR: 2.25e-04 | 7604.2ms/step | 2.6 samples/s
|
| 692 |
+
Step 6580/11700 | Loss: 2.5420 | LR: 2.25e-04 | 7609.9ms/step | 2.6 samples/s
|
| 693 |
+
Step 6590/11700 | Loss: 2.5226 | LR: 2.24e-04 | 7612.7ms/step | 2.6 samples/s
|
| 694 |
+
Step 6600/11700 | Loss: 2.5358 | LR: 2.23e-04 | 7610.5ms/step | 2.6 samples/s
|
| 695 |
+
Step 6610/11700 | Loss: 2.6076 | LR: 2.22e-04 | 7606.7ms/step | 2.6 samples/s
|
| 696 |
+
Step 6620/11700 | Loss: 2.4078 | LR: 2.22e-04 | 7610.6ms/step | 2.6 samples/s
|
| 697 |
+
Step 6630/11700 | Loss: 2.5149 | LR: 2.21e-04 | 7612.1ms/step | 2.6 samples/s
|
| 698 |
+
Step 6640/11700 | Loss: 2.5226 | LR: 2.20e-04 | 7611.9ms/step | 2.6 samples/s
|
| 699 |
+
Step 6650/11700 | Loss: 2.4554 | LR: 2.20e-04 | 7610.1ms/step | 2.6 samples/s
|
| 700 |
+
Step 6660/11700 | Loss: 2.4788 | LR: 2.19e-04 | 7612.5ms/step | 2.6 samples/s
|
| 701 |
+
Step 6670/11700 | Loss: 2.4729 | LR: 2.18e-04 | 7607.7ms/step | 2.6 samples/s
|
| 702 |
+
Step 6680/11700 | Loss: 2.4699 | LR: 2.17e-04 | 7613.3ms/step | 2.6 samples/s
|
| 703 |
+
Step 6690/11700 | Loss: 2.5599 | LR: 2.17e-04 | 7608.4ms/step | 2.6 samples/s
|
| 704 |
+
Step 6700/11700 | Loss: 2.4988 | LR: 2.16e-04 | 7608.2ms/step | 2.6 samples/s
|
| 705 |
+
Step 6710/11700 | Loss: 2.5112 | LR: 2.15e-04 | 7607.6ms/step | 2.6 samples/s
|
| 706 |
+
Step 6720/11700 | Loss: 2.3947 | LR: 2.15e-04 | 7600.6ms/step | 2.6 samples/s
|
| 707 |
+
Step 6730/11700 | Loss: 2.4528 | LR: 2.14e-04 | 7604.1ms/step | 2.6 samples/s
|
| 708 |
+
Step 6740/11700 | Loss: 2.4702 | LR: 2.13e-04 | 7608.0ms/step | 2.6 samples/s
|
| 709 |
+
Step 6750/11700 | Loss: 2.4526 | LR: 2.12e-04 | 7608.3ms/step | 2.6 samples/s
|
| 710 |
+
Validation at step 6750: Loss = 2.5040, Perplexity = 12.23
|
| 711 |
+
Step 6760/11700 | Loss: 2.5270 | LR: 2.12e-04 | 18949.7ms/step | 1.1 samples/s
|
| 712 |
+
Step 6770/11700 | Loss: 2.4926 | LR: 2.11e-04 | 7614.3ms/step | 2.6 samples/s
|
| 713 |
+
Step 6780/11700 | Loss: 2.4326 | LR: 2.10e-04 | 7613.5ms/step | 2.6 samples/s
|
| 714 |
+
Step 6790/11700 | Loss: 2.4363 | LR: 2.10e-04 | 7614.3ms/step | 2.6 samples/s
|
| 715 |
+
Step 6800/11700 | Loss: 2.5190 | LR: 2.09e-04 | 7616.1ms/step | 2.6 samples/s
|
| 716 |
+
Step 6810/11700 | Loss: 2.6866 | LR: 2.08e-04 | 7612.3ms/step | 2.6 samples/s
|
| 717 |
+
Step 6820/11700 | Loss: 2.5277 | LR: 2.08e-04 | 7614.5ms/step | 2.6 samples/s
|
| 718 |
+
Step 6830/11700 | Loss: 2.4387 | LR: 2.07e-04 | 7611.5ms/step | 2.6 samples/s
|
| 719 |
+
Step 6840/11700 | Loss: 2.4580 | LR: 2.06e-04 | 7608.3ms/step | 2.6 samples/s
|
| 720 |
+
Step 6850/11700 | Loss: 2.5311 | LR: 2.05e-04 | 7611.9ms/step | 2.6 samples/s
|
| 721 |
+
Step 6860/11700 | Loss: 2.4716 | LR: 2.05e-04 | 7620.3ms/step | 2.6 samples/s
|
| 722 |
+
Step 6870/11700 | Loss: 2.4594 | LR: 2.04e-04 | 7637.7ms/step | 2.6 samples/s
|
| 723 |
+
Step 6880/11700 | Loss: 2.4605 | LR: 2.03e-04 | 7612.0ms/step | 2.6 samples/s
|
| 724 |
+
Step 6890/11700 | Loss: 2.4814 | LR: 2.03e-04 | 7610.6ms/step | 2.6 samples/s
|
| 725 |
+
Step 6900/11700 | Loss: 2.6037 | LR: 2.02e-04 | 7631.3ms/step | 2.6 samples/s
|
| 726 |
+
Step 6910/11700 | Loss: 2.5723 | LR: 2.01e-04 | 7626.3ms/step | 2.6 samples/s
|
| 727 |
+
Step 6920/11700 | Loss: 2.4288 | LR: 2.00e-04 | 7616.0ms/step | 2.6 samples/s
|
| 728 |
+
Step 6930/11700 | Loss: 2.3774 | LR: 2.00e-04 | 7611.8ms/step | 2.6 samples/s
|
| 729 |
+
Step 6940/11700 | Loss: 2.5882 | LR: 1.99e-04 | 7606.3ms/step | 2.6 samples/s
|
| 730 |
+
Step 6950/11700 | Loss: 2.5361 | LR: 1.98e-04 | 7614.2ms/step | 2.6 samples/s
|
| 731 |
+
Step 6960/11700 | Loss: 2.4846 | LR: 1.98e-04 | 7613.6ms/step | 2.6 samples/s
|
| 732 |
+
Step 6970/11700 | Loss: 2.3327 | LR: 1.97e-04 | 7614.0ms/step | 2.6 samples/s
|
| 733 |
+
Step 6980/11700 | Loss: 2.4929 | LR: 1.96e-04 | 7614.3ms/step | 2.6 samples/s
|
| 734 |
+
Step 6990/11700 | Loss: 2.5604 | LR: 1.96e-04 | 7611.8ms/step | 2.6 samples/s
|
| 735 |
+
Step 7000/11700 | Loss: 2.4927 | LR: 1.95e-04 | 7610.0ms/step | 2.6 samples/s
|
| 736 |
+
Validation at step 7000: Loss = 2.4958, Perplexity = 12.13
|
| 737 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-7000
|
| 738 |
+
Step 7010/11700 | Loss: 2.5957 | LR: 1.94e-04 | 19260.2ms/step | 1.0 samples/s
|
| 739 |
+
Step 7020/11700 | Loss: 2.4771 | LR: 1.93e-04 | 7610.3ms/step | 2.6 samples/s
|
| 740 |
+
Step 7030/11700 | Loss: 2.5690 | LR: 1.93e-04 | 7611.8ms/step | 2.6 samples/s
|
| 741 |
+
Step 7040/11700 | Loss: 2.5455 | LR: 1.92e-04 | 7610.2ms/step | 2.6 samples/s
|
| 742 |
+
Step 7050/11700 | Loss: 2.4176 | LR: 1.91e-04 | 7611.8ms/step | 2.6 samples/s
|
| 743 |
+
Step 7060/11700 | Loss: 2.5393 | LR: 1.91e-04 | 7612.4ms/step | 2.6 samples/s
|
| 744 |
+
Step 7070/11700 | Loss: 2.4671 | LR: 1.90e-04 | 7609.4ms/step | 2.6 samples/s
|
| 745 |
+
Step 7080/11700 | Loss: 2.4823 | LR: 1.89e-04 | 7610.7ms/step | 2.6 samples/s
|
| 746 |
+
Step 7090/11700 | Loss: 2.4852 | LR: 1.89e-04 | 7608.1ms/step | 2.6 samples/s
|
| 747 |
+
Step 7100/11700 | Loss: 2.5228 | LR: 1.88e-04 | 7615.2ms/step | 2.6 samples/s
|
| 748 |
+
Step 7110/11700 | Loss: 2.4640 | LR: 1.87e-04 | 7617.3ms/step | 2.6 samples/s
|
| 749 |
+
Step 7120/11700 | Loss: 2.4083 | LR: 1.87e-04 | 7612.9ms/step | 2.6 samples/s
|
| 750 |
+
Step 7130/11700 | Loss: 2.4064 | LR: 1.86e-04 | 7610.9ms/step | 2.6 samples/s
|
| 751 |
+
Step 7140/11700 | Loss: 2.4667 | LR: 1.85e-04 | 7605.6ms/step | 2.6 samples/s
|
| 752 |
+
Step 7150/11700 | Loss: 2.5480 | LR: 1.84e-04 | 7610.5ms/step | 2.6 samples/s
|
| 753 |
+
Step 7160/11700 | Loss: 2.4613 | LR: 1.84e-04 | 7611.8ms/step | 2.6 samples/s
|
| 754 |
+
Step 7170/11700 | Loss: 2.4995 | LR: 1.83e-04 | 7606.1ms/step | 2.6 samples/s
|
| 755 |
+
Step 7180/11700 | Loss: 2.4060 | LR: 1.82e-04 | 7617.0ms/step | 2.6 samples/s
|
| 756 |
+
Step 7190/11700 | Loss: 2.4091 | LR: 1.82e-04 | 7611.9ms/step | 2.6 samples/s
|
| 757 |
+
Step 7200/11700 | Loss: 2.3739 | LR: 1.81e-04 | 7611.4ms/step | 2.6 samples/s
|
| 758 |
+
Step 7210/11700 | Loss: 2.3184 | LR: 1.80e-04 | 7607.6ms/step | 2.6 samples/s
|
| 759 |
+
Step 7220/11700 | Loss: 2.4571 | LR: 1.80e-04 | 7612.6ms/step | 2.6 samples/s
|
| 760 |
+
Step 7230/11700 | Loss: 2.5005 | LR: 1.79e-04 | 7609.8ms/step | 2.6 samples/s
|
| 761 |
+
Step 7240/11700 | Loss: 2.4589 | LR: 1.78e-04 | 7620.0ms/step | 2.6 samples/s
|
| 762 |
+
Step 7250/11700 | Loss: 2.3302 | LR: 1.78e-04 | 7612.0ms/step | 2.6 samples/s
|
| 763 |
+
Validation at step 7250: Loss = 2.4883, Perplexity = 12.04
|
| 764 |
+
Step 7260/11700 | Loss: 2.4484 | LR: 1.77e-04 | 18932.0ms/step | 1.1 samples/s
|
| 765 |
+
Step 7270/11700 | Loss: 2.5020 | LR: 1.76e-04 | 7602.1ms/step | 2.6 samples/s
|
| 766 |
+
Step 7280/11700 | Loss: 2.5069 | LR: 1.75e-04 | 7600.0ms/step | 2.6 samples/s
|
| 767 |
+
Step 7290/11700 | Loss: 2.5024 | LR: 1.75e-04 | 7600.2ms/step | 2.6 samples/s
|
| 768 |
+
Step 7300/11700 | Loss: 2.4244 | LR: 1.74e-04 | 7599.9ms/step | 2.6 samples/s
|
| 769 |
+
Step 7310/11700 | Loss: 2.5489 | LR: 1.73e-04 | 7600.0ms/step | 2.6 samples/s
|
| 770 |
+
Step 7320/11700 | Loss: 2.4468 | LR: 1.73e-04 | 7600.0ms/step | 2.6 samples/s
|
| 771 |
+
Step 7330/11700 | Loss: 2.5159 | LR: 1.72e-04 | 7600.2ms/step | 2.6 samples/s
|
| 772 |
+
Step 7340/11700 | Loss: 2.4547 | LR: 1.71e-04 | 7599.9ms/step | 2.6 samples/s
|
| 773 |
+
Step 7350/11700 | Loss: 2.4385 | LR: 1.71e-04 | 7600.1ms/step | 2.6 samples/s
|
| 774 |
+
Step 7360/11700 | Loss: 2.3919 | LR: 1.70e-04 | 7600.2ms/step | 2.6 samples/s
|
| 775 |
+
Step 7370/11700 | Loss: 2.4497 | LR: 1.69e-04 | 7601.8ms/step | 2.6 samples/s
|
| 776 |
+
Step 7380/11700 | Loss: 2.4888 | LR: 1.69e-04 | 7600.0ms/step | 2.6 samples/s
|
| 777 |
+
Step 7390/11700 | Loss: 2.4555 | LR: 1.68e-04 | 7601.4ms/step | 2.6 samples/s
|
| 778 |
+
Step 7400/11700 | Loss: 2.4899 | LR: 1.67e-04 | 7600.7ms/step | 2.6 samples/s
|
| 779 |
+
Step 7410/11700 | Loss: 2.4119 | LR: 1.67e-04 | 7600.1ms/step | 2.6 samples/s
|
| 780 |
+
Step 7420/11700 | Loss: 2.3953 | LR: 1.66e-04 | 7606.8ms/step | 2.6 samples/s
|
| 781 |
+
Step 7430/11700 | Loss: 2.4076 | LR: 1.65e-04 | 7601.4ms/step | 2.6 samples/s
|
| 782 |
+
Step 7440/11700 | Loss: 2.5110 | LR: 1.65e-04 | 7599.9ms/step | 2.6 samples/s
|
| 783 |
+
Step 7450/11700 | Loss: 2.4912 | LR: 1.64e-04 | 7599.9ms/step | 2.6 samples/s
|
| 784 |
+
Step 7460/11700 | Loss: 2.4984 | LR: 1.63e-04 | 7600.3ms/step | 2.6 samples/s
|
| 785 |
+
Step 7470/11700 | Loss: 2.4511 | LR: 1.63e-04 | 7600.2ms/step | 2.6 samples/s
|
| 786 |
+
Step 7480/11700 | Loss: 2.5837 | LR: 1.62e-04 | 7599.8ms/step | 2.6 samples/s
|
| 787 |
+
Step 7490/11700 | Loss: 2.4172 | LR: 1.61e-04 | 7600.0ms/step | 2.6 samples/s
|
| 788 |
+
Step 7500/11700 | Loss: 2.4074 | LR: 1.61e-04 | 7600.2ms/step | 2.6 samples/s
|
| 789 |
+
Validation at step 7500: Loss = 2.4824, Perplexity = 11.97
|
| 790 |
+
Step 7510/11700 | Loss: 2.4924 | LR: 1.60e-04 | 18952.5ms/step | 1.1 samples/s
|
| 791 |
+
Step 7520/11700 | Loss: 2.5156 | LR: 1.59e-04 | 7601.8ms/step | 2.6 samples/s
|
| 792 |
+
Step 7530/11700 | Loss: 2.4196 | LR: 1.59e-04 | 7602.0ms/step | 2.6 samples/s
|
| 793 |
+
Step 7540/11700 | Loss: 2.4979 | LR: 1.58e-04 | 7600.2ms/step | 2.6 samples/s
|
| 794 |
+
Step 7550/11700 | Loss: 2.4442 | LR: 1.57e-04 | 7608.3ms/step | 2.6 samples/s
|
| 795 |
+
Step 7560/11700 | Loss: 2.5041 | LR: 1.57e-04 | 7607.7ms/step | 2.6 samples/s
|
| 796 |
+
Step 7570/11700 | Loss: 2.4641 | LR: 1.56e-04 | 7616.2ms/step | 2.6 samples/s
|
| 797 |
+
Step 7580/11700 | Loss: 2.4518 | LR: 1.55e-04 | 7622.0ms/step | 2.6 samples/s
|
| 798 |
+
Step 7590/11700 | Loss: 2.4324 | LR: 1.55e-04 | 7626.3ms/step | 2.6 samples/s
|
| 799 |
+
Step 7600/11700 | Loss: 2.4897 | LR: 1.54e-04 | 7623.0ms/step | 2.6 samples/s
|
| 800 |
+
Step 7610/11700 | Loss: 2.5839 | LR: 1.53e-04 | 7624.8ms/step | 2.6 samples/s
|
| 801 |
+
Step 7620/11700 | Loss: 2.5503 | LR: 1.53e-04 | 7621.8ms/step | 2.6 samples/s
|
| 802 |
+
Step 7630/11700 | Loss: 2.4257 | LR: 1.52e-04 | 7619.3ms/step | 2.6 samples/s
|
| 803 |
+
Step 7640/11700 | Loss: 2.4594 | LR: 1.51e-04 | 7621.3ms/step | 2.6 samples/s
|
| 804 |
+
Step 7650/11700 | Loss: 2.4954 | LR: 1.51e-04 | 7615.8ms/step | 2.6 samples/s
|
| 805 |
+
Step 7660/11700 | Loss: 2.4436 | LR: 1.50e-04 | 7614.2ms/step | 2.6 samples/s
|
| 806 |
+
Step 7670/11700 | Loss: 2.3815 | LR: 1.49e-04 | 7613.6ms/step | 2.6 samples/s
|
| 807 |
+
Step 7680/11700 | Loss: 2.4435 | LR: 1.49e-04 | 7614.8ms/step | 2.6 samples/s
|
| 808 |
+
Step 7690/11700 | Loss: 2.5233 | LR: 1.48e-04 | 7610.2ms/step | 2.6 samples/s
|
| 809 |
+
Step 7700/11700 | Loss: 2.5875 | LR: 1.47e-04 | 7611.5ms/step | 2.6 samples/s
|
| 810 |
+
Step 7710/11700 | Loss: 2.3897 | LR: 1.47e-04 | 7614.3ms/step | 2.6 samples/s
|
| 811 |
+
Step 7720/11700 | Loss: 2.4910 | LR: 1.46e-04 | 7610.0ms/step | 2.6 samples/s
|
| 812 |
+
Step 7730/11700 | Loss: 2.3856 | LR: 1.45e-04 | 7614.1ms/step | 2.6 samples/s
|
| 813 |
+
Step 7740/11700 | Loss: 2.5447 | LR: 1.45e-04 | 7612.1ms/step | 2.6 samples/s
|
| 814 |
+
Step 7750/11700 | Loss: 2.4608 | LR: 1.44e-04 | 7612.9ms/step | 2.6 samples/s
|
| 815 |
+
Validation at step 7750: Loss = 2.4757, Perplexity = 11.89
|
| 816 |
+
Step 7760/11700 | Loss: 2.5773 | LR: 1.43e-04 | 18972.8ms/step | 1.1 samples/s
|
| 817 |
+
Step 7770/11700 | Loss: 2.4405 | LR: 1.43e-04 | 7612.4ms/step | 2.6 samples/s
|
| 818 |
+
Step 7780/11700 | Loss: 2.5363 | LR: 1.42e-04 | 7613.9ms/step | 2.6 samples/s
|
| 819 |
+
Step 7790/11700 | Loss: 2.4366 | LR: 1.41e-04 | 7611.9ms/step | 2.6 samples/s
|
| 820 |
+
Step 7800/11700 | Loss: 2.5219 | LR: 1.41e-04 | 7606.3ms/step | 2.6 samples/s
|
| 821 |
+
Step 7810/11700 | Loss: 2.3639 | LR: 1.40e-04 | 7606.2ms/step | 2.6 samples/s
|
| 822 |
+
Step 7820/11700 | Loss: 2.4076 | LR: 1.40e-04 | 7607.9ms/step | 2.6 samples/s
|
| 823 |
+
Step 7830/11700 | Loss: 2.4099 | LR: 1.39e-04 | 7608.5ms/step | 2.6 samples/s
|
| 824 |
+
Step 7840/11700 | Loss: 2.4722 | LR: 1.38e-04 | 7613.9ms/step | 2.6 samples/s
|
| 825 |
+
Step 7850/11700 | Loss: 2.4925 | LR: 1.38e-04 | 7613.8ms/step | 2.6 samples/s
|
| 826 |
+
Step 7860/11700 | Loss: 2.4905 | LR: 1.37e-04 | 7615.5ms/step | 2.6 samples/s
|
| 827 |
+
Step 7870/11700 | Loss: 2.5307 | LR: 1.36e-04 | 7612.6ms/step | 2.6 samples/s
|
| 828 |
+
Step 7880/11700 | Loss: 2.4855 | LR: 1.36e-04 | 7607.0ms/step | 2.6 samples/s
|
| 829 |
+
Step 7890/11700 | Loss: 2.4431 | LR: 1.35e-04 | 7607.5ms/step | 2.6 samples/s
|
| 830 |
+
Step 7900/11700 | Loss: 2.5452 | LR: 1.34e-04 | 7616.2ms/step | 2.6 samples/s
|
| 831 |
+
Step 7910/11700 | Loss: 2.3239 | LR: 1.34e-04 | 7621.4ms/step | 2.6 samples/s
|
| 832 |
+
Step 7920/11700 | Loss: 2.4277 | LR: 1.33e-04 | 7617.4ms/step | 2.6 samples/s
|
| 833 |
+
Step 7930/11700 | Loss: 2.4371 | LR: 1.33e-04 | 7625.0ms/step | 2.6 samples/s
|
| 834 |
+
Step 7940/11700 | Loss: 2.4883 | LR: 1.32e-04 | 7621.8ms/step | 2.6 samples/s
|
| 835 |
+
Step 7950/11700 | Loss: 2.5104 | LR: 1.31e-04 | 7621.1ms/step | 2.6 samples/s
|
| 836 |
+
Step 7960/11700 | Loss: 2.4369 | LR: 1.31e-04 | 7617.2ms/step | 2.6 samples/s
|
| 837 |
+
Step 7970/11700 | Loss: 2.3847 | LR: 1.30e-04 | 7618.1ms/step | 2.6 samples/s
|
| 838 |
+
Step 7980/11700 | Loss: 2.5393 | LR: 1.29e-04 | 7606.0ms/step | 2.6 samples/s
|
| 839 |
+
Step 7990/11700 | Loss: 2.5148 | LR: 1.29e-04 | 7608.4ms/step | 2.6 samples/s
|
| 840 |
+
Step 8000/11700 | Loss: 2.3985 | LR: 1.28e-04 | 7613.7ms/step | 2.6 samples/s
|
| 841 |
+
Validation at step 8000: Loss = 2.4711, Perplexity = 11.84
|
| 842 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-8000
|
| 843 |
+
Step 8010/11700 | Loss: 2.4858 | LR: 1.27e-04 | 19285.9ms/step | 1.0 samples/s
|
| 844 |
+
Step 8020/11700 | Loss: 2.3606 | LR: 1.27e-04 | 7604.2ms/step | 2.6 samples/s
|
| 845 |
+
Step 8030/11700 | Loss: 2.4512 | LR: 1.26e-04 | 7604.0ms/step | 2.6 samples/s
|
| 846 |
+
Step 8040/11700 | Loss: 2.4588 | LR: 1.26e-04 | 7606.0ms/step | 2.6 samples/s
|
| 847 |
+
Step 8050/11700 | Loss: 2.4208 | LR: 1.25e-04 | 7602.0ms/step | 2.6 samples/s
|
| 848 |
+
Step 8060/11700 | Loss: 2.4477 | LR: 1.24e-04 | 7604.2ms/step | 2.6 samples/s
|
| 849 |
+
Step 8070/11700 | Loss: 2.5032 | LR: 1.24e-04 | 7608.3ms/step | 2.6 samples/s
|
| 850 |
+
Step 8080/11700 | Loss: 2.5011 | LR: 1.23e-04 | 7612.1ms/step | 2.6 samples/s
|
| 851 |
+
Step 8090/11700 | Loss: 2.4319 | LR: 1.23e-04 | 7607.8ms/step | 2.6 samples/s
|
| 852 |
+
Step 8100/11700 | Loss: 2.4687 | LR: 1.22e-04 | 7606.7ms/step | 2.6 samples/s
|
| 853 |
+
Step 8110/11700 | Loss: 2.4155 | LR: 1.21e-04 | 7607.7ms/step | 2.6 samples/s
|
| 854 |
+
Step 8120/11700 | Loss: 2.3963 | LR: 1.21e-04 | 7609.9ms/step | 2.6 samples/s
|
| 855 |
+
Step 8130/11700 | Loss: 2.3743 | LR: 1.20e-04 | 7608.2ms/step | 2.6 samples/s
|
| 856 |
+
Step 8140/11700 | Loss: 2.4296 | LR: 1.19e-04 | 7610.2ms/step | 2.6 samples/s
|
| 857 |
+
Step 8150/11700 | Loss: 2.3140 | LR: 1.19e-04 | 7615.5ms/step | 2.6 samples/s
|
| 858 |
+
Step 8160/11700 | Loss: 2.5428 | LR: 1.18e-04 | 7616.4ms/step | 2.6 samples/s
|
| 859 |
+
Step 8170/11700 | Loss: 2.4384 | LR: 1.18e-04 | 7619.7ms/step | 2.6 samples/s
|
| 860 |
+
Step 8180/11700 | Loss: 2.4834 | LR: 1.17e-04 | 7614.6ms/step | 2.6 samples/s
|
| 861 |
+
Step 8190/11700 | Loss: 2.4023 | LR: 1.16e-04 | 7609.9ms/step | 2.6 samples/s
|
| 862 |
+
Step 8200/11700 | Loss: 2.4544 | LR: 1.16e-04 | 7609.1ms/step | 2.6 samples/s
|
| 863 |
+
Step 8210/11700 | Loss: 2.4875 | LR: 1.15e-04 | 7609.1ms/step | 2.6 samples/s
|
| 864 |
+
Step 8220/11700 | Loss: 2.4880 | LR: 1.15e-04 | 7610.0ms/step | 2.6 samples/s
|
| 865 |
+
Step 8230/11700 | Loss: 2.4703 | LR: 1.14e-04 | 7610.4ms/step | 2.6 samples/s
|
| 866 |
+
Step 8240/11700 | Loss: 2.4412 | LR: 1.13e-04 | 7613.4ms/step | 2.6 samples/s
|
| 867 |
+
Step 8250/11700 | Loss: 2.4716 | LR: 1.13e-04 | 7610.6ms/step | 2.6 samples/s
|
| 868 |
+
Validation at step 8250: Loss = 2.4672, Perplexity = 11.79
|
| 869 |
+
Step 8260/11700 | Loss: 2.4073 | LR: 1.12e-04 | 18951.8ms/step | 1.1 samples/s
|
| 870 |
+
Step 8270/11700 | Loss: 2.4175 | LR: 1.12e-04 | 7612.1ms/step | 2.6 samples/s
|
| 871 |
+
Step 8280/11700 | Loss: 2.5658 | LR: 1.11e-04 | 7642.0ms/step | 2.6 samples/s
|
| 872 |
+
Step 8290/11700 | Loss: 2.4277 | LR: 1.10e-04 | 7621.8ms/step | 2.6 samples/s
|
| 873 |
+
Step 8300/11700 | Loss: 2.4798 | LR: 1.10e-04 | 7618.0ms/step | 2.6 samples/s
|
| 874 |
+
Step 8310/11700 | Loss: 2.4124 | LR: 1.09e-04 | 7616.5ms/step | 2.6 samples/s
|
| 875 |
+
Step 8320/11700 | Loss: 2.4976 | LR: 1.09e-04 | 7616.0ms/step | 2.6 samples/s
|
| 876 |
+
Step 8330/11700 | Loss: 2.5052 | LR: 1.08e-04 | 7614.1ms/step | 2.6 samples/s
|
| 877 |
+
Step 8340/11700 | Loss: 2.4697 | LR: 1.07e-04 | 7614.9ms/step | 2.6 samples/s
|
| 878 |
+
Step 8350/11700 | Loss: 2.4827 | LR: 1.07e-04 | 7613.1ms/step | 2.6 samples/s
|
| 879 |
+
Step 8360/11700 | Loss: 2.4249 | LR: 1.06e-04 | 7614.5ms/step | 2.6 samples/s
|
| 880 |
+
Step 8370/11700 | Loss: 2.4146 | LR: 1.06e-04 | 7613.6ms/step | 2.6 samples/s
|
| 881 |
+
Step 8380/11700 | Loss: 2.4473 | LR: 1.05e-04 | 7615.2ms/step | 2.6 samples/s
|
| 882 |
+
Step 8390/11700 | Loss: 2.4543 | LR: 1.05e-04 | 7608.9ms/step | 2.6 samples/s
|
| 883 |
+
Step 8400/11700 | Loss: 2.5145 | LR: 1.04e-04 | 7611.9ms/step | 2.6 samples/s
|
| 884 |
+
Step 8410/11700 | Loss: 2.6366 | LR: 1.03e-04 | 7610.2ms/step | 2.6 samples/s
|
| 885 |
+
Step 8420/11700 | Loss: 2.3940 | LR: 1.03e-04 | 7614.1ms/step | 2.6 samples/s
|
| 886 |
+
Step 8430/11700 | Loss: 2.5150 | LR: 1.02e-04 | 7613.4ms/step | 2.6 samples/s
|
| 887 |
+
Step 8440/11700 | Loss: 2.5124 | LR: 1.02e-04 | 7615.5ms/step | 2.6 samples/s
|
| 888 |
+
Step 8450/11700 | Loss: 2.4560 | LR: 1.01e-04 | 7614.7ms/step | 2.6 samples/s
|
| 889 |
+
Step 8460/11700 | Loss: 2.4455 | LR: 1.00e-04 | 7614.5ms/step | 2.6 samples/s
|
| 890 |
+
Step 8470/11700 | Loss: 2.4419 | LR: 9.99e-05 | 7614.1ms/step | 2.6 samples/s
|
| 891 |
+
Step 8480/11700 | Loss: 2.4938 | LR: 9.93e-05 | 7615.8ms/step | 2.6 samples/s
|
| 892 |
+
Step 8490/11700 | Loss: 2.4452 | LR: 9.87e-05 | 7612.2ms/step | 2.6 samples/s
|
| 893 |
+
Step 8500/11700 | Loss: 2.3090 | LR: 9.82e-05 | 7612.2ms/step | 2.6 samples/s
|
| 894 |
+
Validation at step 8500: Loss = 2.4631, Perplexity = 11.74
|
| 895 |
+
Step 8510/11700 | Loss: 2.4117 | LR: 9.76e-05 | 18960.3ms/step | 1.1 samples/s
|
| 896 |
+
Step 8520/11700 | Loss: 2.4581 | LR: 9.70e-05 | 7615.7ms/step | 2.6 samples/s
|
| 897 |
+
Step 8530/11700 | Loss: 2.4832 | LR: 9.65e-05 | 7612.4ms/step | 2.6 samples/s
|
| 898 |
+
Step 8540/11700 | Loss: 2.4542 | LR: 9.59e-05 | 7611.7ms/step | 2.6 samples/s
|
| 899 |
+
Step 8550/11700 | Loss: 2.3445 | LR: 9.53e-05 | 7612.1ms/step | 2.6 samples/s
|
| 900 |
+
Step 8560/11700 | Loss: 2.4550 | LR: 9.48e-05 | 7605.8ms/step | 2.6 samples/s
|
| 901 |
+
Step 8570/11700 | Loss: 2.4472 | LR: 9.42e-05 | 7611.9ms/step | 2.6 samples/s
|
| 902 |
+
Step 8580/11700 | Loss: 2.4328 | LR: 9.36e-05 | 7612.2ms/step | 2.6 samples/s
|
| 903 |
+
Step 8590/11700 | Loss: 2.4031 | LR: 9.31e-05 | 7609.7ms/step | 2.6 samples/s
|
| 904 |
+
Step 8600/11700 | Loss: 2.4752 | LR: 9.25e-05 | 7608.3ms/step | 2.6 samples/s
|
| 905 |
+
Step 8610/11700 | Loss: 2.4492 | LR: 9.20e-05 | 7606.8ms/step | 2.6 samples/s
|
| 906 |
+
Step 8620/11700 | Loss: 2.5041 | LR: 9.14e-05 | 7613.9ms/step | 2.6 samples/s
|
| 907 |
+
Step 8630/11700 | Loss: 2.4762 | LR: 9.09e-05 | 7613.3ms/step | 2.6 samples/s
|
| 908 |
+
Step 8640/11700 | Loss: 2.4130 | LR: 9.03e-05 | 7607.9ms/step | 2.6 samples/s
|
| 909 |
+
Step 8650/11700 | Loss: 2.3716 | LR: 8.98e-05 | 7610.4ms/step | 2.6 samples/s
|
| 910 |
+
Step 8660/11700 | Loss: 2.4344 | LR: 8.92e-05 | 7616.1ms/step | 2.6 samples/s
|
| 911 |
+
Step 8670/11700 | Loss: 2.5263 | LR: 8.87e-05 | 7615.9ms/step | 2.6 samples/s
|
| 912 |
+
Step 8680/11700 | Loss: 2.4038 | LR: 8.81e-05 | 7616.2ms/step | 2.6 samples/s
|
| 913 |
+
Step 8690/11700 | Loss: 2.5273 | LR: 8.76e-05 | 7614.3ms/step | 2.6 samples/s
|
| 914 |
+
Step 8700/11700 | Loss: 2.4522 | LR: 8.70e-05 | 7614.4ms/step | 2.6 samples/s
|
| 915 |
+
Step 8710/11700 | Loss: 2.3183 | LR: 8.65e-05 | 7611.4ms/step | 2.6 samples/s
|
| 916 |
+
Step 8720/11700 | Loss: 2.3937 | LR: 8.59e-05 | 7609.9ms/step | 2.6 samples/s
|
| 917 |
+
Step 8730/11700 | Loss: 2.4416 | LR: 8.54e-05 | 7608.0ms/step | 2.6 samples/s
|
| 918 |
+
Step 8740/11700 | Loss: 2.4696 | LR: 8.49e-05 | 7615.2ms/step | 2.6 samples/s
|
| 919 |
+
Step 8750/11700 | Loss: 2.4761 | LR: 8.43e-05 | 7615.0ms/step | 2.6 samples/s
|
| 920 |
+
Validation at step 8750: Loss = 2.4606, Perplexity = 11.71
|
| 921 |
+
Step 8760/11700 | Loss: 2.3846 | LR: 8.38e-05 | 18936.0ms/step | 1.1 samples/s
|
| 922 |
+
Step 8770/11700 | Loss: 2.4421 | LR: 8.33e-05 | 7602.5ms/step | 2.6 samples/s
|
| 923 |
+
Step 8780/11700 | Loss: 2.4413 | LR: 8.27e-05 | 7599.6ms/step | 2.6 samples/s
|
| 924 |
+
Step 8790/11700 | Loss: 2.4803 | LR: 8.22e-05 | 7600.1ms/step | 2.6 samples/s
|
| 925 |
+
Step 8800/11700 | Loss: 2.5086 | LR: 8.17e-05 | 7600.1ms/step | 2.6 samples/s
|
| 926 |
+
Step 8810/11700 | Loss: 2.3626 | LR: 8.11e-05 | 7602.2ms/step | 2.6 samples/s
|
| 927 |
+
Step 8820/11700 | Loss: 2.3586 | LR: 8.06e-05 | 7602.4ms/step | 2.6 samples/s
|
| 928 |
+
Step 8830/11700 | Loss: 2.4181 | LR: 8.01e-05 | 7601.9ms/step | 2.6 samples/s
|
| 929 |
+
Step 8840/11700 | Loss: 2.4983 | LR: 7.95e-05 | 7601.8ms/step | 2.6 samples/s
|
| 930 |
+
Step 8850/11700 | Loss: 2.4658 | LR: 7.90e-05 | 7604.2ms/step | 2.6 samples/s
|
| 931 |
+
Step 8860/11700 | Loss: 2.3445 | LR: 7.85e-05 | 7604.2ms/step | 2.6 samples/s
|
| 932 |
+
Step 8870/11700 | Loss: 2.4008 | LR: 7.80e-05 | 7605.9ms/step | 2.6 samples/s
|
| 933 |
+
Step 8880/11700 | Loss: 2.4861 | LR: 7.75e-05 | 7606.7ms/step | 2.6 samples/s
|
| 934 |
+
Step 8890/11700 | Loss: 2.4481 | LR: 7.69e-05 | 7603.1ms/step | 2.6 samples/s
|
| 935 |
+
Step 8900/11700 | Loss: 2.4261 | LR: 7.64e-05 | 7602.4ms/step | 2.6 samples/s
|
| 936 |
+
Step 8910/11700 | Loss: 2.5236 | LR: 7.59e-05 | 7603.9ms/step | 2.6 samples/s
|
| 937 |
+
Step 8920/11700 | Loss: 2.5018 | LR: 7.54e-05 | 7599.8ms/step | 2.6 samples/s
|
| 938 |
+
Step 8930/11700 | Loss: 2.3859 | LR: 7.49e-05 | 7600.2ms/step | 2.6 samples/s
|
| 939 |
+
Step 8940/11700 | Loss: 2.4857 | LR: 7.44e-05 | 7600.7ms/step | 2.6 samples/s
|
| 940 |
+
Step 8950/11700 | Loss: 2.4159 | LR: 7.39e-05 | 7603.2ms/step | 2.6 samples/s
|
| 941 |
+
Step 8960/11700 | Loss: 2.4252 | LR: 7.34e-05 | 7600.5ms/step | 2.6 samples/s
|
| 942 |
+
Step 8970/11700 | Loss: 2.4335 | LR: 7.28e-05 | 7607.8ms/step | 2.6 samples/s
|
| 943 |
+
Step 8980/11700 | Loss: 2.4233 | LR: 7.23e-05 | 7605.1ms/step | 2.6 samples/s
|
| 944 |
+
Step 8990/11700 | Loss: 2.4610 | LR: 7.18e-05 | 7605.3ms/step | 2.6 samples/s
|
| 945 |
+
Step 9000/11700 | Loss: 2.4376 | LR: 7.13e-05 | 7601.7ms/step | 2.6 samples/s
|
| 946 |
+
Validation at step 9000: Loss = 2.4583, Perplexity = 11.69
|
| 947 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-9000
|
| 948 |
+
Step 9010/11700 | Loss: 2.5255 | LR: 7.08e-05 | 19298.3ms/step | 1.0 samples/s
|
| 949 |
+
Step 9020/11700 | Loss: 2.4385 | LR: 7.03e-05 | 7614.7ms/step | 2.6 samples/s
|
| 950 |
+
Step 9030/11700 | Loss: 2.4030 | LR: 6.98e-05 | 7613.4ms/step | 2.6 samples/s
|
| 951 |
+
Step 9040/11700 | Loss: 2.3766 | LR: 6.93e-05 | 7616.0ms/step | 2.6 samples/s
|
| 952 |
+
Step 9050/11700 | Loss: 2.4717 | LR: 6.88e-05 | 7614.4ms/step | 2.6 samples/s
|
| 953 |
+
Step 9060/11700 | Loss: 2.4862 | LR: 6.83e-05 | 7616.2ms/step | 2.6 samples/s
|
| 954 |
+
Step 9070/11700 | Loss: 2.4996 | LR: 6.79e-05 | 7618.0ms/step | 2.6 samples/s
|
| 955 |
+
Step 9080/11700 | Loss: 2.3957 | LR: 6.74e-05 | 7618.2ms/step | 2.6 samples/s
|
| 956 |
+
Step 9090/11700 | Loss: 2.3826 | LR: 6.69e-05 | 7617.7ms/step | 2.6 samples/s
|
| 957 |
+
Step 9100/11700 | Loss: 2.4701 | LR: 6.64e-05 | 7616.4ms/step | 2.6 samples/s
|
| 958 |
+
Step 9110/11700 | Loss: 2.4690 | LR: 6.59e-05 | 7613.7ms/step | 2.6 samples/s
|
| 959 |
+
Step 9120/11700 | Loss: 2.4997 | LR: 6.54e-05 | 7613.8ms/step | 2.6 samples/s
|
| 960 |
+
Step 9130/11700 | Loss: 2.3659 | LR: 6.49e-05 | 7612.2ms/step | 2.6 samples/s
|
| 961 |
+
Step 9140/11700 | Loss: 2.3443 | LR: 6.45e-05 | 7612.2ms/step | 2.6 samples/s
|
| 962 |
+
Step 9150/11700 | Loss: 2.4553 | LR: 6.40e-05 | 7615.0ms/step | 2.6 samples/s
|
| 963 |
+
Step 9160/11700 | Loss: 2.5446 | LR: 6.35e-05 | 7614.7ms/step | 2.6 samples/s
|
| 964 |
+
Step 9170/11700 | Loss: 2.4946 | LR: 6.30e-05 | 7615.8ms/step | 2.6 samples/s
|
| 965 |
+
Step 9180/11700 | Loss: 2.4301 | LR: 6.25e-05 | 7618.4ms/step | 2.6 samples/s
|
| 966 |
+
Step 9190/11700 | Loss: 2.4538 | LR: 6.21e-05 | 7614.9ms/step | 2.6 samples/s
|
| 967 |
+
Step 9200/11700 | Loss: 2.3359 | LR: 6.16e-05 | 7618.9ms/step | 2.6 samples/s
|
| 968 |
+
Step 9210/11700 | Loss: 2.4414 | LR: 6.11e-05 | 7612.6ms/step | 2.6 samples/s
|
| 969 |
+
Step 9220/11700 | Loss: 2.5241 | LR: 6.07e-05 | 7615.9ms/step | 2.6 samples/s
|
| 970 |
+
Step 9230/11700 | Loss: 2.4303 | LR: 6.02e-05 | 7612.9ms/step | 2.6 samples/s
|
| 971 |
+
Step 9240/11700 | Loss: 2.4171 | LR: 5.97e-05 | 7609.4ms/step | 2.6 samples/s
|
| 972 |
+
Step 9250/11700 | Loss: 2.4482 | LR: 5.93e-05 | 7607.8ms/step | 2.6 samples/s
|
| 973 |
+
Validation at step 9250: Loss = 2.4567, Perplexity = 11.67
|
| 974 |
+
Step 9260/11700 | Loss: 2.3713 | LR: 5.88e-05 | 18959.1ms/step | 1.1 samples/s
|
| 975 |
+
Step 9270/11700 | Loss: 2.4650 | LR: 5.83e-05 | 7611.1ms/step | 2.6 samples/s
|
| 976 |
+
Step 9280/11700 | Loss: 2.3205 | LR: 5.79e-05 | 7631.1ms/step | 2.6 samples/s
|
| 977 |
+
Step 9290/11700 | Loss: 2.4082 | LR: 5.74e-05 | 7638.9ms/step | 2.6 samples/s
|
| 978 |
+
Step 9300/11700 | Loss: 2.4636 | LR: 5.70e-05 | 7608.1ms/step | 2.6 samples/s
|
| 979 |
+
Step 9310/11700 | Loss: 2.3581 | LR: 5.65e-05 | 7625.6ms/step | 2.6 samples/s
|
| 980 |
+
Step 9320/11700 | Loss: 2.4004 | LR: 5.61e-05 | 7610.4ms/step | 2.6 samples/s
|
| 981 |
+
Step 9330/11700 | Loss: 2.3972 | LR: 5.56e-05 | 7616.8ms/step | 2.6 samples/s
|
| 982 |
+
Step 9340/11700 | Loss: 2.5079 | LR: 5.52e-05 | 7631.1ms/step | 2.6 samples/s
|
| 983 |
+
Step 9350/11700 | Loss: 2.3959 | LR: 5.47e-05 | 7621.0ms/step | 2.6 samples/s
|
| 984 |
+
Step 9360/11700 | Loss: 2.4284 | LR: 5.43e-05 | 7607.1ms/step | 2.6 samples/s
|
| 985 |
+
Step 9370/11700 | Loss: 2.4451 | LR: 5.38e-05 | 7606.1ms/step | 2.6 samples/s
|
| 986 |
+
Step 9380/11700 | Loss: 2.5476 | LR: 5.34e-05 | 7615.8ms/step | 2.6 samples/s
|
| 987 |
+
Step 9390/11700 | Loss: 2.4552 | LR: 5.29e-05 | 7616.5ms/step | 2.6 samples/s
|
| 988 |
+
Step 9400/11700 | Loss: 2.4323 | LR: 5.25e-05 | 7630.1ms/step | 2.6 samples/s
|
| 989 |
+
Step 9410/11700 | Loss: 2.4474 | LR: 5.20e-05 | 7612.1ms/step | 2.6 samples/s
|
| 990 |
+
Step 9420/11700 | Loss: 2.5106 | LR: 5.16e-05 | 7617.3ms/step | 2.6 samples/s
|
| 991 |
+
Step 9430/11700 | Loss: 2.3208 | LR: 5.12e-05 | 7612.7ms/step | 2.6 samples/s
|
| 992 |
+
Step 9440/11700 | Loss: 2.5372 | LR: 5.07e-05 | 7607.6ms/step | 2.6 samples/s
|
| 993 |
+
Step 9450/11700 | Loss: 2.4803 | LR: 5.03e-05 | 7613.5ms/step | 2.6 samples/s
|
| 994 |
+
Step 9460/11700 | Loss: 2.4792 | LR: 4.99e-05 | 7620.6ms/step | 2.6 samples/s
|
| 995 |
+
Step 9470/11700 | Loss: 2.4554 | LR: 4.94e-05 | 7600.1ms/step | 2.6 samples/s
|
| 996 |
+
Step 9480/11700 | Loss: 2.4132 | LR: 4.90e-05 | 7618.2ms/step | 2.6 samples/s
|
| 997 |
+
Step 9490/11700 | Loss: 2.5626 | LR: 4.86e-05 | 7614.1ms/step | 2.6 samples/s
|
| 998 |
+
Step 9500/11700 | Loss: 2.3702 | LR: 4.82e-05 | 7613.9ms/step | 2.6 samples/s
|
| 999 |
+
Validation at step 9500: Loss = 2.4554, Perplexity = 11.65
|
| 1000 |
+
Step 9510/11700 | Loss: 2.4578 | LR: 4.77e-05 | 18939.9ms/step | 1.1 samples/s
|
| 1001 |
+
Step 9520/11700 | Loss: 2.4109 | LR: 4.73e-05 | 7601.9ms/step | 2.6 samples/s
|
| 1002 |
+
Step 9530/11700 | Loss: 2.4140 | LR: 4.69e-05 | 7600.1ms/step | 2.6 samples/s
|
| 1003 |
+
Step 9540/11700 | Loss: 2.3761 | LR: 4.65e-05 | 7600.2ms/step | 2.6 samples/s
|
| 1004 |
+
Step 9550/11700 | Loss: 2.5817 | LR: 4.61e-05 | 7599.9ms/step | 2.6 samples/s
|
| 1005 |
+
Step 9560/11700 | Loss: 2.3609 | LR: 4.57e-05 | 7600.1ms/step | 2.6 samples/s
|
| 1006 |
+
Step 9570/11700 | Loss: 2.3992 | LR: 4.52e-05 | 7600.1ms/step | 2.6 samples/s
|
| 1007 |
+
Step 9580/11700 | Loss: 2.5193 | LR: 4.48e-05 | 7600.1ms/step | 2.6 samples/s
|
| 1008 |
+
Step 9590/11700 | Loss: 2.4503 | LR: 4.44e-05 | 7599.9ms/step | 2.6 samples/s
|
| 1009 |
+
Step 9600/11700 | Loss: 2.4596 | LR: 4.40e-05 | 7600.0ms/step | 2.6 samples/s
|
| 1010 |
+
Step 9610/11700 | Loss: 2.4602 | LR: 4.36e-05 | 7600.1ms/step | 2.6 samples/s
|
| 1011 |
+
Step 9620/11700 | Loss: 2.4718 | LR: 4.32e-05 | 7600.0ms/step | 2.6 samples/s
|
| 1012 |
+
Step 9630/11700 | Loss: 2.4188 | LR: 4.28e-05 | 7600.1ms/step | 2.6 samples/s
|
| 1013 |
+
Step 9640/11700 | Loss: 2.4699 | LR: 4.24e-05 | 7600.1ms/step | 2.6 samples/s
|
| 1014 |
+
Step 9650/11700 | Loss: 2.5473 | LR: 4.20e-05 | 7600.3ms/step | 2.6 samples/s
|
| 1015 |
+
Step 9660/11700 | Loss: 2.5540 | LR: 4.16e-05 | 7599.8ms/step | 2.6 samples/s
|
| 1016 |
+
Step 9670/11700 | Loss: 2.4500 | LR: 4.12e-05 | 7600.1ms/step | 2.6 samples/s
|
| 1017 |
+
Step 9680/11700 | Loss: 2.2710 | LR: 4.08e-05 | 7600.1ms/step | 2.6 samples/s
|
| 1018 |
+
Step 9690/11700 | Loss: 2.5106 | LR: 4.04e-05 | 7602.0ms/step | 2.6 samples/s
|
| 1019 |
+
Step 9700/11700 | Loss: 2.3795 | LR: 4.00e-05 | 7600.1ms/step | 2.6 samples/s
|
| 1020 |
+
Step 9710/11700 | Loss: 2.4320 | LR: 3.97e-05 | 7600.0ms/step | 2.6 samples/s
|
| 1021 |
+
Step 9720/11700 | Loss: 2.5059 | LR: 3.93e-05 | 7602.4ms/step | 2.6 samples/s
|
| 1022 |
+
Step 9730/11700 | Loss: 2.3480 | LR: 3.89e-05 | 7599.7ms/step | 2.6 samples/s
|
| 1023 |
+
Step 9740/11700 | Loss: 2.5204 | LR: 3.85e-05 | 7600.1ms/step | 2.6 samples/s
|
| 1024 |
+
Step 9750/11700 | Loss: 2.4430 | LR: 3.81e-05 | 7602.0ms/step | 2.6 samples/s
|
| 1025 |
+
Validation at step 9750: Loss = 2.4544, Perplexity = 11.64
|
| 1026 |
+
Step 9760/11700 | Loss: 2.4282 | LR: 3.77e-05 | 18937.9ms/step | 1.1 samples/s
|
| 1027 |
+
Step 9770/11700 | Loss: 2.3716 | LR: 3.74e-05 | 7600.5ms/step | 2.6 samples/s
|
| 1028 |
+
Step 9780/11700 | Loss: 2.5664 | LR: 3.70e-05 | 7603.6ms/step | 2.6 samples/s
|
| 1029 |
+
Step 9790/11700 | Loss: 2.4246 | LR: 3.66e-05 | 7602.2ms/step | 2.6 samples/s
|
| 1030 |
+
Step 9800/11700 | Loss: 2.3735 | LR: 3.62e-05 | 7602.0ms/step | 2.6 samples/s
|
| 1031 |
+
Step 9810/11700 | Loss: 2.4897 | LR: 3.59e-05 | 7600.0ms/step | 2.6 samples/s
|
| 1032 |
+
Step 9820/11700 | Loss: 2.3394 | LR: 3.55e-05 | 7602.1ms/step | 2.6 samples/s
|
| 1033 |
+
Step 9830/11700 | Loss: 2.3603 | LR: 3.51e-05 | 7601.9ms/step | 2.6 samples/s
|
| 1034 |
+
Step 9840/11700 | Loss: 2.3932 | LR: 3.48e-05 | 7600.2ms/step | 2.6 samples/s
|
| 1035 |
+
Step 9850/11700 | Loss: 2.4696 | LR: 3.44e-05 | 7601.9ms/step | 2.6 samples/s
|
| 1036 |
+
Step 9860/11700 | Loss: 2.4482 | LR: 3.40e-05 | 7600.3ms/step | 2.6 samples/s
|
| 1037 |
+
Step 9870/11700 | Loss: 2.3962 | LR: 3.37e-05 | 7600.0ms/step | 2.6 samples/s
|
| 1038 |
+
Step 9880/11700 | Loss: 2.4365 | LR: 3.33e-05 | 7600.1ms/step | 2.6 samples/s
|
| 1039 |
+
Step 9890/11700 | Loss: 2.3439 | LR: 3.30e-05 | 7602.2ms/step | 2.6 samples/s
|
| 1040 |
+
Step 9900/11700 | Loss: 2.4766 | LR: 3.26e-05 | 7603.7ms/step | 2.6 samples/s
|
| 1041 |
+
Step 9910/11700 | Loss: 2.4588 | LR: 3.22e-05 | 7599.9ms/step | 2.6 samples/s
|
| 1042 |
+
Step 9920/11700 | Loss: 2.4233 | LR: 3.19e-05 | 7602.4ms/step | 2.6 samples/s
|
| 1043 |
+
Step 9930/11700 | Loss: 2.5065 | LR: 3.15e-05 | 7607.9ms/step | 2.6 samples/s
|
| 1044 |
+
Step 9940/11700 | Loss: 2.3181 | LR: 3.12e-05 | 7600.1ms/step | 2.6 samples/s
|
| 1045 |
+
Step 9950/11700 | Loss: 2.4278 | LR: 3.09e-05 | 7600.0ms/step | 2.6 samples/s
|
| 1046 |
+
Step 9960/11700 | Loss: 2.3746 | LR: 3.05e-05 | 7599.9ms/step | 2.6 samples/s
|
| 1047 |
+
Step 9970/11700 | Loss: 2.4510 | LR: 3.02e-05 | 7600.2ms/step | 2.6 samples/s
|
| 1048 |
+
Step 9980/11700 | Loss: 2.3806 | LR: 2.98e-05 | 7599.8ms/step | 2.6 samples/s
|
| 1049 |
+
Step 9990/11700 | Loss: 2.3347 | LR: 2.95e-05 | 7604.2ms/step | 2.6 samples/s
|
| 1050 |
+
Step 10000/11700 | Loss: 2.4492 | LR: 2.92e-05 | 7600.1ms/step | 2.6 samples/s
|
| 1051 |
+
Validation at step 10000: Loss = 2.4540, Perplexity = 11.64
|
| 1052 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-10000
|
| 1053 |
+
Step 10010/11700 | Loss: 2.5458 | LR: 2.88e-05 | 19284.1ms/step | 1.0 samples/s
|
| 1054 |
+
Step 10020/11700 | Loss: 2.5321 | LR: 2.85e-05 | 7600.0ms/step | 2.6 samples/s
|
| 1055 |
+
Step 10030/11700 | Loss: 2.4313 | LR: 2.82e-05 | 7599.9ms/step | 2.6 samples/s
|
| 1056 |
+
Step 10040/11700 | Loss: 2.4829 | LR: 2.78e-05 | 7600.1ms/step | 2.6 samples/s
|
| 1057 |
+
Step 10050/11700 | Loss: 2.5191 | LR: 2.75e-05 | 7600.9ms/step | 2.6 samples/s
|
| 1058 |
+
Step 10060/11700 | Loss: 2.3937 | LR: 2.72e-05 | 7603.5ms/step | 2.6 samples/s
|
| 1059 |
+
Step 10070/11700 | Loss: 2.5121 | LR: 2.68e-05 | 7599.9ms/step | 2.6 samples/s
|
| 1060 |
+
Step 10080/11700 | Loss: 2.3880 | LR: 2.65e-05 | 7600.9ms/step | 2.6 samples/s
|
| 1061 |
+
Step 10090/11700 | Loss: 2.4580 | LR: 2.62e-05 | 7601.4ms/step | 2.6 samples/s
|
| 1062 |
+
Step 10100/11700 | Loss: 2.4204 | LR: 2.59e-05 | 7599.9ms/step | 2.6 samples/s
|
| 1063 |
+
Step 10110/11700 | Loss: 2.4916 | LR: 2.56e-05 | 7601.1ms/step | 2.6 samples/s
|
| 1064 |
+
Step 10120/11700 | Loss: 2.3591 | LR: 2.52e-05 | 7601.0ms/step | 2.6 samples/s
|
| 1065 |
+
Step 10130/11700 | Loss: 2.5268 | LR: 2.49e-05 | 7602.2ms/step | 2.6 samples/s
|
| 1066 |
+
Step 10140/11700 | Loss: 2.5598 | LR: 2.46e-05 | 7600.0ms/step | 2.6 samples/s
|
| 1067 |
+
Step 10150/11700 | Loss: 2.4696 | LR: 2.43e-05 | 7600.0ms/step | 2.6 samples/s
|
| 1068 |
+
Step 10160/11700 | Loss: 2.4546 | LR: 2.40e-05 | 7600.0ms/step | 2.6 samples/s
|
| 1069 |
+
Step 10170/11700 | Loss: 2.4404 | LR: 2.37e-05 | 7600.1ms/step | 2.6 samples/s
|
| 1070 |
+
Step 10180/11700 | Loss: 2.4619 | LR: 2.34e-05 | 7600.1ms/step | 2.6 samples/s
|
| 1071 |
+
Step 10190/11700 | Loss: 2.3682 | LR: 2.31e-05 | 7599.9ms/step | 2.6 samples/s
|
| 1072 |
+
Step 10200/11700 | Loss: 2.3946 | LR: 2.28e-05 | 7600.2ms/step | 2.6 samples/s
|
| 1073 |
+
Step 10210/11700 | Loss: 2.4902 | LR: 2.25e-05 | 7604.2ms/step | 2.6 samples/s
|
| 1074 |
+
Step 10220/11700 | Loss: 2.4472 | LR: 2.22e-05 | 7599.7ms/step | 2.6 samples/s
|
| 1075 |
+
Step 10230/11700 | Loss: 2.5511 | LR: 2.19e-05 | 7600.3ms/step | 2.6 samples/s
|
| 1076 |
+
Step 10240/11700 | Loss: 2.5383 | LR: 2.16e-05 | 7599.9ms/step | 2.6 samples/s
|
| 1077 |
+
Step 10250/11700 | Loss: 2.4876 | LR: 2.13e-05 | 7600.2ms/step | 2.6 samples/s
|
| 1078 |
+
Validation at step 10250: Loss = 2.4537, Perplexity = 11.63
|
| 1079 |
+
Step 10260/11700 | Loss: 2.4953 | LR: 2.10e-05 | 18930.2ms/step | 1.1 samples/s
|
| 1080 |
+
Step 10270/11700 | Loss: 2.3981 | LR: 2.07e-05 | 7601.9ms/step | 2.6 samples/s
|
| 1081 |
+
Step 10280/11700 | Loss: 2.4959 | LR: 2.05e-05 | 7599.9ms/step | 2.6 samples/s
|
| 1082 |
+
Step 10290/11700 | Loss: 2.4799 | LR: 2.02e-05 | 7602.1ms/step | 2.6 samples/s
|
| 1083 |
+
Step 10300/11700 | Loss: 2.3334 | LR: 1.99e-05 | 7602.0ms/step | 2.6 samples/s
|
| 1084 |
+
Step 10310/11700 | Loss: 2.4394 | LR: 1.96e-05 | 7600.0ms/step | 2.6 samples/s
|
| 1085 |
+
Step 10320/11700 | Loss: 2.3862 | LR: 1.93e-05 | 7604.1ms/step | 2.6 samples/s
|
| 1086 |
+
Step 10330/11700 | Loss: 2.3776 | LR: 1.91e-05 | 7600.1ms/step | 2.6 samples/s
|
| 1087 |
+
Step 10340/11700 | Loss: 2.5479 | LR: 1.88e-05 | 7600.1ms/step | 2.6 samples/s
|
| 1088 |
+
Step 10350/11700 | Loss: 2.4774 | LR: 1.85e-05 | 7602.1ms/step | 2.6 samples/s
|
| 1089 |
+
Step 10360/11700 | Loss: 2.4729 | LR: 1.82e-05 | 7599.8ms/step | 2.6 samples/s
|
| 1090 |
+
Step 10370/11700 | Loss: 2.4533 | LR: 1.80e-05 | 7602.1ms/step | 2.6 samples/s
|
| 1091 |
+
Step 10380/11700 | Loss: 2.2901 | LR: 1.77e-05 | 7600.0ms/step | 2.6 samples/s
|
| 1092 |
+
Step 10390/11700 | Loss: 2.5865 | LR: 1.75e-05 | 7600.3ms/step | 2.6 samples/s
|
| 1093 |
+
Step 10400/11700 | Loss: 2.4950 | LR: 1.72e-05 | 7599.7ms/step | 2.6 samples/s
|
| 1094 |
+
Step 10410/11700 | Loss: 2.4454 | LR: 1.69e-05 | 7602.1ms/step | 2.6 samples/s
|
| 1095 |
+
Step 10420/11700 | Loss: 2.4234 | LR: 1.67e-05 | 7600.6ms/step | 2.6 samples/s
|
| 1096 |
+
Step 10430/11700 | Loss: 2.4170 | LR: 1.64e-05 | 7601.4ms/step | 2.6 samples/s
|
| 1097 |
+
Step 10440/11700 | Loss: 2.3904 | LR: 1.62e-05 | 7600.2ms/step | 2.6 samples/s
|
| 1098 |
+
Step 10450/11700 | Loss: 2.4663 | LR: 1.59e-05 | 7599.8ms/step | 2.6 samples/s
|
| 1099 |
+
Step 10460/11700 | Loss: 2.4901 | LR: 1.57e-05 | 7600.5ms/step | 2.6 samples/s
|
| 1100 |
+
Step 10470/11700 | Loss: 2.3774 | LR: 1.54e-05 | 7600.5ms/step | 2.6 samples/s
|
| 1101 |
+
Step 10480/11700 | Loss: 2.4175 | LR: 1.52e-05 | 7609.4ms/step | 2.6 samples/s
|
| 1102 |
+
Step 10490/11700 | Loss: 2.3597 | LR: 1.49e-05 | 7600.3ms/step | 2.6 samples/s
|
| 1103 |
+
Step 10500/11700 | Loss: 2.4478 | LR: 1.47e-05 | 7600.1ms/step | 2.6 samples/s
|
| 1104 |
+
Validation at step 10500: Loss = 2.4536, Perplexity = 11.63
|
| 1105 |
+
Step 10510/11700 | Loss: 2.5715 | LR: 1.44e-05 | 18946.2ms/step | 1.1 samples/s
|
| 1106 |
+
Step 10520/11700 | Loss: 2.4730 | LR: 1.42e-05 | 7608.0ms/step | 2.6 samples/s
|
| 1107 |
+
Step 10530/11700 | Loss: 2.4598 | LR: 1.40e-05 | 7618.8ms/step | 2.6 samples/s
|
| 1108 |
+
Step 10540/11700 | Loss: 2.3088 | LR: 1.37e-05 | 7611.3ms/step | 2.6 samples/s
|
| 1109 |
+
Step 10550/11700 | Loss: 2.4844 | LR: 1.35e-05 | 7615.1ms/step | 2.6 samples/s
|
| 1110 |
+
Step 10560/11700 | Loss: 2.5850 | LR: 1.33e-05 | 7603.0ms/step | 2.6 samples/s
|
| 1111 |
+
Step 10570/11700 | Loss: 2.3412 | LR: 1.30e-05 | 7604.3ms/step | 2.6 samples/s
|
| 1112 |
+
Step 10580/11700 | Loss: 2.5177 | LR: 1.28e-05 | 7607.9ms/step | 2.6 samples/s
|
| 1113 |
+
Step 10590/11700 | Loss: 2.3795 | LR: 1.26e-05 | 7607.7ms/step | 2.6 samples/s
|
| 1114 |
+
Step 10600/11700 | Loss: 2.4693 | LR: 1.23e-05 | 7612.3ms/step | 2.6 samples/s
|
| 1115 |
+
Step 10610/11700 | Loss: 2.4381 | LR: 1.21e-05 | 7605.9ms/step | 2.6 samples/s
|
| 1116 |
+
Step 10620/11700 | Loss: 2.3561 | LR: 1.19e-05 | 7604.8ms/step | 2.6 samples/s
|
| 1117 |
+
Step 10630/11700 | Loss: 2.4259 | LR: 1.17e-05 | 7607.5ms/step | 2.6 samples/s
|
| 1118 |
+
Step 10640/11700 | Loss: 2.4098 | LR: 1.15e-05 | 7601.9ms/step | 2.6 samples/s
|
| 1119 |
+
Step 10650/11700 | Loss: 2.4633 | LR: 1.13e-05 | 7606.1ms/step | 2.6 samples/s
|
| 1120 |
+
Step 10660/11700 | Loss: 2.4150 | LR: 1.10e-05 | 7604.2ms/step | 2.6 samples/s
|
| 1121 |
+
Step 10670/11700 | Loss: 2.4020 | LR: 1.08e-05 | 7607.8ms/step | 2.6 samples/s
|
| 1122 |
+
Step 10680/11700 | Loss: 2.4774 | LR: 1.06e-05 | 7604.2ms/step | 2.6 samples/s
|
| 1123 |
+
Step 10690/11700 | Loss: 2.4662 | LR: 1.04e-05 | 7607.8ms/step | 2.6 samples/s
|
| 1124 |
+
Step 10700/11700 | Loss: 2.4347 | LR: 1.02e-05 | 7600.2ms/step | 2.6 samples/s
|
| 1125 |
+
Step 10710/11700 | Loss: 2.4674 | LR: 1.00e-05 | 7604.2ms/step | 2.6 samples/s
|
| 1126 |
+
Step 10720/11700 | Loss: 2.4029 | LR: 9.82e-06 | 7604.0ms/step | 2.6 samples/s
|
| 1127 |
+
Step 10730/11700 | Loss: 2.5051 | LR: 9.62e-06 | 7606.1ms/step | 2.6 samples/s
|
| 1128 |
+
Step 10740/11700 | Loss: 2.3322 | LR: 9.42e-06 | 7611.8ms/step | 2.6 samples/s
|
| 1129 |
+
Step 10750/11700 | Loss: 2.3614 | LR: 9.23e-06 | 7608.2ms/step | 2.6 samples/s
|
| 1130 |
+
Validation at step 10750: Loss = 2.4533, Perplexity = 11.63
|
| 1131 |
+
Step 10760/11700 | Loss: 2.5300 | LR: 9.04e-06 | 18932.2ms/step | 1.1 samples/s
|
| 1132 |
+
Step 10770/11700 | Loss: 2.5045 | LR: 8.85e-06 | 7604.1ms/step | 2.6 samples/s
|
| 1133 |
+
Step 10780/11700 | Loss: 2.5332 | LR: 8.66e-06 | 7600.0ms/step | 2.6 samples/s
|
| 1134 |
+
Step 10790/11700 | Loss: 2.4433 | LR: 8.47e-06 | 7601.8ms/step | 2.6 samples/s
|
| 1135 |
+
Step 10800/11700 | Loss: 2.5369 | LR: 8.29e-06 | 7604.4ms/step | 2.6 samples/s
|
| 1136 |
+
Step 10810/11700 | Loss: 2.4351 | LR: 8.11e-06 | 7607.7ms/step | 2.6 samples/s
|
| 1137 |
+
Step 10820/11700 | Loss: 2.5222 | LR: 7.93e-06 | 7601.9ms/step | 2.6 samples/s
|
| 1138 |
+
Step 10830/11700 | Loss: 2.4210 | LR: 7.75e-06 | 7600.1ms/step | 2.6 samples/s
|
| 1139 |
+
Step 10840/11700 | Loss: 2.4038 | LR: 7.57e-06 | 7600.0ms/step | 2.6 samples/s
|
| 1140 |
+
Step 10850/11700 | Loss: 2.3539 | LR: 7.40e-06 | 7600.1ms/step | 2.6 samples/s
|
| 1141 |
+
Step 10860/11700 | Loss: 2.3557 | LR: 7.22e-06 | 7600.2ms/step | 2.6 samples/s
|
| 1142 |
+
Step 10870/11700 | Loss: 2.3831 | LR: 7.05e-06 | 7599.9ms/step | 2.6 samples/s
|
| 1143 |
+
Step 10880/11700 | Loss: 2.3627 | LR: 6.89e-06 | 7602.3ms/step | 2.6 samples/s
|
| 1144 |
+
Step 10890/11700 | Loss: 2.3494 | LR: 6.72e-06 | 7600.2ms/step | 2.6 samples/s
|
| 1145 |
+
Step 10900/11700 | Loss: 2.4546 | LR: 6.56e-06 | 7601.8ms/step | 2.6 samples/s
|
| 1146 |
+
Step 10910/11700 | Loss: 2.4948 | LR: 6.39e-06 | 7604.0ms/step | 2.6 samples/s
|
| 1147 |
+
Step 10920/11700 | Loss: 2.4178 | LR: 6.23e-06 | 7608.1ms/step | 2.6 samples/s
|
| 1148 |
+
Step 10930/11700 | Loss: 2.3399 | LR: 6.08e-06 | 7600.0ms/step | 2.6 samples/s
|
| 1149 |
+
Step 10940/11700 | Loss: 2.3624 | LR: 5.92e-06 | 7600.0ms/step | 2.6 samples/s
|
| 1150 |
+
Step 10950/11700 | Loss: 2.3693 | LR: 5.77e-06 | 7600.2ms/step | 2.6 samples/s
|
| 1151 |
+
Step 10960/11700 | Loss: 2.5184 | LR: 5.61e-06 | 7600.0ms/step | 2.6 samples/s
|
| 1152 |
+
Step 10970/11700 | Loss: 2.4757 | LR: 5.46e-06 | 7606.2ms/step | 2.6 samples/s
|
| 1153 |
+
Step 10980/11700 | Loss: 2.4721 | LR: 5.31e-06 | 7599.9ms/step | 2.6 samples/s
|
| 1154 |
+
Step 10990/11700 | Loss: 2.4103 | LR: 5.17e-06 | 7600.0ms/step | 2.6 samples/s
|
| 1155 |
+
Step 11000/11700 | Loss: 2.3903 | LR: 5.02e-06 | 7600.1ms/step | 2.6 samples/s
|
| 1156 |
+
Validation at step 11000: Loss = 2.4533, Perplexity = 11.63
|
| 1157 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-11000
|
| 1158 |
+
Step 11010/11700 | Loss: 2.3799 | LR: 4.88e-06 | 19262.2ms/step | 1.0 samples/s
|
| 1159 |
+
Step 11020/11700 | Loss: 2.4480 | LR: 4.74e-06 | 7606.2ms/step | 2.6 samples/s
|
| 1160 |
+
Step 11030/11700 | Loss: 2.4817 | LR: 4.60e-06 | 7606.0ms/step | 2.6 samples/s
|
| 1161 |
+
Step 11040/11700 | Loss: 2.4731 | LR: 4.47e-06 | 7611.9ms/step | 2.6 samples/s
|
| 1162 |
+
Step 11050/11700 | Loss: 2.4967 | LR: 4.33e-06 | 7604.8ms/step | 2.6 samples/s
|
| 1163 |
+
Step 11060/11700 | Loss: 2.4293 | LR: 4.20e-06 | 7607.4ms/step | 2.6 samples/s
|
| 1164 |
+
Step 11070/11700 | Loss: 2.3917 | LR: 4.07e-06 | 7604.3ms/step | 2.6 samples/s
|
| 1165 |
+
Step 11080/11700 | Loss: 2.4512 | LR: 3.94e-06 | 7609.9ms/step | 2.6 samples/s
|
| 1166 |
+
Step 11090/11700 | Loss: 2.4358 | LR: 3.82e-06 | 7608.0ms/step | 2.6 samples/s
|
| 1167 |
+
Step 11100/11700 | Loss: 2.4819 | LR: 3.69e-06 | 7608.2ms/step | 2.6 samples/s
|
| 1168 |
+
Step 11110/11700 | Loss: 2.4274 | LR: 3.57e-06 | 7608.7ms/step | 2.6 samples/s
|
| 1169 |
+
Step 11120/11700 | Loss: 2.4722 | LR: 3.45e-06 | 7607.2ms/step | 2.6 samples/s
|
| 1170 |
+
Step 11130/11700 | Loss: 2.5189 | LR: 3.34e-06 | 7610.1ms/step | 2.6 samples/s
|
| 1171 |
+
Step 11140/11700 | Loss: 2.4225 | LR: 3.22e-06 | 7605.8ms/step | 2.6 samples/s
|
| 1172 |
+
Step 11150/11700 | Loss: 2.4454 | LR: 3.11e-06 | 7606.3ms/step | 2.6 samples/s
|
| 1173 |
+
Step 11160/11700 | Loss: 2.3913 | LR: 2.99e-06 | 7608.1ms/step | 2.6 samples/s
|
| 1174 |
+
Step 11170/11700 | Loss: 2.5045 | LR: 2.88e-06 | 7607.9ms/step | 2.6 samples/s
|
| 1175 |
+
Step 11180/11700 | Loss: 2.4156 | LR: 2.78e-06 | 7602.3ms/step | 2.6 samples/s
|
| 1176 |
+
Step 11190/11700 | Loss: 2.4551 | LR: 2.67e-06 | 7600.0ms/step | 2.6 samples/s
|
| 1177 |
+
Step 11200/11700 | Loss: 2.4489 | LR: 2.57e-06 | 7608.1ms/step | 2.6 samples/s
|
| 1178 |
+
Step 11210/11700 | Loss: 2.4983 | LR: 2.47e-06 | 7603.6ms/step | 2.6 samples/s
|
| 1179 |
+
Step 11220/11700 | Loss: 2.3905 | LR: 2.37e-06 | 7600.2ms/step | 2.6 samples/s
|
| 1180 |
+
Step 11230/11700 | Loss: 2.3179 | LR: 2.27e-06 | 7600.1ms/step | 2.6 samples/s
|
| 1181 |
+
Step 11240/11700 | Loss: 2.3857 | LR: 2.17e-06 | 7608.0ms/step | 2.6 samples/s
|
| 1182 |
+
Step 11250/11700 | Loss: 2.4019 | LR: 2.08e-06 | 7600.4ms/step | 2.6 samples/s
|
| 1183 |
+
Validation at step 11250: Loss = 2.4531, Perplexity = 11.62
|
| 1184 |
+
Step 11260/11700 | Loss: 2.4338 | LR: 1.99e-06 | 18931.7ms/step | 1.1 samples/s
|
| 1185 |
+
Step 11270/11700 | Loss: 2.4790 | LR: 1.90e-06 | 7601.9ms/step | 2.6 samples/s
|
| 1186 |
+
Step 11280/11700 | Loss: 2.4201 | LR: 1.81e-06 | 7600.3ms/step | 2.6 samples/s
|
| 1187 |
+
Step 11290/11700 | Loss: 2.4886 | LR: 1.73e-06 | 7600.1ms/step | 2.6 samples/s
|
| 1188 |
+
Step 11300/11700 | Loss: 2.4558 | LR: 1.64e-06 | 7599.9ms/step | 2.6 samples/s
|
| 1189 |
+
Step 11310/11700 | Loss: 2.4240 | LR: 1.56e-06 | 7600.2ms/step | 2.6 samples/s
|
| 1190 |
+
Step 11320/11700 | Loss: 2.4550 | LR: 1.48e-06 | 7602.3ms/step | 2.6 samples/s
|
| 1191 |
+
Step 11330/11700 | Loss: 2.3810 | LR: 1.41e-06 | 7600.1ms/step | 2.6 samples/s
|
| 1192 |
+
Step 11340/11700 | Loss: 2.3584 | LR: 1.33e-06 | 7601.7ms/step | 2.6 samples/s
|
| 1193 |
+
Step 11350/11700 | Loss: 2.4822 | LR: 1.26e-06 | 7606.1ms/step | 2.6 samples/s
|
| 1194 |
+
Step 11360/11700 | Loss: 2.4748 | LR: 1.19e-06 | 7600.1ms/step | 2.6 samples/s
|
| 1195 |
+
Step 11370/11700 | Loss: 2.4345 | LR: 1.12e-06 | 7602.4ms/step | 2.6 samples/s
|
| 1196 |
+
Step 11380/11700 | Loss: 2.4208 | LR: 1.05e-06 | 7601.6ms/step | 2.6 samples/s
|
| 1197 |
+
Step 11390/11700 | Loss: 2.4389 | LR: 9.88e-07 | 7600.1ms/step | 2.6 samples/s
|
| 1198 |
+
Step 11400/11700 | Loss: 2.4666 | LR: 9.25e-07 | 7600.0ms/step | 2.6 samples/s
|
| 1199 |
+
Step 11410/11700 | Loss: 2.4413 | LR: 8.65e-07 | 7604.2ms/step | 2.6 samples/s
|
| 1200 |
+
Step 11420/11700 | Loss: 2.2909 | LR: 8.06e-07 | 7604.8ms/step | 2.6 samples/s
|
| 1201 |
+
Step 11430/11700 | Loss: 2.3937 | LR: 7.50e-07 | 7605.4ms/step | 2.6 samples/s
|
| 1202 |
+
Step 11440/11700 | Loss: 2.4832 | LR: 6.95e-07 | 7604.2ms/step | 2.6 samples/s
|
| 1203 |
+
Step 11450/11700 | Loss: 2.3296 | LR: 6.43e-07 | 7599.9ms/step | 2.6 samples/s
|
| 1204 |
+
Step 11460/11700 | Loss: 2.4556 | LR: 5.92e-07 | 7599.9ms/step | 2.6 samples/s
|
| 1205 |
+
Step 11470/11700 | Loss: 2.4999 | LR: 5.44e-07 | 7602.0ms/step | 2.6 samples/s
|
| 1206 |
+
Step 11480/11700 | Loss: 2.3700 | LR: 4.98e-07 | 7602.1ms/step | 2.6 samples/s
|
| 1207 |
+
Step 11490/11700 | Loss: 2.4707 | LR: 4.54e-07 | 7600.1ms/step | 2.6 samples/s
|
| 1208 |
+
Step 11500/11700 | Loss: 2.4053 | LR: 4.11e-07 | 7604.0ms/step | 2.6 samples/s
|
| 1209 |
+
Validation at step 11500: Loss = 2.4533, Perplexity = 11.63
|
| 1210 |
+
Step 11510/11700 | Loss: 2.3911 | LR: 3.71e-07 | 18934.2ms/step | 1.1 samples/s
|
| 1211 |
+
Step 11520/11700 | Loss: 2.4649 | LR: 3.33e-07 | 7600.3ms/step | 2.6 samples/s
|
| 1212 |
+
Step 11530/11700 | Loss: 2.4093 | LR: 2.97e-07 | 7601.8ms/step | 2.6 samples/s
|
| 1213 |
+
Step 11540/11700 | Loss: 2.3863 | LR: 2.63e-07 | 7599.9ms/step | 2.6 samples/s
|
| 1214 |
+
Step 11550/11700 | Loss: 2.5006 | LR: 2.31e-07 | 7600.2ms/step | 2.6 samples/s
|
| 1215 |
+
Step 11560/11700 | Loss: 2.4048 | LR: 2.02e-07 | 7599.9ms/step | 2.6 samples/s
|
| 1216 |
+
Step 11570/11700 | Loss: 2.5185 | LR: 1.74e-07 | 7603.9ms/step | 2.6 samples/s
|
| 1217 |
+
Step 11580/11700 | Loss: 2.3550 | LR: 1.48e-07 | 7602.2ms/step | 2.6 samples/s
|
| 1218 |
+
Step 11590/11700 | Loss: 2.4716 | LR: 1.24e-07 | 7600.1ms/step | 2.6 samples/s
|
| 1219 |
+
Step 11600/11700 | Loss: 2.4740 | LR: 1.03e-07 | 7600.0ms/step | 2.6 samples/s
|
| 1220 |
+
Step 11610/11700 | Loss: 2.4743 | LR: 8.33e-08 | 7602.1ms/step | 2.6 samples/s
|
| 1221 |
+
Step 11620/11700 | Loss: 2.3911 | LR: 6.58e-08 | 7602.1ms/step | 2.6 samples/s
|
| 1222 |
+
Step 11630/11700 | Loss: 2.3382 | LR: 5.04e-08 | 7606.9ms/step | 2.6 samples/s
|
| 1223 |
+
Step 11640/11700 | Loss: 2.4731 | LR: 3.70e-08 | 7603.7ms/step | 2.6 samples/s
|
| 1224 |
+
Step 11650/11700 | Loss: 2.5252 | LR: 2.57e-08 | 7603.7ms/step | 2.6 samples/s
|
| 1225 |
+
Step 11660/11700 | Loss: 2.2996 | LR: 1.65e-08 | 7599.9ms/step | 2.6 samples/s
|
| 1226 |
+
Step 11670/11700 | Loss: 2.4539 | LR: 9.26e-09 | 7604.3ms/step | 2.6 samples/s
|
| 1227 |
+
Step 11680/11700 | Loss: 2.4658 | LR: 4.12e-09 | 7602.2ms/step | 2.6 samples/s
|
| 1228 |
+
Step 11690/11700 | Loss: 2.3702 | LR: 1.03e-09 | 7604.0ms/step | 2.6 samples/s
|
| 1229 |
+
Step 11700/11700 | Loss: 2.4349 | LR: 0.00e+00 | 7599.5ms/step | 2.6 samples/s
|
| 1230 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753/checkpoint-11700
|
epfl_final/SparseAtt/wandb/run-20260707_035835-sh1m3lmh/files/requirements.txt
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
epfl_final/SparseAtt/wandb/run-20260707_035835-sh1m3lmh/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-07-07T03:58:35.279877Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"configs/ladder30k/resampled.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/TokenPrune.git",
|
| 14 |
+
"commit": "88298c7a418622530ca5426b22b1e5ebd10ca0cb"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "h100-train-4-gpu-0-0",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 23 |
+
"gpu_count": 4,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "9374931091456",
|
| 27 |
+
"used": "4873970151424"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1623005732864"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-85844f64-d144-1c84-306d-f88b5f232a77"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 43 |
+
"memoryTotal": "85520809984",
|
| 44 |
+
"cudaCores": 16896,
|
| 45 |
+
"architecture": "Hopper",
|
| 46 |
+
"uuid": "GPU-4e4c2343-4f5b-a2ff-312e-03d402a27803"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 50 |
+
"memoryTotal": "85520809984",
|
| 51 |
+
"cudaCores": 16896,
|
| 52 |
+
"architecture": "Hopper",
|
| 53 |
+
"uuid": "GPU-fbf0ccd1-8bd4-5f57-6e74-6c0cf7eb70c9"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 57 |
+
"memoryTotal": "85520809984",
|
| 58 |
+
"cudaCores": 16896,
|
| 59 |
+
"architecture": "Hopper",
|
| 60 |
+
"uuid": "GPU-410cba62-f098-3454-c960-ebc55bd081af"
|
| 61 |
+
}
|
| 62 |
+
],
|
| 63 |
+
"cudaVersion": "13.0",
|
| 64 |
+
"writerId": "pkavbxq1hkh38trzbvq50jk6zamishzu"
|
| 65 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260707_035835-sh1m3lmh/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"eval/uses_dense_attention":0,"eval/perplexity":11.627101743631949,"_timestamp":1.7834910602941074e+09,"train/loss":2.434876452870667,"_step":11700,"eval/dense_interval_end":-1,"eval/uses_sparse_attention":1,"eval/dense_interval_active":0,"train/learning_rate":0,"speed/steps_per_sec":0.1315875030899959,"train/global_step":11700,"speed/samples_per_sec":2.631750061799918,"train/epoch":0.3122998078155029,"eval/loss":2.4533387302830816,"eval/dense_interval_start":-1,"speed/ms_per_step":7599.505853652954,"_wandb":{"runtime":94347},"_runtime":94347.283324786}
|
epfl_final/SparseAtt/wandb/run-20260707_035835-sh1m3lmh/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260707_035835-sh1m3lmh/logs/debug.log
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-07-07 03:58:35,299 INFO MainThread:2494 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-07-07 03:58:35,299 INFO MainThread:2494 [wandb_setup.py:_flush():81] Configure stats pid to 2494
|
| 3 |
+
2026-07-07 03:58:35,299 INFO MainThread:2494 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-07-07 03:58:35,299 INFO MainThread:2494 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260707_035835-sh1m3lmh/logs/debug.log
|
| 5 |
+
2026-07-07 03:58:35,299 INFO MainThread:2494 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260707_035835-sh1m3lmh/logs/debug-internal.log
|
| 6 |
+
2026-07-07 03:58:35,299 INFO MainThread:2494 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-07-07 03:58:35,299 INFO MainThread:2494 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/longctx30k/tokenized', 'val_path': None, 'max_seq_length': 30720, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l30k365m_resampled/run_20260707_035753', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 11700, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 20, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m resampled 30k nrb120', 'wandb_tags': ['qwen3', '365m', '30k', 'scaleladder', 'resampled'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 120, 'sparse_mask_pool_size': 1000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_30k_b128_nrb120', 'sparse_mask_seed': 0, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-07-07 03:58:35,300 INFO MainThread:2494 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-07-07 03:58:35,546 INFO MainThread:2494 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-07-07 03:58:35,866 INFO MainThread:2494 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-07-07 03:58:35,868 INFO MainThread:2494 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-07-07 03:58:35,881 INFO MainThread:2494 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-07-07 03:58:36,400 INFO MainThread:2494 [wandb_init.py:init():1056] starting run threads in backend
|
| 15 |
+
2026-07-07 03:58:36,603 INFO MainThread:2494 [wandb_run.py:_console_start():2554] atexit reg
|
| 16 |
+
2026-07-07 03:58:36,603 INFO MainThread:2494 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 17 |
+
2026-07-07 03:58:36,603 INFO MainThread:2494 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 18 |
+
2026-07-07 03:58:36,603 INFO MainThread:2494 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 19 |
+
2026-07-07 03:58:36,610 INFO MainThread:2494 [wandb_init.py:init():1094] run started, returning control to user process
|
| 20 |
+
2026-07-08 06:11:03,672 INFO MainThread:2494 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/sh1m3lmh
|
| 21 |
+
2026-07-08 06:11:03,675 INFO MainThread:2494 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
|
| 22 |
+
2026-07-08 06:11:03,675 INFO MainThread:2494 [wandb_run.py:_restore():2502] restore
|
| 23 |
+
2026-07-08 06:11:03,675 INFO MainThread:2494 [wandb_run.py:_restore():2508] restore done
|
| 24 |
+
2026-07-08 06:11:05,027 INFO MainThread:2494 [wandb_run.py:_footer_sync_info():3889] logging synced files
|
epfl_final/SparseAtt/wandb/run-20260707_035835-sh1m3lmh/run-sh1m3lmh.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ec725f02572751b32faab86949a5ec25d97485c28e7e4ee4b5b05069ad9e5669
|
| 3 |
+
size 29906706
|
epfl_final/SparseAtt/wandb/run-20260707_170941-qlqhc3gy/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-07-07T17:09:41.399772Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"configs/ladder8k/dense.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/TokenPrune.git",
|
| 14 |
+
"commit": "88298c7a418622530ca5426b22b1e5ebd10ca0cb"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "h100-workspace-4-gpu-0-4",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 23 |
+
"gpu_count": 4,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "9371709865984",
|
| 27 |
+
"used": "5840433963008"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1623005732864"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-98029aa4-c589-b65b-3ab7-6495256f31ed"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 43 |
+
"memoryTotal": "85520809984",
|
| 44 |
+
"cudaCores": 16896,
|
| 45 |
+
"architecture": "Hopper",
|
| 46 |
+
"uuid": "GPU-37a1c6c2-d3df-3064-44df-33fa82304bd0"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 50 |
+
"memoryTotal": "85520809984",
|
| 51 |
+
"cudaCores": 16896,
|
| 52 |
+
"architecture": "Hopper",
|
| 53 |
+
"uuid": "GPU-b87e8111-63b6-b364-28b9-c4ce747aec08"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 57 |
+
"memoryTotal": "85520809984",
|
| 58 |
+
"cudaCores": 16896,
|
| 59 |
+
"architecture": "Hopper",
|
| 60 |
+
"uuid": "GPU-bbbaaf81-9234-a1c6-0093-6d87b4e6af4c"
|
| 61 |
+
}
|
| 62 |
+
],
|
| 63 |
+
"cudaVersion": "13.0",
|
| 64 |
+
"writerId": "0uhgdi87cehnjefqhto3h58d11zxvenc"
|
| 65 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260707_170941-qlqhc3gy/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|