EPFL final checkpoints and W&B history 751-800 of 1463
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +7 -0
- epfl_final/SparseAtt/wandb/run-20260820_041559-ysa9iyov/files/output.log +456 -0
- epfl_final/SparseAtt/wandb/run-20260820_041559-ysa9iyov/files/requirements.txt +142 -0
- epfl_final/SparseAtt/wandb/run-20260820_041559-ysa9iyov/files/wandb-metadata.json +72 -0
- epfl_final/SparseAtt/wandb/run-20260820_041559-ysa9iyov/logs/debug-internal.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260820_041559-ysa9iyov/run-ysa9iyov.wandb +3 -0
- epfl_final/SparseAtt/wandb/run-20260820_140401-jqstnvej/files/output.log +655 -0
- epfl_final/SparseAtt/wandb/run-20260820_140401-jqstnvej/files/requirements.txt +142 -0
- epfl_final/SparseAtt/wandb/run-20260820_140401-jqstnvej/files/wandb-metadata.json +65 -0
- epfl_final/SparseAtt/wandb/run-20260820_140401-jqstnvej/logs/debug-internal.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260820_140401-jqstnvej/logs/debug.log +20 -0
- epfl_final/SparseAtt/wandb/run-20260820_140401-jqstnvej/run-jqstnvej.wandb +3 -0
- epfl_final/SparseAtt/wandb/run-20260820_140401-ysa9iyov/files/output.log +646 -0
- epfl_final/SparseAtt/wandb/run-20260820_140401-ysa9iyov/files/requirements.txt +142 -0
- epfl_final/SparseAtt/wandb/run-20260820_140401-ysa9iyov/files/wandb-metadata.json +65 -0
- epfl_final/SparseAtt/wandb/run-20260820_140401-ysa9iyov/logs/debug-internal.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260820_140401-ysa9iyov/logs/debug.log +20 -0
- epfl_final/SparseAtt/wandb/run-20260820_140401-ysa9iyov/run-ysa9iyov.wandb +3 -0
- epfl_final/SparseAtt/wandb/run-20260820_140811-xfnxdn24/files/config.yaml +200 -0
- epfl_final/SparseAtt/wandb/run-20260820_140811-xfnxdn24/files/output.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260820_140811-xfnxdn24/files/requirements.txt +142 -0
- epfl_final/SparseAtt/wandb/run-20260820_140811-xfnxdn24/files/wandb-metadata.json +65 -0
- epfl_final/SparseAtt/wandb/run-20260820_140811-xfnxdn24/files/wandb-summary.json +1 -0
- epfl_final/SparseAtt/wandb/run-20260820_140811-xfnxdn24/logs/debug-internal.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260820_140811-xfnxdn24/logs/debug.log +25 -0
- epfl_final/SparseAtt/wandb/run-20260820_140811-xfnxdn24/run-xfnxdn24.wandb +3 -0
- epfl_final/SparseAtt/wandb/run-20260820_140814-bajm6b15/files/config.yaml +200 -0
- epfl_final/SparseAtt/wandb/run-20260820_140814-bajm6b15/files/output.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260820_140814-bajm6b15/files/requirements.txt +142 -0
- epfl_final/SparseAtt/wandb/run-20260820_140814-bajm6b15/files/wandb-metadata.json +65 -0
- epfl_final/SparseAtt/wandb/run-20260820_140814-bajm6b15/files/wandb-summary.json +1 -0
- epfl_final/SparseAtt/wandb/run-20260820_140814-bajm6b15/logs/debug-internal.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260820_140814-bajm6b15/logs/debug.log +25 -0
- epfl_final/SparseAtt/wandb/run-20260820_140814-bajm6b15/run-bajm6b15.wandb +3 -0
- epfl_final/SparseAtt/wandb/run-20260820_144758-zovy63wq/files/config.yaml +184 -0
- epfl_final/SparseAtt/wandb/run-20260820_144758-zovy63wq/files/output.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260820_144758-zovy63wq/files/requirements.txt +142 -0
- epfl_final/SparseAtt/wandb/run-20260820_144758-zovy63wq/files/wandb-metadata.json +44 -0
- epfl_final/SparseAtt/wandb/run-20260820_144758-zovy63wq/files/wandb-summary.json +1 -0
- epfl_final/SparseAtt/wandb/run-20260820_144758-zovy63wq/logs/debug-internal.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260820_144758-zovy63wq/logs/debug.log +25 -0
- epfl_final/SparseAtt/wandb/run-20260820_144758-zovy63wq/run-zovy63wq.wandb +3 -0
- epfl_final/SparseAtt/wandb/run-20260820_194209-jqstnvej/files/config.yaml +199 -0
- epfl_final/SparseAtt/wandb/run-20260820_194209-jqstnvej/files/output.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260820_194209-jqstnvej/files/requirements.txt +142 -0
- epfl_final/SparseAtt/wandb/run-20260820_194209-jqstnvej/files/wandb-metadata.json +65 -0
- epfl_final/SparseAtt/wandb/run-20260820_194209-jqstnvej/files/wandb-summary.json +1 -0
- epfl_final/SparseAtt/wandb/run-20260820_194209-jqstnvej/logs/debug-internal.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260820_194209-jqstnvej/logs/debug.log +25 -0
- epfl_final/SparseAtt/wandb/run-20260820_194209-jqstnvej/run-jqstnvej.wandb +3 -0
.gitattributes
CHANGED
|
@@ -97,3 +97,10 @@ epfl_final/SparseAtt/wandb/run-20260820_041558-bajm6b15/run-bajm6b15.wandb filte
|
|
| 97 |
epfl_final/SparseAtt/wandb/run-20260820_041559-xfnxdn24/run-xfnxdn24.wandb filter=lfs diff=lfs merge=lfs -text
|
| 98 |
epfl_final/SparseAtt/wandb/run-20260820_041559-zovy63wq/run-zovy63wq.wandb filter=lfs diff=lfs merge=lfs -text
|
| 99 |
epfl_final/SparseAtt/wandb/run-20260820_041559-jqstnvej/run-jqstnvej.wandb filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
epfl_final/SparseAtt/wandb/run-20260820_041559-xfnxdn24/run-xfnxdn24.wandb filter=lfs diff=lfs merge=lfs -text
|
| 98 |
epfl_final/SparseAtt/wandb/run-20260820_041559-zovy63wq/run-zovy63wq.wandb filter=lfs diff=lfs merge=lfs -text
|
| 99 |
epfl_final/SparseAtt/wandb/run-20260820_041559-jqstnvej/run-jqstnvej.wandb filter=lfs diff=lfs merge=lfs -text
|
| 100 |
+
epfl_final/SparseAtt/wandb/run-20260820_041559-ysa9iyov/run-ysa9iyov.wandb filter=lfs diff=lfs merge=lfs -text
|
| 101 |
+
epfl_final/SparseAtt/wandb/run-20260820_140401-jqstnvej/run-jqstnvej.wandb filter=lfs diff=lfs merge=lfs -text
|
| 102 |
+
epfl_final/SparseAtt/wandb/run-20260820_140401-ysa9iyov/run-ysa9iyov.wandb filter=lfs diff=lfs merge=lfs -text
|
| 103 |
+
epfl_final/SparseAtt/wandb/run-20260820_140811-xfnxdn24/run-xfnxdn24.wandb filter=lfs diff=lfs merge=lfs -text
|
| 104 |
+
epfl_final/SparseAtt/wandb/run-20260820_140814-bajm6b15/run-bajm6b15.wandb filter=lfs diff=lfs merge=lfs -text
|
| 105 |
+
epfl_final/SparseAtt/wandb/run-20260820_144758-zovy63wq/run-zovy63wq.wandb filter=lfs diff=lfs merge=lfs -text
|
| 106 |
+
epfl_final/SparseAtt/wandb/run-20260820_194209-jqstnvej/run-jqstnvej.wandb filter=lfs diff=lfs merge=lfs -text
|
epfl_final/SparseAtt/wandb/run-20260820_041559-ysa9iyov/files/output.log
ADDED
|
@@ -0,0 +1,456 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Starting training...
|
| 2 |
+
Validation at step 14000: Loss = 3.0315, Perplexity = 20.73
|
| 3 |
+
Switching attention mode for step 14001: dense (dense step interval)
|
| 4 |
+
Step 14010/44000 | Loss: 3.0521 | LR: 3.93e-04 | 30764.5ms/step | 0.7 samples/s
|
| 5 |
+
Step 14020/44000 | Loss: 3.0747 | LR: 3.93e-04 | 1400.4ms/step | 14.3 samples/s
|
| 6 |
+
Step 14030/44000 | Loss: 3.0141 | LR: 3.92e-04 | 1401.7ms/step | 14.3 samples/s
|
| 7 |
+
Step 14040/44000 | Loss: 3.0482 | LR: 3.92e-04 | 1404.4ms/step | 14.2 samples/s
|
| 8 |
+
Step 14050/44000 | Loss: 3.0244 | LR: 3.92e-04 | 1402.5ms/step | 14.3 samples/s
|
| 9 |
+
Step 14060/44000 | Loss: 2.9957 | LR: 3.92e-04 | 1405.5ms/step | 14.2 samples/s
|
| 10 |
+
Step 14070/44000 | Loss: 3.0557 | LR: 3.92e-04 | 1404.4ms/step | 14.2 samples/s
|
| 11 |
+
Step 14080/44000 | Loss: 3.0433 | LR: 3.92e-04 | 1407.5ms/step | 14.2 samples/s
|
| 12 |
+
Step 14090/44000 | Loss: 3.0255 | LR: 3.92e-04 | 1406.2ms/step | 14.2 samples/s
|
| 13 |
+
Step 14100/44000 | Loss: 3.0300 | LR: 3.91e-04 | 1401.6ms/step | 14.3 samples/s
|
| 14 |
+
Step 14110/44000 | Loss: 3.0152 | LR: 3.91e-04 | 1400.1ms/step | 14.3 samples/s
|
| 15 |
+
Step 14120/44000 | Loss: 3.0562 | LR: 3.91e-04 | 1400.1ms/step | 14.3 samples/s
|
| 16 |
+
Step 14130/44000 | Loss: 3.0295 | LR: 3.91e-04 | 1399.9ms/step | 14.3 samples/s
|
| 17 |
+
Step 14140/44000 | Loss: 3.0463 | LR: 3.91e-04 | 1400.0ms/step | 14.3 samples/s
|
| 18 |
+
Step 14150/44000 | Loss: 3.0376 | LR: 3.91e-04 | 1400.0ms/step | 14.3 samples/s
|
| 19 |
+
Step 14160/44000 | Loss: 3.0425 | LR: 3.90e-04 | 1399.9ms/step | 14.3 samples/s
|
| 20 |
+
Step 14170/44000 | Loss: 3.0176 | LR: 3.90e-04 | 1400.3ms/step | 14.3 samples/s
|
| 21 |
+
Step 14180/44000 | Loss: 3.0226 | LR: 3.90e-04 | 1399.5ms/step | 14.3 samples/s
|
| 22 |
+
Step 14190/44000 | Loss: 3.0413 | LR: 3.90e-04 | 1400.6ms/step | 14.3 samples/s
|
| 23 |
+
Step 14200/44000 | Loss: 3.0287 | LR: 3.90e-04 | 1399.7ms/step | 14.3 samples/s
|
| 24 |
+
Step 14210/44000 | Loss: 3.0243 | LR: 3.90e-04 | 1399.9ms/step | 14.3 samples/s
|
| 25 |
+
Step 14220/44000 | Loss: 3.0587 | LR: 3.90e-04 | 1400.2ms/step | 14.3 samples/s
|
| 26 |
+
Step 14230/44000 | Loss: 3.0377 | LR: 3.89e-04 | 1399.9ms/step | 14.3 samples/s
|
| 27 |
+
Step 14240/44000 | Loss: 3.0638 | LR: 3.89e-04 | 1400.3ms/step | 14.3 samples/s
|
| 28 |
+
Step 14250/44000 | Loss: 3.0836 | LR: 3.89e-04 | 1401.7ms/step | 14.3 samples/s
|
| 29 |
+
Step 14260/44000 | Loss: 3.0346 | LR: 3.89e-04 | 1400.8ms/step | 14.3 samples/s
|
| 30 |
+
Step 14270/44000 | Loss: 3.0117 | LR: 3.89e-04 | 1401.1ms/step | 14.3 samples/s
|
| 31 |
+
Step 14280/44000 | Loss: 3.0250 | LR: 3.89e-04 | 1400.0ms/step | 14.3 samples/s
|
| 32 |
+
Step 14290/44000 | Loss: 3.0364 | LR: 3.89e-04 | 1399.8ms/step | 14.3 samples/s
|
| 33 |
+
Step 14300/44000 | Loss: 3.0342 | LR: 3.88e-04 | 1400.3ms/step | 14.3 samples/s
|
| 34 |
+
Step 14310/44000 | Loss: 3.0343 | LR: 3.88e-04 | 1399.9ms/step | 14.3 samples/s
|
| 35 |
+
Step 14320/44000 | Loss: 3.0531 | LR: 3.88e-04 | 1399.9ms/step | 14.3 samples/s
|
| 36 |
+
Step 14330/44000 | Loss: 3.0523 | LR: 3.88e-04 | 1400.1ms/step | 14.3 samples/s
|
| 37 |
+
Step 14340/44000 | Loss: 3.0429 | LR: 3.88e-04 | 1399.9ms/step | 14.3 samples/s
|
| 38 |
+
Step 14350/44000 | Loss: 3.0559 | LR: 3.88e-04 | 1400.1ms/step | 14.3 samples/s
|
| 39 |
+
Step 14360/44000 | Loss: 3.0261 | LR: 3.87e-04 | 1399.9ms/step | 14.3 samples/s
|
| 40 |
+
Step 14370/44000 | Loss: 3.0138 | LR: 3.87e-04 | 1400.5ms/step | 14.3 samples/s
|
| 41 |
+
Step 14380/44000 | Loss: 3.0530 | LR: 3.87e-04 | 1399.5ms/step | 14.3 samples/s
|
| 42 |
+
Step 14390/44000 | Loss: 3.0273 | LR: 3.87e-04 | 1400.0ms/step | 14.3 samples/s
|
| 43 |
+
Step 14400/44000 | Loss: 3.0345 | LR: 3.87e-04 | 1399.9ms/step | 14.3 samples/s
|
| 44 |
+
Step 14410/44000 | Loss: 3.0357 | LR: 3.87e-04 | 1400.2ms/step | 14.3 samples/s
|
| 45 |
+
Step 14420/44000 | Loss: 3.0597 | LR: 3.87e-04 | 1399.8ms/step | 14.3 samples/s
|
| 46 |
+
Step 14430/44000 | Loss: 3.0228 | LR: 3.86e-04 | 1400.1ms/step | 14.3 samples/s
|
| 47 |
+
Step 14440/44000 | Loss: 3.0280 | LR: 3.86e-04 | 1399.9ms/step | 14.3 samples/s
|
| 48 |
+
Step 14450/44000 | Loss: 3.0255 | LR: 3.86e-04 | 1400.3ms/step | 14.3 samples/s
|
| 49 |
+
Step 14460/44000 | Loss: 3.0530 | LR: 3.86e-04 | 1399.8ms/step | 14.3 samples/s
|
| 50 |
+
Step 14470/44000 | Loss: 3.0447 | LR: 3.86e-04 | 1399.9ms/step | 14.3 samples/s
|
| 51 |
+
Step 14480/44000 | Loss: 3.0540 | LR: 3.86e-04 | 1400.0ms/step | 14.3 samples/s
|
| 52 |
+
Step 14490/44000 | Loss: 3.0544 | LR: 3.85e-04 | 1400.3ms/step | 14.3 samples/s
|
| 53 |
+
Step 14500/44000 | Loss: 2.9941 | LR: 3.85e-04 | 1400.0ms/step | 14.3 samples/s
|
| 54 |
+
Step 14510/44000 | Loss: 3.0325 | LR: 3.85e-04 | 1399.8ms/step | 14.3 samples/s
|
| 55 |
+
Step 14520/44000 | Loss: 3.0357 | LR: 3.85e-04 | 1399.9ms/step | 14.3 samples/s
|
| 56 |
+
Step 14530/44000 | Loss: 3.0294 | LR: 3.85e-04 | 1400.3ms/step | 14.3 samples/s
|
| 57 |
+
Step 14540/44000 | Loss: 3.0313 | LR: 3.85e-04 | 1399.9ms/step | 14.3 samples/s
|
| 58 |
+
Step 14550/44000 | Loss: 3.0350 | LR: 3.85e-04 | 1399.7ms/step | 14.3 samples/s
|
| 59 |
+
Step 14560/44000 | Loss: 3.0122 | LR: 3.84e-04 | 1400.1ms/step | 14.3 samples/s
|
| 60 |
+
Step 14570/44000 | Loss: 3.0366 | LR: 3.84e-04 | 1400.1ms/step | 14.3 samples/s
|
| 61 |
+
Step 14580/44000 | Loss: 3.0213 | LR: 3.84e-04 | 1400.4ms/step | 14.3 samples/s
|
| 62 |
+
Step 14590/44000 | Loss: 3.0545 | LR: 3.84e-04 | 1399.4ms/step | 14.3 samples/s
|
| 63 |
+
Step 14600/44000 | Loss: 3.0290 | LR: 3.84e-04 | 1400.4ms/step | 14.3 samples/s
|
| 64 |
+
Step 14610/44000 | Loss: 3.0532 | LR: 3.84e-04 | 1400.1ms/step | 14.3 samples/s
|
| 65 |
+
Step 14620/44000 | Loss: 3.0001 | LR: 3.83e-04 | 1399.6ms/step | 14.3 samples/s
|
| 66 |
+
Step 14630/44000 | Loss: 3.0206 | LR: 3.83e-04 | 1399.9ms/step | 14.3 samples/s
|
| 67 |
+
Step 14640/44000 | Loss: 3.0220 | LR: 3.83e-04 | 1400.1ms/step | 14.3 samples/s
|
| 68 |
+
Step 14650/44000 | Loss: 3.0496 | LR: 3.83e-04 | 1400.2ms/step | 14.3 samples/s
|
| 69 |
+
Step 14660/44000 | Loss: 3.0342 | LR: 3.83e-04 | 1399.8ms/step | 14.3 samples/s
|
| 70 |
+
Step 14670/44000 | Loss: 3.0729 | LR: 3.83e-04 | 1399.9ms/step | 14.3 samples/s
|
| 71 |
+
Step 14680/44000 | Loss: 3.0262 | LR: 3.83e-04 | 1400.2ms/step | 14.3 samples/s
|
| 72 |
+
Step 14690/44000 | Loss: 3.0151 | LR: 3.82e-04 | 1400.0ms/step | 14.3 samples/s
|
| 73 |
+
Step 14700/44000 | Loss: 3.0294 | LR: 3.82e-04 | 1399.7ms/step | 14.3 samples/s
|
| 74 |
+
Step 14710/44000 | Loss: 3.0133 | LR: 3.82e-04 | 1400.0ms/step | 14.3 samples/s
|
| 75 |
+
Step 14720/44000 | Loss: 3.0168 | LR: 3.82e-04 | 1400.4ms/step | 14.3 samples/s
|
| 76 |
+
Step 14730/44000 | Loss: 3.0168 | LR: 3.82e-04 | 1400.1ms/step | 14.3 samples/s
|
| 77 |
+
Step 14740/44000 | Loss: 3.0555 | LR: 3.82e-04 | 1400.0ms/step | 14.3 samples/s
|
| 78 |
+
Step 14750/44000 | Loss: 3.0453 | LR: 3.81e-04 | 1399.5ms/step | 14.3 samples/s
|
| 79 |
+
Step 14760/44000 | Loss: 3.0187 | LR: 3.81e-04 | 1400.1ms/step | 14.3 samples/s
|
| 80 |
+
Step 14770/44000 | Loss: 3.0488 | LR: 3.81e-04 | 1400.2ms/step | 14.3 samples/s
|
| 81 |
+
Step 14780/44000 | Loss: 3.0276 | LR: 3.81e-04 | 1400.0ms/step | 14.3 samples/s
|
| 82 |
+
Step 14790/44000 | Loss: 3.0210 | LR: 3.81e-04 | 1399.7ms/step | 14.3 samples/s
|
| 83 |
+
Step 14800/44000 | Loss: 3.0516 | LR: 3.81e-04 | 1400.1ms/step | 14.3 samples/s
|
| 84 |
+
Step 14810/44000 | Loss: 3.0319 | LR: 3.81e-04 | 1399.9ms/step | 14.3 samples/s
|
| 85 |
+
Step 14820/44000 | Loss: 3.0442 | LR: 3.80e-04 | 1400.0ms/step | 14.3 samples/s
|
| 86 |
+
Step 14830/44000 | Loss: 3.0371 | LR: 3.80e-04 | 1400.1ms/step | 14.3 samples/s
|
| 87 |
+
Step 14840/44000 | Loss: 3.0045 | LR: 3.80e-04 | 1399.8ms/step | 14.3 samples/s
|
| 88 |
+
Step 14850/44000 | Loss: 2.9777 | LR: 3.80e-04 | 1400.3ms/step | 14.3 samples/s
|
| 89 |
+
Step 14860/44000 | Loss: 3.0043 | LR: 3.80e-04 | 1399.7ms/step | 14.3 samples/s
|
| 90 |
+
Step 14870/44000 | Loss: 3.0675 | LR: 3.80e-04 | 1400.5ms/step | 14.3 samples/s
|
| 91 |
+
Step 14880/44000 | Loss: 3.0157 | LR: 3.79e-04 | 1399.5ms/step | 14.3 samples/s
|
| 92 |
+
Step 14890/44000 | Loss: 3.0207 | LR: 3.79e-04 | 1400.1ms/step | 14.3 samples/s
|
| 93 |
+
Step 14900/44000 | Loss: 3.0087 | LR: 3.79e-04 | 1400.0ms/step | 14.3 samples/s
|
| 94 |
+
Step 14910/44000 | Loss: 3.0392 | LR: 3.79e-04 | 1399.9ms/step | 14.3 samples/s
|
| 95 |
+
Step 14920/44000 | Loss: 3.0563 | LR: 3.79e-04 | 1400.3ms/step | 14.3 samples/s
|
| 96 |
+
Step 14930/44000 | Loss: 3.0244 | LR: 3.79e-04 | 1400.0ms/step | 14.3 samples/s
|
| 97 |
+
Step 14940/44000 | Loss: 3.0206 | LR: 3.79e-04 | 1402.0ms/step | 14.3 samples/s
|
| 98 |
+
Step 14950/44000 | Loss: 3.0400 | LR: 3.78e-04 | 1399.8ms/step | 14.3 samples/s
|
| 99 |
+
Step 14960/44000 | Loss: 3.0309 | LR: 3.78e-04 | 1400.3ms/step | 14.3 samples/s
|
| 100 |
+
Step 14970/44000 | Loss: 3.0373 | LR: 3.78e-04 | 1401.5ms/step | 14.3 samples/s
|
| 101 |
+
Step 14980/44000 | Loss: 3.0326 | LR: 3.78e-04 | 1400.2ms/step | 14.3 samples/s
|
| 102 |
+
Step 14990/44000 | Loss: 3.0105 | LR: 3.78e-04 | 1400.1ms/step | 14.3 samples/s
|
| 103 |
+
Step 15000/44000 | Loss: 3.0185 | LR: 3.78e-04 | 1400.4ms/step | 14.3 samples/s
|
| 104 |
+
Validation at step 15000: Loss = 3.0168, Perplexity = 20.43
|
| 105 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-15000
|
| 106 |
+
Step 15010/44000 | Loss: 3.0053 | LR: 3.77e-04 | 3941.5ms/step | 5.1 samples/s
|
| 107 |
+
Step 15020/44000 | Loss: 3.0368 | LR: 3.77e-04 | 1402.4ms/step | 14.3 samples/s
|
| 108 |
+
Step 15030/44000 | Loss: 3.0373 | LR: 3.77e-04 | 1409.6ms/step | 14.2 samples/s
|
| 109 |
+
Step 15040/44000 | Loss: 3.0137 | LR: 3.77e-04 | 1410.2ms/step | 14.2 samples/s
|
| 110 |
+
Step 15050/44000 | Loss: 3.0237 | LR: 3.77e-04 | 1411.9ms/step | 14.2 samples/s
|
| 111 |
+
Step 15060/44000 | Loss: 3.0238 | LR: 3.77e-04 | 1409.0ms/step | 14.2 samples/s
|
| 112 |
+
Step 15070/44000 | Loss: 3.0276 | LR: 3.77e-04 | 1412.4ms/step | 14.2 samples/s
|
| 113 |
+
Step 15080/44000 | Loss: 3.0443 | LR: 3.76e-04 | 1411.0ms/step | 14.2 samples/s
|
| 114 |
+
Step 15090/44000 | Loss: 3.0242 | LR: 3.76e-04 | 1410.4ms/step | 14.2 samples/s
|
| 115 |
+
Step 15100/44000 | Loss: 3.0022 | LR: 3.76e-04 | 1411.3ms/step | 14.2 samples/s
|
| 116 |
+
Step 15110/44000 | Loss: 3.0292 | LR: 3.76e-04 | 1412.4ms/step | 14.2 samples/s
|
| 117 |
+
Step 15120/44000 | Loss: 3.0013 | LR: 3.76e-04 | 1414.7ms/step | 14.1 samples/s
|
| 118 |
+
Step 15130/44000 | Loss: 3.0519 | LR: 3.76e-04 | 1409.6ms/step | 14.2 samples/s
|
| 119 |
+
Step 15140/44000 | Loss: 3.0067 | LR: 3.75e-04 | 1411.2ms/step | 14.2 samples/s
|
| 120 |
+
Step 15150/44000 | Loss: 3.0282 | LR: 3.75e-04 | 1412.2ms/step | 14.2 samples/s
|
| 121 |
+
Step 15160/44000 | Loss: 3.0164 | LR: 3.75e-04 | 1411.7ms/step | 14.2 samples/s
|
| 122 |
+
Step 15170/44000 | Loss: 3.0404 | LR: 3.75e-04 | 1410.0ms/step | 14.2 samples/s
|
| 123 |
+
Step 15180/44000 | Loss: 2.9985 | LR: 3.75e-04 | 1408.0ms/step | 14.2 samples/s
|
| 124 |
+
Step 15190/44000 | Loss: 3.0056 | LR: 3.75e-04 | 1412.1ms/step | 14.2 samples/s
|
| 125 |
+
Step 15200/44000 | Loss: 3.0177 | LR: 3.74e-04 | 1413.8ms/step | 14.1 samples/s
|
| 126 |
+
Step 15210/44000 | Loss: 3.0339 | LR: 3.74e-04 | 1409.1ms/step | 14.2 samples/s
|
| 127 |
+
Step 15220/44000 | Loss: 3.0289 | LR: 3.74e-04 | 1409.4ms/step | 14.2 samples/s
|
| 128 |
+
Step 15230/44000 | Loss: 3.0372 | LR: 3.74e-04 | 1409.8ms/step | 14.2 samples/s
|
| 129 |
+
Step 15240/44000 | Loss: 3.0108 | LR: 3.74e-04 | 1412.0ms/step | 14.2 samples/s
|
| 130 |
+
Step 15250/44000 | Loss: 3.0425 | LR: 3.74e-04 | 1408.8ms/step | 14.2 samples/s
|
| 131 |
+
Step 15260/44000 | Loss: 3.0209 | LR: 3.74e-04 | 1408.6ms/step | 14.2 samples/s
|
| 132 |
+
Step 15270/44000 | Loss: 3.0247 | LR: 3.73e-04 | 1412.5ms/step | 14.2 samples/s
|
| 133 |
+
Step 15280/44000 | Loss: 3.0363 | LR: 3.73e-04 | 1412.3ms/step | 14.2 samples/s
|
| 134 |
+
Step 15290/44000 | Loss: 3.0178 | LR: 3.73e-04 | 1412.4ms/step | 14.2 samples/s
|
| 135 |
+
Step 15300/44000 | Loss: 3.0352 | LR: 3.73e-04 | 1410.6ms/step | 14.2 samples/s
|
| 136 |
+
Step 15310/44000 | Loss: 3.0114 | LR: 3.73e-04 | 1413.1ms/step | 14.2 samples/s
|
| 137 |
+
Step 15320/44000 | Loss: 3.0030 | LR: 3.73e-04 | 1411.7ms/step | 14.2 samples/s
|
| 138 |
+
Step 15330/44000 | Loss: 3.0128 | LR: 3.72e-04 | 1411.8ms/step | 14.2 samples/s
|
| 139 |
+
Step 15340/44000 | Loss: 3.0430 | LR: 3.72e-04 | 1410.2ms/step | 14.2 samples/s
|
| 140 |
+
Step 15350/44000 | Loss: 3.0171 | LR: 3.72e-04 | 1410.0ms/step | 14.2 samples/s
|
| 141 |
+
Step 15360/44000 | Loss: 3.0435 | LR: 3.72e-04 | 1407.8ms/step | 14.2 samples/s
|
| 142 |
+
Step 15370/44000 | Loss: 3.0452 | LR: 3.72e-04 | 1409.5ms/step | 14.2 samples/s
|
| 143 |
+
Step 15380/44000 | Loss: 3.0110 | LR: 3.72e-04 | 1408.6ms/step | 14.2 samples/s
|
| 144 |
+
Step 15390/44000 | Loss: 3.0187 | LR: 3.71e-04 | 1410.5ms/step | 14.2 samples/s
|
| 145 |
+
Step 15400/44000 | Loss: 3.0359 | LR: 3.71e-04 | 1410.7ms/step | 14.2 samples/s
|
| 146 |
+
Step 15410/44000 | Loss: 3.0216 | LR: 3.71e-04 | 1411.2ms/step | 14.2 samples/s
|
| 147 |
+
Step 15420/44000 | Loss: 3.0334 | LR: 3.71e-04 | 1412.5ms/step | 14.2 samples/s
|
| 148 |
+
Step 15430/44000 | Loss: 3.0338 | LR: 3.71e-04 | 1411.6ms/step | 14.2 samples/s
|
| 149 |
+
Step 15440/44000 | Loss: 3.0213 | LR: 3.71e-04 | 1411.4ms/step | 14.2 samples/s
|
| 150 |
+
Step 15450/44000 | Loss: 3.0234 | LR: 3.71e-04 | 1410.6ms/step | 14.2 samples/s
|
| 151 |
+
Step 15460/44000 | Loss: 3.0322 | LR: 3.70e-04 | 1411.4ms/step | 14.2 samples/s
|
| 152 |
+
Step 15470/44000 | Loss: 3.0304 | LR: 3.70e-04 | 1412.0ms/step | 14.2 samples/s
|
| 153 |
+
Step 15480/44000 | Loss: 3.0119 | LR: 3.70e-04 | 1412.8ms/step | 14.2 samples/s
|
| 154 |
+
Step 15490/44000 | Loss: 3.0070 | LR: 3.70e-04 | 1409.2ms/step | 14.2 samples/s
|
| 155 |
+
Step 15500/44000 | Loss: 3.0243 | LR: 3.70e-04 | 1410.9ms/step | 14.2 samples/s
|
| 156 |
+
Step 15510/44000 | Loss: 3.0337 | LR: 3.70e-04 | 1410.5ms/step | 14.2 samples/s
|
| 157 |
+
Step 15520/44000 | Loss: 3.0178 | LR: 3.69e-04 | 1408.4ms/step | 14.2 samples/s
|
| 158 |
+
Step 15530/44000 | Loss: 2.9951 | LR: 3.69e-04 | 1409.1ms/step | 14.2 samples/s
|
| 159 |
+
Step 15540/44000 | Loss: 3.0524 | LR: 3.69e-04 | 1411.5ms/step | 14.2 samples/s
|
| 160 |
+
Step 15550/44000 | Loss: 3.0217 | LR: 3.69e-04 | 1408.2ms/step | 14.2 samples/s
|
| 161 |
+
Step 15560/44000 | Loss: 3.0148 | LR: 3.69e-04 | 1409.3ms/step | 14.2 samples/s
|
| 162 |
+
Step 15570/44000 | Loss: 2.9971 | LR: 3.69e-04 | 1410.2ms/step | 14.2 samples/s
|
| 163 |
+
Step 15580/44000 | Loss: 3.0258 | LR: 3.68e-04 | 1410.1ms/step | 14.2 samples/s
|
| 164 |
+
Step 15590/44000 | Loss: 3.0436 | LR: 3.68e-04 | 1412.1ms/step | 14.2 samples/s
|
| 165 |
+
Step 15600/44000 | Loss: 3.0186 | LR: 3.68e-04 | 1407.6ms/step | 14.2 samples/s
|
| 166 |
+
Step 15610/44000 | Loss: 2.9948 | LR: 3.68e-04 | 1410.4ms/step | 14.2 samples/s
|
| 167 |
+
Step 15620/44000 | Loss: 3.0137 | LR: 3.68e-04 | 1412.1ms/step | 14.2 samples/s
|
| 168 |
+
Step 15630/44000 | Loss: 3.0320 | LR: 3.68e-04 | 1412.2ms/step | 14.2 samples/s
|
| 169 |
+
Step 15640/44000 | Loss: 3.0359 | LR: 3.67e-04 | 1411.8ms/step | 14.2 samples/s
|
| 170 |
+
Step 15650/44000 | Loss: 3.0122 | LR: 3.67e-04 | 1410.7ms/step | 14.2 samples/s
|
| 171 |
+
Step 15660/44000 | Loss: 3.0295 | LR: 3.67e-04 | 1411.1ms/step | 14.2 samples/s
|
| 172 |
+
Step 15670/44000 | Loss: 3.0124 | LR: 3.67e-04 | 1411.2ms/step | 14.2 samples/s
|
| 173 |
+
Step 15680/44000 | Loss: 3.0503 | LR: 3.67e-04 | 1411.2ms/step | 14.2 samples/s
|
| 174 |
+
Step 15690/44000 | Loss: 3.0024 | LR: 3.67e-04 | 1409.5ms/step | 14.2 samples/s
|
| 175 |
+
Step 15700/44000 | Loss: 3.0032 | LR: 3.67e-04 | 1408.5ms/step | 14.2 samples/s
|
| 176 |
+
Step 15710/44000 | Loss: 3.0169 | LR: 3.66e-04 | 1412.2ms/step | 14.2 samples/s
|
| 177 |
+
Step 15720/44000 | Loss: 2.9977 | LR: 3.66e-04 | 1411.1ms/step | 14.2 samples/s
|
| 178 |
+
Step 15730/44000 | Loss: 3.0203 | LR: 3.66e-04 | 1408.1ms/step | 14.2 samples/s
|
| 179 |
+
Step 15740/44000 | Loss: 3.0099 | LR: 3.66e-04 | 1412.6ms/step | 14.2 samples/s
|
| 180 |
+
Step 15750/44000 | Loss: 3.0203 | LR: 3.66e-04 | 1409.6ms/step | 14.2 samples/s
|
| 181 |
+
Step 15760/44000 | Loss: 3.0059 | LR: 3.66e-04 | 1411.6ms/step | 14.2 samples/s
|
| 182 |
+
Step 15770/44000 | Loss: 3.0506 | LR: 3.65e-04 | 1412.3ms/step | 14.2 samples/s
|
| 183 |
+
Step 15780/44000 | Loss: 2.9989 | LR: 3.65e-04 | 1412.1ms/step | 14.2 samples/s
|
| 184 |
+
Step 15790/44000 | Loss: 3.0394 | LR: 3.65e-04 | 1412.1ms/step | 14.2 samples/s
|
| 185 |
+
Step 15800/44000 | Loss: 3.0058 | LR: 3.65e-04 | 1414.2ms/step | 14.1 samples/s
|
| 186 |
+
Step 15810/44000 | Loss: 3.0214 | LR: 3.65e-04 | 1411.8ms/step | 14.2 samples/s
|
| 187 |
+
Step 15820/44000 | Loss: 2.9922 | LR: 3.65e-04 | 1408.5ms/step | 14.2 samples/s
|
| 188 |
+
Step 15830/44000 | Loss: 2.9982 | LR: 3.64e-04 | 1407.2ms/step | 14.2 samples/s
|
| 189 |
+
Step 15840/44000 | Loss: 3.0272 | LR: 3.64e-04 | 1411.9ms/step | 14.2 samples/s
|
| 190 |
+
Step 15850/44000 | Loss: 3.0119 | LR: 3.64e-04 | 1411.0ms/step | 14.2 samples/s
|
| 191 |
+
Step 15860/44000 | Loss: 3.0130 | LR: 3.64e-04 | 1411.3ms/step | 14.2 samples/s
|
| 192 |
+
Step 15870/44000 | Loss: 3.0085 | LR: 3.64e-04 | 1409.9ms/step | 14.2 samples/s
|
| 193 |
+
Step 15880/44000 | Loss: 3.0284 | LR: 3.64e-04 | 1412.5ms/step | 14.2 samples/s
|
| 194 |
+
Step 15890/44000 | Loss: 2.9768 | LR: 3.63e-04 | 1411.4ms/step | 14.2 samples/s
|
| 195 |
+
Step 15900/44000 | Loss: 3.0116 | LR: 3.63e-04 | 1410.3ms/step | 14.2 samples/s
|
| 196 |
+
Step 15910/44000 | Loss: 3.0028 | LR: 3.63e-04 | 1408.9ms/step | 14.2 samples/s
|
| 197 |
+
Step 15920/44000 | Loss: 3.0251 | LR: 3.63e-04 | 1407.7ms/step | 14.2 samples/s
|
| 198 |
+
Step 15930/44000 | Loss: 3.0078 | LR: 3.63e-04 | 1409.8ms/step | 14.2 samples/s
|
| 199 |
+
Step 15940/44000 | Loss: 3.0058 | LR: 3.63e-04 | 1410.9ms/step | 14.2 samples/s
|
| 200 |
+
Step 15950/44000 | Loss: 3.0315 | LR: 3.62e-04 | 1410.2ms/step | 14.2 samples/s
|
| 201 |
+
Step 15960/44000 | Loss: 2.9918 | LR: 3.62e-04 | 1410.2ms/step | 14.2 samples/s
|
| 202 |
+
Step 15970/44000 | Loss: 2.9993 | LR: 3.62e-04 | 1410.8ms/step | 14.2 samples/s
|
| 203 |
+
Step 15980/44000 | Loss: 3.0220 | LR: 3.62e-04 | 1409.8ms/step | 14.2 samples/s
|
| 204 |
+
Step 15990/44000 | Loss: 3.0168 | LR: 3.62e-04 | 1408.6ms/step | 14.2 samples/s
|
| 205 |
+
Step 16000/44000 | Loss: 3.0033 | LR: 3.62e-04 | 1407.0ms/step | 14.2 samples/s
|
| 206 |
+
Validation at step 16000: Loss = 3.0032, Perplexity = 20.15
|
| 207 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-16000
|
| 208 |
+
Step 16010/44000 | Loss: 3.0004 | LR: 3.62e-04 | 3962.0ms/step | 5.0 samples/s
|
| 209 |
+
Step 16020/44000 | Loss: 3.0341 | LR: 3.61e-04 | 1403.8ms/step | 14.2 samples/s
|
| 210 |
+
Step 16030/44000 | Loss: 2.9863 | LR: 3.61e-04 | 1406.9ms/step | 14.2 samples/s
|
| 211 |
+
Step 16040/44000 | Loss: 3.0186 | LR: 3.61e-04 | 1400.9ms/step | 14.3 samples/s
|
| 212 |
+
Step 16050/44000 | Loss: 3.0309 | LR: 3.61e-04 | 1402.2ms/step | 14.3 samples/s
|
| 213 |
+
Step 16060/44000 | Loss: 3.0071 | LR: 3.61e-04 | 1404.0ms/step | 14.2 samples/s
|
| 214 |
+
Step 16070/44000 | Loss: 2.9967 | LR: 3.61e-04 | 1401.9ms/step | 14.3 samples/s
|
| 215 |
+
Step 16080/44000 | Loss: 3.0232 | LR: 3.60e-04 | 1400.0ms/step | 14.3 samples/s
|
| 216 |
+
Step 16090/44000 | Loss: 3.0047 | LR: 3.60e-04 | 1400.3ms/step | 14.3 samples/s
|
| 217 |
+
Step 16100/44000 | Loss: 3.0289 | LR: 3.60e-04 | 1401.8ms/step | 14.3 samples/s
|
| 218 |
+
Step 16110/44000 | Loss: 2.9921 | LR: 3.60e-04 | 1399.6ms/step | 14.3 samples/s
|
| 219 |
+
Step 16120/44000 | Loss: 3.0093 | LR: 3.60e-04 | 1400.4ms/step | 14.3 samples/s
|
| 220 |
+
Step 16130/44000 | Loss: 2.9896 | LR: 3.60e-04 | 1400.0ms/step | 14.3 samples/s
|
| 221 |
+
Step 16140/44000 | Loss: 2.9900 | LR: 3.59e-04 | 1402.5ms/step | 14.3 samples/s
|
| 222 |
+
Step 16150/44000 | Loss: 3.0304 | LR: 3.59e-04 | 1404.2ms/step | 14.2 samples/s
|
| 223 |
+
Step 16160/44000 | Loss: 3.0094 | LR: 3.59e-04 | 1403.2ms/step | 14.3 samples/s
|
| 224 |
+
Step 16170/44000 | Loss: 2.9771 | LR: 3.59e-04 | 1402.0ms/step | 14.3 samples/s
|
| 225 |
+
Step 16180/44000 | Loss: 2.9958 | LR: 3.59e-04 | 1404.3ms/step | 14.2 samples/s
|
| 226 |
+
Step 16190/44000 | Loss: 2.9992 | LR: 3.59e-04 | 1401.7ms/step | 14.3 samples/s
|
| 227 |
+
Step 16200/44000 | Loss: 3.0221 | LR: 3.58e-04 | 1403.7ms/step | 14.2 samples/s
|
| 228 |
+
Step 16210/44000 | Loss: 3.0118 | LR: 3.58e-04 | 1400.2ms/step | 14.3 samples/s
|
| 229 |
+
Step 16220/44000 | Loss: 3.0274 | LR: 3.58e-04 | 1402.1ms/step | 14.3 samples/s
|
| 230 |
+
Step 16230/44000 | Loss: 3.0129 | LR: 3.58e-04 | 1402.1ms/step | 14.3 samples/s
|
| 231 |
+
Step 16240/44000 | Loss: 2.9960 | LR: 3.58e-04 | 1404.5ms/step | 14.2 samples/s
|
| 232 |
+
Step 16250/44000 | Loss: 3.0097 | LR: 3.58e-04 | 1400.9ms/step | 14.3 samples/s
|
| 233 |
+
Step 16260/44000 | Loss: 3.0072 | LR: 3.57e-04 | 1404.0ms/step | 14.2 samples/s
|
| 234 |
+
Step 16270/44000 | Loss: 2.9829 | LR: 3.57e-04 | 1402.4ms/step | 14.3 samples/s
|
| 235 |
+
Step 16280/44000 | Loss: 2.9928 | LR: 3.57e-04 | 1401.7ms/step | 14.3 samples/s
|
| 236 |
+
Step 16290/44000 | Loss: 3.0136 | LR: 3.57e-04 | 1400.4ms/step | 14.3 samples/s
|
| 237 |
+
Step 16300/44000 | Loss: 3.0019 | LR: 3.57e-04 | 1401.5ms/step | 14.3 samples/s
|
| 238 |
+
Step 16310/44000 | Loss: 3.0136 | LR: 3.57e-04 | 1402.7ms/step | 14.3 samples/s
|
| 239 |
+
Step 16320/44000 | Loss: 3.0140 | LR: 3.56e-04 | 1401.6ms/step | 14.3 samples/s
|
| 240 |
+
Step 16330/44000 | Loss: 3.0250 | LR: 3.56e-04 | 1401.0ms/step | 14.3 samples/s
|
| 241 |
+
Step 16340/44000 | Loss: 2.9840 | LR: 3.56e-04 | 1401.0ms/step | 14.3 samples/s
|
| 242 |
+
Step 16350/44000 | Loss: 3.0017 | LR: 3.56e-04 | 1399.6ms/step | 14.3 samples/s
|
| 243 |
+
Step 16360/44000 | Loss: 3.0169 | LR: 3.56e-04 | 1401.0ms/step | 14.3 samples/s
|
| 244 |
+
Step 16370/44000 | Loss: 3.0185 | LR: 3.56e-04 | 1399.2ms/step | 14.3 samples/s
|
| 245 |
+
Step 16380/44000 | Loss: 2.9950 | LR: 3.55e-04 | 1402.1ms/step | 14.3 samples/s
|
| 246 |
+
Step 16390/44000 | Loss: 3.0218 | LR: 3.55e-04 | 1400.4ms/step | 14.3 samples/s
|
| 247 |
+
Step 16400/44000 | Loss: 2.9841 | LR: 3.55e-04 | 1399.6ms/step | 14.3 samples/s
|
| 248 |
+
Step 16410/44000 | Loss: 3.0440 | LR: 3.55e-04 | 1400.1ms/step | 14.3 samples/s
|
| 249 |
+
Step 16420/44000 | Loss: 2.9912 | LR: 3.55e-04 | 1402.4ms/step | 14.3 samples/s
|
| 250 |
+
Step 16430/44000 | Loss: 2.9887 | LR: 3.55e-04 | 1403.3ms/step | 14.3 samples/s
|
| 251 |
+
Step 16440/44000 | Loss: 3.0123 | LR: 3.54e-04 | 1400.6ms/step | 14.3 samples/s
|
| 252 |
+
Step 16450/44000 | Loss: 2.9977 | LR: 3.54e-04 | 1402.2ms/step | 14.3 samples/s
|
| 253 |
+
Step 16460/44000 | Loss: 2.9785 | LR: 3.54e-04 | 1402.2ms/step | 14.3 samples/s
|
| 254 |
+
Step 16470/44000 | Loss: 2.9969 | LR: 3.54e-04 | 1401.2ms/step | 14.3 samples/s
|
| 255 |
+
Step 16480/44000 | Loss: 3.0249 | LR: 3.54e-04 | 1400.0ms/step | 14.3 samples/s
|
| 256 |
+
Step 16490/44000 | Loss: 3.0014 | LR: 3.54e-04 | 1400.1ms/step | 14.3 samples/s
|
| 257 |
+
Step 16500/44000 | Loss: 3.0037 | LR: 3.53e-04 | 1400.3ms/step | 14.3 samples/s
|
| 258 |
+
Step 16510/44000 | Loss: 3.0069 | LR: 3.53e-04 | 1399.6ms/step | 14.3 samples/s
|
| 259 |
+
Step 16520/44000 | Loss: 2.9786 | LR: 3.53e-04 | 1400.2ms/step | 14.3 samples/s
|
| 260 |
+
Step 16530/44000 | Loss: 3.0031 | LR: 3.53e-04 | 1402.0ms/step | 14.3 samples/s
|
| 261 |
+
Step 16540/44000 | Loss: 3.0183 | LR: 3.53e-04 | 1400.4ms/step | 14.3 samples/s
|
| 262 |
+
Step 16550/44000 | Loss: 3.0102 | LR: 3.53e-04 | 1401.2ms/step | 14.3 samples/s
|
| 263 |
+
Step 16560/44000 | Loss: 3.0114 | LR: 3.52e-04 | 1400.0ms/step | 14.3 samples/s
|
| 264 |
+
Step 16570/44000 | Loss: 2.9928 | LR: 3.52e-04 | 1400.5ms/step | 14.3 samples/s
|
| 265 |
+
Step 16580/44000 | Loss: 2.9866 | LR: 3.52e-04 | 1400.1ms/step | 14.3 samples/s
|
| 266 |
+
Step 16590/44000 | Loss: 3.0028 | LR: 3.52e-04 | 1402.4ms/step | 14.3 samples/s
|
| 267 |
+
Step 16600/44000 | Loss: 2.9975 | LR: 3.52e-04 | 1403.3ms/step | 14.3 samples/s
|
| 268 |
+
Step 16610/44000 | Loss: 3.0065 | LR: 3.52e-04 | 1402.0ms/step | 14.3 samples/s
|
| 269 |
+
Step 16620/44000 | Loss: 3.0008 | LR: 3.51e-04 | 1403.8ms/step | 14.2 samples/s
|
| 270 |
+
Step 16630/44000 | Loss: 3.0111 | LR: 3.51e-04 | 1402.6ms/step | 14.3 samples/s
|
| 271 |
+
Step 16640/44000 | Loss: 2.9828 | LR: 3.51e-04 | 1405.3ms/step | 14.2 samples/s
|
| 272 |
+
Step 16650/44000 | Loss: 3.0010 | LR: 3.51e-04 | 1404.1ms/step | 14.2 samples/s
|
| 273 |
+
Step 16660/44000 | Loss: 3.0140 | LR: 3.51e-04 | 1404.2ms/step | 14.2 samples/s
|
| 274 |
+
Step 16670/44000 | Loss: 2.9953 | LR: 3.51e-04 | 1402.2ms/step | 14.3 samples/s
|
| 275 |
+
Step 16680/44000 | Loss: 3.0067 | LR: 3.50e-04 | 1401.5ms/step | 14.3 samples/s
|
| 276 |
+
Step 16690/44000 | Loss: 3.0189 | LR: 3.50e-04 | 1402.4ms/step | 14.3 samples/s
|
| 277 |
+
Step 16700/44000 | Loss: 3.0106 | LR: 3.50e-04 | 1406.0ms/step | 14.2 samples/s
|
| 278 |
+
Step 16710/44000 | Loss: 3.0117 | LR: 3.50e-04 | 1403.8ms/step | 14.2 samples/s
|
| 279 |
+
Step 16720/44000 | Loss: 3.0164 | LR: 3.50e-04 | 1404.1ms/step | 14.2 samples/s
|
| 280 |
+
Step 16730/44000 | Loss: 3.0528 | LR: 3.50e-04 | 1403.9ms/step | 14.2 samples/s
|
| 281 |
+
Step 16740/44000 | Loss: 2.9955 | LR: 3.49e-04 | 1402.1ms/step | 14.3 samples/s
|
| 282 |
+
Step 16750/44000 | Loss: 3.0083 | LR: 3.49e-04 | 1399.5ms/step | 14.3 samples/s
|
| 283 |
+
Step 16760/44000 | Loss: 2.9814 | LR: 3.49e-04 | 1400.8ms/step | 14.3 samples/s
|
| 284 |
+
Step 16770/44000 | Loss: 2.9909 | LR: 3.49e-04 | 1403.7ms/step | 14.2 samples/s
|
| 285 |
+
Step 16780/44000 | Loss: 2.9880 | LR: 3.49e-04 | 1402.2ms/step | 14.3 samples/s
|
| 286 |
+
Step 16790/44000 | Loss: 3.0199 | LR: 3.49e-04 | 1404.3ms/step | 14.2 samples/s
|
| 287 |
+
Step 16800/44000 | Loss: 2.9944 | LR: 3.48e-04 | 1404.6ms/step | 14.2 samples/s
|
| 288 |
+
Step 16810/44000 | Loss: 3.0162 | LR: 3.48e-04 | 1400.8ms/step | 14.3 samples/s
|
| 289 |
+
Step 16820/44000 | Loss: 2.9954 | LR: 3.48e-04 | 1401.6ms/step | 14.3 samples/s
|
| 290 |
+
Step 16830/44000 | Loss: 3.0058 | LR: 3.48e-04 | 1400.2ms/step | 14.3 samples/s
|
| 291 |
+
Step 16840/44000 | Loss: 3.0278 | LR: 3.48e-04 | 1402.3ms/step | 14.3 samples/s
|
| 292 |
+
Step 16850/44000 | Loss: 3.0013 | LR: 3.48e-04 | 1401.7ms/step | 14.3 samples/s
|
| 293 |
+
Step 16860/44000 | Loss: 3.0208 | LR: 3.47e-04 | 1400.3ms/step | 14.3 samples/s
|
| 294 |
+
Step 16870/44000 | Loss: 3.0287 | LR: 3.47e-04 | 1403.6ms/step | 14.2 samples/s
|
| 295 |
+
Step 16880/44000 | Loss: 2.9802 | LR: 3.47e-04 | 1400.0ms/step | 14.3 samples/s
|
| 296 |
+
Step 16890/44000 | Loss: 3.0092 | LR: 3.47e-04 | 1402.5ms/step | 14.3 samples/s
|
| 297 |
+
Step 16900/44000 | Loss: 3.0112 | LR: 3.47e-04 | 1399.4ms/step | 14.3 samples/s
|
| 298 |
+
Step 16910/44000 | Loss: 2.9900 | LR: 3.47e-04 | 1402.6ms/step | 14.3 samples/s
|
| 299 |
+
Step 16920/44000 | Loss: 3.0025 | LR: 3.46e-04 | 1400.0ms/step | 14.3 samples/s
|
| 300 |
+
Step 16930/44000 | Loss: 3.0420 | LR: 3.46e-04 | 1402.6ms/step | 14.3 samples/s
|
| 301 |
+
Step 16940/44000 | Loss: 2.9802 | LR: 3.46e-04 | 1403.0ms/step | 14.3 samples/s
|
| 302 |
+
Step 16950/44000 | Loss: 3.0236 | LR: 3.46e-04 | 1402.2ms/step | 14.3 samples/s
|
| 303 |
+
Step 16960/44000 | Loss: 2.9748 | LR: 3.46e-04 | 1399.9ms/step | 14.3 samples/s
|
| 304 |
+
Step 16970/44000 | Loss: 2.9966 | LR: 3.46e-04 | 1402.0ms/step | 14.3 samples/s
|
| 305 |
+
Step 16980/44000 | Loss: 3.0043 | LR: 3.45e-04 | 1401.6ms/step | 14.3 samples/s
|
| 306 |
+
Step 16990/44000 | Loss: 3.0047 | LR: 3.45e-04 | 1400.0ms/step | 14.3 samples/s
|
| 307 |
+
Step 17000/44000 | Loss: 3.0066 | LR: 3.45e-04 | 1399.9ms/step | 14.3 samples/s
|
| 308 |
+
Validation at step 17000: Loss = 2.9912, Perplexity = 19.91
|
| 309 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-17000
|
| 310 |
+
Step 17010/44000 | Loss: 2.9859 | LR: 3.45e-04 | 3948.6ms/step | 5.1 samples/s
|
| 311 |
+
Step 17020/44000 | Loss: 3.0082 | LR: 3.45e-04 | 1403.8ms/step | 14.2 samples/s
|
| 312 |
+
Step 17030/44000 | Loss: 2.9998 | LR: 3.45e-04 | 1401.9ms/step | 14.3 samples/s
|
| 313 |
+
Step 17040/44000 | Loss: 3.0025 | LR: 3.44e-04 | 1407.1ms/step | 14.2 samples/s
|
| 314 |
+
Step 17050/44000 | Loss: 3.0099 | LR: 3.44e-04 | 1408.8ms/step | 14.2 samples/s
|
| 315 |
+
Step 17060/44000 | Loss: 3.0021 | LR: 3.44e-04 | 1409.4ms/step | 14.2 samples/s
|
| 316 |
+
Step 17070/44000 | Loss: 2.9977 | LR: 3.44e-04 | 1410.9ms/step | 14.2 samples/s
|
| 317 |
+
Step 17080/44000 | Loss: 3.0071 | LR: 3.44e-04 | 1409.5ms/step | 14.2 samples/s
|
| 318 |
+
Step 17090/44000 | Loss: 2.9980 | LR: 3.44e-04 | 1407.8ms/step | 14.2 samples/s
|
| 319 |
+
Step 17100/44000 | Loss: 2.9998 | LR: 3.43e-04 | 1408.5ms/step | 14.2 samples/s
|
| 320 |
+
Step 17110/44000 | Loss: 3.0134 | LR: 3.43e-04 | 1408.3ms/step | 14.2 samples/s
|
| 321 |
+
Step 17120/44000 | Loss: 3.0028 | LR: 3.43e-04 | 1412.4ms/step | 14.2 samples/s
|
| 322 |
+
Step 17130/44000 | Loss: 3.0126 | LR: 3.43e-04 | 1409.1ms/step | 14.2 samples/s
|
| 323 |
+
Step 17140/44000 | Loss: 2.9939 | LR: 3.43e-04 | 1408.1ms/step | 14.2 samples/s
|
| 324 |
+
Step 17150/44000 | Loss: 3.0095 | LR: 3.43e-04 | 1409.9ms/step | 14.2 samples/s
|
| 325 |
+
Step 17160/44000 | Loss: 2.9964 | LR: 3.42e-04 | 1410.6ms/step | 14.2 samples/s
|
| 326 |
+
Step 17170/44000 | Loss: 2.9783 | LR: 3.42e-04 | 1411.3ms/step | 14.2 samples/s
|
| 327 |
+
Step 17180/44000 | Loss: 3.0097 | LR: 3.42e-04 | 1410.4ms/step | 14.2 samples/s
|
| 328 |
+
Step 17190/44000 | Loss: 2.9733 | LR: 3.42e-04 | 1407.8ms/step | 14.2 samples/s
|
| 329 |
+
Step 17200/44000 | Loss: 3.0141 | LR: 3.42e-04 | 1409.9ms/step | 14.2 samples/s
|
| 330 |
+
Step 17210/44000 | Loss: 3.0139 | LR: 3.42e-04 | 1406.3ms/step | 14.2 samples/s
|
| 331 |
+
Step 17220/44000 | Loss: 2.9842 | LR: 3.41e-04 | 1405.9ms/step | 14.2 samples/s
|
| 332 |
+
Step 17230/44000 | Loss: 2.9903 | LR: 3.41e-04 | 1406.2ms/step | 14.2 samples/s
|
| 333 |
+
Step 17240/44000 | Loss: 3.0228 | LR: 3.41e-04 | 1410.1ms/step | 14.2 samples/s
|
| 334 |
+
Step 17250/44000 | Loss: 3.0173 | LR: 3.41e-04 | 1410.3ms/step | 14.2 samples/s
|
| 335 |
+
Step 17260/44000 | Loss: 2.9775 | LR: 3.41e-04 | 1413.2ms/step | 14.2 samples/s
|
| 336 |
+
Step 17270/44000 | Loss: 2.9755 | LR: 3.41e-04 | 1408.4ms/step | 14.2 samples/s
|
| 337 |
+
Step 17280/44000 | Loss: 3.0070 | LR: 3.40e-04 | 1410.0ms/step | 14.2 samples/s
|
| 338 |
+
Step 17290/44000 | Loss: 2.9774 | LR: 3.40e-04 | 1409.8ms/step | 14.2 samples/s
|
| 339 |
+
Step 17300/44000 | Loss: 2.9916 | LR: 3.40e-04 | 1406.9ms/step | 14.2 samples/s
|
| 340 |
+
Step 17310/44000 | Loss: 3.0065 | LR: 3.40e-04 | 1406.8ms/step | 14.2 samples/s
|
| 341 |
+
Step 17320/44000 | Loss: 3.0215 | LR: 3.40e-04 | 1410.0ms/step | 14.2 samples/s
|
| 342 |
+
Step 17330/44000 | Loss: 3.0171 | LR: 3.40e-04 | 1409.5ms/step | 14.2 samples/s
|
| 343 |
+
Step 17340/44000 | Loss: 3.0105 | LR: 3.39e-04 | 1409.0ms/step | 14.2 samples/s
|
| 344 |
+
Step 17350/44000 | Loss: 3.0224 | LR: 3.39e-04 | 1405.4ms/step | 14.2 samples/s
|
| 345 |
+
Step 17360/44000 | Loss: 2.9547 | LR: 3.39e-04 | 1399.9ms/step | 14.3 samples/s
|
| 346 |
+
Step 17370/44000 | Loss: 3.0315 | LR: 3.39e-04 | 1402.9ms/step | 14.3 samples/s
|
| 347 |
+
Step 17380/44000 | Loss: 2.9934 | LR: 3.39e-04 | 1401.7ms/step | 14.3 samples/s
|
| 348 |
+
Step 17390/44000 | Loss: 3.0234 | LR: 3.39e-04 | 1402.0ms/step | 14.3 samples/s
|
| 349 |
+
Step 17400/44000 | Loss: 3.0135 | LR: 3.38e-04 | 1401.7ms/step | 14.3 samples/s
|
| 350 |
+
Step 17410/44000 | Loss: 2.9675 | LR: 3.38e-04 | 1401.7ms/step | 14.3 samples/s
|
| 351 |
+
Step 17420/44000 | Loss: 2.9944 | LR: 3.38e-04 | 1402.1ms/step | 14.3 samples/s
|
| 352 |
+
Step 17430/44000 | Loss: 2.9855 | LR: 3.38e-04 | 1400.3ms/step | 14.3 samples/s
|
| 353 |
+
Step 17440/44000 | Loss: 3.0193 | LR: 3.38e-04 | 1400.6ms/step | 14.3 samples/s
|
| 354 |
+
Step 17450/44000 | Loss: 3.0119 | LR: 3.38e-04 | 1401.8ms/step | 14.3 samples/s
|
| 355 |
+
Step 17460/44000 | Loss: 2.9935 | LR: 3.37e-04 | 1401.1ms/step | 14.3 samples/s
|
| 356 |
+
Step 17470/44000 | Loss: 2.9749 | LR: 3.37e-04 | 1399.9ms/step | 14.3 samples/s
|
| 357 |
+
Step 17480/44000 | Loss: 3.0113 | LR: 3.37e-04 | 1400.2ms/step | 14.3 samples/s
|
| 358 |
+
Step 17490/44000 | Loss: 2.9957 | LR: 3.37e-04 | 1406.0ms/step | 14.2 samples/s
|
| 359 |
+
Step 17500/44000 | Loss: 2.9999 | LR: 3.37e-04 | 1402.3ms/step | 14.3 samples/s
|
| 360 |
+
Step 17510/44000 | Loss: 2.9788 | LR: 3.37e-04 | 1401.7ms/step | 14.3 samples/s
|
| 361 |
+
Step 17520/44000 | Loss: 2.9929 | LR: 3.36e-04 | 1404.1ms/step | 14.2 samples/s
|
| 362 |
+
Step 17530/44000 | Loss: 2.9834 | LR: 3.36e-04 | 1404.5ms/step | 14.2 samples/s
|
| 363 |
+
Step 17540/44000 | Loss: 2.9702 | LR: 3.36e-04 | 1402.0ms/step | 14.3 samples/s
|
| 364 |
+
Step 17550/44000 | Loss: 2.9899 | LR: 3.36e-04 | 1402.8ms/step | 14.3 samples/s
|
| 365 |
+
Step 17560/44000 | Loss: 2.9560 | LR: 3.36e-04 | 1402.6ms/step | 14.3 samples/s
|
| 366 |
+
Step 17570/44000 | Loss: 2.9599 | LR: 3.35e-04 | 1404.3ms/step | 14.2 samples/s
|
| 367 |
+
Step 17580/44000 | Loss: 2.9956 | LR: 3.35e-04 | 1402.1ms/step | 14.3 samples/s
|
| 368 |
+
Step 17590/44000 | Loss: 2.9956 | LR: 3.35e-04 | 1399.8ms/step | 14.3 samples/s
|
| 369 |
+
Step 17600/44000 | Loss: 3.0255 | LR: 3.35e-04 | 1402.0ms/step | 14.3 samples/s
|
| 370 |
+
Step 17610/44000 | Loss: 2.9929 | LR: 3.35e-04 | 1403.8ms/step | 14.2 samples/s
|
| 371 |
+
Step 17620/44000 | Loss: 2.9550 | LR: 3.35e-04 | 1400.6ms/step | 14.3 samples/s
|
| 372 |
+
Step 17630/44000 | Loss: 3.0227 | LR: 3.34e-04 | 1403.5ms/step | 14.3 samples/s
|
| 373 |
+
Step 17640/44000 | Loss: 2.9923 | LR: 3.34e-04 | 1402.5ms/step | 14.3 samples/s
|
| 374 |
+
Step 17650/44000 | Loss: 2.9923 | LR: 3.34e-04 | 1401.3ms/step | 14.3 samples/s
|
| 375 |
+
Step 17660/44000 | Loss: 2.9837 | LR: 3.34e-04 | 1402.2ms/step | 14.3 samples/s
|
| 376 |
+
Step 17670/44000 | Loss: 2.9928 | LR: 3.34e-04 | 1402.8ms/step | 14.3 samples/s
|
| 377 |
+
Step 17680/44000 | Loss: 2.9856 | LR: 3.34e-04 | 1401.3ms/step | 14.3 samples/s
|
| 378 |
+
Step 17690/44000 | Loss: 2.9979 | LR: 3.33e-04 | 1400.2ms/step | 14.3 samples/s
|
| 379 |
+
Step 17700/44000 | Loss: 2.9534 | LR: 3.33e-04 | 1401.7ms/step | 14.3 samples/s
|
| 380 |
+
Step 17710/44000 | Loss: 2.9818 | LR: 3.33e-04 | 1402.3ms/step | 14.3 samples/s
|
| 381 |
+
Step 17720/44000 | Loss: 2.9715 | LR: 3.33e-04 | 1399.6ms/step | 14.3 samples/s
|
| 382 |
+
Step 17730/44000 | Loss: 2.9720 | LR: 3.33e-04 | 1400.6ms/step | 14.3 samples/s
|
| 383 |
+
Step 17740/44000 | Loss: 2.9461 | LR: 3.33e-04 | 1401.3ms/step | 14.3 samples/s
|
| 384 |
+
Step 17750/44000 | Loss: 3.0006 | LR: 3.32e-04 | 1400.1ms/step | 14.3 samples/s
|
| 385 |
+
Step 17760/44000 | Loss: 2.9835 | LR: 3.32e-04 | 1399.8ms/step | 14.3 samples/s
|
| 386 |
+
Step 17770/44000 | Loss: 2.9829 | LR: 3.32e-04 | 1402.0ms/step | 14.3 samples/s
|
| 387 |
+
Step 17780/44000 | Loss: 2.9943 | LR: 3.32e-04 | 1402.2ms/step | 14.3 samples/s
|
| 388 |
+
Step 17790/44000 | Loss: 3.0138 | LR: 3.32e-04 | 1400.1ms/step | 14.3 samples/s
|
| 389 |
+
Step 17800/44000 | Loss: 2.9886 | LR: 3.32e-04 | 1401.8ms/step | 14.3 samples/s
|
| 390 |
+
Step 17810/44000 | Loss: 2.9793 | LR: 3.31e-04 | 1400.0ms/step | 14.3 samples/s
|
| 391 |
+
Step 17820/44000 | Loss: 2.9885 | LR: 3.31e-04 | 1402.2ms/step | 14.3 samples/s
|
| 392 |
+
Step 17830/44000 | Loss: 2.9953 | LR: 3.31e-04 | 1402.1ms/step | 14.3 samples/s
|
| 393 |
+
Step 17840/44000 | Loss: 3.0073 | LR: 3.31e-04 | 1401.9ms/step | 14.3 samples/s
|
| 394 |
+
Step 17850/44000 | Loss: 3.0087 | LR: 3.31e-04 | 1402.1ms/step | 14.3 samples/s
|
| 395 |
+
Step 17860/44000 | Loss: 3.0400 | LR: 3.31e-04 | 1402.3ms/step | 14.3 samples/s
|
| 396 |
+
Step 17870/44000 | Loss: 2.9774 | LR: 3.30e-04 | 1401.5ms/step | 14.3 samples/s
|
| 397 |
+
Step 17880/44000 | Loss: 2.9930 | LR: 3.30e-04 | 1401.9ms/step | 14.3 samples/s
|
| 398 |
+
Step 17890/44000 | Loss: 2.9595 | LR: 3.30e-04 | 1401.9ms/step | 14.3 samples/s
|
| 399 |
+
Step 17900/44000 | Loss: 2.9887 | LR: 3.30e-04 | 1402.2ms/step | 14.3 samples/s
|
| 400 |
+
Step 17910/44000 | Loss: 2.9930 | LR: 3.30e-04 | 1404.0ms/step | 14.2 samples/s
|
| 401 |
+
Step 17920/44000 | Loss: 2.9938 | LR: 3.29e-04 | 1404.1ms/step | 14.2 samples/s
|
| 402 |
+
Step 17930/44000 | Loss: 2.9668 | LR: 3.29e-04 | 1402.2ms/step | 14.3 samples/s
|
| 403 |
+
Step 17940/44000 | Loss: 3.0049 | LR: 3.29e-04 | 1401.7ms/step | 14.3 samples/s
|
| 404 |
+
Step 17950/44000 | Loss: 2.9615 | LR: 3.29e-04 | 1400.1ms/step | 14.3 samples/s
|
| 405 |
+
Step 17960/44000 | Loss: 2.9870 | LR: 3.29e-04 | 1401.7ms/step | 14.3 samples/s
|
| 406 |
+
Step 17970/44000 | Loss: 2.9579 | LR: 3.29e-04 | 1402.6ms/step | 14.3 samples/s
|
| 407 |
+
Step 17980/44000 | Loss: 3.0065 | LR: 3.28e-04 | 1405.4ms/step | 14.2 samples/s
|
| 408 |
+
Step 17990/44000 | Loss: 2.9547 | LR: 3.28e-04 | 1409.2ms/step | 14.2 samples/s
|
| 409 |
+
Step 18000/44000 | Loss: 3.0120 | LR: 3.28e-04 | 1408.9ms/step | 14.2 samples/s
|
| 410 |
+
Validation at step 18000: Loss = 2.9788, Perplexity = 19.66
|
| 411 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-18000
|
| 412 |
+
Step 18010/44000 | Loss: 3.0195 | LR: 3.28e-04 | 4022.0ms/step | 5.0 samples/s
|
| 413 |
+
Step 18020/44000 | Loss: 2.9689 | LR: 3.28e-04 | 1405.7ms/step | 14.2 samples/s
|
| 414 |
+
Step 18030/44000 | Loss: 2.9515 | LR: 3.28e-04 | 1410.2ms/step | 14.2 samples/s
|
| 415 |
+
Step 18040/44000 | Loss: 2.9922 | LR: 3.27e-04 | 1410.7ms/step | 14.2 samples/s
|
| 416 |
+
Step 18050/44000 | Loss: 2.9694 | LR: 3.27e-04 | 1409.9ms/step | 14.2 samples/s
|
| 417 |
+
Step 18060/44000 | Loss: 2.9700 | LR: 3.27e-04 | 1412.4ms/step | 14.2 samples/s
|
| 418 |
+
Step 18070/44000 | Loss: 3.0065 | LR: 3.27e-04 | 1412.9ms/step | 14.2 samples/s
|
| 419 |
+
Step 18080/44000 | Loss: 3.0038 | LR: 3.27e-04 | 1410.6ms/step | 14.2 samples/s
|
| 420 |
+
Step 18090/44000 | Loss: 2.9937 | LR: 3.27e-04 | 1411.3ms/step | 14.2 samples/s
|
| 421 |
+
Step 18100/44000 | Loss: 2.9676 | LR: 3.26e-04 | 1409.3ms/step | 14.2 samples/s
|
| 422 |
+
Step 18110/44000 | Loss: 2.9604 | LR: 3.26e-04 | 1410.5ms/step | 14.2 samples/s
|
| 423 |
+
Step 18120/44000 | Loss: 2.9780 | LR: 3.26e-04 | 1410.3ms/step | 14.2 samples/s
|
| 424 |
+
Step 18130/44000 | Loss: 2.9988 | LR: 3.26e-04 | 1410.0ms/step | 14.2 samples/s
|
| 425 |
+
Step 18140/44000 | Loss: 2.9800 | LR: 3.26e-04 | 1409.8ms/step | 14.2 samples/s
|
| 426 |
+
Step 18150/44000 | Loss: 3.0062 | LR: 3.26e-04 | 1411.2ms/step | 14.2 samples/s
|
| 427 |
+
Step 18160/44000 | Loss: 2.9385 | LR: 3.25e-04 | 1409.3ms/step | 14.2 samples/s
|
| 428 |
+
Step 18170/44000 | Loss: 2.9684 | LR: 3.25e-04 | 1409.8ms/step | 14.2 samples/s
|
| 429 |
+
Step 18180/44000 | Loss: 3.0144 | LR: 3.25e-04 | 1411.7ms/step | 14.2 samples/s
|
| 430 |
+
Step 18190/44000 | Loss: 2.9787 | LR: 3.25e-04 | 1408.5ms/step | 14.2 samples/s
|
| 431 |
+
Step 18200/44000 | Loss: 3.0038 | LR: 3.25e-04 | 1410.3ms/step | 14.2 samples/s
|
| 432 |
+
Step 18210/44000 | Loss: 2.9708 | LR: 3.24e-04 | 1409.8ms/step | 14.2 samples/s
|
| 433 |
+
Step 18220/44000 | Loss: 2.9810 | LR: 3.24e-04 | 1411.5ms/step | 14.2 samples/s
|
| 434 |
+
Step 18230/44000 | Loss: 2.9815 | LR: 3.24e-04 | 1408.3ms/step | 14.2 samples/s
|
| 435 |
+
Step 18240/44000 | Loss: 3.0291 | LR: 3.24e-04 | 1409.5ms/step | 14.2 samples/s
|
| 436 |
+
Step 18250/44000 | Loss: 2.9642 | LR: 3.24e-04 | 1407.2ms/step | 14.2 samples/s
|
| 437 |
+
Step 18260/44000 | Loss: 3.0108 | LR: 3.24e-04 | 1408.9ms/step | 14.2 samples/s
|
| 438 |
+
Step 18270/44000 | Loss: 2.9636 | LR: 3.23e-04 | 1410.6ms/step | 14.2 samples/s
|
| 439 |
+
Step 18280/44000 | Loss: 2.9813 | LR: 3.23e-04 | 1408.1ms/step | 14.2 samples/s
|
| 440 |
+
Step 18290/44000 | Loss: 3.0170 | LR: 3.23e-04 | 1410.9ms/step | 14.2 samples/s
|
| 441 |
+
Step 18300/44000 | Loss: 3.0146 | LR: 3.23e-04 | 1410.5ms/step | 14.2 samples/s
|
| 442 |
+
Step 18310/44000 | Loss: 2.9883 | LR: 3.23e-04 | 1409.9ms/step | 14.2 samples/s
|
| 443 |
+
Step 18320/44000 | Loss: 2.9947 | LR: 3.23e-04 | 1408.7ms/step | 14.2 samples/s
|
| 444 |
+
Step 18330/44000 | Loss: 2.9648 | LR: 3.22e-04 | 1411.8ms/step | 14.2 samples/s
|
| 445 |
+
Step 18340/44000 | Loss: 3.0022 | LR: 3.22e-04 | 1413.6ms/step | 14.1 samples/s
|
| 446 |
+
Step 18350/44000 | Loss: 2.9705 | LR: 3.22e-04 | 1412.3ms/step | 14.2 samples/s
|
| 447 |
+
Step 18360/44000 | Loss: 3.0002 | LR: 3.22e-04 | 1409.5ms/step | 14.2 samples/s
|
| 448 |
+
Step 18370/44000 | Loss: 2.9743 | LR: 3.22e-04 | 1410.3ms/step | 14.2 samples/s
|
| 449 |
+
Step 18380/44000 | Loss: 3.0053 | LR: 3.22e-04 | 1407.6ms/step | 14.2 samples/s
|
| 450 |
+
Step 18390/44000 | Loss: 2.9929 | LR: 3.21e-04 | 1410.0ms/step | 14.2 samples/s
|
| 451 |
+
Step 18400/44000 | Loss: 2.9745 | LR: 3.21e-04 | 1408.1ms/step | 14.2 samples/s
|
| 452 |
+
Step 18410/44000 | Loss: 2.9821 | LR: 3.21e-04 | 1409.8ms/step | 14.2 samples/s
|
| 453 |
+
Step 18420/44000 | Loss: 2.9775 | LR: 3.21e-04 | 1406.0ms/step | 14.2 samples/s
|
| 454 |
+
Step 18430/44000 | Loss: 2.9899 | LR: 3.21e-04 | 1410.0ms/step | 14.2 samples/s
|
| 455 |
+
Step 18440/44000 | Loss: 2.9887 | LR: 3.20e-04 | 1408.6ms/step | 14.2 samples/s
|
| 456 |
+
Step 18450/44000 | Loss: 2.9978 | LR: 3.20e-04 | 1410.2ms/step | 14.2 samples/s
|
epfl_final/SparseAtt/wandb/run-20260820_041559-ysa9iyov/files/requirements.txt
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
| 142 |
+
entmax==1.3
|
epfl_final/SparseAtt/wandb/run-20260820_041559-ysa9iyov/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-08-20T04:15:59.505687Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"/mloscratch/homes/navasard/sparse/configs_ropectl/resume_a1p0_wandb/seed43.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/TokenPrune.git",
|
| 14 |
+
"commit": "5637886f0cc760009989b14e68c6ae06d4827418"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "h100-workspace-5-gpu-0-2",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 23 |
+
"gpu_count": 5,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "9374931091456",
|
| 27 |
+
"used": "5285341044736"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1623005626368"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-6fb12770-6b51-9595-91c9-0d20801e3896"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 43 |
+
"memoryTotal": "85520809984",
|
| 44 |
+
"cudaCores": 16896,
|
| 45 |
+
"architecture": "Hopper",
|
| 46 |
+
"uuid": "GPU-5cb9f8b7-4a13-4f85-d73e-6bcbebebefb8"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 50 |
+
"memoryTotal": "85520809984",
|
| 51 |
+
"cudaCores": 16896,
|
| 52 |
+
"architecture": "Hopper",
|
| 53 |
+
"uuid": "GPU-3512802a-d501-0a3c-9812-4ab9deb3a3a9"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 57 |
+
"memoryTotal": "85520809984",
|
| 58 |
+
"cudaCores": 16896,
|
| 59 |
+
"architecture": "Hopper",
|
| 60 |
+
"uuid": "GPU-24436def-beee-c9f1-bd05-c97df556de08"
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 64 |
+
"memoryTotal": "85520809984",
|
| 65 |
+
"cudaCores": 16896,
|
| 66 |
+
"architecture": "Hopper",
|
| 67 |
+
"uuid": "GPU-7b827c86-854a-af51-b5d3-6aaa43307d4a"
|
| 68 |
+
}
|
| 69 |
+
],
|
| 70 |
+
"cudaVersion": "13.0",
|
| 71 |
+
"writerId": "lulxoi41qgvmsgddgfxcdtjuei6ncl3k"
|
| 72 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260820_041559-ysa9iyov/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260820_041559-ysa9iyov/run-ysa9iyov.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7ba65f437bbc42fc7a03fadd46502da1762884271e8bd7bc27cb5d774ac16e8c
|
| 3 |
+
size 2785280
|
epfl_final/SparseAtt/wandb/run-20260820_140401-jqstnvej/files/output.log
ADDED
|
@@ -0,0 +1,655 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Starting training...
|
| 2 |
+
Validation at step 17000: Loss = 3.0074, Perplexity = 20.23
|
| 3 |
+
wandb: WARNING Tried to log to step 17000 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 4 |
+
Switching attention mode for step 17001: dense (dense step interval)
|
| 5 |
+
Step 17010/44000 | Loss: 2.9953 | LR: 3.45e-04 | 42298.8ms/step | 0.5 samples/s
|
| 6 |
+
wandb: WARNING Tried to log to step 17010 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 7 |
+
Step 17020/44000 | Loss: 3.0000 | LR: 3.45e-04 | 1764.7ms/step | 11.3 samples/s
|
| 8 |
+
wandb: WARNING Tried to log to step 17020 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 9 |
+
Step 17030/44000 | Loss: 2.9989 | LR: 3.45e-04 | 1765.3ms/step | 11.3 samples/s
|
| 10 |
+
wandb: WARNING Tried to log to step 17030 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 11 |
+
Step 17040/44000 | Loss: 2.9807 | LR: 3.44e-04 | 1767.9ms/step | 11.3 samples/s
|
| 12 |
+
wandb: WARNING Tried to log to step 17040 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 13 |
+
Step 17050/44000 | Loss: 2.9804 | LR: 3.44e-04 | 1767.1ms/step | 11.3 samples/s
|
| 14 |
+
wandb: WARNING Tried to log to step 17050 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 15 |
+
Step 17060/44000 | Loss: 3.0394 | LR: 3.44e-04 | 1766.2ms/step | 11.3 samples/s
|
| 16 |
+
wandb: WARNING Tried to log to step 17060 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 17 |
+
Step 17070/44000 | Loss: 3.0309 | LR: 3.44e-04 | 1768.2ms/step | 11.3 samples/s
|
| 18 |
+
wandb: WARNING Tried to log to step 17070 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 19 |
+
Step 17080/44000 | Loss: 3.0213 | LR: 3.44e-04 | 1768.4ms/step | 11.3 samples/s
|
| 20 |
+
wandb: WARNING Tried to log to step 17080 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 21 |
+
Step 17090/44000 | Loss: 3.0357 | LR: 3.44e-04 | 1769.2ms/step | 11.3 samples/s
|
| 22 |
+
wandb: WARNING Tried to log to step 17090 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 23 |
+
Step 17100/44000 | Loss: 2.9922 | LR: 3.43e-04 | 1769.5ms/step | 11.3 samples/s
|
| 24 |
+
wandb: WARNING Tried to log to step 17100 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 25 |
+
Step 17110/44000 | Loss: 2.9951 | LR: 3.43e-04 | 1769.3ms/step | 11.3 samples/s
|
| 26 |
+
wandb: WARNING Tried to log to step 17110 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 27 |
+
Step 17120/44000 | Loss: 2.9471 | LR: 3.43e-04 | 1768.7ms/step | 11.3 samples/s
|
| 28 |
+
wandb: WARNING Tried to log to step 17120 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 29 |
+
Step 17130/44000 | Loss: 2.9969 | LR: 3.43e-04 | 1767.5ms/step | 11.3 samples/s
|
| 30 |
+
wandb: WARNING Tried to log to step 17130 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 31 |
+
Step 17140/44000 | Loss: 2.9777 | LR: 3.43e-04 | 1767.6ms/step | 11.3 samples/s
|
| 32 |
+
wandb: WARNING Tried to log to step 17140 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 33 |
+
Step 17150/44000 | Loss: 2.9867 | LR: 3.43e-04 | 1768.0ms/step | 11.3 samples/s
|
| 34 |
+
wandb: WARNING Tried to log to step 17150 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 35 |
+
Step 17160/44000 | Loss: 2.9971 | LR: 3.42e-04 | 1768.9ms/step | 11.3 samples/s
|
| 36 |
+
wandb: WARNING Tried to log to step 17160 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 37 |
+
Step 17170/44000 | Loss: 2.9694 | LR: 3.42e-04 | 1768.6ms/step | 11.3 samples/s
|
| 38 |
+
wandb: WARNING Tried to log to step 17170 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 39 |
+
Step 17180/44000 | Loss: 2.9811 | LR: 3.42e-04 | 1768.6ms/step | 11.3 samples/s
|
| 40 |
+
wandb: WARNING Tried to log to step 17180 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 41 |
+
Step 17190/44000 | Loss: 3.0168 | LR: 3.42e-04 | 1768.7ms/step | 11.3 samples/s
|
| 42 |
+
wandb: WARNING Tried to log to step 17190 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 43 |
+
Step 17200/44000 | Loss: 2.9846 | LR: 3.42e-04 | 1769.9ms/step | 11.3 samples/s
|
| 44 |
+
wandb: WARNING Tried to log to step 17200 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 45 |
+
Step 17210/44000 | Loss: 2.9550 | LR: 3.42e-04 | 1768.8ms/step | 11.3 samples/s
|
| 46 |
+
wandb: WARNING Tried to log to step 17210 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 47 |
+
Step 17220/44000 | Loss: 2.9769 | LR: 3.41e-04 | 1769.4ms/step | 11.3 samples/s
|
| 48 |
+
wandb: WARNING Tried to log to step 17220 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 49 |
+
Step 17230/44000 | Loss: 3.0017 | LR: 3.41e-04 | 1769.2ms/step | 11.3 samples/s
|
| 50 |
+
wandb: WARNING Tried to log to step 17230 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 51 |
+
Step 17240/44000 | Loss: 2.9902 | LR: 3.41e-04 | 1769.0ms/step | 11.3 samples/s
|
| 52 |
+
wandb: WARNING Tried to log to step 17240 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 53 |
+
Step 17250/44000 | Loss: 3.0014 | LR: 3.41e-04 | 1768.4ms/step | 11.3 samples/s
|
| 54 |
+
wandb: WARNING Tried to log to step 17250 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 55 |
+
Step 17260/44000 | Loss: 3.0338 | LR: 3.41e-04 | 1769.7ms/step | 11.3 samples/s
|
| 56 |
+
wandb: WARNING Tried to log to step 17260 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 57 |
+
Step 17270/44000 | Loss: 3.0320 | LR: 3.41e-04 | 1769.6ms/step | 11.3 samples/s
|
| 58 |
+
wandb: WARNING Tried to log to step 17270 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 59 |
+
Step 17280/44000 | Loss: 2.9712 | LR: 3.40e-04 | 1769.3ms/step | 11.3 samples/s
|
| 60 |
+
wandb: WARNING Tried to log to step 17280 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 61 |
+
Step 17290/44000 | Loss: 3.0376 | LR: 3.40e-04 | 1770.7ms/step | 11.3 samples/s
|
| 62 |
+
wandb: WARNING Tried to log to step 17290 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 63 |
+
Step 17300/44000 | Loss: 3.0024 | LR: 3.40e-04 | 1770.3ms/step | 11.3 samples/s
|
| 64 |
+
wandb: WARNING Tried to log to step 17300 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 65 |
+
Step 17310/44000 | Loss: 2.9896 | LR: 3.40e-04 | 1770.0ms/step | 11.3 samples/s
|
| 66 |
+
wandb: WARNING Tried to log to step 17310 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 67 |
+
Step 17320/44000 | Loss: 3.0202 | LR: 3.40e-04 | 1772.8ms/step | 11.3 samples/s
|
| 68 |
+
wandb: WARNING Tried to log to step 17320 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 69 |
+
Step 17330/44000 | Loss: 2.9877 | LR: 3.40e-04 | 1769.8ms/step | 11.3 samples/s
|
| 70 |
+
wandb: WARNING Tried to log to step 17330 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 71 |
+
Step 17340/44000 | Loss: 3.0100 | LR: 3.39e-04 | 1768.7ms/step | 11.3 samples/s
|
| 72 |
+
wandb: WARNING Tried to log to step 17340 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 73 |
+
Step 17350/44000 | Loss: 2.9876 | LR: 3.39e-04 | 1768.4ms/step | 11.3 samples/s
|
| 74 |
+
wandb: WARNING Tried to log to step 17350 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 75 |
+
Step 17360/44000 | Loss: 2.9364 | LR: 3.39e-04 | 1768.5ms/step | 11.3 samples/s
|
| 76 |
+
wandb: WARNING Tried to log to step 17360 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 77 |
+
Step 17370/44000 | Loss: 3.0425 | LR: 3.39e-04 | 1767.9ms/step | 11.3 samples/s
|
| 78 |
+
wandb: WARNING Tried to log to step 17370 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 79 |
+
Step 17380/44000 | Loss: 2.9622 | LR: 3.39e-04 | 1768.4ms/step | 11.3 samples/s
|
| 80 |
+
wandb: WARNING Tried to log to step 17380 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 81 |
+
Step 17390/44000 | Loss: 3.0031 | LR: 3.39e-04 | 1767.7ms/step | 11.3 samples/s
|
| 82 |
+
wandb: WARNING Tried to log to step 17390 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 83 |
+
Step 17400/44000 | Loss: 2.9896 | LR: 3.38e-04 | 1768.1ms/step | 11.3 samples/s
|
| 84 |
+
wandb: WARNING Tried to log to step 17400 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 85 |
+
Step 17410/44000 | Loss: 2.9699 | LR: 3.38e-04 | 1769.6ms/step | 11.3 samples/s
|
| 86 |
+
wandb: WARNING Tried to log to step 17410 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 87 |
+
Step 17420/44000 | Loss: 2.9919 | LR: 3.38e-04 | 1769.2ms/step | 11.3 samples/s
|
| 88 |
+
wandb: WARNING Tried to log to step 17420 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 89 |
+
Step 17430/44000 | Loss: 2.9849 | LR: 3.38e-04 | 1767.6ms/step | 11.3 samples/s
|
| 90 |
+
wandb: WARNING Tried to log to step 17430 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 91 |
+
Step 17440/44000 | Loss: 2.9565 | LR: 3.38e-04 | 1767.9ms/step | 11.3 samples/s
|
| 92 |
+
wandb: WARNING Tried to log to step 17440 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 93 |
+
Step 17450/44000 | Loss: 3.0010 | LR: 3.38e-04 | 1767.9ms/step | 11.3 samples/s
|
| 94 |
+
wandb: WARNING Tried to log to step 17450 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 95 |
+
Step 17460/44000 | Loss: 2.9859 | LR: 3.37e-04 | 1768.2ms/step | 11.3 samples/s
|
| 96 |
+
wandb: WARNING Tried to log to step 17460 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 97 |
+
Step 17470/44000 | Loss: 2.9710 | LR: 3.37e-04 | 1769.0ms/step | 11.3 samples/s
|
| 98 |
+
wandb: WARNING Tried to log to step 17470 that is less than the current step 17471. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 99 |
+
Step 17480/44000 | Loss: 2.9962 | LR: 3.37e-04 | 1769.1ms/step | 11.3 samples/s
|
| 100 |
+
Step 17490/44000 | Loss: 3.0029 | LR: 3.37e-04 | 1770.1ms/step | 11.3 samples/s
|
| 101 |
+
Step 17500/44000 | Loss: 2.9515 | LR: 3.37e-04 | 1769.6ms/step | 11.3 samples/s
|
| 102 |
+
Step 17510/44000 | Loss: 2.9864 | LR: 3.37e-04 | 1769.0ms/step | 11.3 samples/s
|
| 103 |
+
Step 17520/44000 | Loss: 3.0140 | LR: 3.36e-04 | 1769.8ms/step | 11.3 samples/s
|
| 104 |
+
Step 17530/44000 | Loss: 2.9341 | LR: 3.36e-04 | 1769.9ms/step | 11.3 samples/s
|
| 105 |
+
Step 17540/44000 | Loss: 2.9562 | LR: 3.36e-04 | 1768.0ms/step | 11.3 samples/s
|
| 106 |
+
Step 17550/44000 | Loss: 2.9230 | LR: 3.36e-04 | 1768.3ms/step | 11.3 samples/s
|
| 107 |
+
Step 17560/44000 | Loss: 2.9470 | LR: 3.36e-04 | 1768.6ms/step | 11.3 samples/s
|
| 108 |
+
Step 17570/44000 | Loss: 2.9793 | LR: 3.35e-04 | 1768.6ms/step | 11.3 samples/s
|
| 109 |
+
Step 17580/44000 | Loss: 2.9684 | LR: 3.35e-04 | 1769.8ms/step | 11.3 samples/s
|
| 110 |
+
Step 17590/44000 | Loss: 3.0173 | LR: 3.35e-04 | 1771.3ms/step | 11.3 samples/s
|
| 111 |
+
Step 17600/44000 | Loss: 3.0286 | LR: 3.35e-04 | 1770.7ms/step | 11.3 samples/s
|
| 112 |
+
Step 17610/44000 | Loss: 3.0003 | LR: 3.35e-04 | 1769.6ms/step | 11.3 samples/s
|
| 113 |
+
Step 17620/44000 | Loss: 2.9908 | LR: 3.35e-04 | 1768.5ms/step | 11.3 samples/s
|
| 114 |
+
Step 17630/44000 | Loss: 3.0167 | LR: 3.34e-04 | 1770.4ms/step | 11.3 samples/s
|
| 115 |
+
Step 17640/44000 | Loss: 3.0106 | LR: 3.34e-04 | 1769.1ms/step | 11.3 samples/s
|
| 116 |
+
Step 17650/44000 | Loss: 2.9579 | LR: 3.34e-04 | 1768.7ms/step | 11.3 samples/s
|
| 117 |
+
Step 17660/44000 | Loss: 2.9935 | LR: 3.34e-04 | 1769.7ms/step | 11.3 samples/s
|
| 118 |
+
Step 17670/44000 | Loss: 3.0074 | LR: 3.34e-04 | 1770.4ms/step | 11.3 samples/s
|
| 119 |
+
Step 17680/44000 | Loss: 2.9855 | LR: 3.34e-04 | 1769.7ms/step | 11.3 samples/s
|
| 120 |
+
Step 17690/44000 | Loss: 2.9490 | LR: 3.33e-04 | 1769.4ms/step | 11.3 samples/s
|
| 121 |
+
Step 17700/44000 | Loss: 3.0226 | LR: 3.33e-04 | 1769.8ms/step | 11.3 samples/s
|
| 122 |
+
Step 17710/44000 | Loss: 3.0278 | LR: 3.33e-04 | 1772.0ms/step | 11.3 samples/s
|
| 123 |
+
Step 17720/44000 | Loss: 2.9615 | LR: 3.33e-04 | 1770.4ms/step | 11.3 samples/s
|
| 124 |
+
Step 17730/44000 | Loss: 3.0084 | LR: 3.33e-04 | 1771.7ms/step | 11.3 samples/s
|
| 125 |
+
Step 17740/44000 | Loss: 2.9796 | LR: 3.33e-04 | 1769.3ms/step | 11.3 samples/s
|
| 126 |
+
Step 17750/44000 | Loss: 2.9873 | LR: 3.32e-04 | 1770.7ms/step | 11.3 samples/s
|
| 127 |
+
Step 17760/44000 | Loss: 2.9708 | LR: 3.32e-04 | 1773.3ms/step | 11.3 samples/s
|
| 128 |
+
Step 17770/44000 | Loss: 2.9688 | LR: 3.32e-04 | 1772.8ms/step | 11.3 samples/s
|
| 129 |
+
Step 17780/44000 | Loss: 2.9390 | LR: 3.32e-04 | 1774.8ms/step | 11.3 samples/s
|
| 130 |
+
Step 17790/44000 | Loss: 2.9552 | LR: 3.32e-04 | 1770.7ms/step | 11.3 samples/s
|
| 131 |
+
Step 17800/44000 | Loss: 3.0171 | LR: 3.32e-04 | 1769.5ms/step | 11.3 samples/s
|
| 132 |
+
Step 17810/44000 | Loss: 3.0365 | LR: 3.31e-04 | 1771.1ms/step | 11.3 samples/s
|
| 133 |
+
Step 17820/44000 | Loss: 3.0011 | LR: 3.31e-04 | 1770.9ms/step | 11.3 samples/s
|
| 134 |
+
Step 17830/44000 | Loss: 3.0162 | LR: 3.31e-04 | 1768.5ms/step | 11.3 samples/s
|
| 135 |
+
Step 17840/44000 | Loss: 2.9913 | LR: 3.31e-04 | 1772.6ms/step | 11.3 samples/s
|
| 136 |
+
Step 17850/44000 | Loss: 2.9622 | LR: 3.31e-04 | 1772.4ms/step | 11.3 samples/s
|
| 137 |
+
Step 17860/44000 | Loss: 3.0152 | LR: 3.31e-04 | 1770.3ms/step | 11.3 samples/s
|
| 138 |
+
Step 17870/44000 | Loss: 2.9732 | LR: 3.30e-04 | 1769.4ms/step | 11.3 samples/s
|
| 139 |
+
Step 17880/44000 | Loss: 2.9904 | LR: 3.30e-04 | 1768.9ms/step | 11.3 samples/s
|
| 140 |
+
Step 17890/44000 | Loss: 2.9925 | LR: 3.30e-04 | 1770.4ms/step | 11.3 samples/s
|
| 141 |
+
Step 17900/44000 | Loss: 2.9819 | LR: 3.30e-04 | 1769.9ms/step | 11.3 samples/s
|
| 142 |
+
Step 17910/44000 | Loss: 2.9529 | LR: 3.30e-04 | 1770.6ms/step | 11.3 samples/s
|
| 143 |
+
Step 17920/44000 | Loss: 3.0158 | LR: 3.29e-04 | 1769.9ms/step | 11.3 samples/s
|
| 144 |
+
Step 17930/44000 | Loss: 2.9927 | LR: 3.29e-04 | 1770.5ms/step | 11.3 samples/s
|
| 145 |
+
Step 17940/44000 | Loss: 2.9965 | LR: 3.29e-04 | 1769.9ms/step | 11.3 samples/s
|
| 146 |
+
Step 17950/44000 | Loss: 2.9844 | LR: 3.29e-04 | 1767.6ms/step | 11.3 samples/s
|
| 147 |
+
Step 17960/44000 | Loss: 2.9831 | LR: 3.29e-04 | 1769.0ms/step | 11.3 samples/s
|
| 148 |
+
Step 17970/44000 | Loss: 2.9568 | LR: 3.29e-04 | 1770.8ms/step | 11.3 samples/s
|
| 149 |
+
Step 17980/44000 | Loss: 2.9801 | LR: 3.28e-04 | 1771.5ms/step | 11.3 samples/s
|
| 150 |
+
Step 17990/44000 | Loss: 2.9367 | LR: 3.28e-04 | 1771.4ms/step | 11.3 samples/s
|
| 151 |
+
Step 18000/44000 | Loss: 3.0193 | LR: 3.28e-04 | 1769.5ms/step | 11.3 samples/s
|
| 152 |
+
Validation at step 18000: Loss = 2.9956, Perplexity = 20.00
|
| 153 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-18000
|
| 154 |
+
Step 18010/44000 | Loss: 2.9897 | LR: 3.28e-04 | 3629.0ms/step | 5.5 samples/s
|
| 155 |
+
Step 18020/44000 | Loss: 2.9957 | LR: 3.28e-04 | 1768.1ms/step | 11.3 samples/s
|
| 156 |
+
Step 18030/44000 | Loss: 2.9941 | LR: 3.28e-04 | 1769.8ms/step | 11.3 samples/s
|
| 157 |
+
Step 18040/44000 | Loss: 2.9955 | LR: 3.27e-04 | 1772.3ms/step | 11.3 samples/s
|
| 158 |
+
Step 18050/44000 | Loss: 3.0196 | LR: 3.27e-04 | 1769.6ms/step | 11.3 samples/s
|
| 159 |
+
Step 18060/44000 | Loss: 2.9550 | LR: 3.27e-04 | 1768.9ms/step | 11.3 samples/s
|
| 160 |
+
Step 18070/44000 | Loss: 2.9920 | LR: 3.27e-04 | 1770.0ms/step | 11.3 samples/s
|
| 161 |
+
Step 18080/44000 | Loss: 2.9590 | LR: 3.27e-04 | 1771.4ms/step | 11.3 samples/s
|
| 162 |
+
Step 18090/44000 | Loss: 2.9606 | LR: 3.27e-04 | 1770.9ms/step | 11.3 samples/s
|
| 163 |
+
Step 18100/44000 | Loss: 2.9588 | LR: 3.26e-04 | 1771.8ms/step | 11.3 samples/s
|
| 164 |
+
Step 18110/44000 | Loss: 2.9402 | LR: 3.26e-04 | 1772.2ms/step | 11.3 samples/s
|
| 165 |
+
Step 18120/44000 | Loss: 3.0048 | LR: 3.26e-04 | 1772.8ms/step | 11.3 samples/s
|
| 166 |
+
Step 18130/44000 | Loss: 2.9780 | LR: 3.26e-04 | 1773.0ms/step | 11.3 samples/s
|
| 167 |
+
Step 18140/44000 | Loss: 2.9664 | LR: 3.26e-04 | 1769.4ms/step | 11.3 samples/s
|
| 168 |
+
Step 18150/44000 | Loss: 2.9470 | LR: 3.26e-04 | 1769.9ms/step | 11.3 samples/s
|
| 169 |
+
Step 18160/44000 | Loss: 2.9995 | LR: 3.25e-04 | 1770.8ms/step | 11.3 samples/s
|
| 170 |
+
Step 18170/44000 | Loss: 2.9699 | LR: 3.25e-04 | 1769.8ms/step | 11.3 samples/s
|
| 171 |
+
Step 18180/44000 | Loss: 2.9892 | LR: 3.25e-04 | 1769.9ms/step | 11.3 samples/s
|
| 172 |
+
Step 18190/44000 | Loss: 2.9809 | LR: 3.25e-04 | 1772.5ms/step | 11.3 samples/s
|
| 173 |
+
Step 18200/44000 | Loss: 2.9980 | LR: 3.25e-04 | 1771.5ms/step | 11.3 samples/s
|
| 174 |
+
Step 18210/44000 | Loss: 2.9697 | LR: 3.24e-04 | 1771.8ms/step | 11.3 samples/s
|
| 175 |
+
Step 18220/44000 | Loss: 2.9545 | LR: 3.24e-04 | 1770.8ms/step | 11.3 samples/s
|
| 176 |
+
Step 18230/44000 | Loss: 2.9375 | LR: 3.24e-04 | 1770.8ms/step | 11.3 samples/s
|
| 177 |
+
Step 18240/44000 | Loss: 2.9661 | LR: 3.24e-04 | 1772.6ms/step | 11.3 samples/s
|
| 178 |
+
Step 18250/44000 | Loss: 2.9907 | LR: 3.24e-04 | 1771.9ms/step | 11.3 samples/s
|
| 179 |
+
Step 18260/44000 | Loss: 2.9708 | LR: 3.24e-04 | 1772.0ms/step | 11.3 samples/s
|
| 180 |
+
Step 18270/44000 | Loss: 2.9797 | LR: 3.23e-04 | 1771.2ms/step | 11.3 samples/s
|
| 181 |
+
Step 18280/44000 | Loss: 2.9829 | LR: 3.23e-04 | 1771.5ms/step | 11.3 samples/s
|
| 182 |
+
Step 18290/44000 | Loss: 2.9986 | LR: 3.23e-04 | 1770.7ms/step | 11.3 samples/s
|
| 183 |
+
Step 18300/44000 | Loss: 2.9809 | LR: 3.23e-04 | 1771.9ms/step | 11.3 samples/s
|
| 184 |
+
Step 18310/44000 | Loss: 2.9801 | LR: 3.23e-04 | 1772.4ms/step | 11.3 samples/s
|
| 185 |
+
Step 18320/44000 | Loss: 2.9896 | LR: 3.23e-04 | 1772.6ms/step | 11.3 samples/s
|
| 186 |
+
Step 18330/44000 | Loss: 2.9907 | LR: 3.22e-04 | 1772.7ms/step | 11.3 samples/s
|
| 187 |
+
Step 18340/44000 | Loss: 3.0383 | LR: 3.22e-04 | 1770.9ms/step | 11.3 samples/s
|
| 188 |
+
Step 18350/44000 | Loss: 2.9696 | LR: 3.22e-04 | 1769.9ms/step | 11.3 samples/s
|
| 189 |
+
Step 18360/44000 | Loss: 2.9360 | LR: 3.22e-04 | 1770.6ms/step | 11.3 samples/s
|
| 190 |
+
Step 18370/44000 | Loss: 2.9669 | LR: 3.22e-04 | 1768.4ms/step | 11.3 samples/s
|
| 191 |
+
Step 18380/44000 | Loss: 2.9547 | LR: 3.22e-04 | 1769.3ms/step | 11.3 samples/s
|
| 192 |
+
Step 18390/44000 | Loss: 2.9836 | LR: 3.21e-04 | 1769.7ms/step | 11.3 samples/s
|
| 193 |
+
Step 18400/44000 | Loss: 2.9594 | LR: 3.21e-04 | 1770.6ms/step | 11.3 samples/s
|
| 194 |
+
Step 18410/44000 | Loss: 2.9419 | LR: 3.21e-04 | 1771.6ms/step | 11.3 samples/s
|
| 195 |
+
Step 18420/44000 | Loss: 3.0278 | LR: 3.21e-04 | 1769.8ms/step | 11.3 samples/s
|
| 196 |
+
Step 18430/44000 | Loss: 3.0356 | LR: 3.21e-04 | 1770.7ms/step | 11.3 samples/s
|
| 197 |
+
Step 18440/44000 | Loss: 2.9890 | LR: 3.20e-04 | 1770.9ms/step | 11.3 samples/s
|
| 198 |
+
Step 18450/44000 | Loss: 2.9739 | LR: 3.20e-04 | 1770.5ms/step | 11.3 samples/s
|
| 199 |
+
Step 18460/44000 | Loss: 2.9957 | LR: 3.20e-04 | 1772.4ms/step | 11.3 samples/s
|
| 200 |
+
Step 18470/44000 | Loss: 2.9845 | LR: 3.20e-04 | 1770.9ms/step | 11.3 samples/s
|
| 201 |
+
Step 18480/44000 | Loss: 2.9698 | LR: 3.20e-04 | 1771.4ms/step | 11.3 samples/s
|
| 202 |
+
Step 18490/44000 | Loss: 2.9566 | LR: 3.20e-04 | 1770.4ms/step | 11.3 samples/s
|
| 203 |
+
Step 18500/44000 | Loss: 2.9605 | LR: 3.19e-04 | 1771.3ms/step | 11.3 samples/s
|
| 204 |
+
Step 18510/44000 | Loss: 2.9494 | LR: 3.19e-04 | 1771.1ms/step | 11.3 samples/s
|
| 205 |
+
Step 18520/44000 | Loss: 2.9192 | LR: 3.19e-04 | 1770.7ms/step | 11.3 samples/s
|
| 206 |
+
Step 18530/44000 | Loss: 2.9784 | LR: 3.19e-04 | 1770.9ms/step | 11.3 samples/s
|
| 207 |
+
Step 18540/44000 | Loss: 2.9874 | LR: 3.19e-04 | 1771.8ms/step | 11.3 samples/s
|
| 208 |
+
Step 18550/44000 | Loss: 2.9844 | LR: 3.19e-04 | 1769.3ms/step | 11.3 samples/s
|
| 209 |
+
Step 18560/44000 | Loss: 2.9353 | LR: 3.18e-04 | 1769.7ms/step | 11.3 samples/s
|
| 210 |
+
Step 18570/44000 | Loss: 2.9711 | LR: 3.18e-04 | 1770.9ms/step | 11.3 samples/s
|
| 211 |
+
Step 18580/44000 | Loss: 3.0260 | LR: 3.18e-04 | 1770.8ms/step | 11.3 samples/s
|
| 212 |
+
Step 18590/44000 | Loss: 2.9768 | LR: 3.18e-04 | 1772.7ms/step | 11.3 samples/s
|
| 213 |
+
Step 18600/44000 | Loss: 2.9523 | LR: 3.18e-04 | 1770.3ms/step | 11.3 samples/s
|
| 214 |
+
Step 18610/44000 | Loss: 3.0193 | LR: 3.18e-04 | 1771.4ms/step | 11.3 samples/s
|
| 215 |
+
Step 18620/44000 | Loss: 2.9781 | LR: 3.17e-04 | 1769.3ms/step | 11.3 samples/s
|
| 216 |
+
Step 18630/44000 | Loss: 2.9890 | LR: 3.17e-04 | 1770.0ms/step | 11.3 samples/s
|
| 217 |
+
Step 18640/44000 | Loss: 2.9644 | LR: 3.17e-04 | 1770.3ms/step | 11.3 samples/s
|
| 218 |
+
Step 18650/44000 | Loss: 3.0036 | LR: 3.17e-04 | 1771.1ms/step | 11.3 samples/s
|
| 219 |
+
Step 18660/44000 | Loss: 2.9546 | LR: 3.17e-04 | 1769.6ms/step | 11.3 samples/s
|
| 220 |
+
Step 18670/44000 | Loss: 2.9598 | LR: 3.16e-04 | 1771.4ms/step | 11.3 samples/s
|
| 221 |
+
Step 18680/44000 | Loss: 3.0120 | LR: 3.16e-04 | 1770.7ms/step | 11.3 samples/s
|
| 222 |
+
Step 18690/44000 | Loss: 2.9864 | LR: 3.16e-04 | 1769.8ms/step | 11.3 samples/s
|
| 223 |
+
Step 18700/44000 | Loss: 2.9835 | LR: 3.16e-04 | 1770.4ms/step | 11.3 samples/s
|
| 224 |
+
Step 18710/44000 | Loss: 2.9747 | LR: 3.16e-04 | 1770.7ms/step | 11.3 samples/s
|
| 225 |
+
Step 18720/44000 | Loss: 2.9969 | LR: 3.16e-04 | 1771.5ms/step | 11.3 samples/s
|
| 226 |
+
Step 18730/44000 | Loss: 2.9643 | LR: 3.15e-04 | 1771.6ms/step | 11.3 samples/s
|
| 227 |
+
Step 18740/44000 | Loss: 2.9840 | LR: 3.15e-04 | 1771.7ms/step | 11.3 samples/s
|
| 228 |
+
Step 18750/44000 | Loss: 2.9541 | LR: 3.15e-04 | 1771.9ms/step | 11.3 samples/s
|
| 229 |
+
Step 18760/44000 | Loss: 2.9484 | LR: 3.15e-04 | 1768.8ms/step | 11.3 samples/s
|
| 230 |
+
Step 18770/44000 | Loss: 2.9171 | LR: 3.15e-04 | 1770.0ms/step | 11.3 samples/s
|
| 231 |
+
Step 18780/44000 | Loss: 2.9861 | LR: 3.15e-04 | 1769.6ms/step | 11.3 samples/s
|
| 232 |
+
Step 18790/44000 | Loss: 2.9754 | LR: 3.14e-04 | 1769.6ms/step | 11.3 samples/s
|
| 233 |
+
Step 18800/44000 | Loss: 2.9667 | LR: 3.14e-04 | 1770.9ms/step | 11.3 samples/s
|
| 234 |
+
Step 18810/44000 | Loss: 2.9540 | LR: 3.14e-04 | 1770.4ms/step | 11.3 samples/s
|
| 235 |
+
Step 18820/44000 | Loss: 2.9444 | LR: 3.14e-04 | 1771.2ms/step | 11.3 samples/s
|
| 236 |
+
Step 18830/44000 | Loss: 2.9815 | LR: 3.14e-04 | 1770.9ms/step | 11.3 samples/s
|
| 237 |
+
Step 18840/44000 | Loss: 2.9418 | LR: 3.13e-04 | 1770.7ms/step | 11.3 samples/s
|
| 238 |
+
Step 18850/44000 | Loss: 2.9511 | LR: 3.13e-04 | 1770.5ms/step | 11.3 samples/s
|
| 239 |
+
Step 18860/44000 | Loss: 2.9654 | LR: 3.13e-04 | 1770.4ms/step | 11.3 samples/s
|
| 240 |
+
Step 18870/44000 | Loss: 2.9814 | LR: 3.13e-04 | 1770.9ms/step | 11.3 samples/s
|
| 241 |
+
Step 18880/44000 | Loss: 2.9764 | LR: 3.13e-04 | 1769.8ms/step | 11.3 samples/s
|
| 242 |
+
Step 18890/44000 | Loss: 2.9910 | LR: 3.13e-04 | 1769.6ms/step | 11.3 samples/s
|
| 243 |
+
Step 18900/44000 | Loss: 3.0031 | LR: 3.12e-04 | 1770.1ms/step | 11.3 samples/s
|
| 244 |
+
Step 18910/44000 | Loss: 2.9752 | LR: 3.12e-04 | 1771.0ms/step | 11.3 samples/s
|
| 245 |
+
Step 18920/44000 | Loss: 3.0025 | LR: 3.12e-04 | 1770.4ms/step | 11.3 samples/s
|
| 246 |
+
Step 18930/44000 | Loss: 3.0012 | LR: 3.12e-04 | 1771.5ms/step | 11.3 samples/s
|
| 247 |
+
Step 18940/44000 | Loss: 2.9471 | LR: 3.12e-04 | 1771.2ms/step | 11.3 samples/s
|
| 248 |
+
Step 18950/44000 | Loss: 2.9523 | LR: 3.12e-04 | 1772.1ms/step | 11.3 samples/s
|
| 249 |
+
Step 18960/44000 | Loss: 2.9595 | LR: 3.11e-04 | 1769.2ms/step | 11.3 samples/s
|
| 250 |
+
Step 18970/44000 | Loss: 2.9815 | LR: 3.11e-04 | 1770.6ms/step | 11.3 samples/s
|
| 251 |
+
Step 18980/44000 | Loss: 2.9753 | LR: 3.11e-04 | 1769.9ms/step | 11.3 samples/s
|
| 252 |
+
Step 18990/44000 | Loss: 3.0137 | LR: 3.11e-04 | 1769.1ms/step | 11.3 samples/s
|
| 253 |
+
Step 19000/44000 | Loss: 3.0280 | LR: 3.11e-04 | 1769.0ms/step | 11.3 samples/s
|
| 254 |
+
Validation at step 19000: Loss = 2.9844, Perplexity = 19.77
|
| 255 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-19000
|
| 256 |
+
Step 19010/44000 | Loss: 2.9494 | LR: 3.11e-04 | 3549.4ms/step | 5.6 samples/s
|
| 257 |
+
Step 19020/44000 | Loss: 2.9590 | LR: 3.10e-04 | 1768.8ms/step | 11.3 samples/s
|
| 258 |
+
Step 19030/44000 | Loss: 2.9950 | LR: 3.10e-04 | 1771.1ms/step | 11.3 samples/s
|
| 259 |
+
Step 19040/44000 | Loss: 2.9828 | LR: 3.10e-04 | 1770.4ms/step | 11.3 samples/s
|
| 260 |
+
Step 19050/44000 | Loss: 2.9433 | LR: 3.10e-04 | 1771.6ms/step | 11.3 samples/s
|
| 261 |
+
Step 19060/44000 | Loss: 3.0023 | LR: 3.10e-04 | 1771.6ms/step | 11.3 samples/s
|
| 262 |
+
Step 19070/44000 | Loss: 2.9470 | LR: 3.09e-04 | 1771.7ms/step | 11.3 samples/s
|
| 263 |
+
Step 19080/44000 | Loss: 2.9568 | LR: 3.09e-04 | 1771.3ms/step | 11.3 samples/s
|
| 264 |
+
Step 19090/44000 | Loss: 2.9839 | LR: 3.09e-04 | 1773.3ms/step | 11.3 samples/s
|
| 265 |
+
Step 19100/44000 | Loss: 2.9568 | LR: 3.09e-04 | 1772.0ms/step | 11.3 samples/s
|
| 266 |
+
Step 19110/44000 | Loss: 2.9778 | LR: 3.09e-04 | 1771.4ms/step | 11.3 samples/s
|
| 267 |
+
Step 19120/44000 | Loss: 2.9881 | LR: 3.09e-04 | 1771.3ms/step | 11.3 samples/s
|
| 268 |
+
Step 19130/44000 | Loss: 2.9526 | LR: 3.08e-04 | 1772.5ms/step | 11.3 samples/s
|
| 269 |
+
Step 19140/44000 | Loss: 2.9738 | LR: 3.08e-04 | 1771.2ms/step | 11.3 samples/s
|
| 270 |
+
Step 19150/44000 | Loss: 2.9667 | LR: 3.08e-04 | 1772.0ms/step | 11.3 samples/s
|
| 271 |
+
Step 19160/44000 | Loss: 2.9943 | LR: 3.08e-04 | 1769.6ms/step | 11.3 samples/s
|
| 272 |
+
Step 19170/44000 | Loss: 2.9402 | LR: 3.08e-04 | 1770.2ms/step | 11.3 samples/s
|
| 273 |
+
Step 19180/44000 | Loss: 2.9953 | LR: 3.08e-04 | 1771.5ms/step | 11.3 samples/s
|
| 274 |
+
Step 19190/44000 | Loss: 3.0067 | LR: 3.07e-04 | 1771.0ms/step | 11.3 samples/s
|
| 275 |
+
Step 19200/44000 | Loss: 2.9986 | LR: 3.07e-04 | 1770.2ms/step | 11.3 samples/s
|
| 276 |
+
Step 19210/44000 | Loss: 2.9489 | LR: 3.07e-04 | 1770.3ms/step | 11.3 samples/s
|
| 277 |
+
Step 19220/44000 | Loss: 2.9948 | LR: 3.07e-04 | 1770.9ms/step | 11.3 samples/s
|
| 278 |
+
Step 19230/44000 | Loss: 3.0039 | LR: 3.07e-04 | 1768.9ms/step | 11.3 samples/s
|
| 279 |
+
Step 19240/44000 | Loss: 2.9916 | LR: 3.06e-04 | 1770.4ms/step | 11.3 samples/s
|
| 280 |
+
Step 19250/44000 | Loss: 2.9411 | LR: 3.06e-04 | 1770.0ms/step | 11.3 samples/s
|
| 281 |
+
Step 19260/44000 | Loss: 2.9537 | LR: 3.06e-04 | 1770.7ms/step | 11.3 samples/s
|
| 282 |
+
Step 19270/44000 | Loss: 2.9491 | LR: 3.06e-04 | 1770.5ms/step | 11.3 samples/s
|
| 283 |
+
Step 19280/44000 | Loss: 2.9620 | LR: 3.06e-04 | 1771.3ms/step | 11.3 samples/s
|
| 284 |
+
Step 19290/44000 | Loss: 2.9807 | LR: 3.06e-04 | 1770.6ms/step | 11.3 samples/s
|
| 285 |
+
Step 19300/44000 | Loss: 2.9847 | LR: 3.05e-04 | 1770.7ms/step | 11.3 samples/s
|
| 286 |
+
Step 19310/44000 | Loss: 2.9413 | LR: 3.05e-04 | 1772.0ms/step | 11.3 samples/s
|
| 287 |
+
Step 19320/44000 | Loss: 3.0060 | LR: 3.05e-04 | 1772.5ms/step | 11.3 samples/s
|
| 288 |
+
Step 19330/44000 | Loss: 2.9936 | LR: 3.05e-04 | 1772.4ms/step | 11.3 samples/s
|
| 289 |
+
Step 19340/44000 | Loss: 2.9193 | LR: 3.05e-04 | 1771.9ms/step | 11.3 samples/s
|
| 290 |
+
Step 19350/44000 | Loss: 3.0031 | LR: 3.04e-04 | 1771.0ms/step | 11.3 samples/s
|
| 291 |
+
Step 19360/44000 | Loss: 2.9996 | LR: 3.04e-04 | 1770.7ms/step | 11.3 samples/s
|
| 292 |
+
Step 19370/44000 | Loss: 2.9783 | LR: 3.04e-04 | 1769.4ms/step | 11.3 samples/s
|
| 293 |
+
Step 19380/44000 | Loss: 2.9958 | LR: 3.04e-04 | 1769.1ms/step | 11.3 samples/s
|
| 294 |
+
Step 19390/44000 | Loss: 2.9684 | LR: 3.04e-04 | 1770.7ms/step | 11.3 samples/s
|
| 295 |
+
Step 19400/44000 | Loss: 2.9573 | LR: 3.04e-04 | 1771.1ms/step | 11.3 samples/s
|
| 296 |
+
Step 19410/44000 | Loss: 2.9659 | LR: 3.03e-04 | 1771.0ms/step | 11.3 samples/s
|
| 297 |
+
Step 19420/44000 | Loss: 2.9652 | LR: 3.03e-04 | 1771.9ms/step | 11.3 samples/s
|
| 298 |
+
Step 19430/44000 | Loss: 2.9947 | LR: 3.03e-04 | 1771.3ms/step | 11.3 samples/s
|
| 299 |
+
Step 19440/44000 | Loss: 2.9503 | LR: 3.03e-04 | 1772.2ms/step | 11.3 samples/s
|
| 300 |
+
Step 19450/44000 | Loss: 2.9506 | LR: 3.03e-04 | 1770.6ms/step | 11.3 samples/s
|
| 301 |
+
Step 19460/44000 | Loss: 2.9538 | LR: 3.03e-04 | 1770.8ms/step | 11.3 samples/s
|
| 302 |
+
Step 19470/44000 | Loss: 2.9783 | LR: 3.02e-04 | 1770.5ms/step | 11.3 samples/s
|
| 303 |
+
Step 19480/44000 | Loss: 3.0068 | LR: 3.02e-04 | 1770.4ms/step | 11.3 samples/s
|
| 304 |
+
Step 19490/44000 | Loss: 2.9531 | LR: 3.02e-04 | 1769.5ms/step | 11.3 samples/s
|
| 305 |
+
Step 19500/44000 | Loss: 2.9721 | LR: 3.02e-04 | 1770.5ms/step | 11.3 samples/s
|
| 306 |
+
Step 19510/44000 | Loss: 2.9562 | LR: 3.02e-04 | 1773.0ms/step | 11.3 samples/s
|
| 307 |
+
Step 19520/44000 | Loss: 2.9827 | LR: 3.01e-04 | 1771.8ms/step | 11.3 samples/s
|
| 308 |
+
Step 19530/44000 | Loss: 2.9471 | LR: 3.01e-04 | 1772.0ms/step | 11.3 samples/s
|
| 309 |
+
Step 19540/44000 | Loss: 2.9769 | LR: 3.01e-04 | 1770.6ms/step | 11.3 samples/s
|
| 310 |
+
Step 19550/44000 | Loss: 2.9495 | LR: 3.01e-04 | 1771.2ms/step | 11.3 samples/s
|
| 311 |
+
Step 19560/44000 | Loss: 2.9559 | LR: 3.01e-04 | 1772.4ms/step | 11.3 samples/s
|
| 312 |
+
Step 19570/44000 | Loss: 2.9899 | LR: 3.01e-04 | 1771.6ms/step | 11.3 samples/s
|
| 313 |
+
Step 19580/44000 | Loss: 2.9869 | LR: 3.00e-04 | 1771.4ms/step | 11.3 samples/s
|
| 314 |
+
Step 19590/44000 | Loss: 2.9162 | LR: 3.00e-04 | 1771.4ms/step | 11.3 samples/s
|
| 315 |
+
Step 19600/44000 | Loss: 2.9708 | LR: 3.00e-04 | 1772.5ms/step | 11.3 samples/s
|
| 316 |
+
Step 19610/44000 | Loss: 2.9644 | LR: 3.00e-04 | 1771.6ms/step | 11.3 samples/s
|
| 317 |
+
Step 19620/44000 | Loss: 2.9560 | LR: 3.00e-04 | 1771.7ms/step | 11.3 samples/s
|
| 318 |
+
Step 19630/44000 | Loss: 2.9680 | LR: 3.00e-04 | 1770.0ms/step | 11.3 samples/s
|
| 319 |
+
Step 19640/44000 | Loss: 2.9741 | LR: 2.99e-04 | 1770.5ms/step | 11.3 samples/s
|
| 320 |
+
Step 19650/44000 | Loss: 2.9759 | LR: 2.99e-04 | 1769.5ms/step | 11.3 samples/s
|
| 321 |
+
Step 19660/44000 | Loss: 2.9623 | LR: 2.99e-04 | 1770.7ms/step | 11.3 samples/s
|
| 322 |
+
Step 19670/44000 | Loss: 2.9420 | LR: 2.99e-04 | 1770.9ms/step | 11.3 samples/s
|
| 323 |
+
Step 19680/44000 | Loss: 2.9692 | LR: 2.99e-04 | 1770.8ms/step | 11.3 samples/s
|
| 324 |
+
Step 19690/44000 | Loss: 2.9437 | LR: 2.98e-04 | 1769.9ms/step | 11.3 samples/s
|
| 325 |
+
Step 19700/44000 | Loss: 2.9444 | LR: 2.98e-04 | 1770.1ms/step | 11.3 samples/s
|
| 326 |
+
Step 19710/44000 | Loss: 2.9681 | LR: 2.98e-04 | 1771.0ms/step | 11.3 samples/s
|
| 327 |
+
Step 19720/44000 | Loss: 2.9803 | LR: 2.98e-04 | 1771.2ms/step | 11.3 samples/s
|
| 328 |
+
Step 19730/44000 | Loss: 2.9701 | LR: 2.98e-04 | 1770.4ms/step | 11.3 samples/s
|
| 329 |
+
Step 19740/44000 | Loss: 2.9696 | LR: 2.98e-04 | 1770.8ms/step | 11.3 samples/s
|
| 330 |
+
Step 19750/44000 | Loss: 2.9536 | LR: 2.97e-04 | 1771.4ms/step | 11.3 samples/s
|
| 331 |
+
Step 19760/44000 | Loss: 2.9481 | LR: 2.97e-04 | 1771.4ms/step | 11.3 samples/s
|
| 332 |
+
Step 19770/44000 | Loss: 2.9471 | LR: 2.97e-04 | 1770.4ms/step | 11.3 samples/s
|
| 333 |
+
Step 19780/44000 | Loss: 2.9547 | LR: 2.97e-04 | 1769.1ms/step | 11.3 samples/s
|
| 334 |
+
Step 19790/44000 | Loss: 2.9828 | LR: 2.97e-04 | 1769.1ms/step | 11.3 samples/s
|
| 335 |
+
Step 19800/44000 | Loss: 2.9386 | LR: 2.96e-04 | 1769.8ms/step | 11.3 samples/s
|
| 336 |
+
Step 19810/44000 | Loss: 2.9634 | LR: 2.96e-04 | 1771.0ms/step | 11.3 samples/s
|
| 337 |
+
Step 19820/44000 | Loss: 2.9613 | LR: 2.96e-04 | 1769.8ms/step | 11.3 samples/s
|
| 338 |
+
Step 19830/44000 | Loss: 3.0163 | LR: 2.96e-04 | 1770.3ms/step | 11.3 samples/s
|
| 339 |
+
Step 19840/44000 | Loss: 2.9451 | LR: 2.96e-04 | 1770.3ms/step | 11.3 samples/s
|
| 340 |
+
Step 19850/44000 | Loss: 2.9251 | LR: 2.96e-04 | 1772.6ms/step | 11.3 samples/s
|
| 341 |
+
Step 19860/44000 | Loss: 2.9628 | LR: 2.95e-04 | 1771.3ms/step | 11.3 samples/s
|
| 342 |
+
Step 19870/44000 | Loss: 2.9556 | LR: 2.95e-04 | 1771.5ms/step | 11.3 samples/s
|
| 343 |
+
Step 19880/44000 | Loss: 3.0099 | LR: 2.95e-04 | 1770.1ms/step | 11.3 samples/s
|
| 344 |
+
Step 19890/44000 | Loss: 2.9379 | LR: 2.95e-04 | 1776.5ms/step | 11.3 samples/s
|
| 345 |
+
Step 19900/44000 | Loss: 3.0089 | LR: 2.95e-04 | 1773.8ms/step | 11.3 samples/s
|
| 346 |
+
Step 19910/44000 | Loss: 2.9912 | LR: 2.95e-04 | 1770.9ms/step | 11.3 samples/s
|
| 347 |
+
Step 19920/44000 | Loss: 2.9984 | LR: 2.94e-04 | 1772.6ms/step | 11.3 samples/s
|
| 348 |
+
Step 19930/44000 | Loss: 2.9582 | LR: 2.94e-04 | 1771.5ms/step | 11.3 samples/s
|
| 349 |
+
Step 19940/44000 | Loss: 2.9479 | LR: 2.94e-04 | 1772.7ms/step | 11.3 samples/s
|
| 350 |
+
Step 19950/44000 | Loss: 2.9565 | LR: 2.94e-04 | 1772.1ms/step | 11.3 samples/s
|
| 351 |
+
Step 19960/44000 | Loss: 2.9890 | LR: 2.94e-04 | 1773.3ms/step | 11.3 samples/s
|
| 352 |
+
Step 19970/44000 | Loss: 2.9651 | LR: 2.93e-04 | 1772.6ms/step | 11.3 samples/s
|
| 353 |
+
Step 19980/44000 | Loss: 2.9537 | LR: 2.93e-04 | 1770.7ms/step | 11.3 samples/s
|
| 354 |
+
Step 19990/44000 | Loss: 2.9003 | LR: 2.93e-04 | 1770.5ms/step | 11.3 samples/s
|
| 355 |
+
Step 20000/44000 | Loss: 2.9248 | LR: 2.93e-04 | 1772.2ms/step | 11.3 samples/s
|
| 356 |
+
Validation at step 20000: Loss = 2.9722, Perplexity = 19.53
|
| 357 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-20000
|
| 358 |
+
Step 20010/44000 | Loss: 2.9816 | LR: 2.93e-04 | 3625.9ms/step | 5.5 samples/s
|
| 359 |
+
Step 20020/44000 | Loss: 2.9778 | LR: 2.93e-04 | 1770.1ms/step | 11.3 samples/s
|
| 360 |
+
Step 20030/44000 | Loss: 2.9212 | LR: 2.92e-04 | 1770.6ms/step | 11.3 samples/s
|
| 361 |
+
Step 20040/44000 | Loss: 2.9739 | LR: 2.92e-04 | 1770.1ms/step | 11.3 samples/s
|
| 362 |
+
Step 20050/44000 | Loss: 2.9567 | LR: 2.92e-04 | 1770.6ms/step | 11.3 samples/s
|
| 363 |
+
Step 20060/44000 | Loss: 2.9815 | LR: 2.92e-04 | 1769.9ms/step | 11.3 samples/s
|
| 364 |
+
Step 20070/44000 | Loss: 2.9755 | LR: 2.92e-04 | 1769.5ms/step | 11.3 samples/s
|
| 365 |
+
Step 20080/44000 | Loss: 2.9633 | LR: 2.91e-04 | 1770.1ms/step | 11.3 samples/s
|
| 366 |
+
Step 20090/44000 | Loss: 2.9590 | LR: 2.91e-04 | 1771.7ms/step | 11.3 samples/s
|
| 367 |
+
Step 20100/44000 | Loss: 2.9741 | LR: 2.91e-04 | 1769.4ms/step | 11.3 samples/s
|
| 368 |
+
Step 20110/44000 | Loss: 2.9806 | LR: 2.91e-04 | 1770.4ms/step | 11.3 samples/s
|
| 369 |
+
Step 20120/44000 | Loss: 2.9475 | LR: 2.91e-04 | 1770.5ms/step | 11.3 samples/s
|
| 370 |
+
Step 20130/44000 | Loss: 2.9405 | LR: 2.91e-04 | 1771.7ms/step | 11.3 samples/s
|
| 371 |
+
Step 20140/44000 | Loss: 2.9664 | LR: 2.90e-04 | 1772.1ms/step | 11.3 samples/s
|
| 372 |
+
Step 20150/44000 | Loss: 2.9486 | LR: 2.90e-04 | 1771.5ms/step | 11.3 samples/s
|
| 373 |
+
Step 20160/44000 | Loss: 3.0032 | LR: 2.90e-04 | 1772.0ms/step | 11.3 samples/s
|
| 374 |
+
Step 20170/44000 | Loss: 2.9958 | LR: 2.90e-04 | 1771.3ms/step | 11.3 samples/s
|
| 375 |
+
Step 20180/44000 | Loss: 2.9919 | LR: 2.90e-04 | 1768.5ms/step | 11.3 samples/s
|
| 376 |
+
Step 20190/44000 | Loss: 2.9245 | LR: 2.90e-04 | 1768.7ms/step | 11.3 samples/s
|
| 377 |
+
Step 20200/44000 | Loss: 2.9556 | LR: 2.89e-04 | 1769.2ms/step | 11.3 samples/s
|
| 378 |
+
Step 20210/44000 | Loss: 2.9703 | LR: 2.89e-04 | 1769.3ms/step | 11.3 samples/s
|
| 379 |
+
Step 20220/44000 | Loss: 2.9727 | LR: 2.89e-04 | 1772.1ms/step | 11.3 samples/s
|
| 380 |
+
Step 20230/44000 | Loss: 2.9500 | LR: 2.89e-04 | 1771.0ms/step | 11.3 samples/s
|
| 381 |
+
Step 20240/44000 | Loss: 2.9612 | LR: 2.89e-04 | 1771.3ms/step | 11.3 samples/s
|
| 382 |
+
Step 20250/44000 | Loss: 2.9833 | LR: 2.88e-04 | 1771.6ms/step | 11.3 samples/s
|
| 383 |
+
Step 20260/44000 | Loss: 3.0002 | LR: 2.88e-04 | 1770.6ms/step | 11.3 samples/s
|
| 384 |
+
Step 20270/44000 | Loss: 2.9567 | LR: 2.88e-04 | 1770.1ms/step | 11.3 samples/s
|
| 385 |
+
Step 20280/44000 | Loss: 2.9705 | LR: 2.88e-04 | 1770.6ms/step | 11.3 samples/s
|
| 386 |
+
Step 20290/44000 | Loss: 2.9384 | LR: 2.88e-04 | 1770.4ms/step | 11.3 samples/s
|
| 387 |
+
Step 20300/44000 | Loss: 2.9583 | LR: 2.88e-04 | 1770.1ms/step | 11.3 samples/s
|
| 388 |
+
Step 20310/44000 | Loss: 2.9347 | LR: 2.87e-04 | 1770.4ms/step | 11.3 samples/s
|
| 389 |
+
Step 20320/44000 | Loss: 2.9550 | LR: 2.87e-04 | 1769.8ms/step | 11.3 samples/s
|
| 390 |
+
Step 20330/44000 | Loss: 2.9434 | LR: 2.87e-04 | 1770.9ms/step | 11.3 samples/s
|
| 391 |
+
Step 20340/44000 | Loss: 2.9719 | LR: 2.87e-04 | 1771.2ms/step | 11.3 samples/s
|
| 392 |
+
Step 20350/44000 | Loss: 2.9250 | LR: 2.87e-04 | 1772.5ms/step | 11.3 samples/s
|
| 393 |
+
Step 20360/44000 | Loss: 2.9488 | LR: 2.86e-04 | 1771.2ms/step | 11.3 samples/s
|
| 394 |
+
Step 20370/44000 | Loss: 2.9819 | LR: 2.86e-04 | 1771.0ms/step | 11.3 samples/s
|
| 395 |
+
Step 20380/44000 | Loss: 2.9286 | LR: 2.86e-04 | 1769.5ms/step | 11.3 samples/s
|
| 396 |
+
Step 20390/44000 | Loss: 3.0053 | LR: 2.86e-04 | 1771.1ms/step | 11.3 samples/s
|
| 397 |
+
Step 20400/44000 | Loss: 2.9318 | LR: 2.86e-04 | 1769.1ms/step | 11.3 samples/s
|
| 398 |
+
Step 20410/44000 | Loss: 2.9306 | LR: 2.86e-04 | 1768.1ms/step | 11.3 samples/s
|
| 399 |
+
Step 20420/44000 | Loss: 2.9649 | LR: 2.85e-04 | 1767.8ms/step | 11.3 samples/s
|
| 400 |
+
Step 20430/44000 | Loss: 2.9844 | LR: 2.85e-04 | 1769.2ms/step | 11.3 samples/s
|
| 401 |
+
Step 20440/44000 | Loss: 2.9624 | LR: 2.85e-04 | 1769.8ms/step | 11.3 samples/s
|
| 402 |
+
Step 20450/44000 | Loss: 3.0124 | LR: 2.85e-04 | 1769.3ms/step | 11.3 samples/s
|
| 403 |
+
Step 20460/44000 | Loss: 3.0169 | LR: 2.85e-04 | 1769.4ms/step | 11.3 samples/s
|
| 404 |
+
Step 20470/44000 | Loss: 2.9604 | LR: 2.84e-04 | 1770.5ms/step | 11.3 samples/s
|
| 405 |
+
Step 20480/44000 | Loss: 2.9868 | LR: 2.84e-04 | 1771.8ms/step | 11.3 samples/s
|
| 406 |
+
Step 20490/44000 | Loss: 2.9644 | LR: 2.84e-04 | 1770.6ms/step | 11.3 samples/s
|
| 407 |
+
Step 20500/44000 | Loss: 2.9266 | LR: 2.84e-04 | 1771.9ms/step | 11.3 samples/s
|
| 408 |
+
Step 20510/44000 | Loss: 2.9403 | LR: 2.84e-04 | 1771.8ms/step | 11.3 samples/s
|
| 409 |
+
Step 20520/44000 | Loss: 2.9423 | LR: 2.84e-04 | 1771.4ms/step | 11.3 samples/s
|
| 410 |
+
Step 20530/44000 | Loss: 2.9933 | LR: 2.83e-04 | 1770.2ms/step | 11.3 samples/s
|
| 411 |
+
Step 20540/44000 | Loss: 2.9370 | LR: 2.83e-04 | 1770.2ms/step | 11.3 samples/s
|
| 412 |
+
Step 20550/44000 | Loss: 2.9665 | LR: 2.83e-04 | 1770.3ms/step | 11.3 samples/s
|
| 413 |
+
Step 20560/44000 | Loss: 2.9544 | LR: 2.83e-04 | 1770.3ms/step | 11.3 samples/s
|
| 414 |
+
Step 20570/44000 | Loss: 2.9111 | LR: 2.83e-04 | 1772.4ms/step | 11.3 samples/s
|
| 415 |
+
Step 20580/44000 | Loss: 2.9822 | LR: 2.83e-04 | 1771.8ms/step | 11.3 samples/s
|
| 416 |
+
Step 20590/44000 | Loss: 2.9426 | LR: 2.82e-04 | 1770.4ms/step | 11.3 samples/s
|
| 417 |
+
Step 20600/44000 | Loss: 2.9467 | LR: 2.82e-04 | 1771.4ms/step | 11.3 samples/s
|
| 418 |
+
Step 20610/44000 | Loss: 2.9570 | LR: 2.82e-04 | 1769.2ms/step | 11.3 samples/s
|
| 419 |
+
Step 20620/44000 | Loss: 2.9540 | LR: 2.82e-04 | 1770.1ms/step | 11.3 samples/s
|
| 420 |
+
Step 20630/44000 | Loss: 2.9329 | LR: 2.82e-04 | 1769.8ms/step | 11.3 samples/s
|
| 421 |
+
Step 20640/44000 | Loss: 2.9869 | LR: 2.81e-04 | 1769.6ms/step | 11.3 samples/s
|
| 422 |
+
Step 20650/44000 | Loss: 2.9165 | LR: 2.81e-04 | 1770.8ms/step | 11.3 samples/s
|
| 423 |
+
Step 20660/44000 | Loss: 2.9320 | LR: 2.81e-04 | 1770.4ms/step | 11.3 samples/s
|
| 424 |
+
Step 20670/44000 | Loss: 2.9362 | LR: 2.81e-04 | 1771.0ms/step | 11.3 samples/s
|
| 425 |
+
Step 20680/44000 | Loss: 2.9663 | LR: 2.81e-04 | 1770.9ms/step | 11.3 samples/s
|
| 426 |
+
Step 20690/44000 | Loss: 2.8882 | LR: 2.81e-04 | 1770.5ms/step | 11.3 samples/s
|
| 427 |
+
Step 20700/44000 | Loss: 2.9963 | LR: 2.80e-04 | 1769.9ms/step | 11.3 samples/s
|
| 428 |
+
Step 20710/44000 | Loss: 2.9678 | LR: 2.80e-04 | 1771.0ms/step | 11.3 samples/s
|
| 429 |
+
Step 20720/44000 | Loss: 2.9527 | LR: 2.80e-04 | 1771.0ms/step | 11.3 samples/s
|
| 430 |
+
Step 20730/44000 | Loss: 2.9973 | LR: 2.80e-04 | 1771.2ms/step | 11.3 samples/s
|
| 431 |
+
Step 20740/44000 | Loss: 3.0062 | LR: 2.80e-04 | 1770.5ms/step | 11.3 samples/s
|
| 432 |
+
Step 20750/44000 | Loss: 2.9608 | LR: 2.79e-04 | 1772.0ms/step | 11.3 samples/s
|
| 433 |
+
Step 20760/44000 | Loss: 2.9559 | LR: 2.79e-04 | 1771.7ms/step | 11.3 samples/s
|
| 434 |
+
Step 20770/44000 | Loss: 2.9498 | LR: 2.79e-04 | 1772.1ms/step | 11.3 samples/s
|
| 435 |
+
Step 20780/44000 | Loss: 2.9675 | LR: 2.79e-04 | 1771.2ms/step | 11.3 samples/s
|
| 436 |
+
Step 20790/44000 | Loss: 2.9515 | LR: 2.79e-04 | 1769.0ms/step | 11.3 samples/s
|
| 437 |
+
Step 20800/44000 | Loss: 2.9370 | LR: 2.79e-04 | 1769.6ms/step | 11.3 samples/s
|
| 438 |
+
Step 20810/44000 | Loss: 2.9494 | LR: 2.78e-04 | 1770.3ms/step | 11.3 samples/s
|
| 439 |
+
Step 20820/44000 | Loss: 2.9558 | LR: 2.78e-04 | 1770.0ms/step | 11.3 samples/s
|
| 440 |
+
Step 20830/44000 | Loss: 2.9733 | LR: 2.78e-04 | 1770.6ms/step | 11.3 samples/s
|
| 441 |
+
Step 20840/44000 | Loss: 2.9305 | LR: 2.78e-04 | 1770.8ms/step | 11.3 samples/s
|
| 442 |
+
Step 20850/44000 | Loss: 2.9401 | LR: 2.78e-04 | 1773.0ms/step | 11.3 samples/s
|
| 443 |
+
Step 20860/44000 | Loss: 2.9898 | LR: 2.77e-04 | 1770.0ms/step | 11.3 samples/s
|
| 444 |
+
Step 20870/44000 | Loss: 2.9470 | LR: 2.77e-04 | 1771.4ms/step | 11.3 samples/s
|
| 445 |
+
Step 20880/44000 | Loss: 2.9657 | LR: 2.77e-04 | 1772.3ms/step | 11.3 samples/s
|
| 446 |
+
Step 20890/44000 | Loss: 2.9494 | LR: 2.77e-04 | 1770.9ms/step | 11.3 samples/s
|
| 447 |
+
Step 20900/44000 | Loss: 2.9441 | LR: 2.77e-04 | 1771.1ms/step | 11.3 samples/s
|
| 448 |
+
Step 20910/44000 | Loss: 2.9343 | LR: 2.77e-04 | 1771.2ms/step | 11.3 samples/s
|
| 449 |
+
Step 20920/44000 | Loss: 2.9755 | LR: 2.76e-04 | 1771.0ms/step | 11.3 samples/s
|
| 450 |
+
Step 20930/44000 | Loss: 2.9439 | LR: 2.76e-04 | 1771.4ms/step | 11.3 samples/s
|
| 451 |
+
Step 20940/44000 | Loss: 2.9537 | LR: 2.76e-04 | 1770.7ms/step | 11.3 samples/s
|
| 452 |
+
Step 20950/44000 | Loss: 2.9685 | LR: 2.76e-04 | 1772.1ms/step | 11.3 samples/s
|
| 453 |
+
Step 20960/44000 | Loss: 2.9687 | LR: 2.76e-04 | 1772.2ms/step | 11.3 samples/s
|
| 454 |
+
Step 20970/44000 | Loss: 2.9550 | LR: 2.75e-04 | 1772.4ms/step | 11.3 samples/s
|
| 455 |
+
Step 20980/44000 | Loss: 2.9290 | LR: 2.75e-04 | 1773.5ms/step | 11.3 samples/s
|
| 456 |
+
Step 20990/44000 | Loss: 2.9885 | LR: 2.75e-04 | 1770.9ms/step | 11.3 samples/s
|
| 457 |
+
Step 21000/44000 | Loss: 2.9418 | LR: 2.75e-04 | 1769.5ms/step | 11.3 samples/s
|
| 458 |
+
Validation at step 21000: Loss = 2.9617, Perplexity = 19.33
|
| 459 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-21000
|
| 460 |
+
Step 21010/44000 | Loss: 2.9727 | LR: 2.75e-04 | 3547.8ms/step | 5.6 samples/s
|
| 461 |
+
Step 21020/44000 | Loss: 2.9407 | LR: 2.75e-04 | 1769.0ms/step | 11.3 samples/s
|
| 462 |
+
Step 21030/44000 | Loss: 2.9414 | LR: 2.74e-04 | 1770.3ms/step | 11.3 samples/s
|
| 463 |
+
Step 21040/44000 | Loss: 2.9263 | LR: 2.74e-04 | 1771.0ms/step | 11.3 samples/s
|
| 464 |
+
Step 21050/44000 | Loss: 2.9586 | LR: 2.74e-04 | 1770.9ms/step | 11.3 samples/s
|
| 465 |
+
Step 21060/44000 | Loss: 2.9516 | LR: 2.74e-04 | 1769.1ms/step | 11.3 samples/s
|
| 466 |
+
Step 21070/44000 | Loss: 2.8871 | LR: 2.74e-04 | 1769.7ms/step | 11.3 samples/s
|
| 467 |
+
Step 21080/44000 | Loss: 2.9939 | LR: 2.73e-04 | 1770.1ms/step | 11.3 samples/s
|
| 468 |
+
Step 21090/44000 | Loss: 2.9073 | LR: 2.73e-04 | 1770.5ms/step | 11.3 samples/s
|
| 469 |
+
Step 21100/44000 | Loss: 2.9544 | LR: 2.73e-04 | 1771.9ms/step | 11.3 samples/s
|
| 470 |
+
Step 21110/44000 | Loss: 2.9470 | LR: 2.73e-04 | 1771.6ms/step | 11.3 samples/s
|
| 471 |
+
Step 21120/44000 | Loss: 2.9368 | LR: 2.73e-04 | 1769.9ms/step | 11.3 samples/s
|
| 472 |
+
Step 21130/44000 | Loss: 2.9874 | LR: 2.73e-04 | 1770.6ms/step | 11.3 samples/s
|
| 473 |
+
Step 21140/44000 | Loss: 2.8958 | LR: 2.72e-04 | 1771.4ms/step | 11.3 samples/s
|
| 474 |
+
Step 21150/44000 | Loss: 2.9277 | LR: 2.72e-04 | 1770.2ms/step | 11.3 samples/s
|
| 475 |
+
Step 21160/44000 | Loss: 2.9177 | LR: 2.72e-04 | 1772.2ms/step | 11.3 samples/s
|
| 476 |
+
Step 21170/44000 | Loss: 2.9782 | LR: 2.72e-04 | 1772.7ms/step | 11.3 samples/s
|
| 477 |
+
Step 21180/44000 | Loss: 2.9245 | LR: 2.72e-04 | 1773.4ms/step | 11.3 samples/s
|
| 478 |
+
Step 21190/44000 | Loss: 2.9490 | LR: 2.71e-04 | 1769.9ms/step | 11.3 samples/s
|
| 479 |
+
Step 21200/44000 | Loss: 2.9321 | LR: 2.71e-04 | 1770.4ms/step | 11.3 samples/s
|
| 480 |
+
Step 21210/44000 | Loss: 2.9134 | LR: 2.71e-04 | 1770.8ms/step | 11.3 samples/s
|
| 481 |
+
Step 21220/44000 | Loss: 2.9325 | LR: 2.71e-04 | 1771.3ms/step | 11.3 samples/s
|
| 482 |
+
Step 21230/44000 | Loss: 2.9484 | LR: 2.71e-04 | 1772.9ms/step | 11.3 samples/s
|
| 483 |
+
Step 21240/44000 | Loss: 2.9883 | LR: 2.71e-04 | 1771.5ms/step | 11.3 samples/s
|
| 484 |
+
Step 21250/44000 | Loss: 2.9579 | LR: 2.70e-04 | 1771.2ms/step | 11.3 samples/s
|
| 485 |
+
Step 21260/44000 | Loss: 2.9837 | LR: 2.70e-04 | 1770.7ms/step | 11.3 samples/s
|
| 486 |
+
Step 21270/44000 | Loss: 2.9694 | LR: 2.70e-04 | 1770.3ms/step | 11.3 samples/s
|
| 487 |
+
Step 21280/44000 | Loss: 2.9666 | LR: 2.70e-04 | 1771.0ms/step | 11.3 samples/s
|
| 488 |
+
Step 21290/44000 | Loss: 2.9292 | LR: 2.70e-04 | 1771.3ms/step | 11.3 samples/s
|
| 489 |
+
Step 21300/44000 | Loss: 2.9695 | LR: 2.70e-04 | 1771.5ms/step | 11.3 samples/s
|
| 490 |
+
Step 21310/44000 | Loss: 2.9305 | LR: 2.69e-04 | 1771.9ms/step | 11.3 samples/s
|
| 491 |
+
Step 21320/44000 | Loss: 2.9512 | LR: 2.69e-04 | 1772.1ms/step | 11.3 samples/s
|
| 492 |
+
Step 21330/44000 | Loss: 2.9623 | LR: 2.69e-04 | 1772.0ms/step | 11.3 samples/s
|
| 493 |
+
Step 21340/44000 | Loss: 2.9600 | LR: 2.69e-04 | 1772.6ms/step | 11.3 samples/s
|
| 494 |
+
Step 21350/44000 | Loss: 2.9760 | LR: 2.69e-04 | 1773.5ms/step | 11.3 samples/s
|
| 495 |
+
Step 21360/44000 | Loss: 2.9180 | LR: 2.68e-04 | 1771.9ms/step | 11.3 samples/s
|
| 496 |
+
Step 21370/44000 | Loss: 2.9145 | LR: 2.68e-04 | 1772.0ms/step | 11.3 samples/s
|
| 497 |
+
Step 21380/44000 | Loss: 2.9765 | LR: 2.68e-04 | 1772.4ms/step | 11.3 samples/s
|
| 498 |
+
Step 21390/44000 | Loss: 2.9132 | LR: 2.68e-04 | 1771.6ms/step | 11.3 samples/s
|
| 499 |
+
Step 21400/44000 | Loss: 2.9095 | LR: 2.68e-04 | 1769.1ms/step | 11.3 samples/s
|
| 500 |
+
Step 21410/44000 | Loss: 2.9186 | LR: 2.68e-04 | 1768.8ms/step | 11.3 samples/s
|
| 501 |
+
Step 21420/44000 | Loss: 2.9608 | LR: 2.67e-04 | 1769.8ms/step | 11.3 samples/s
|
| 502 |
+
Step 21430/44000 | Loss: 2.9342 | LR: 2.67e-04 | 1770.2ms/step | 11.3 samples/s
|
| 503 |
+
Step 21440/44000 | Loss: 2.9320 | LR: 2.67e-04 | 1770.1ms/step | 11.3 samples/s
|
| 504 |
+
Step 21450/44000 | Loss: 2.9778 | LR: 2.67e-04 | 1769.6ms/step | 11.3 samples/s
|
| 505 |
+
Step 21460/44000 | Loss: 2.9842 | LR: 2.67e-04 | 1769.1ms/step | 11.3 samples/s
|
| 506 |
+
Step 21470/44000 | Loss: 2.9218 | LR: 2.66e-04 | 1769.9ms/step | 11.3 samples/s
|
| 507 |
+
Step 21480/44000 | Loss: 2.9486 | LR: 2.66e-04 | 1769.7ms/step | 11.3 samples/s
|
| 508 |
+
Step 21490/44000 | Loss: 2.9668 | LR: 2.66e-04 | 1769.3ms/step | 11.3 samples/s
|
| 509 |
+
Step 21500/44000 | Loss: 2.9292 | LR: 2.66e-04 | 1770.5ms/step | 11.3 samples/s
|
| 510 |
+
Step 21510/44000 | Loss: 2.9646 | LR: 2.66e-04 | 1770.9ms/step | 11.3 samples/s
|
| 511 |
+
Step 21520/44000 | Loss: 2.9676 | LR: 2.66e-04 | 1770.2ms/step | 11.3 samples/s
|
| 512 |
+
Step 21530/44000 | Loss: 2.9028 | LR: 2.65e-04 | 1771.7ms/step | 11.3 samples/s
|
| 513 |
+
Step 21540/44000 | Loss: 2.9628 | LR: 2.65e-04 | 1771.6ms/step | 11.3 samples/s
|
| 514 |
+
Step 21550/44000 | Loss: 2.9413 | LR: 2.65e-04 | 1771.8ms/step | 11.3 samples/s
|
| 515 |
+
Step 21560/44000 | Loss: 2.9478 | LR: 2.65e-04 | 1771.2ms/step | 11.3 samples/s
|
| 516 |
+
Step 21570/44000 | Loss: 2.9693 | LR: 2.65e-04 | 1771.1ms/step | 11.3 samples/s
|
| 517 |
+
Step 21580/44000 | Loss: 2.9467 | LR: 2.64e-04 | 1772.8ms/step | 11.3 samples/s
|
| 518 |
+
Step 21590/44000 | Loss: 2.9269 | LR: 2.64e-04 | 1773.4ms/step | 11.3 samples/s
|
| 519 |
+
Step 21600/44000 | Loss: 2.9671 | LR: 2.64e-04 | 1771.1ms/step | 11.3 samples/s
|
| 520 |
+
Step 21610/44000 | Loss: 2.9265 | LR: 2.64e-04 | 1771.8ms/step | 11.3 samples/s
|
| 521 |
+
Step 21620/44000 | Loss: 2.9045 | LR: 2.64e-04 | 1770.5ms/step | 11.3 samples/s
|
| 522 |
+
Step 21630/44000 | Loss: 2.9044 | LR: 2.64e-04 | 1771.2ms/step | 11.3 samples/s
|
| 523 |
+
Step 21640/44000 | Loss: 2.9569 | LR: 2.63e-04 | 1771.3ms/step | 11.3 samples/s
|
| 524 |
+
Step 21650/44000 | Loss: 2.9823 | LR: 2.63e-04 | 1773.1ms/step | 11.3 samples/s
|
| 525 |
+
Step 21660/44000 | Loss: 2.9459 | LR: 2.63e-04 | 1773.0ms/step | 11.3 samples/s
|
| 526 |
+
Step 21670/44000 | Loss: 2.9741 | LR: 2.63e-04 | 1772.1ms/step | 11.3 samples/s
|
| 527 |
+
Step 21680/44000 | Loss: 2.9790 | LR: 2.63e-04 | 1770.7ms/step | 11.3 samples/s
|
| 528 |
+
Step 21690/44000 | Loss: 2.9701 | LR: 2.62e-04 | 1771.1ms/step | 11.3 samples/s
|
| 529 |
+
Step 21700/44000 | Loss: 2.9565 | LR: 2.62e-04 | 1771.6ms/step | 11.3 samples/s
|
| 530 |
+
Step 21710/44000 | Loss: 2.9762 | LR: 2.62e-04 | 1770.4ms/step | 11.3 samples/s
|
| 531 |
+
Step 21720/44000 | Loss: 2.9282 | LR: 2.62e-04 | 1770.9ms/step | 11.3 samples/s
|
| 532 |
+
Step 21730/44000 | Loss: 2.9703 | LR: 2.62e-04 | 1772.2ms/step | 11.3 samples/s
|
| 533 |
+
Step 21740/44000 | Loss: 2.9507 | LR: 2.62e-04 | 1771.4ms/step | 11.3 samples/s
|
| 534 |
+
Step 21750/44000 | Loss: 2.9642 | LR: 2.61e-04 | 1770.8ms/step | 11.3 samples/s
|
| 535 |
+
Step 21760/44000 | Loss: 2.9588 | LR: 2.61e-04 | 1772.7ms/step | 11.3 samples/s
|
| 536 |
+
Step 21770/44000 | Loss: 2.8906 | LR: 2.61e-04 | 1773.6ms/step | 11.3 samples/s
|
| 537 |
+
Step 21780/44000 | Loss: 2.9805 | LR: 2.61e-04 | 1772.7ms/step | 11.3 samples/s
|
| 538 |
+
Step 21790/44000 | Loss: 2.8840 | LR: 2.61e-04 | 1773.2ms/step | 11.3 samples/s
|
| 539 |
+
Step 21800/44000 | Loss: 2.9755 | LR: 2.60e-04 | 1772.4ms/step | 11.3 samples/s
|
| 540 |
+
Step 21810/44000 | Loss: 2.9098 | LR: 2.60e-04 | 1770.4ms/step | 11.3 samples/s
|
| 541 |
+
Step 21820/44000 | Loss: 2.9524 | LR: 2.60e-04 | 1771.0ms/step | 11.3 samples/s
|
| 542 |
+
Step 21830/44000 | Loss: 2.9297 | LR: 2.60e-04 | 1770.7ms/step | 11.3 samples/s
|
| 543 |
+
Step 21840/44000 | Loss: 2.9537 | LR: 2.60e-04 | 1769.3ms/step | 11.3 samples/s
|
| 544 |
+
Step 21850/44000 | Loss: 2.9244 | LR: 2.60e-04 | 1768.5ms/step | 11.3 samples/s
|
| 545 |
+
Step 21860/44000 | Loss: 2.9163 | LR: 2.59e-04 | 1771.6ms/step | 11.3 samples/s
|
| 546 |
+
Step 21870/44000 | Loss: 2.9120 | LR: 2.59e-04 | 1771.2ms/step | 11.3 samples/s
|
| 547 |
+
Step 21880/44000 | Loss: 2.9423 | LR: 2.59e-04 | 1771.1ms/step | 11.3 samples/s
|
| 548 |
+
Step 21890/44000 | Loss: 2.9102 | LR: 2.59e-04 | 1773.4ms/step | 11.3 samples/s
|
| 549 |
+
Step 21900/44000 | Loss: 2.9573 | LR: 2.59e-04 | 1774.7ms/step | 11.3 samples/s
|
| 550 |
+
Step 21910/44000 | Loss: 2.9441 | LR: 2.58e-04 | 1772.7ms/step | 11.3 samples/s
|
| 551 |
+
Step 21920/44000 | Loss: 2.9309 | LR: 2.58e-04 | 1770.5ms/step | 11.3 samples/s
|
| 552 |
+
Step 21930/44000 | Loss: 2.9375 | LR: 2.58e-04 | 1770.8ms/step | 11.3 samples/s
|
| 553 |
+
Step 21940/44000 | Loss: 2.9143 | LR: 2.58e-04 | 1771.7ms/step | 11.3 samples/s
|
| 554 |
+
Step 21950/44000 | Loss: 2.9301 | LR: 2.58e-04 | 1771.5ms/step | 11.3 samples/s
|
| 555 |
+
Step 21960/44000 | Loss: 2.9244 | LR: 2.58e-04 | 1772.3ms/step | 11.3 samples/s
|
| 556 |
+
Step 21970/44000 | Loss: 2.9328 | LR: 2.57e-04 | 1772.3ms/step | 11.3 samples/s
|
| 557 |
+
Step 21980/44000 | Loss: 2.9116 | LR: 2.57e-04 | 1772.6ms/step | 11.3 samples/s
|
| 558 |
+
Step 21990/44000 | Loss: 2.9667 | LR: 2.57e-04 | 1773.9ms/step | 11.3 samples/s
|
| 559 |
+
Step 22000/44000 | Loss: 2.9800 | LR: 2.57e-04 | 1772.9ms/step | 11.3 samples/s
|
| 560 |
+
Validation at step 22000: Loss = 2.9528, Perplexity = 19.16
|
| 561 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-22000
|
| 562 |
+
Step 22010/44000 | Loss: 2.9338 | LR: 2.57e-04 | 3560.1ms/step | 5.6 samples/s
|
| 563 |
+
Step 22020/44000 | Loss: 2.8921 | LR: 2.56e-04 | 1771.7ms/step | 11.3 samples/s
|
| 564 |
+
Step 22030/44000 | Loss: 2.9259 | LR: 2.56e-04 | 1772.4ms/step | 11.3 samples/s
|
| 565 |
+
Step 22040/44000 | Loss: 2.9486 | LR: 2.56e-04 | 1772.0ms/step | 11.3 samples/s
|
| 566 |
+
Step 22050/44000 | Loss: 2.9976 | LR: 2.56e-04 | 1771.1ms/step | 11.3 samples/s
|
| 567 |
+
Step 22060/44000 | Loss: 2.9513 | LR: 2.56e-04 | 1771.6ms/step | 11.3 samples/s
|
| 568 |
+
Step 22070/44000 | Loss: 2.9246 | LR: 2.56e-04 | 1772.0ms/step | 11.3 samples/s
|
| 569 |
+
Step 22080/44000 | Loss: 2.9267 | LR: 2.55e-04 | 1773.3ms/step | 11.3 samples/s
|
| 570 |
+
Step 22090/44000 | Loss: 2.9569 | LR: 2.55e-04 | 1772.1ms/step | 11.3 samples/s
|
| 571 |
+
Step 22100/44000 | Loss: 2.9295 | LR: 2.55e-04 | 1774.0ms/step | 11.3 samples/s
|
| 572 |
+
Step 22110/44000 | Loss: 2.9712 | LR: 2.55e-04 | 1773.3ms/step | 11.3 samples/s
|
| 573 |
+
Step 22120/44000 | Loss: 2.9615 | LR: 2.55e-04 | 1772.8ms/step | 11.3 samples/s
|
| 574 |
+
Step 22130/44000 | Loss: 2.9275 | LR: 2.54e-04 | 1771.5ms/step | 11.3 samples/s
|
| 575 |
+
Step 22140/44000 | Loss: 2.9286 | LR: 2.54e-04 | 1771.4ms/step | 11.3 samples/s
|
| 576 |
+
Step 22150/44000 | Loss: 2.9220 | LR: 2.54e-04 | 1771.9ms/step | 11.3 samples/s
|
| 577 |
+
Step 22160/44000 | Loss: 2.9725 | LR: 2.54e-04 | 1772.6ms/step | 11.3 samples/s
|
| 578 |
+
Step 22170/44000 | Loss: 3.0177 | LR: 2.54e-04 | 1773.2ms/step | 11.3 samples/s
|
| 579 |
+
Step 22180/44000 | Loss: 2.9142 | LR: 2.54e-04 | 1773.9ms/step | 11.3 samples/s
|
| 580 |
+
Step 22190/44000 | Loss: 2.9659 | LR: 2.53e-04 | 1773.6ms/step | 11.3 samples/s
|
| 581 |
+
Step 22200/44000 | Loss: 2.9481 | LR: 2.53e-04 | 1775.0ms/step | 11.3 samples/s
|
| 582 |
+
Step 22210/44000 | Loss: 2.9754 | LR: 2.53e-04 | 1771.7ms/step | 11.3 samples/s
|
| 583 |
+
Step 22220/44000 | Loss: 2.8976 | LR: 2.53e-04 | 1772.3ms/step | 11.3 samples/s
|
| 584 |
+
Step 22230/44000 | Loss: 2.9833 | LR: 2.53e-04 | 1773.7ms/step | 11.3 samples/s
|
| 585 |
+
Step 22240/44000 | Loss: 2.9172 | LR: 2.52e-04 | 1772.4ms/step | 11.3 samples/s
|
| 586 |
+
Step 22250/44000 | Loss: 2.9282 | LR: 2.52e-04 | 1772.6ms/step | 11.3 samples/s
|
| 587 |
+
Step 22260/44000 | Loss: 2.9525 | LR: 2.52e-04 | 1772.5ms/step | 11.3 samples/s
|
| 588 |
+
Step 22270/44000 | Loss: 2.9295 | LR: 2.52e-04 | 1772.0ms/step | 11.3 samples/s
|
| 589 |
+
Step 22280/44000 | Loss: 2.9373 | LR: 2.52e-04 | 1772.4ms/step | 11.3 samples/s
|
| 590 |
+
Step 22290/44000 | Loss: 2.8460 | LR: 2.52e-04 | 1772.9ms/step | 11.3 samples/s
|
| 591 |
+
Step 22300/44000 | Loss: 2.8991 | LR: 2.51e-04 | 1770.6ms/step | 11.3 samples/s
|
| 592 |
+
Step 22310/44000 | Loss: 2.9836 | LR: 2.51e-04 | 1771.5ms/step | 11.3 samples/s
|
| 593 |
+
Step 22320/44000 | Loss: 2.9604 | LR: 2.51e-04 | 1772.5ms/step | 11.3 samples/s
|
| 594 |
+
Step 22330/44000 | Loss: 2.8921 | LR: 2.51e-04 | 1771.1ms/step | 11.3 samples/s
|
| 595 |
+
Step 22340/44000 | Loss: 2.9545 | LR: 2.51e-04 | 1770.8ms/step | 11.3 samples/s
|
| 596 |
+
Step 22350/44000 | Loss: 2.9822 | LR: 2.50e-04 | 1771.8ms/step | 11.3 samples/s
|
| 597 |
+
Step 22360/44000 | Loss: 2.9276 | LR: 2.50e-04 | 1771.7ms/step | 11.3 samples/s
|
| 598 |
+
Step 22370/44000 | Loss: 2.9170 | LR: 2.50e-04 | 1772.4ms/step | 11.3 samples/s
|
| 599 |
+
Step 22380/44000 | Loss: 2.9609 | LR: 2.50e-04 | 1773.5ms/step | 11.3 samples/s
|
| 600 |
+
Step 22390/44000 | Loss: 2.9126 | LR: 2.50e-04 | 1773.8ms/step | 11.3 samples/s
|
| 601 |
+
Step 22400/44000 | Loss: 2.9267 | LR: 2.50e-04 | 1773.5ms/step | 11.3 samples/s
|
| 602 |
+
Step 22410/44000 | Loss: 2.9545 | LR: 2.49e-04 | 1773.4ms/step | 11.3 samples/s
|
| 603 |
+
Step 22420/44000 | Loss: 2.9243 | LR: 2.49e-04 | 1771.0ms/step | 11.3 samples/s
|
| 604 |
+
Step 22430/44000 | Loss: 2.9332 | LR: 2.49e-04 | 1771.4ms/step | 11.3 samples/s
|
| 605 |
+
Step 22440/44000 | Loss: 2.9683 | LR: 2.49e-04 | 1771.3ms/step | 11.3 samples/s
|
| 606 |
+
Step 22450/44000 | Loss: 2.9830 | LR: 2.49e-04 | 1770.6ms/step | 11.3 samples/s
|
| 607 |
+
Step 22460/44000 | Loss: 2.8939 | LR: 2.48e-04 | 1770.3ms/step | 11.3 samples/s
|
| 608 |
+
Step 22470/44000 | Loss: 2.9308 | LR: 2.48e-04 | 1770.8ms/step | 11.3 samples/s
|
| 609 |
+
Step 22480/44000 | Loss: 2.9259 | LR: 2.48e-04 | 1771.5ms/step | 11.3 samples/s
|
| 610 |
+
Step 22490/44000 | Loss: 2.9606 | LR: 2.48e-04 | 1774.1ms/step | 11.3 samples/s
|
| 611 |
+
Step 22500/44000 | Loss: 2.9183 | LR: 2.48e-04 | 1773.6ms/step | 11.3 samples/s
|
| 612 |
+
Step 22510/44000 | Loss: 2.9498 | LR: 2.48e-04 | 1771.7ms/step | 11.3 samples/s
|
| 613 |
+
Step 22520/44000 | Loss: 2.9181 | LR: 2.47e-04 | 1773.5ms/step | 11.3 samples/s
|
| 614 |
+
Step 22530/44000 | Loss: 2.9146 | LR: 2.47e-04 | 1772.8ms/step | 11.3 samples/s
|
| 615 |
+
Step 22540/44000 | Loss: 2.9389 | LR: 2.47e-04 | 1773.3ms/step | 11.3 samples/s
|
| 616 |
+
Step 22550/44000 | Loss: 2.9228 | LR: 2.47e-04 | 1773.0ms/step | 11.3 samples/s
|
| 617 |
+
Step 22560/44000 | Loss: 2.9578 | LR: 2.47e-04 | 1772.6ms/step | 11.3 samples/s
|
| 618 |
+
Step 22570/44000 | Loss: 2.9510 | LR: 2.46e-04 | 1773.7ms/step | 11.3 samples/s
|
| 619 |
+
Step 22580/44000 | Loss: 2.9165 | LR: 2.46e-04 | 1772.8ms/step | 11.3 samples/s
|
| 620 |
+
Step 22590/44000 | Loss: 2.9103 | LR: 2.46e-04 | 1774.3ms/step | 11.3 samples/s
|
| 621 |
+
Step 22600/44000 | Loss: 2.9526 | LR: 2.46e-04 | 1773.1ms/step | 11.3 samples/s
|
| 622 |
+
Step 22610/44000 | Loss: 2.8903 | LR: 2.46e-04 | 1774.3ms/step | 11.3 samples/s
|
| 623 |
+
Step 22620/44000 | Loss: 2.9446 | LR: 2.46e-04 | 1772.4ms/step | 11.3 samples/s
|
| 624 |
+
Step 22630/44000 | Loss: 2.9834 | LR: 2.45e-04 | 1771.9ms/step | 11.3 samples/s
|
| 625 |
+
Step 22640/44000 | Loss: 2.9538 | LR: 2.45e-04 | 1771.7ms/step | 11.3 samples/s
|
| 626 |
+
Step 22650/44000 | Loss: 2.9311 | LR: 2.45e-04 | 1772.9ms/step | 11.3 samples/s
|
| 627 |
+
Step 22660/44000 | Loss: 2.9542 | LR: 2.45e-04 | 1771.9ms/step | 11.3 samples/s
|
| 628 |
+
Step 22670/44000 | Loss: 2.9533 | LR: 2.45e-04 | 1771.4ms/step | 11.3 samples/s
|
| 629 |
+
Step 22680/44000 | Loss: 2.8716 | LR: 2.44e-04 | 1771.6ms/step | 11.3 samples/s
|
| 630 |
+
Step 22690/44000 | Loss: 2.9154 | LR: 2.44e-04 | 1772.4ms/step | 11.3 samples/s
|
| 631 |
+
Step 22700/44000 | Loss: 2.9791 | LR: 2.44e-04 | 1772.3ms/step | 11.3 samples/s
|
| 632 |
+
Step 22710/44000 | Loss: 2.9177 | LR: 2.44e-04 | 1772.2ms/step | 11.3 samples/s
|
| 633 |
+
Step 22720/44000 | Loss: 2.9510 | LR: 2.44e-04 | 1773.0ms/step | 11.3 samples/s
|
| 634 |
+
Step 22730/44000 | Loss: 2.9412 | LR: 2.44e-04 | 1772.1ms/step | 11.3 samples/s
|
| 635 |
+
Step 22740/44000 | Loss: 2.9281 | LR: 2.43e-04 | 1772.3ms/step | 11.3 samples/s
|
| 636 |
+
Step 22750/44000 | Loss: 2.8959 | LR: 2.43e-04 | 1772.1ms/step | 11.3 samples/s
|
| 637 |
+
Step 22760/44000 | Loss: 2.9371 | LR: 2.43e-04 | 1772.2ms/step | 11.3 samples/s
|
| 638 |
+
Step 22770/44000 | Loss: 2.9628 | LR: 2.43e-04 | 1771.5ms/step | 11.3 samples/s
|
| 639 |
+
Step 22780/44000 | Loss: 2.9025 | LR: 2.43e-04 | 1770.6ms/step | 11.3 samples/s
|
| 640 |
+
Step 22790/44000 | Loss: 2.9476 | LR: 2.42e-04 | 1771.8ms/step | 11.3 samples/s
|
| 641 |
+
Step 22800/44000 | Loss: 2.9513 | LR: 2.42e-04 | 1772.6ms/step | 11.3 samples/s
|
| 642 |
+
Step 22810/44000 | Loss: 2.9639 | LR: 2.42e-04 | 1771.8ms/step | 11.3 samples/s
|
| 643 |
+
Step 22820/44000 | Loss: 2.9381 | LR: 2.42e-04 | 1773.2ms/step | 11.3 samples/s
|
| 644 |
+
Step 22830/44000 | Loss: 2.9468 | LR: 2.42e-04 | 1770.6ms/step | 11.3 samples/s
|
| 645 |
+
Step 22840/44000 | Loss: 2.9237 | LR: 2.42e-04 | 1771.5ms/step | 11.3 samples/s
|
| 646 |
+
Step 22850/44000 | Loss: 2.9807 | LR: 2.41e-04 | 1771.2ms/step | 11.3 samples/s
|
| 647 |
+
Step 22860/44000 | Loss: 2.9457 | LR: 2.41e-04 | 1770.2ms/step | 11.3 samples/s
|
| 648 |
+
Step 22870/44000 | Loss: 2.9746 | LR: 2.41e-04 | 1770.5ms/step | 11.3 samples/s
|
| 649 |
+
Step 22880/44000 | Loss: 2.9520 | LR: 2.41e-04 | 1770.1ms/step | 11.3 samples/s
|
| 650 |
+
Step 22890/44000 | Loss: 2.9710 | LR: 2.41e-04 | 1772.0ms/step | 11.3 samples/s
|
| 651 |
+
Step 22900/44000 | Loss: 2.9509 | LR: 2.40e-04 | 1771.6ms/step | 11.3 samples/s
|
| 652 |
+
Step 22910/44000 | Loss: 2.9411 | LR: 2.40e-04 | 1771.8ms/step | 11.3 samples/s
|
| 653 |
+
Step 22920/44000 | Loss: 2.9636 | LR: 2.40e-04 | 1771.6ms/step | 11.3 samples/s
|
| 654 |
+
Step 22930/44000 | Loss: 2.9357 | LR: 2.40e-04 | 1772.1ms/step | 11.3 samples/s
|
| 655 |
+
Step 22940/44000 | Loss: 2.9113 | LR: 2.40e-04 | 1771.4ms/step | 11.3 samples/s
|
epfl_final/SparseAtt/wandb/run-20260820_140401-jqstnvej/files/requirements.txt
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
| 142 |
+
entmax==1.3
|
epfl_final/SparseAtt/wandb/run-20260820_140401-jqstnvej/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-08-20T14:04:01.676908Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"/mloscratch/homes/navasard/sparse/configs_ropectl/resume_a1p0_ddp2_a100/seed42.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/TokenPrune.git",
|
| 14 |
+
"commit": "5637886f0cc760009989b14e68c6ae06d4827418"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "a100-train-4-gpu-3-0-12",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
|
| 20 |
+
"cpu_count": 64,
|
| 21 |
+
"cpu_count_logical": 128,
|
| 22 |
+
"gpu": "NVIDIA A100-SXM4-80GB",
|
| 23 |
+
"gpu_count": 4,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "7679362727936",
|
| 27 |
+
"used": "4570677764096"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1081884934144"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 36 |
+
"memoryTotal": "85899345920",
|
| 37 |
+
"cudaCores": 6912,
|
| 38 |
+
"architecture": "Ampere",
|
| 39 |
+
"uuid": "GPU-acb0c1f0-24c3-b247-3a34-5a388f299636"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 43 |
+
"memoryTotal": "85899345920",
|
| 44 |
+
"cudaCores": 6912,
|
| 45 |
+
"architecture": "Ampere",
|
| 46 |
+
"uuid": "GPU-e671cf21-d915-5ffb-9a1d-d5b78ed74769"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 50 |
+
"memoryTotal": "85899345920",
|
| 51 |
+
"cudaCores": 6912,
|
| 52 |
+
"architecture": "Ampere",
|
| 53 |
+
"uuid": "GPU-6adf3f18-ae44-1e33-62a1-898a55bd9925"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 57 |
+
"memoryTotal": "85899345920",
|
| 58 |
+
"cudaCores": 6912,
|
| 59 |
+
"architecture": "Ampere",
|
| 60 |
+
"uuid": "GPU-bd9e5b3c-f1d9-ad77-5085-33fdd309a690"
|
| 61 |
+
}
|
| 62 |
+
],
|
| 63 |
+
"cudaVersion": "13.0",
|
| 64 |
+
"writerId": "movb1z0xenpxede7z73p3o5oq6pwib5k"
|
| 65 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260820_140401-jqstnvej/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260820_140401-jqstnvej/logs/debug.log
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-08-20 14:04:01,737 INFO MainThread:376 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-08-20 14:04:01,737 INFO MainThread:376 [wandb_setup.py:_flush():81] Configure stats pid to 376
|
| 3 |
+
2026-08-20 14:04:01,737 INFO MainThread:376 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-08-20 14:04:01,737 INFO MainThread:376 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_140401-jqstnvej/logs/debug.log
|
| 5 |
+
2026-08-20 14:04:01,737 INFO MainThread:376 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_140401-jqstnvej/logs/debug-internal.log
|
| 6 |
+
2026-08-20 14:04:01,737 INFO MainThread:376 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-08-20 14:04:01,737 INFO MainThread:376 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-17000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 5, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 1000, 'eval_batches': 50, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'l8k365m_dense_ropectl_a1p0_seed42', 'wandb_tags': ['qwen3', '365m', '8k', 'dense', 'rope-control', 'ddp2-a100'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': 0.0, 'rope_train_position_scale_set': [1.0, 0.9896339488176222, 0.9692159692159692, 0.9420289855072463, 0.8730158730158729], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-08-20 14:04:01,737 INFO MainThread:376 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-08-20 14:04:02,493 INFO MainThread:376 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-08-20 14:04:02,940 INFO MainThread:376 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-08-20 14:04:02,943 INFO MainThread:376 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-08-20 14:04:02,974 INFO MainThread:376 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-08-20 14:04:03,686 INFO MainThread:376 [wandb_init.py:init():1051] run resumed
|
| 15 |
+
2026-08-20 14:04:03,687 INFO MainThread:376 [wandb_init.py:init():1056] starting run threads in backend
|
| 16 |
+
2026-08-20 14:04:03,853 INFO MainThread:376 [wandb_run.py:_console_start():2554] atexit reg
|
| 17 |
+
2026-08-20 14:04:03,854 INFO MainThread:376 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 18 |
+
2026-08-20 14:04:03,854 INFO MainThread:376 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 19 |
+
2026-08-20 14:04:03,854 INFO MainThread:376 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 20 |
+
2026-08-20 14:04:03,857 INFO MainThread:376 [wandb_init.py:init():1094] run started, returning control to user process
|
epfl_final/SparseAtt/wandb/run-20260820_140401-jqstnvej/run-jqstnvej.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f1e0046f8a37a4251cdadf7f909d2db1d258ce2ac9b1ef0c3bbad3935ac7a4a0
|
| 3 |
+
size 2228224
|
epfl_final/SparseAtt/wandb/run-20260820_140401-ysa9iyov/files/output.log
ADDED
|
@@ -0,0 +1,646 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Starting training...
|
| 2 |
+
Validation at step 18000: Loss = 2.9728, Perplexity = 19.55
|
| 3 |
+
wandb: WARNING Tried to log to step 18000 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 4 |
+
Switching attention mode for step 18001: dense (dense step interval)
|
| 5 |
+
Step 18010/44000 | Loss: 2.9971 | LR: 3.28e-04 | 42307.5ms/step | 0.5 samples/s
|
| 6 |
+
wandb: WARNING Tried to log to step 18010 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 7 |
+
Step 18020/44000 | Loss: 2.9907 | LR: 3.28e-04 | 1773.9ms/step | 11.3 samples/s
|
| 8 |
+
wandb: WARNING Tried to log to step 18020 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 9 |
+
Step 18030/44000 | Loss: 3.0001 | LR: 3.28e-04 | 1777.2ms/step | 11.3 samples/s
|
| 10 |
+
wandb: WARNING Tried to log to step 18030 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 11 |
+
Step 18040/44000 | Loss: 2.9561 | LR: 3.27e-04 | 1778.8ms/step | 11.2 samples/s
|
| 12 |
+
wandb: WARNING Tried to log to step 18040 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 13 |
+
Step 18050/44000 | Loss: 2.9921 | LR: 3.27e-04 | 1779.0ms/step | 11.2 samples/s
|
| 14 |
+
wandb: WARNING Tried to log to step 18050 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 15 |
+
Step 18060/44000 | Loss: 3.0097 | LR: 3.27e-04 | 1781.8ms/step | 11.2 samples/s
|
| 16 |
+
wandb: WARNING Tried to log to step 18060 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 17 |
+
Step 18070/44000 | Loss: 2.9773 | LR: 3.27e-04 | 1781.6ms/step | 11.2 samples/s
|
| 18 |
+
wandb: WARNING Tried to log to step 18070 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 19 |
+
Step 18080/44000 | Loss: 2.9883 | LR: 3.27e-04 | 1781.0ms/step | 11.2 samples/s
|
| 20 |
+
wandb: WARNING Tried to log to step 18080 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 21 |
+
Step 18090/44000 | Loss: 2.9542 | LR: 3.27e-04 | 1780.5ms/step | 11.2 samples/s
|
| 22 |
+
wandb: WARNING Tried to log to step 18090 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 23 |
+
Step 18100/44000 | Loss: 2.9393 | LR: 3.26e-04 | 1781.5ms/step | 11.2 samples/s
|
| 24 |
+
wandb: WARNING Tried to log to step 18100 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 25 |
+
Step 18110/44000 | Loss: 3.0064 | LR: 3.26e-04 | 1781.2ms/step | 11.2 samples/s
|
| 26 |
+
wandb: WARNING Tried to log to step 18110 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 27 |
+
Step 18120/44000 | Loss: 2.9451 | LR: 3.26e-04 | 1782.0ms/step | 11.2 samples/s
|
| 28 |
+
wandb: WARNING Tried to log to step 18120 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 29 |
+
Step 18130/44000 | Loss: 2.9749 | LR: 3.26e-04 | 1781.3ms/step | 11.2 samples/s
|
| 30 |
+
wandb: WARNING Tried to log to step 18130 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 31 |
+
Step 18140/44000 | Loss: 3.0306 | LR: 3.26e-04 | 1781.8ms/step | 11.2 samples/s
|
| 32 |
+
wandb: WARNING Tried to log to step 18140 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 33 |
+
Step 18150/44000 | Loss: 3.0022 | LR: 3.26e-04 | 1781.8ms/step | 11.2 samples/s
|
| 34 |
+
wandb: WARNING Tried to log to step 18150 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 35 |
+
Step 18160/44000 | Loss: 2.9571 | LR: 3.25e-04 | 1782.0ms/step | 11.2 samples/s
|
| 36 |
+
wandb: WARNING Tried to log to step 18160 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 37 |
+
Step 18170/44000 | Loss: 2.9630 | LR: 3.25e-04 | 1783.4ms/step | 11.2 samples/s
|
| 38 |
+
wandb: WARNING Tried to log to step 18170 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 39 |
+
Step 18180/44000 | Loss: 2.9619 | LR: 3.25e-04 | 1783.4ms/step | 11.2 samples/s
|
| 40 |
+
wandb: WARNING Tried to log to step 18180 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 41 |
+
Step 18190/44000 | Loss: 3.0241 | LR: 3.25e-04 | 1782.4ms/step | 11.2 samples/s
|
| 42 |
+
wandb: WARNING Tried to log to step 18190 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 43 |
+
Step 18200/44000 | Loss: 2.9990 | LR: 3.25e-04 | 1785.0ms/step | 11.2 samples/s
|
| 44 |
+
wandb: WARNING Tried to log to step 18200 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 45 |
+
Step 18210/44000 | Loss: 3.0164 | LR: 3.24e-04 | 1783.6ms/step | 11.2 samples/s
|
| 46 |
+
wandb: WARNING Tried to log to step 18210 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 47 |
+
Step 18220/44000 | Loss: 2.9772 | LR: 3.24e-04 | 1782.7ms/step | 11.2 samples/s
|
| 48 |
+
wandb: WARNING Tried to log to step 18220 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 49 |
+
Step 18230/44000 | Loss: 2.9828 | LR: 3.24e-04 | 1782.7ms/step | 11.2 samples/s
|
| 50 |
+
wandb: WARNING Tried to log to step 18230 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 51 |
+
Step 18240/44000 | Loss: 3.0113 | LR: 3.24e-04 | 1783.1ms/step | 11.2 samples/s
|
| 52 |
+
wandb: WARNING Tried to log to step 18240 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 53 |
+
Step 18250/44000 | Loss: 2.9762 | LR: 3.24e-04 | 1783.6ms/step | 11.2 samples/s
|
| 54 |
+
wandb: WARNING Tried to log to step 18250 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 55 |
+
Step 18260/44000 | Loss: 3.0124 | LR: 3.24e-04 | 1783.3ms/step | 11.2 samples/s
|
| 56 |
+
wandb: WARNING Tried to log to step 18260 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 57 |
+
Step 18270/44000 | Loss: 2.9906 | LR: 3.23e-04 | 1783.6ms/step | 11.2 samples/s
|
| 58 |
+
wandb: WARNING Tried to log to step 18270 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 59 |
+
Step 18280/44000 | Loss: 2.9492 | LR: 3.23e-04 | 1784.0ms/step | 11.2 samples/s
|
| 60 |
+
wandb: WARNING Tried to log to step 18280 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 61 |
+
Step 18290/44000 | Loss: 3.0016 | LR: 3.23e-04 | 1781.8ms/step | 11.2 samples/s
|
| 62 |
+
wandb: WARNING Tried to log to step 18290 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 63 |
+
Step 18300/44000 | Loss: 3.0241 | LR: 3.23e-04 | 1782.7ms/step | 11.2 samples/s
|
| 64 |
+
wandb: WARNING Tried to log to step 18300 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 65 |
+
Step 18310/44000 | Loss: 3.0117 | LR: 3.23e-04 | 1782.7ms/step | 11.2 samples/s
|
| 66 |
+
wandb: WARNING Tried to log to step 18310 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 67 |
+
Step 18320/44000 | Loss: 3.0382 | LR: 3.23e-04 | 1783.2ms/step | 11.2 samples/s
|
| 68 |
+
wandb: WARNING Tried to log to step 18320 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 69 |
+
Step 18330/44000 | Loss: 3.0040 | LR: 3.22e-04 | 1782.2ms/step | 11.2 samples/s
|
| 70 |
+
wandb: WARNING Tried to log to step 18330 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 71 |
+
Step 18340/44000 | Loss: 2.9568 | LR: 3.22e-04 | 1782.7ms/step | 11.2 samples/s
|
| 72 |
+
wandb: WARNING Tried to log to step 18340 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 73 |
+
Step 18350/44000 | Loss: 2.9752 | LR: 3.22e-04 | 1782.5ms/step | 11.2 samples/s
|
| 74 |
+
wandb: WARNING Tried to log to step 18350 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 75 |
+
Step 18360/44000 | Loss: 2.9717 | LR: 3.22e-04 | 1782.6ms/step | 11.2 samples/s
|
| 76 |
+
wandb: WARNING Tried to log to step 18360 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 77 |
+
Step 18370/44000 | Loss: 3.0086 | LR: 3.22e-04 | 1781.7ms/step | 11.2 samples/s
|
| 78 |
+
wandb: WARNING Tried to log to step 18370 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 79 |
+
Step 18380/44000 | Loss: 2.9834 | LR: 3.22e-04 | 1782.6ms/step | 11.2 samples/s
|
| 80 |
+
wandb: WARNING Tried to log to step 18380 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 81 |
+
Step 18390/44000 | Loss: 3.0108 | LR: 3.21e-04 | 1782.5ms/step | 11.2 samples/s
|
| 82 |
+
wandb: WARNING Tried to log to step 18390 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 83 |
+
Step 18400/44000 | Loss: 3.0106 | LR: 3.21e-04 | 1783.0ms/step | 11.2 samples/s
|
| 84 |
+
wandb: WARNING Tried to log to step 18400 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 85 |
+
Step 18410/44000 | Loss: 2.9893 | LR: 3.21e-04 | 1781.9ms/step | 11.2 samples/s
|
| 86 |
+
wandb: WARNING Tried to log to step 18410 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 87 |
+
Step 18420/44000 | Loss: 2.9824 | LR: 3.21e-04 | 1781.9ms/step | 11.2 samples/s
|
| 88 |
+
wandb: WARNING Tried to log to step 18420 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 89 |
+
Step 18430/44000 | Loss: 2.9528 | LR: 3.21e-04 | 1780.9ms/step | 11.2 samples/s
|
| 90 |
+
wandb: WARNING Tried to log to step 18430 that is less than the current step 18431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 91 |
+
Step 18440/44000 | Loss: 2.9364 | LR: 3.20e-04 | 1781.7ms/step | 11.2 samples/s
|
| 92 |
+
Step 18450/44000 | Loss: 2.9737 | LR: 3.20e-04 | 1783.2ms/step | 11.2 samples/s
|
| 93 |
+
Step 18460/44000 | Loss: 2.9931 | LR: 3.20e-04 | 1782.8ms/step | 11.2 samples/s
|
| 94 |
+
Step 18470/44000 | Loss: 3.0209 | LR: 3.20e-04 | 1782.0ms/step | 11.2 samples/s
|
| 95 |
+
Step 18480/44000 | Loss: 2.9502 | LR: 3.20e-04 | 1782.9ms/step | 11.2 samples/s
|
| 96 |
+
Step 18490/44000 | Loss: 2.9597 | LR: 3.20e-04 | 1783.0ms/step | 11.2 samples/s
|
| 97 |
+
Step 18500/44000 | Loss: 2.9759 | LR: 3.19e-04 | 1783.1ms/step | 11.2 samples/s
|
| 98 |
+
Step 18510/44000 | Loss: 2.9690 | LR: 3.19e-04 | 1783.1ms/step | 11.2 samples/s
|
| 99 |
+
Step 18520/44000 | Loss: 2.9974 | LR: 3.19e-04 | 1782.9ms/step | 11.2 samples/s
|
| 100 |
+
Step 18530/44000 | Loss: 3.0094 | LR: 3.19e-04 | 1783.1ms/step | 11.2 samples/s
|
| 101 |
+
Step 18540/44000 | Loss: 3.0191 | LR: 3.19e-04 | 1784.2ms/step | 11.2 samples/s
|
| 102 |
+
Step 18550/44000 | Loss: 2.9463 | LR: 3.19e-04 | 1782.2ms/step | 11.2 samples/s
|
| 103 |
+
Step 18560/44000 | Loss: 3.0022 | LR: 3.18e-04 | 1782.6ms/step | 11.2 samples/s
|
| 104 |
+
Step 18570/44000 | Loss: 3.0116 | LR: 3.18e-04 | 1781.9ms/step | 11.2 samples/s
|
| 105 |
+
Step 18580/44000 | Loss: 3.0194 | LR: 3.18e-04 | 1783.9ms/step | 11.2 samples/s
|
| 106 |
+
Step 18590/44000 | Loss: 2.9816 | LR: 3.18e-04 | 1783.1ms/step | 11.2 samples/s
|
| 107 |
+
Step 18600/44000 | Loss: 2.9943 | LR: 3.18e-04 | 1783.7ms/step | 11.2 samples/s
|
| 108 |
+
Step 18610/44000 | Loss: 2.9607 | LR: 3.18e-04 | 1782.4ms/step | 11.2 samples/s
|
| 109 |
+
Step 18620/44000 | Loss: 3.0022 | LR: 3.17e-04 | 1782.5ms/step | 11.2 samples/s
|
| 110 |
+
Step 18630/44000 | Loss: 3.0042 | LR: 3.17e-04 | 1782.9ms/step | 11.2 samples/s
|
| 111 |
+
Step 18640/44000 | Loss: 3.0192 | LR: 3.17e-04 | 1782.6ms/step | 11.2 samples/s
|
| 112 |
+
Step 18650/44000 | Loss: 2.9810 | LR: 3.17e-04 | 1782.3ms/step | 11.2 samples/s
|
| 113 |
+
Step 18660/44000 | Loss: 2.9998 | LR: 3.17e-04 | 1782.8ms/step | 11.2 samples/s
|
| 114 |
+
Step 18670/44000 | Loss: 2.9850 | LR: 3.16e-04 | 1782.9ms/step | 11.2 samples/s
|
| 115 |
+
Step 18680/44000 | Loss: 3.0212 | LR: 3.16e-04 | 1783.8ms/step | 11.2 samples/s
|
| 116 |
+
Step 18690/44000 | Loss: 2.9495 | LR: 3.16e-04 | 1783.3ms/step | 11.2 samples/s
|
| 117 |
+
Step 18700/44000 | Loss: 2.9738 | LR: 3.16e-04 | 1781.9ms/step | 11.2 samples/s
|
| 118 |
+
Step 18710/44000 | Loss: 3.0018 | LR: 3.16e-04 | 1783.0ms/step | 11.2 samples/s
|
| 119 |
+
Step 18720/44000 | Loss: 2.9478 | LR: 3.16e-04 | 1782.1ms/step | 11.2 samples/s
|
| 120 |
+
Step 18730/44000 | Loss: 3.0106 | LR: 3.15e-04 | 1782.9ms/step | 11.2 samples/s
|
| 121 |
+
Step 18740/44000 | Loss: 2.9539 | LR: 3.15e-04 | 1781.9ms/step | 11.2 samples/s
|
| 122 |
+
Step 18750/44000 | Loss: 2.9888 | LR: 3.15e-04 | 1786.6ms/step | 11.2 samples/s
|
| 123 |
+
Step 18760/44000 | Loss: 2.9621 | LR: 3.15e-04 | 1784.1ms/step | 11.2 samples/s
|
| 124 |
+
Step 18770/44000 | Loss: 2.9700 | LR: 3.15e-04 | 1784.5ms/step | 11.2 samples/s
|
| 125 |
+
Step 18780/44000 | Loss: 2.9620 | LR: 3.15e-04 | 1783.6ms/step | 11.2 samples/s
|
| 126 |
+
Step 18790/44000 | Loss: 2.9962 | LR: 3.14e-04 | 1783.3ms/step | 11.2 samples/s
|
| 127 |
+
Step 18800/44000 | Loss: 2.9913 | LR: 3.14e-04 | 1786.7ms/step | 11.2 samples/s
|
| 128 |
+
Step 18810/44000 | Loss: 2.9776 | LR: 3.14e-04 | 1786.7ms/step | 11.2 samples/s
|
| 129 |
+
Step 18820/44000 | Loss: 2.9986 | LR: 3.14e-04 | 1782.5ms/step | 11.2 samples/s
|
| 130 |
+
Step 18830/44000 | Loss: 2.9464 | LR: 3.14e-04 | 1783.4ms/step | 11.2 samples/s
|
| 131 |
+
Step 18840/44000 | Loss: 2.9397 | LR: 3.13e-04 | 1783.2ms/step | 11.2 samples/s
|
| 132 |
+
Step 18850/44000 | Loss: 2.9791 | LR: 3.13e-04 | 1783.6ms/step | 11.2 samples/s
|
| 133 |
+
Step 18860/44000 | Loss: 2.9596 | LR: 3.13e-04 | 1783.5ms/step | 11.2 samples/s
|
| 134 |
+
Step 18870/44000 | Loss: 2.9497 | LR: 3.13e-04 | 1783.7ms/step | 11.2 samples/s
|
| 135 |
+
Step 18880/44000 | Loss: 2.9688 | LR: 3.13e-04 | 1783.8ms/step | 11.2 samples/s
|
| 136 |
+
Step 18890/44000 | Loss: 2.9681 | LR: 3.13e-04 | 1784.2ms/step | 11.2 samples/s
|
| 137 |
+
Step 18900/44000 | Loss: 2.9671 | LR: 3.12e-04 | 1784.4ms/step | 11.2 samples/s
|
| 138 |
+
Step 18910/44000 | Loss: 2.9530 | LR: 3.12e-04 | 1784.7ms/step | 11.2 samples/s
|
| 139 |
+
Step 18920/44000 | Loss: 2.9778 | LR: 3.12e-04 | 1784.4ms/step | 11.2 samples/s
|
| 140 |
+
Step 18930/44000 | Loss: 3.0043 | LR: 3.12e-04 | 1783.4ms/step | 11.2 samples/s
|
| 141 |
+
Step 18940/44000 | Loss: 2.9871 | LR: 3.12e-04 | 1782.7ms/step | 11.2 samples/s
|
| 142 |
+
Step 18950/44000 | Loss: 2.9887 | LR: 3.12e-04 | 1782.5ms/step | 11.2 samples/s
|
| 143 |
+
Step 18960/44000 | Loss: 2.9920 | LR: 3.11e-04 | 1783.0ms/step | 11.2 samples/s
|
| 144 |
+
Step 18970/44000 | Loss: 2.9774 | LR: 3.11e-04 | 1783.8ms/step | 11.2 samples/s
|
| 145 |
+
Step 18980/44000 | Loss: 2.9757 | LR: 3.11e-04 | 1782.9ms/step | 11.2 samples/s
|
| 146 |
+
Step 18990/44000 | Loss: 2.9514 | LR: 3.11e-04 | 1783.4ms/step | 11.2 samples/s
|
| 147 |
+
Step 19000/44000 | Loss: 2.9648 | LR: 3.11e-04 | 1782.4ms/step | 11.2 samples/s
|
| 148 |
+
Validation at step 19000: Loss = 2.9626, Perplexity = 19.35
|
| 149 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-19000
|
| 150 |
+
Step 19010/44000 | Loss: 2.9803 | LR: 3.11e-04 | 3584.0ms/step | 5.6 samples/s
|
| 151 |
+
Step 19020/44000 | Loss: 2.9681 | LR: 3.10e-04 | 1785.4ms/step | 11.2 samples/s
|
| 152 |
+
Step 19030/44000 | Loss: 2.9647 | LR: 3.10e-04 | 1786.4ms/step | 11.2 samples/s
|
| 153 |
+
Step 19040/44000 | Loss: 2.9520 | LR: 3.10e-04 | 1786.3ms/step | 11.2 samples/s
|
| 154 |
+
Step 19050/44000 | Loss: 3.0237 | LR: 3.10e-04 | 1787.6ms/step | 11.2 samples/s
|
| 155 |
+
Step 19060/44000 | Loss: 2.9959 | LR: 3.10e-04 | 1785.9ms/step | 11.2 samples/s
|
| 156 |
+
Step 19070/44000 | Loss: 2.9645 | LR: 3.09e-04 | 1787.2ms/step | 11.2 samples/s
|
| 157 |
+
Step 19080/44000 | Loss: 3.0128 | LR: 3.09e-04 | 1784.9ms/step | 11.2 samples/s
|
| 158 |
+
Step 19090/44000 | Loss: 2.9736 | LR: 3.09e-04 | 1786.8ms/step | 11.2 samples/s
|
| 159 |
+
Step 19100/44000 | Loss: 3.0043 | LR: 3.09e-04 | 1786.2ms/step | 11.2 samples/s
|
| 160 |
+
Step 19110/44000 | Loss: 3.0010 | LR: 3.09e-04 | 1787.1ms/step | 11.2 samples/s
|
| 161 |
+
Step 19120/44000 | Loss: 3.0102 | LR: 3.09e-04 | 1786.1ms/step | 11.2 samples/s
|
| 162 |
+
Step 19130/44000 | Loss: 3.0016 | LR: 3.08e-04 | 1786.9ms/step | 11.2 samples/s
|
| 163 |
+
Step 19140/44000 | Loss: 3.0037 | LR: 3.08e-04 | 1785.6ms/step | 11.2 samples/s
|
| 164 |
+
Step 19150/44000 | Loss: 2.9993 | LR: 3.08e-04 | 1787.7ms/step | 11.2 samples/s
|
| 165 |
+
Step 19160/44000 | Loss: 2.9615 | LR: 3.08e-04 | 1787.0ms/step | 11.2 samples/s
|
| 166 |
+
Step 19170/44000 | Loss: 3.0020 | LR: 3.08e-04 | 1786.2ms/step | 11.2 samples/s
|
| 167 |
+
Step 19180/44000 | Loss: 3.0121 | LR: 3.08e-04 | 1785.0ms/step | 11.2 samples/s
|
| 168 |
+
Step 19190/44000 | Loss: 2.9320 | LR: 3.07e-04 | 1786.2ms/step | 11.2 samples/s
|
| 169 |
+
Step 19200/44000 | Loss: 2.9542 | LR: 3.07e-04 | 1786.1ms/step | 11.2 samples/s
|
| 170 |
+
Step 19210/44000 | Loss: 2.9760 | LR: 3.07e-04 | 1785.5ms/step | 11.2 samples/s
|
| 171 |
+
Step 19220/44000 | Loss: 3.0029 | LR: 3.07e-04 | 1785.0ms/step | 11.2 samples/s
|
| 172 |
+
Step 19230/44000 | Loss: 2.9765 | LR: 3.07e-04 | 1786.1ms/step | 11.2 samples/s
|
| 173 |
+
Step 19240/44000 | Loss: 2.9915 | LR: 3.06e-04 | 1786.2ms/step | 11.2 samples/s
|
| 174 |
+
Step 19250/44000 | Loss: 2.9664 | LR: 3.06e-04 | 1786.1ms/step | 11.2 samples/s
|
| 175 |
+
Step 19260/44000 | Loss: 2.9532 | LR: 3.06e-04 | 1785.8ms/step | 11.2 samples/s
|
| 176 |
+
Step 19270/44000 | Loss: 2.9631 | LR: 3.06e-04 | 1784.7ms/step | 11.2 samples/s
|
| 177 |
+
Step 19280/44000 | Loss: 3.0000 | LR: 3.06e-04 | 1786.2ms/step | 11.2 samples/s
|
| 178 |
+
Step 19290/44000 | Loss: 2.9515 | LR: 3.06e-04 | 1784.9ms/step | 11.2 samples/s
|
| 179 |
+
Step 19300/44000 | Loss: 2.9979 | LR: 3.05e-04 | 1786.6ms/step | 11.2 samples/s
|
| 180 |
+
Step 19310/44000 | Loss: 2.9852 | LR: 3.05e-04 | 1786.5ms/step | 11.2 samples/s
|
| 181 |
+
Step 19320/44000 | Loss: 2.9740 | LR: 3.05e-04 | 1786.5ms/step | 11.2 samples/s
|
| 182 |
+
Step 19330/44000 | Loss: 2.9537 | LR: 3.05e-04 | 1786.3ms/step | 11.2 samples/s
|
| 183 |
+
Step 19340/44000 | Loss: 2.9845 | LR: 3.05e-04 | 1786.3ms/step | 11.2 samples/s
|
| 184 |
+
Step 19350/44000 | Loss: 2.9603 | LR: 3.04e-04 | 1785.3ms/step | 11.2 samples/s
|
| 185 |
+
Step 19360/44000 | Loss: 2.9405 | LR: 3.04e-04 | 1784.9ms/step | 11.2 samples/s
|
| 186 |
+
Step 19370/44000 | Loss: 2.9557 | LR: 3.04e-04 | 1784.2ms/step | 11.2 samples/s
|
| 187 |
+
Step 19380/44000 | Loss: 2.9809 | LR: 3.04e-04 | 1785.4ms/step | 11.2 samples/s
|
| 188 |
+
Step 19390/44000 | Loss: 2.9811 | LR: 3.04e-04 | 1784.9ms/step | 11.2 samples/s
|
| 189 |
+
Step 19400/44000 | Loss: 2.9769 | LR: 3.04e-04 | 1787.2ms/step | 11.2 samples/s
|
| 190 |
+
Step 19410/44000 | Loss: 2.9840 | LR: 3.03e-04 | 1786.3ms/step | 11.2 samples/s
|
| 191 |
+
Step 19420/44000 | Loss: 2.9710 | LR: 3.03e-04 | 1786.6ms/step | 11.2 samples/s
|
| 192 |
+
Step 19430/44000 | Loss: 2.9264 | LR: 3.03e-04 | 1784.7ms/step | 11.2 samples/s
|
| 193 |
+
Step 19440/44000 | Loss: 2.9834 | LR: 3.03e-04 | 1785.9ms/step | 11.2 samples/s
|
| 194 |
+
Step 19450/44000 | Loss: 3.0116 | LR: 3.03e-04 | 1785.7ms/step | 11.2 samples/s
|
| 195 |
+
Step 19460/44000 | Loss: 2.9537 | LR: 3.03e-04 | 1787.3ms/step | 11.2 samples/s
|
| 196 |
+
Step 19470/44000 | Loss: 2.9745 | LR: 3.02e-04 | 1788.1ms/step | 11.2 samples/s
|
| 197 |
+
Step 19480/44000 | Loss: 2.9615 | LR: 3.02e-04 | 1786.2ms/step | 11.2 samples/s
|
| 198 |
+
Step 19490/44000 | Loss: 2.9910 | LR: 3.02e-04 | 1786.4ms/step | 11.2 samples/s
|
| 199 |
+
Step 19500/44000 | Loss: 3.0047 | LR: 3.02e-04 | 1787.2ms/step | 11.2 samples/s
|
| 200 |
+
Step 19510/44000 | Loss: 2.9671 | LR: 3.02e-04 | 1785.6ms/step | 11.2 samples/s
|
| 201 |
+
Step 19520/44000 | Loss: 2.9678 | LR: 3.01e-04 | 1785.5ms/step | 11.2 samples/s
|
| 202 |
+
Step 19530/44000 | Loss: 2.9600 | LR: 3.01e-04 | 1785.8ms/step | 11.2 samples/s
|
| 203 |
+
Step 19540/44000 | Loss: 2.9711 | LR: 3.01e-04 | 1785.8ms/step | 11.2 samples/s
|
| 204 |
+
Step 19550/44000 | Loss: 2.9762 | LR: 3.01e-04 | 1786.3ms/step | 11.2 samples/s
|
| 205 |
+
Step 19560/44000 | Loss: 2.9826 | LR: 3.01e-04 | 1785.5ms/step | 11.2 samples/s
|
| 206 |
+
Step 19570/44000 | Loss: 2.9861 | LR: 3.01e-04 | 1785.6ms/step | 11.2 samples/s
|
| 207 |
+
Step 19580/44000 | Loss: 2.9780 | LR: 3.00e-04 | 1785.7ms/step | 11.2 samples/s
|
| 208 |
+
Step 19590/44000 | Loss: 2.9711 | LR: 3.00e-04 | 1785.8ms/step | 11.2 samples/s
|
| 209 |
+
Step 19600/44000 | Loss: 2.9793 | LR: 3.00e-04 | 1785.5ms/step | 11.2 samples/s
|
| 210 |
+
Step 19610/44000 | Loss: 3.0148 | LR: 3.00e-04 | 1785.2ms/step | 11.2 samples/s
|
| 211 |
+
Step 19620/44000 | Loss: 3.0110 | LR: 3.00e-04 | 1782.6ms/step | 11.2 samples/s
|
| 212 |
+
Step 19630/44000 | Loss: 2.9517 | LR: 3.00e-04 | 1785.6ms/step | 11.2 samples/s
|
| 213 |
+
Step 19640/44000 | Loss: 2.9868 | LR: 2.99e-04 | 1785.3ms/step | 11.2 samples/s
|
| 214 |
+
Step 19650/44000 | Loss: 2.9841 | LR: 2.99e-04 | 1786.1ms/step | 11.2 samples/s
|
| 215 |
+
Step 19660/44000 | Loss: 2.9842 | LR: 2.99e-04 | 1784.9ms/step | 11.2 samples/s
|
| 216 |
+
Step 19670/44000 | Loss: 3.0168 | LR: 2.99e-04 | 1786.1ms/step | 11.2 samples/s
|
| 217 |
+
Step 19680/44000 | Loss: 2.9891 | LR: 2.99e-04 | 1785.7ms/step | 11.2 samples/s
|
| 218 |
+
Step 19690/44000 | Loss: 2.9710 | LR: 2.98e-04 | 1786.8ms/step | 11.2 samples/s
|
| 219 |
+
Step 19700/44000 | Loss: 2.9777 | LR: 2.98e-04 | 1786.5ms/step | 11.2 samples/s
|
| 220 |
+
Step 19710/44000 | Loss: 2.9780 | LR: 2.98e-04 | 1787.1ms/step | 11.2 samples/s
|
| 221 |
+
Step 19720/44000 | Loss: 3.0133 | LR: 2.98e-04 | 1787.6ms/step | 11.2 samples/s
|
| 222 |
+
Step 19730/44000 | Loss: 2.9665 | LR: 2.98e-04 | 1786.6ms/step | 11.2 samples/s
|
| 223 |
+
Step 19740/44000 | Loss: 2.9740 | LR: 2.98e-04 | 1785.4ms/step | 11.2 samples/s
|
| 224 |
+
Step 19750/44000 | Loss: 2.9987 | LR: 2.97e-04 | 1786.4ms/step | 11.2 samples/s
|
| 225 |
+
Step 19760/44000 | Loss: 2.9649 | LR: 2.97e-04 | 1785.2ms/step | 11.2 samples/s
|
| 226 |
+
Step 19770/44000 | Loss: 2.9672 | LR: 2.97e-04 | 1786.0ms/step | 11.2 samples/s
|
| 227 |
+
Step 19780/44000 | Loss: 2.9803 | LR: 2.97e-04 | 1785.9ms/step | 11.2 samples/s
|
| 228 |
+
Step 19790/44000 | Loss: 3.0131 | LR: 2.97e-04 | 1785.7ms/step | 11.2 samples/s
|
| 229 |
+
Step 19800/44000 | Loss: 2.9232 | LR: 2.96e-04 | 1785.6ms/step | 11.2 samples/s
|
| 230 |
+
Step 19810/44000 | Loss: 2.9741 | LR: 2.96e-04 | 1785.9ms/step | 11.2 samples/s
|
| 231 |
+
Step 19820/44000 | Loss: 3.0011 | LR: 2.96e-04 | 1785.4ms/step | 11.2 samples/s
|
| 232 |
+
Step 19830/44000 | Loss: 2.9685 | LR: 2.96e-04 | 1785.9ms/step | 11.2 samples/s
|
| 233 |
+
Step 19840/44000 | Loss: 3.0102 | LR: 2.96e-04 | 1785.9ms/step | 11.2 samples/s
|
| 234 |
+
Step 19850/44000 | Loss: 2.9582 | LR: 2.96e-04 | 1786.0ms/step | 11.2 samples/s
|
| 235 |
+
Step 19860/44000 | Loss: 3.0023 | LR: 2.95e-04 | 1786.2ms/step | 11.2 samples/s
|
| 236 |
+
Step 19870/44000 | Loss: 2.9845 | LR: 2.95e-04 | 1785.1ms/step | 11.2 samples/s
|
| 237 |
+
Step 19880/44000 | Loss: 3.0251 | LR: 2.95e-04 | 1785.3ms/step | 11.2 samples/s
|
| 238 |
+
Step 19890/44000 | Loss: 2.9631 | LR: 2.95e-04 | 1785.0ms/step | 11.2 samples/s
|
| 239 |
+
Step 19900/44000 | Loss: 2.9846 | LR: 2.95e-04 | 1786.8ms/step | 11.2 samples/s
|
| 240 |
+
Step 19910/44000 | Loss: 2.9572 | LR: 2.95e-04 | 1785.3ms/step | 11.2 samples/s
|
| 241 |
+
Step 19920/44000 | Loss: 2.9981 | LR: 2.94e-04 | 1784.7ms/step | 11.2 samples/s
|
| 242 |
+
Step 19930/44000 | Loss: 2.9439 | LR: 2.94e-04 | 1784.0ms/step | 11.2 samples/s
|
| 243 |
+
Step 19940/44000 | Loss: 2.9729 | LR: 2.94e-04 | 1785.8ms/step | 11.2 samples/s
|
| 244 |
+
Step 19950/44000 | Loss: 2.9531 | LR: 2.94e-04 | 1784.2ms/step | 11.2 samples/s
|
| 245 |
+
Step 19960/44000 | Loss: 2.9860 | LR: 2.94e-04 | 1784.4ms/step | 11.2 samples/s
|
| 246 |
+
Step 19970/44000 | Loss: 2.9534 | LR: 2.93e-04 | 1783.5ms/step | 11.2 samples/s
|
| 247 |
+
Step 19980/44000 | Loss: 2.9586 | LR: 2.93e-04 | 1784.1ms/step | 11.2 samples/s
|
| 248 |
+
Step 19990/44000 | Loss: 2.9507 | LR: 2.93e-04 | 1784.3ms/step | 11.2 samples/s
|
| 249 |
+
Step 20000/44000 | Loss: 2.9806 | LR: 2.93e-04 | 1784.9ms/step | 11.2 samples/s
|
| 250 |
+
Validation at step 20000: Loss = 2.9514, Perplexity = 19.13
|
| 251 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-20000
|
| 252 |
+
Step 20010/44000 | Loss: 2.9525 | LR: 2.93e-04 | 3588.2ms/step | 5.6 samples/s
|
| 253 |
+
Step 20020/44000 | Loss: 2.9913 | LR: 2.93e-04 | 1784.6ms/step | 11.2 samples/s
|
| 254 |
+
Step 20030/44000 | Loss: 2.9715 | LR: 2.92e-04 | 1785.8ms/step | 11.2 samples/s
|
| 255 |
+
Step 20040/44000 | Loss: 2.9938 | LR: 2.92e-04 | 1784.2ms/step | 11.2 samples/s
|
| 256 |
+
Step 20050/44000 | Loss: 2.9420 | LR: 2.92e-04 | 1784.0ms/step | 11.2 samples/s
|
| 257 |
+
Step 20060/44000 | Loss: 2.9639 | LR: 2.92e-04 | 1783.9ms/step | 11.2 samples/s
|
| 258 |
+
Step 20070/44000 | Loss: 3.0013 | LR: 2.92e-04 | 1784.7ms/step | 11.2 samples/s
|
| 259 |
+
Step 20080/44000 | Loss: 2.9357 | LR: 2.91e-04 | 1784.5ms/step | 11.2 samples/s
|
| 260 |
+
Step 20090/44000 | Loss: 2.9849 | LR: 2.91e-04 | 1784.5ms/step | 11.2 samples/s
|
| 261 |
+
Step 20100/44000 | Loss: 2.9498 | LR: 2.91e-04 | 1784.7ms/step | 11.2 samples/s
|
| 262 |
+
Step 20110/44000 | Loss: 2.9376 | LR: 2.91e-04 | 1785.1ms/step | 11.2 samples/s
|
| 263 |
+
Step 20120/44000 | Loss: 2.9287 | LR: 2.91e-04 | 1784.4ms/step | 11.2 samples/s
|
| 264 |
+
Step 20130/44000 | Loss: 2.9613 | LR: 2.91e-04 | 1784.2ms/step | 11.2 samples/s
|
| 265 |
+
Step 20140/44000 | Loss: 2.9598 | LR: 2.90e-04 | 1786.0ms/step | 11.2 samples/s
|
| 266 |
+
Step 20150/44000 | Loss: 2.9421 | LR: 2.90e-04 | 1785.0ms/step | 11.2 samples/s
|
| 267 |
+
Step 20160/44000 | Loss: 3.0087 | LR: 2.90e-04 | 1784.8ms/step | 11.2 samples/s
|
| 268 |
+
Step 20170/44000 | Loss: 2.9875 | LR: 2.90e-04 | 1784.1ms/step | 11.2 samples/s
|
| 269 |
+
Step 20180/44000 | Loss: 2.9414 | LR: 2.90e-04 | 1784.9ms/step | 11.2 samples/s
|
| 270 |
+
Step 20190/44000 | Loss: 2.9603 | LR: 2.90e-04 | 1784.0ms/step | 11.2 samples/s
|
| 271 |
+
Step 20200/44000 | Loss: 2.9277 | LR: 2.89e-04 | 1786.8ms/step | 11.2 samples/s
|
| 272 |
+
Step 20210/44000 | Loss: 2.9626 | LR: 2.89e-04 | 1784.3ms/step | 11.2 samples/s
|
| 273 |
+
Step 20220/44000 | Loss: 2.9394 | LR: 2.89e-04 | 1788.5ms/step | 11.2 samples/s
|
| 274 |
+
Step 20230/44000 | Loss: 2.9337 | LR: 2.89e-04 | 1784.7ms/step | 11.2 samples/s
|
| 275 |
+
Step 20240/44000 | Loss: 2.9535 | LR: 2.89e-04 | 1786.2ms/step | 11.2 samples/s
|
| 276 |
+
Step 20250/44000 | Loss: 2.9984 | LR: 2.88e-04 | 1785.6ms/step | 11.2 samples/s
|
| 277 |
+
Step 20260/44000 | Loss: 2.9629 | LR: 2.88e-04 | 1787.2ms/step | 11.2 samples/s
|
| 278 |
+
Step 20270/44000 | Loss: 2.9640 | LR: 2.88e-04 | 1784.1ms/step | 11.2 samples/s
|
| 279 |
+
Step 20280/44000 | Loss: 2.9570 | LR: 2.88e-04 | 1786.5ms/step | 11.2 samples/s
|
| 280 |
+
Step 20290/44000 | Loss: 2.9456 | LR: 2.88e-04 | 1786.6ms/step | 11.2 samples/s
|
| 281 |
+
Step 20300/44000 | Loss: 2.9794 | LR: 2.88e-04 | 1784.1ms/step | 11.2 samples/s
|
| 282 |
+
Step 20310/44000 | Loss: 2.9673 | LR: 2.87e-04 | 1785.3ms/step | 11.2 samples/s
|
| 283 |
+
Step 20320/44000 | Loss: 2.9821 | LR: 2.87e-04 | 1785.3ms/step | 11.2 samples/s
|
| 284 |
+
Step 20330/44000 | Loss: 2.9740 | LR: 2.87e-04 | 1784.6ms/step | 11.2 samples/s
|
| 285 |
+
Step 20340/44000 | Loss: 2.9713 | LR: 2.87e-04 | 1784.0ms/step | 11.2 samples/s
|
| 286 |
+
Step 20350/44000 | Loss: 2.9644 | LR: 2.87e-04 | 1786.9ms/step | 11.2 samples/s
|
| 287 |
+
Step 20360/44000 | Loss: 2.9474 | LR: 2.86e-04 | 1785.9ms/step | 11.2 samples/s
|
| 288 |
+
Step 20370/44000 | Loss: 2.9757 | LR: 2.86e-04 | 1786.2ms/step | 11.2 samples/s
|
| 289 |
+
Step 20380/44000 | Loss: 2.9547 | LR: 2.86e-04 | 1784.9ms/step | 11.2 samples/s
|
| 290 |
+
Step 20390/44000 | Loss: 2.9301 | LR: 2.86e-04 | 1784.8ms/step | 11.2 samples/s
|
| 291 |
+
Step 20400/44000 | Loss: 2.9666 | LR: 2.86e-04 | 1784.2ms/step | 11.2 samples/s
|
| 292 |
+
Step 20410/44000 | Loss: 2.9617 | LR: 2.86e-04 | 1785.0ms/step | 11.2 samples/s
|
| 293 |
+
Step 20420/44000 | Loss: 2.9757 | LR: 2.85e-04 | 1786.5ms/step | 11.2 samples/s
|
| 294 |
+
Step 20430/44000 | Loss: 2.9885 | LR: 2.85e-04 | 1785.8ms/step | 11.2 samples/s
|
| 295 |
+
Step 20440/44000 | Loss: 2.9902 | LR: 2.85e-04 | 1784.2ms/step | 11.2 samples/s
|
| 296 |
+
Step 20450/44000 | Loss: 2.9591 | LR: 2.85e-04 | 1785.5ms/step | 11.2 samples/s
|
| 297 |
+
Step 20460/44000 | Loss: 2.9288 | LR: 2.85e-04 | 1784.3ms/step | 11.2 samples/s
|
| 298 |
+
Step 20470/44000 | Loss: 2.9821 | LR: 2.84e-04 | 1785.2ms/step | 11.2 samples/s
|
| 299 |
+
Step 20480/44000 | Loss: 2.9736 | LR: 2.84e-04 | 1784.5ms/step | 11.2 samples/s
|
| 300 |
+
Step 20490/44000 | Loss: 2.9740 | LR: 2.84e-04 | 1785.8ms/step | 11.2 samples/s
|
| 301 |
+
Step 20500/44000 | Loss: 2.9178 | LR: 2.84e-04 | 1784.6ms/step | 11.2 samples/s
|
| 302 |
+
Step 20510/44000 | Loss: 3.0172 | LR: 2.84e-04 | 1785.1ms/step | 11.2 samples/s
|
| 303 |
+
Step 20520/44000 | Loss: 2.9671 | LR: 2.84e-04 | 1784.5ms/step | 11.2 samples/s
|
| 304 |
+
Step 20530/44000 | Loss: 2.9320 | LR: 2.83e-04 | 1784.8ms/step | 11.2 samples/s
|
| 305 |
+
Step 20540/44000 | Loss: 2.9589 | LR: 2.83e-04 | 1785.2ms/step | 11.2 samples/s
|
| 306 |
+
Step 20550/44000 | Loss: 2.9659 | LR: 2.83e-04 | 1783.8ms/step | 11.2 samples/s
|
| 307 |
+
Step 20560/44000 | Loss: 2.9420 | LR: 2.83e-04 | 1784.0ms/step | 11.2 samples/s
|
| 308 |
+
Step 20570/44000 | Loss: 2.9842 | LR: 2.83e-04 | 1783.3ms/step | 11.2 samples/s
|
| 309 |
+
Step 20580/44000 | Loss: 2.9633 | LR: 2.83e-04 | 1784.6ms/step | 11.2 samples/s
|
| 310 |
+
Step 20590/44000 | Loss: 2.9772 | LR: 2.82e-04 | 1784.5ms/step | 11.2 samples/s
|
| 311 |
+
Step 20600/44000 | Loss: 2.9824 | LR: 2.82e-04 | 1785.4ms/step | 11.2 samples/s
|
| 312 |
+
Step 20610/44000 | Loss: 2.9429 | LR: 2.82e-04 | 1784.3ms/step | 11.2 samples/s
|
| 313 |
+
Step 20620/44000 | Loss: 2.9572 | LR: 2.82e-04 | 1784.1ms/step | 11.2 samples/s
|
| 314 |
+
Step 20630/44000 | Loss: 2.9799 | LR: 2.82e-04 | 1783.6ms/step | 11.2 samples/s
|
| 315 |
+
Step 20640/44000 | Loss: 2.9444 | LR: 2.81e-04 | 1784.2ms/step | 11.2 samples/s
|
| 316 |
+
Step 20650/44000 | Loss: 2.9568 | LR: 2.81e-04 | 1783.0ms/step | 11.2 samples/s
|
| 317 |
+
Step 20660/44000 | Loss: 2.9536 | LR: 2.81e-04 | 1784.0ms/step | 11.2 samples/s
|
| 318 |
+
Step 20670/44000 | Loss: 2.9776 | LR: 2.81e-04 | 1783.2ms/step | 11.2 samples/s
|
| 319 |
+
Step 20680/44000 | Loss: 2.9186 | LR: 2.81e-04 | 1783.7ms/step | 11.2 samples/s
|
| 320 |
+
Step 20690/44000 | Loss: 2.9539 | LR: 2.81e-04 | 1783.3ms/step | 11.2 samples/s
|
| 321 |
+
Step 20700/44000 | Loss: 2.9371 | LR: 2.80e-04 | 1785.0ms/step | 11.2 samples/s
|
| 322 |
+
Step 20710/44000 | Loss: 2.9530 | LR: 2.80e-04 | 1784.1ms/step | 11.2 samples/s
|
| 323 |
+
Step 20720/44000 | Loss: 2.9832 | LR: 2.80e-04 | 1786.5ms/step | 11.2 samples/s
|
| 324 |
+
Step 20730/44000 | Loss: 2.9093 | LR: 2.80e-04 | 1786.4ms/step | 11.2 samples/s
|
| 325 |
+
Step 20740/44000 | Loss: 2.9929 | LR: 2.80e-04 | 1784.8ms/step | 11.2 samples/s
|
| 326 |
+
Step 20750/44000 | Loss: 2.9719 | LR: 2.79e-04 | 1784.1ms/step | 11.2 samples/s
|
| 327 |
+
Step 20760/44000 | Loss: 3.0116 | LR: 2.79e-04 | 1783.6ms/step | 11.2 samples/s
|
| 328 |
+
Step 20770/44000 | Loss: 2.9461 | LR: 2.79e-04 | 1783.8ms/step | 11.2 samples/s
|
| 329 |
+
Step 20780/44000 | Loss: 2.9501 | LR: 2.79e-04 | 1783.9ms/step | 11.2 samples/s
|
| 330 |
+
Step 20790/44000 | Loss: 2.9585 | LR: 2.79e-04 | 1783.8ms/step | 11.2 samples/s
|
| 331 |
+
Step 20800/44000 | Loss: 2.9744 | LR: 2.79e-04 | 1783.7ms/step | 11.2 samples/s
|
| 332 |
+
Step 20810/44000 | Loss: 2.9439 | LR: 2.78e-04 | 1784.1ms/step | 11.2 samples/s
|
| 333 |
+
Step 20820/44000 | Loss: 2.9059 | LR: 2.78e-04 | 1783.4ms/step | 11.2 samples/s
|
| 334 |
+
Step 20830/44000 | Loss: 2.9289 | LR: 2.78e-04 | 1785.1ms/step | 11.2 samples/s
|
| 335 |
+
Step 20840/44000 | Loss: 2.9486 | LR: 2.78e-04 | 1783.8ms/step | 11.2 samples/s
|
| 336 |
+
Step 20850/44000 | Loss: 2.9122 | LR: 2.78e-04 | 1785.2ms/step | 11.2 samples/s
|
| 337 |
+
Step 20860/44000 | Loss: 3.0248 | LR: 2.77e-04 | 1784.2ms/step | 11.2 samples/s
|
| 338 |
+
Step 20870/44000 | Loss: 2.9592 | LR: 2.77e-04 | 1792.6ms/step | 11.2 samples/s
|
| 339 |
+
Step 20880/44000 | Loss: 2.9901 | LR: 2.77e-04 | 1788.2ms/step | 11.2 samples/s
|
| 340 |
+
Step 20890/44000 | Loss: 2.9332 | LR: 2.77e-04 | 1785.5ms/step | 11.2 samples/s
|
| 341 |
+
Step 20900/44000 | Loss: 2.9794 | LR: 2.77e-04 | 1784.2ms/step | 11.2 samples/s
|
| 342 |
+
Step 20910/44000 | Loss: 2.9616 | LR: 2.77e-04 | 1784.8ms/step | 11.2 samples/s
|
| 343 |
+
Step 20920/44000 | Loss: 2.9739 | LR: 2.76e-04 | 1784.1ms/step | 11.2 samples/s
|
| 344 |
+
Step 20930/44000 | Loss: 2.9450 | LR: 2.76e-04 | 1785.6ms/step | 11.2 samples/s
|
| 345 |
+
Step 20940/44000 | Loss: 2.9973 | LR: 2.76e-04 | 1786.0ms/step | 11.2 samples/s
|
| 346 |
+
Step 20950/44000 | Loss: 2.9661 | LR: 2.76e-04 | 1787.1ms/step | 11.2 samples/s
|
| 347 |
+
Step 20960/44000 | Loss: 2.9288 | LR: 2.76e-04 | 1784.9ms/step | 11.2 samples/s
|
| 348 |
+
Step 20970/44000 | Loss: 2.9681 | LR: 2.75e-04 | 1785.0ms/step | 11.2 samples/s
|
| 349 |
+
Step 20980/44000 | Loss: 2.9672 | LR: 2.75e-04 | 1785.5ms/step | 11.2 samples/s
|
| 350 |
+
Step 20990/44000 | Loss: 2.9756 | LR: 2.75e-04 | 1784.5ms/step | 11.2 samples/s
|
| 351 |
+
Step 21000/44000 | Loss: 2.9553 | LR: 2.75e-04 | 1785.9ms/step | 11.2 samples/s
|
| 352 |
+
Validation at step 21000: Loss = 2.9396, Perplexity = 18.91
|
| 353 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-21000
|
| 354 |
+
Step 21010/44000 | Loss: 2.9004 | LR: 2.75e-04 | 3591.0ms/step | 5.6 samples/s
|
| 355 |
+
Step 21020/44000 | Loss: 2.9762 | LR: 2.75e-04 | 1782.9ms/step | 11.2 samples/s
|
| 356 |
+
Step 21030/44000 | Loss: 2.9528 | LR: 2.74e-04 | 1783.8ms/step | 11.2 samples/s
|
| 357 |
+
Step 21040/44000 | Loss: 2.9749 | LR: 2.74e-04 | 1784.6ms/step | 11.2 samples/s
|
| 358 |
+
Step 21050/44000 | Loss: 2.9377 | LR: 2.74e-04 | 1784.3ms/step | 11.2 samples/s
|
| 359 |
+
Step 21060/44000 | Loss: 2.9398 | LR: 2.74e-04 | 1783.0ms/step | 11.2 samples/s
|
| 360 |
+
Step 21070/44000 | Loss: 2.9675 | LR: 2.74e-04 | 1784.6ms/step | 11.2 samples/s
|
| 361 |
+
Step 21080/44000 | Loss: 2.9737 | LR: 2.73e-04 | 1783.6ms/step | 11.2 samples/s
|
| 362 |
+
Step 21090/44000 | Loss: 3.0046 | LR: 2.73e-04 | 1784.6ms/step | 11.2 samples/s
|
| 363 |
+
Step 21100/44000 | Loss: 2.9257 | LR: 2.73e-04 | 1783.4ms/step | 11.2 samples/s
|
| 364 |
+
Step 21110/44000 | Loss: 2.9474 | LR: 2.73e-04 | 1784.4ms/step | 11.2 samples/s
|
| 365 |
+
Step 21120/44000 | Loss: 2.9563 | LR: 2.73e-04 | 1783.2ms/step | 11.2 samples/s
|
| 366 |
+
Step 21130/44000 | Loss: 2.9624 | LR: 2.73e-04 | 1783.9ms/step | 11.2 samples/s
|
| 367 |
+
Step 21140/44000 | Loss: 2.9489 | LR: 2.72e-04 | 1784.2ms/step | 11.2 samples/s
|
| 368 |
+
Step 21150/44000 | Loss: 2.9244 | LR: 2.72e-04 | 1785.5ms/step | 11.2 samples/s
|
| 369 |
+
Step 21160/44000 | Loss: 2.9642 | LR: 2.72e-04 | 1784.0ms/step | 11.2 samples/s
|
| 370 |
+
Step 21170/44000 | Loss: 2.9359 | LR: 2.72e-04 | 1783.6ms/step | 11.2 samples/s
|
| 371 |
+
Step 21180/44000 | Loss: 2.9125 | LR: 2.72e-04 | 1782.5ms/step | 11.2 samples/s
|
| 372 |
+
Step 21190/44000 | Loss: 2.9541 | LR: 2.71e-04 | 1783.9ms/step | 11.2 samples/s
|
| 373 |
+
Step 21200/44000 | Loss: 2.9062 | LR: 2.71e-04 | 1784.2ms/step | 11.2 samples/s
|
| 374 |
+
Step 21210/44000 | Loss: 2.9329 | LR: 2.71e-04 | 1784.6ms/step | 11.2 samples/s
|
| 375 |
+
Step 21220/44000 | Loss: 2.9098 | LR: 2.71e-04 | 1784.3ms/step | 11.2 samples/s
|
| 376 |
+
Step 21230/44000 | Loss: 2.9342 | LR: 2.71e-04 | 1785.1ms/step | 11.2 samples/s
|
| 377 |
+
Step 21240/44000 | Loss: 2.9464 | LR: 2.71e-04 | 1785.3ms/step | 11.2 samples/s
|
| 378 |
+
Step 21250/44000 | Loss: 2.9395 | LR: 2.70e-04 | 1786.7ms/step | 11.2 samples/s
|
| 379 |
+
Step 21260/44000 | Loss: 2.9534 | LR: 2.70e-04 | 1785.1ms/step | 11.2 samples/s
|
| 380 |
+
Step 21270/44000 | Loss: 2.9706 | LR: 2.70e-04 | 1784.1ms/step | 11.2 samples/s
|
| 381 |
+
Step 21280/44000 | Loss: 2.9487 | LR: 2.70e-04 | 1784.4ms/step | 11.2 samples/s
|
| 382 |
+
Step 21290/44000 | Loss: 2.9144 | LR: 2.70e-04 | 1783.1ms/step | 11.2 samples/s
|
| 383 |
+
Step 21300/44000 | Loss: 2.9522 | LR: 2.70e-04 | 1784.1ms/step | 11.2 samples/s
|
| 384 |
+
Step 21310/44000 | Loss: 2.9817 | LR: 2.69e-04 | 1783.2ms/step | 11.2 samples/s
|
| 385 |
+
Step 21320/44000 | Loss: 2.9461 | LR: 2.69e-04 | 1785.1ms/step | 11.2 samples/s
|
| 386 |
+
Step 21330/44000 | Loss: 2.9692 | LR: 2.69e-04 | 1782.8ms/step | 11.2 samples/s
|
| 387 |
+
Step 21340/44000 | Loss: 2.9745 | LR: 2.69e-04 | 1783.7ms/step | 11.2 samples/s
|
| 388 |
+
Step 21350/44000 | Loss: 2.9756 | LR: 2.69e-04 | 1783.4ms/step | 11.2 samples/s
|
| 389 |
+
Step 21360/44000 | Loss: 2.9168 | LR: 2.68e-04 | 1784.6ms/step | 11.2 samples/s
|
| 390 |
+
Step 21370/44000 | Loss: 2.9945 | LR: 2.68e-04 | 1784.1ms/step | 11.2 samples/s
|
| 391 |
+
Step 21380/44000 | Loss: 2.9426 | LR: 2.68e-04 | 1782.8ms/step | 11.2 samples/s
|
| 392 |
+
Step 21390/44000 | Loss: 2.9864 | LR: 2.68e-04 | 1782.4ms/step | 11.2 samples/s
|
| 393 |
+
Step 21400/44000 | Loss: 3.0251 | LR: 2.68e-04 | 1782.9ms/step | 11.2 samples/s
|
| 394 |
+
Step 21410/44000 | Loss: 2.9505 | LR: 2.68e-04 | 1783.3ms/step | 11.2 samples/s
|
| 395 |
+
Step 21420/44000 | Loss: 2.9903 | LR: 2.67e-04 | 1784.0ms/step | 11.2 samples/s
|
| 396 |
+
Step 21430/44000 | Loss: 2.9451 | LR: 2.67e-04 | 1783.5ms/step | 11.2 samples/s
|
| 397 |
+
Step 21440/44000 | Loss: 2.9615 | LR: 2.67e-04 | 1784.3ms/step | 11.2 samples/s
|
| 398 |
+
Step 21450/44000 | Loss: 2.9595 | LR: 2.67e-04 | 1783.8ms/step | 11.2 samples/s
|
| 399 |
+
Step 21460/44000 | Loss: 2.9730 | LR: 2.67e-04 | 1783.3ms/step | 11.2 samples/s
|
| 400 |
+
Step 21470/44000 | Loss: 2.9147 | LR: 2.66e-04 | 1784.8ms/step | 11.2 samples/s
|
| 401 |
+
Step 21480/44000 | Loss: 2.9483 | LR: 2.66e-04 | 1783.7ms/step | 11.2 samples/s
|
| 402 |
+
Step 21490/44000 | Loss: 2.9476 | LR: 2.66e-04 | 1783.9ms/step | 11.2 samples/s
|
| 403 |
+
Step 21500/44000 | Loss: 2.9503 | LR: 2.66e-04 | 1783.3ms/step | 11.2 samples/s
|
| 404 |
+
Step 21510/44000 | Loss: 2.9316 | LR: 2.66e-04 | 1783.8ms/step | 11.2 samples/s
|
| 405 |
+
Step 21520/44000 | Loss: 2.9278 | LR: 2.66e-04 | 1783.0ms/step | 11.2 samples/s
|
| 406 |
+
Step 21530/44000 | Loss: 2.9478 | LR: 2.65e-04 | 1784.2ms/step | 11.2 samples/s
|
| 407 |
+
Step 21540/44000 | Loss: 2.9481 | LR: 2.65e-04 | 1783.2ms/step | 11.2 samples/s
|
| 408 |
+
Step 21550/44000 | Loss: 2.9387 | LR: 2.65e-04 | 1784.6ms/step | 11.2 samples/s
|
| 409 |
+
Step 21560/44000 | Loss: 2.9281 | LR: 2.65e-04 | 1784.4ms/step | 11.2 samples/s
|
| 410 |
+
Step 21570/44000 | Loss: 2.9686 | LR: 2.65e-04 | 1783.5ms/step | 11.2 samples/s
|
| 411 |
+
Step 21580/44000 | Loss: 2.9503 | LR: 2.64e-04 | 1783.2ms/step | 11.2 samples/s
|
| 412 |
+
Step 21590/44000 | Loss: 2.9576 | LR: 2.64e-04 | 1784.0ms/step | 11.2 samples/s
|
| 413 |
+
Step 21600/44000 | Loss: 2.9265 | LR: 2.64e-04 | 1783.0ms/step | 11.2 samples/s
|
| 414 |
+
Step 21610/44000 | Loss: 2.9480 | LR: 2.64e-04 | 1783.4ms/step | 11.2 samples/s
|
| 415 |
+
Step 21620/44000 | Loss: 2.9793 | LR: 2.64e-04 | 1782.8ms/step | 11.2 samples/s
|
| 416 |
+
Step 21630/44000 | Loss: 2.9569 | LR: 2.64e-04 | 1782.6ms/step | 11.2 samples/s
|
| 417 |
+
Step 21640/44000 | Loss: 2.9689 | LR: 2.63e-04 | 1782.5ms/step | 11.2 samples/s
|
| 418 |
+
Step 21650/44000 | Loss: 2.9492 | LR: 2.63e-04 | 1782.6ms/step | 11.2 samples/s
|
| 419 |
+
Step 21660/44000 | Loss: 2.9155 | LR: 2.63e-04 | 1784.1ms/step | 11.2 samples/s
|
| 420 |
+
Step 21670/44000 | Loss: 2.9601 | LR: 2.63e-04 | 1783.5ms/step | 11.2 samples/s
|
| 421 |
+
Step 21680/44000 | Loss: 2.9736 | LR: 2.63e-04 | 1783.7ms/step | 11.2 samples/s
|
| 422 |
+
Step 21690/44000 | Loss: 2.9557 | LR: 2.62e-04 | 1783.2ms/step | 11.2 samples/s
|
| 423 |
+
Step 21700/44000 | Loss: 2.9446 | LR: 2.62e-04 | 1784.6ms/step | 11.2 samples/s
|
| 424 |
+
Step 21710/44000 | Loss: 2.9369 | LR: 2.62e-04 | 1784.9ms/step | 11.2 samples/s
|
| 425 |
+
Step 21720/44000 | Loss: 3.0092 | LR: 2.62e-04 | 1784.3ms/step | 11.2 samples/s
|
| 426 |
+
Step 21730/44000 | Loss: 2.9173 | LR: 2.62e-04 | 1784.2ms/step | 11.2 samples/s
|
| 427 |
+
Step 21740/44000 | Loss: 2.9280 | LR: 2.62e-04 | 1784.7ms/step | 11.2 samples/s
|
| 428 |
+
Step 21750/44000 | Loss: 2.9393 | LR: 2.61e-04 | 1784.1ms/step | 11.2 samples/s
|
| 429 |
+
Step 21760/44000 | Loss: 2.9398 | LR: 2.61e-04 | 1785.0ms/step | 11.2 samples/s
|
| 430 |
+
Step 21770/44000 | Loss: 2.9675 | LR: 2.61e-04 | 1785.3ms/step | 11.2 samples/s
|
| 431 |
+
Step 21780/44000 | Loss: 2.9116 | LR: 2.61e-04 | 1785.5ms/step | 11.2 samples/s
|
| 432 |
+
Step 21790/44000 | Loss: 2.9578 | LR: 2.61e-04 | 1784.4ms/step | 11.2 samples/s
|
| 433 |
+
Step 21800/44000 | Loss: 2.8972 | LR: 2.60e-04 | 1785.4ms/step | 11.2 samples/s
|
| 434 |
+
Step 21810/44000 | Loss: 2.9899 | LR: 2.60e-04 | 1784.3ms/step | 11.2 samples/s
|
| 435 |
+
Step 21820/44000 | Loss: 2.9564 | LR: 2.60e-04 | 1784.9ms/step | 11.2 samples/s
|
| 436 |
+
Step 21830/44000 | Loss: 2.9341 | LR: 2.60e-04 | 1784.9ms/step | 11.2 samples/s
|
| 437 |
+
Step 21840/44000 | Loss: 2.9223 | LR: 2.60e-04 | 1784.8ms/step | 11.2 samples/s
|
| 438 |
+
Step 21850/44000 | Loss: 2.9339 | LR: 2.60e-04 | 1784.1ms/step | 11.2 samples/s
|
| 439 |
+
Step 21860/44000 | Loss: 2.9618 | LR: 2.59e-04 | 1784.8ms/step | 11.2 samples/s
|
| 440 |
+
Step 21870/44000 | Loss: 2.9161 | LR: 2.59e-04 | 1785.1ms/step | 11.2 samples/s
|
| 441 |
+
Step 21880/44000 | Loss: 2.9389 | LR: 2.59e-04 | 1784.0ms/step | 11.2 samples/s
|
| 442 |
+
Step 21890/44000 | Loss: 2.9089 | LR: 2.59e-04 | 1784.0ms/step | 11.2 samples/s
|
| 443 |
+
Step 21900/44000 | Loss: 2.9641 | LR: 2.59e-04 | 1784.0ms/step | 11.2 samples/s
|
| 444 |
+
Step 21910/44000 | Loss: 2.9514 | LR: 2.58e-04 | 1784.2ms/step | 11.2 samples/s
|
| 445 |
+
Step 21920/44000 | Loss: 2.9250 | LR: 2.58e-04 | 1783.8ms/step | 11.2 samples/s
|
| 446 |
+
Step 21930/44000 | Loss: 2.9058 | LR: 2.58e-04 | 1784.1ms/step | 11.2 samples/s
|
| 447 |
+
Step 21940/44000 | Loss: 2.9537 | LR: 2.58e-04 | 1784.4ms/step | 11.2 samples/s
|
| 448 |
+
Step 21950/44000 | Loss: 2.9575 | LR: 2.58e-04 | 1785.0ms/step | 11.2 samples/s
|
| 449 |
+
Step 21960/44000 | Loss: 2.9640 | LR: 2.58e-04 | 1784.0ms/step | 11.2 samples/s
|
| 450 |
+
Step 21970/44000 | Loss: 2.9483 | LR: 2.57e-04 | 1784.1ms/step | 11.2 samples/s
|
| 451 |
+
Step 21980/44000 | Loss: 2.9637 | LR: 2.57e-04 | 1783.8ms/step | 11.2 samples/s
|
| 452 |
+
Step 21990/44000 | Loss: 2.9575 | LR: 2.57e-04 | 1783.3ms/step | 11.2 samples/s
|
| 453 |
+
Step 22000/44000 | Loss: 2.9742 | LR: 2.57e-04 | 1782.5ms/step | 11.2 samples/s
|
| 454 |
+
Validation at step 22000: Loss = 2.9299, Perplexity = 18.73
|
| 455 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-22000
|
| 456 |
+
Step 22010/44000 | Loss: 2.9529 | LR: 2.57e-04 | 3598.0ms/step | 5.6 samples/s
|
| 457 |
+
Step 22020/44000 | Loss: 2.9475 | LR: 2.56e-04 | 1782.7ms/step | 11.2 samples/s
|
| 458 |
+
Step 22030/44000 | Loss: 2.9249 | LR: 2.56e-04 | 1783.3ms/step | 11.2 samples/s
|
| 459 |
+
Step 22040/44000 | Loss: 2.9355 | LR: 2.56e-04 | 1783.9ms/step | 11.2 samples/s
|
| 460 |
+
Step 22050/44000 | Loss: 2.9719 | LR: 2.56e-04 | 1783.8ms/step | 11.2 samples/s
|
| 461 |
+
Step 22060/44000 | Loss: 2.9282 | LR: 2.56e-04 | 1784.5ms/step | 11.2 samples/s
|
| 462 |
+
Step 22070/44000 | Loss: 2.9475 | LR: 2.56e-04 | 1783.7ms/step | 11.2 samples/s
|
| 463 |
+
Step 22080/44000 | Loss: 2.9194 | LR: 2.55e-04 | 1784.2ms/step | 11.2 samples/s
|
| 464 |
+
Step 22090/44000 | Loss: 2.9284 | LR: 2.55e-04 | 1784.9ms/step | 11.2 samples/s
|
| 465 |
+
Step 22100/44000 | Loss: 2.9592 | LR: 2.55e-04 | 1784.2ms/step | 11.2 samples/s
|
| 466 |
+
Step 22110/44000 | Loss: 2.9502 | LR: 2.55e-04 | 1784.3ms/step | 11.2 samples/s
|
| 467 |
+
Step 22120/44000 | Loss: 2.9309 | LR: 2.55e-04 | 1784.9ms/step | 11.2 samples/s
|
| 468 |
+
Step 22130/44000 | Loss: 2.9673 | LR: 2.54e-04 | 1785.1ms/step | 11.2 samples/s
|
| 469 |
+
Step 22140/44000 | Loss: 2.9595 | LR: 2.54e-04 | 1786.3ms/step | 11.2 samples/s
|
| 470 |
+
Step 22150/44000 | Loss: 2.9657 | LR: 2.54e-04 | 1785.5ms/step | 11.2 samples/s
|
| 471 |
+
Step 22160/44000 | Loss: 2.9435 | LR: 2.54e-04 | 1783.9ms/step | 11.2 samples/s
|
| 472 |
+
Step 22170/44000 | Loss: 2.9544 | LR: 2.54e-04 | 1784.4ms/step | 11.2 samples/s
|
| 473 |
+
Step 22180/44000 | Loss: 2.9317 | LR: 2.54e-04 | 1785.0ms/step | 11.2 samples/s
|
| 474 |
+
Step 22190/44000 | Loss: 2.9142 | LR: 2.53e-04 | 1785.3ms/step | 11.2 samples/s
|
| 475 |
+
Step 22200/44000 | Loss: 2.9581 | LR: 2.53e-04 | 1784.3ms/step | 11.2 samples/s
|
| 476 |
+
Step 22210/44000 | Loss: 2.8973 | LR: 2.53e-04 | 1784.9ms/step | 11.2 samples/s
|
| 477 |
+
Step 22220/44000 | Loss: 2.9223 | LR: 2.53e-04 | 1784.1ms/step | 11.2 samples/s
|
| 478 |
+
Step 22230/44000 | Loss: 2.9138 | LR: 2.53e-04 | 1784.9ms/step | 11.2 samples/s
|
| 479 |
+
Step 22240/44000 | Loss: 2.9175 | LR: 2.52e-04 | 1783.4ms/step | 11.2 samples/s
|
| 480 |
+
Step 22250/44000 | Loss: 2.9396 | LR: 2.52e-04 | 1784.4ms/step | 11.2 samples/s
|
| 481 |
+
Step 22260/44000 | Loss: 2.9605 | LR: 2.52e-04 | 1783.5ms/step | 11.2 samples/s
|
| 482 |
+
Step 22270/44000 | Loss: 2.9301 | LR: 2.52e-04 | 1785.1ms/step | 11.2 samples/s
|
| 483 |
+
Step 22280/44000 | Loss: 2.9358 | LR: 2.52e-04 | 1785.0ms/step | 11.2 samples/s
|
| 484 |
+
Step 22290/44000 | Loss: 2.9451 | LR: 2.52e-04 | 1785.5ms/step | 11.2 samples/s
|
| 485 |
+
Step 22300/44000 | Loss: 2.9260 | LR: 2.51e-04 | 1785.0ms/step | 11.2 samples/s
|
| 486 |
+
Step 22310/44000 | Loss: 2.9436 | LR: 2.51e-04 | 1785.3ms/step | 11.2 samples/s
|
| 487 |
+
Step 22320/44000 | Loss: 2.8736 | LR: 2.51e-04 | 1786.8ms/step | 11.2 samples/s
|
| 488 |
+
Step 22330/44000 | Loss: 2.9177 | LR: 2.51e-04 | 1786.5ms/step | 11.2 samples/s
|
| 489 |
+
Step 22340/44000 | Loss: 2.9895 | LR: 2.51e-04 | 1786.3ms/step | 11.2 samples/s
|
| 490 |
+
Step 22350/44000 | Loss: 2.9402 | LR: 2.50e-04 | 1786.7ms/step | 11.2 samples/s
|
| 491 |
+
Step 22360/44000 | Loss: 2.8859 | LR: 2.50e-04 | 1787.8ms/step | 11.2 samples/s
|
| 492 |
+
Step 22370/44000 | Loss: 2.9543 | LR: 2.50e-04 | 1784.5ms/step | 11.2 samples/s
|
| 493 |
+
Step 22380/44000 | Loss: 2.9313 | LR: 2.50e-04 | 1784.7ms/step | 11.2 samples/s
|
| 494 |
+
Step 22390/44000 | Loss: 2.9229 | LR: 2.50e-04 | 1785.1ms/step | 11.2 samples/s
|
| 495 |
+
Step 22400/44000 | Loss: 2.9187 | LR: 2.50e-04 | 1785.9ms/step | 11.2 samples/s
|
| 496 |
+
Step 22410/44000 | Loss: 2.9282 | LR: 2.49e-04 | 1783.9ms/step | 11.2 samples/s
|
| 497 |
+
Step 22420/44000 | Loss: 2.9421 | LR: 2.49e-04 | 1783.1ms/step | 11.2 samples/s
|
| 498 |
+
Step 22430/44000 | Loss: 2.9235 | LR: 2.49e-04 | 1783.7ms/step | 11.2 samples/s
|
| 499 |
+
Step 22440/44000 | Loss: 2.9476 | LR: 2.49e-04 | 1784.7ms/step | 11.2 samples/s
|
| 500 |
+
Step 22450/44000 | Loss: 2.9239 | LR: 2.49e-04 | 1785.0ms/step | 11.2 samples/s
|
| 501 |
+
Step 22460/44000 | Loss: 2.9868 | LR: 2.48e-04 | 1784.7ms/step | 11.2 samples/s
|
| 502 |
+
Step 22470/44000 | Loss: 2.9359 | LR: 2.48e-04 | 1785.0ms/step | 11.2 samples/s
|
| 503 |
+
Step 22480/44000 | Loss: 2.9391 | LR: 2.48e-04 | 1786.1ms/step | 11.2 samples/s
|
| 504 |
+
Step 22490/44000 | Loss: 2.9371 | LR: 2.48e-04 | 1784.2ms/step | 11.2 samples/s
|
| 505 |
+
Step 22500/44000 | Loss: 2.9759 | LR: 2.48e-04 | 1784.7ms/step | 11.2 samples/s
|
| 506 |
+
Step 22510/44000 | Loss: 2.9478 | LR: 2.48e-04 | 1784.6ms/step | 11.2 samples/s
|
| 507 |
+
Step 22520/44000 | Loss: 2.9331 | LR: 2.47e-04 | 1784.4ms/step | 11.2 samples/s
|
| 508 |
+
Step 22530/44000 | Loss: 2.9508 | LR: 2.47e-04 | 1784.3ms/step | 11.2 samples/s
|
| 509 |
+
Step 22540/44000 | Loss: 2.9616 | LR: 2.47e-04 | 1785.1ms/step | 11.2 samples/s
|
| 510 |
+
Step 22550/44000 | Loss: 2.9633 | LR: 2.47e-04 | 1786.1ms/step | 11.2 samples/s
|
| 511 |
+
Step 22560/44000 | Loss: 2.9919 | LR: 2.47e-04 | 1784.5ms/step | 11.2 samples/s
|
| 512 |
+
Step 22570/44000 | Loss: 2.9654 | LR: 2.46e-04 | 1786.2ms/step | 11.2 samples/s
|
| 513 |
+
Step 22580/44000 | Loss: 2.9429 | LR: 2.46e-04 | 1786.2ms/step | 11.2 samples/s
|
| 514 |
+
Step 22590/44000 | Loss: 3.0019 | LR: 2.46e-04 | 1788.8ms/step | 11.2 samples/s
|
| 515 |
+
Step 22600/44000 | Loss: 2.9478 | LR: 2.46e-04 | 1785.6ms/step | 11.2 samples/s
|
| 516 |
+
Step 22610/44000 | Loss: 2.9274 | LR: 2.46e-04 | 1785.8ms/step | 11.2 samples/s
|
| 517 |
+
Step 22620/44000 | Loss: 2.9556 | LR: 2.46e-04 | 1785.0ms/step | 11.2 samples/s
|
| 518 |
+
Step 22630/44000 | Loss: 2.9437 | LR: 2.45e-04 | 1785.8ms/step | 11.2 samples/s
|
| 519 |
+
Step 22640/44000 | Loss: 2.9011 | LR: 2.45e-04 | 1785.5ms/step | 11.2 samples/s
|
| 520 |
+
Step 22650/44000 | Loss: 2.9521 | LR: 2.45e-04 | 1787.7ms/step | 11.2 samples/s
|
| 521 |
+
Step 22660/44000 | Loss: 2.9198 | LR: 2.45e-04 | 1786.1ms/step | 11.2 samples/s
|
| 522 |
+
Step 22670/44000 | Loss: 2.9658 | LR: 2.45e-04 | 1785.7ms/step | 11.2 samples/s
|
| 523 |
+
Step 22680/44000 | Loss: 2.9353 | LR: 2.44e-04 | 1785.8ms/step | 11.2 samples/s
|
| 524 |
+
Step 22690/44000 | Loss: 2.9288 | LR: 2.44e-04 | 1785.4ms/step | 11.2 samples/s
|
| 525 |
+
Step 22700/44000 | Loss: 2.9867 | LR: 2.44e-04 | 1784.9ms/step | 11.2 samples/s
|
| 526 |
+
Step 22710/44000 | Loss: 2.9050 | LR: 2.44e-04 | 1784.9ms/step | 11.2 samples/s
|
| 527 |
+
Step 22720/44000 | Loss: 3.0138 | LR: 2.44e-04 | 1785.8ms/step | 11.2 samples/s
|
| 528 |
+
Step 22730/44000 | Loss: 2.9125 | LR: 2.44e-04 | 1786.2ms/step | 11.2 samples/s
|
| 529 |
+
Step 22740/44000 | Loss: 2.9592 | LR: 2.43e-04 | 1784.8ms/step | 11.2 samples/s
|
| 530 |
+
Step 22750/44000 | Loss: 2.9658 | LR: 2.43e-04 | 1785.7ms/step | 11.2 samples/s
|
| 531 |
+
Step 22760/44000 | Loss: 2.9370 | LR: 2.43e-04 | 1785.6ms/step | 11.2 samples/s
|
| 532 |
+
Step 22770/44000 | Loss: 2.9834 | LR: 2.43e-04 | 1784.4ms/step | 11.2 samples/s
|
| 533 |
+
Step 22780/44000 | Loss: 2.9316 | LR: 2.43e-04 | 1783.8ms/step | 11.2 samples/s
|
| 534 |
+
Step 22790/44000 | Loss: 2.9402 | LR: 2.42e-04 | 1783.8ms/step | 11.2 samples/s
|
| 535 |
+
Step 22800/44000 | Loss: 2.9216 | LR: 2.42e-04 | 1782.9ms/step | 11.2 samples/s
|
| 536 |
+
Step 22810/44000 | Loss: 2.9279 | LR: 2.42e-04 | 1782.6ms/step | 11.2 samples/s
|
| 537 |
+
Step 22820/44000 | Loss: 2.9985 | LR: 2.42e-04 | 1785.0ms/step | 11.2 samples/s
|
| 538 |
+
Step 22830/44000 | Loss: 2.9255 | LR: 2.42e-04 | 1784.7ms/step | 11.2 samples/s
|
| 539 |
+
Step 22840/44000 | Loss: 2.9068 | LR: 2.42e-04 | 1785.3ms/step | 11.2 samples/s
|
| 540 |
+
Step 22850/44000 | Loss: 2.9259 | LR: 2.41e-04 | 1785.6ms/step | 11.2 samples/s
|
| 541 |
+
Step 22860/44000 | Loss: 2.9009 | LR: 2.41e-04 | 1790.5ms/step | 11.2 samples/s
|
| 542 |
+
Step 22870/44000 | Loss: 2.9616 | LR: 2.41e-04 | 1788.9ms/step | 11.2 samples/s
|
| 543 |
+
Step 22880/44000 | Loss: 2.9366 | LR: 2.41e-04 | 1786.5ms/step | 11.2 samples/s
|
| 544 |
+
Step 22890/44000 | Loss: 2.9706 | LR: 2.41e-04 | 1784.9ms/step | 11.2 samples/s
|
| 545 |
+
Step 22900/44000 | Loss: 2.9555 | LR: 2.40e-04 | 1787.0ms/step | 11.2 samples/s
|
| 546 |
+
Step 22910/44000 | Loss: 2.9683 | LR: 2.40e-04 | 1786.4ms/step | 11.2 samples/s
|
| 547 |
+
Step 22920/44000 | Loss: 2.9725 | LR: 2.40e-04 | 1785.6ms/step | 11.2 samples/s
|
| 548 |
+
Step 22930/44000 | Loss: 2.9169 | LR: 2.40e-04 | 1784.8ms/step | 11.2 samples/s
|
| 549 |
+
Step 22940/44000 | Loss: 2.9324 | LR: 2.40e-04 | 1785.0ms/step | 11.2 samples/s
|
| 550 |
+
Step 22950/44000 | Loss: 2.9440 | LR: 2.40e-04 | 1785.0ms/step | 11.2 samples/s
|
| 551 |
+
Step 22960/44000 | Loss: 2.9002 | LR: 2.39e-04 | 1785.5ms/step | 11.2 samples/s
|
| 552 |
+
Step 22970/44000 | Loss: 2.8904 | LR: 2.39e-04 | 1786.8ms/step | 11.2 samples/s
|
| 553 |
+
Step 22980/44000 | Loss: 2.9205 | LR: 2.39e-04 | 1786.6ms/step | 11.2 samples/s
|
| 554 |
+
Step 22990/44000 | Loss: 2.9192 | LR: 2.39e-04 | 1785.3ms/step | 11.2 samples/s
|
| 555 |
+
Step 23000/44000 | Loss: 2.9285 | LR: 2.39e-04 | 1785.5ms/step | 11.2 samples/s
|
| 556 |
+
Validation at step 23000: Loss = 2.9212, Perplexity = 18.56
|
| 557 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-23000
|
| 558 |
+
Step 23010/44000 | Loss: 2.9101 | LR: 2.38e-04 | 3602.2ms/step | 5.6 samples/s
|
| 559 |
+
Step 23020/44000 | Loss: 2.8920 | LR: 2.38e-04 | 1785.4ms/step | 11.2 samples/s
|
| 560 |
+
Step 23030/44000 | Loss: 2.8955 | LR: 2.38e-04 | 1784.7ms/step | 11.2 samples/s
|
| 561 |
+
Step 23040/44000 | Loss: 2.9346 | LR: 2.38e-04 | 1786.1ms/step | 11.2 samples/s
|
| 562 |
+
Step 23050/44000 | Loss: 2.9574 | LR: 2.38e-04 | 1786.2ms/step | 11.2 samples/s
|
| 563 |
+
Step 23060/44000 | Loss: 2.9709 | LR: 2.38e-04 | 1786.1ms/step | 11.2 samples/s
|
| 564 |
+
Step 23070/44000 | Loss: 2.9376 | LR: 2.37e-04 | 1786.7ms/step | 11.2 samples/s
|
| 565 |
+
Step 23080/44000 | Loss: 2.9551 | LR: 2.37e-04 | 1786.4ms/step | 11.2 samples/s
|
| 566 |
+
Step 23090/44000 | Loss: 2.9186 | LR: 2.37e-04 | 1786.1ms/step | 11.2 samples/s
|
| 567 |
+
Step 23100/44000 | Loss: 2.9119 | LR: 2.37e-04 | 1786.6ms/step | 11.2 samples/s
|
| 568 |
+
Step 23110/44000 | Loss: 2.9158 | LR: 2.37e-04 | 1787.6ms/step | 11.2 samples/s
|
| 569 |
+
Step 23120/44000 | Loss: 2.9540 | LR: 2.36e-04 | 1787.1ms/step | 11.2 samples/s
|
| 570 |
+
Step 23130/44000 | Loss: 2.9618 | LR: 2.36e-04 | 1788.0ms/step | 11.2 samples/s
|
| 571 |
+
Step 23140/44000 | Loss: 2.9273 | LR: 2.36e-04 | 1786.1ms/step | 11.2 samples/s
|
| 572 |
+
Step 23150/44000 | Loss: 2.9128 | LR: 2.36e-04 | 1786.9ms/step | 11.2 samples/s
|
| 573 |
+
Step 23160/44000 | Loss: 2.9295 | LR: 2.36e-04 | 1786.9ms/step | 11.2 samples/s
|
| 574 |
+
Step 23170/44000 | Loss: 2.9265 | LR: 2.36e-04 | 1788.0ms/step | 11.2 samples/s
|
| 575 |
+
Step 23180/44000 | Loss: 2.9517 | LR: 2.35e-04 | 1785.6ms/step | 11.2 samples/s
|
| 576 |
+
Step 23190/44000 | Loss: 2.9260 | LR: 2.35e-04 | 1784.9ms/step | 11.2 samples/s
|
| 577 |
+
Step 23200/44000 | Loss: 2.9469 | LR: 2.35e-04 | 1787.5ms/step | 11.2 samples/s
|
| 578 |
+
Step 23210/44000 | Loss: 2.9487 | LR: 2.35e-04 | 1787.4ms/step | 11.2 samples/s
|
| 579 |
+
Step 23220/44000 | Loss: 2.9608 | LR: 2.35e-04 | 1786.7ms/step | 11.2 samples/s
|
| 580 |
+
Step 23230/44000 | Loss: 2.9214 | LR: 2.34e-04 | 1786.5ms/step | 11.2 samples/s
|
| 581 |
+
Step 23240/44000 | Loss: 2.9871 | LR: 2.34e-04 | 1787.2ms/step | 11.2 samples/s
|
| 582 |
+
Step 23250/44000 | Loss: 2.9293 | LR: 2.34e-04 | 1787.2ms/step | 11.2 samples/s
|
| 583 |
+
Step 23260/44000 | Loss: 2.9407 | LR: 2.34e-04 | 1786.7ms/step | 11.2 samples/s
|
| 584 |
+
Step 23270/44000 | Loss: 2.9158 | LR: 2.34e-04 | 1787.1ms/step | 11.2 samples/s
|
| 585 |
+
Step 23280/44000 | Loss: 2.8751 | LR: 2.34e-04 | 1786.5ms/step | 11.2 samples/s
|
| 586 |
+
Step 23290/44000 | Loss: 2.8713 | LR: 2.33e-04 | 1786.0ms/step | 11.2 samples/s
|
| 587 |
+
Step 23300/44000 | Loss: 2.8930 | LR: 2.33e-04 | 1786.0ms/step | 11.2 samples/s
|
| 588 |
+
Step 23310/44000 | Loss: 2.9444 | LR: 2.33e-04 | 1787.5ms/step | 11.2 samples/s
|
| 589 |
+
Step 23320/44000 | Loss: 2.9697 | LR: 2.33e-04 | 1786.9ms/step | 11.2 samples/s
|
| 590 |
+
Step 23330/44000 | Loss: 2.9259 | LR: 2.33e-04 | 1786.5ms/step | 11.2 samples/s
|
| 591 |
+
Step 23340/44000 | Loss: 2.8984 | LR: 2.32e-04 | 1786.5ms/step | 11.2 samples/s
|
| 592 |
+
Step 23350/44000 | Loss: 2.9234 | LR: 2.32e-04 | 1787.3ms/step | 11.2 samples/s
|
| 593 |
+
Step 23360/44000 | Loss: 2.8995 | LR: 2.32e-04 | 1785.6ms/step | 11.2 samples/s
|
| 594 |
+
Step 23370/44000 | Loss: 2.9135 | LR: 2.32e-04 | 1787.1ms/step | 11.2 samples/s
|
| 595 |
+
Step 23380/44000 | Loss: 2.9199 | LR: 2.32e-04 | 1786.7ms/step | 11.2 samples/s
|
| 596 |
+
Step 23390/44000 | Loss: 2.9365 | LR: 2.32e-04 | 1787.1ms/step | 11.2 samples/s
|
| 597 |
+
Step 23400/44000 | Loss: 2.9408 | LR: 2.31e-04 | 1784.5ms/step | 11.2 samples/s
|
| 598 |
+
Step 23410/44000 | Loss: 2.9225 | LR: 2.31e-04 | 1785.2ms/step | 11.2 samples/s
|
| 599 |
+
Step 23420/44000 | Loss: 2.9611 | LR: 2.31e-04 | 1786.1ms/step | 11.2 samples/s
|
| 600 |
+
Step 23430/44000 | Loss: 2.9693 | LR: 2.31e-04 | 1787.0ms/step | 11.2 samples/s
|
| 601 |
+
Step 23440/44000 | Loss: 2.9681 | LR: 2.31e-04 | 1785.6ms/step | 11.2 samples/s
|
| 602 |
+
Step 23450/44000 | Loss: 2.9283 | LR: 2.30e-04 | 1786.4ms/step | 11.2 samples/s
|
| 603 |
+
Step 23460/44000 | Loss: 2.8931 | LR: 2.30e-04 | 1786.5ms/step | 11.2 samples/s
|
| 604 |
+
Step 23470/44000 | Loss: 2.8925 | LR: 2.30e-04 | 1786.6ms/step | 11.2 samples/s
|
| 605 |
+
Step 23480/44000 | Loss: 2.9322 | LR: 2.30e-04 | 1785.6ms/step | 11.2 samples/s
|
| 606 |
+
Step 23490/44000 | Loss: 2.9570 | LR: 2.30e-04 | 1786.6ms/step | 11.2 samples/s
|
| 607 |
+
Step 23500/44000 | Loss: 2.9036 | LR: 2.30e-04 | 1787.3ms/step | 11.2 samples/s
|
| 608 |
+
Step 23510/44000 | Loss: 2.9557 | LR: 2.29e-04 | 1787.3ms/step | 11.2 samples/s
|
| 609 |
+
Step 23520/44000 | Loss: 2.9466 | LR: 2.29e-04 | 1788.2ms/step | 11.2 samples/s
|
| 610 |
+
Step 23530/44000 | Loss: 2.9419 | LR: 2.29e-04 | 1787.3ms/step | 11.2 samples/s
|
| 611 |
+
Step 23540/44000 | Loss: 2.9325 | LR: 2.29e-04 | 1787.0ms/step | 11.2 samples/s
|
| 612 |
+
Step 23550/44000 | Loss: 2.9084 | LR: 2.29e-04 | 1786.5ms/step | 11.2 samples/s
|
| 613 |
+
Step 23560/44000 | Loss: 2.9623 | LR: 2.29e-04 | 1787.9ms/step | 11.2 samples/s
|
| 614 |
+
Step 23570/44000 | Loss: 2.9347 | LR: 2.28e-04 | 1786.7ms/step | 11.2 samples/s
|
| 615 |
+
Step 23580/44000 | Loss: 2.9231 | LR: 2.28e-04 | 1787.4ms/step | 11.2 samples/s
|
| 616 |
+
Step 23590/44000 | Loss: 2.9382 | LR: 2.28e-04 | 1786.6ms/step | 11.2 samples/s
|
| 617 |
+
Step 23600/44000 | Loss: 2.9333 | LR: 2.28e-04 | 1786.1ms/step | 11.2 samples/s
|
| 618 |
+
Step 23610/44000 | Loss: 2.9002 | LR: 2.28e-04 | 1786.1ms/step | 11.2 samples/s
|
| 619 |
+
Step 23620/44000 | Loss: 2.9147 | LR: 2.27e-04 | 1786.8ms/step | 11.2 samples/s
|
| 620 |
+
Step 23630/44000 | Loss: 2.9571 | LR: 2.27e-04 | 1786.0ms/step | 11.2 samples/s
|
| 621 |
+
Step 23640/44000 | Loss: 2.9667 | LR: 2.27e-04 | 1788.0ms/step | 11.2 samples/s
|
| 622 |
+
Step 23650/44000 | Loss: 2.9068 | LR: 2.27e-04 | 1787.4ms/step | 11.2 samples/s
|
| 623 |
+
Step 23660/44000 | Loss: 2.9729 | LR: 2.27e-04 | 1789.0ms/step | 11.2 samples/s
|
| 624 |
+
Step 23670/44000 | Loss: 2.9478 | LR: 2.27e-04 | 1787.2ms/step | 11.2 samples/s
|
| 625 |
+
Step 23680/44000 | Loss: 2.9451 | LR: 2.26e-04 | 1787.1ms/step | 11.2 samples/s
|
| 626 |
+
Step 23690/44000 | Loss: 2.9159 | LR: 2.26e-04 | 1787.7ms/step | 11.2 samples/s
|
| 627 |
+
Step 23700/44000 | Loss: 2.9361 | LR: 2.26e-04 | 1788.2ms/step | 11.2 samples/s
|
| 628 |
+
Step 23710/44000 | Loss: 2.9220 | LR: 2.26e-04 | 1787.5ms/step | 11.2 samples/s
|
| 629 |
+
Step 23720/44000 | Loss: 2.9507 | LR: 2.26e-04 | 1786.4ms/step | 11.2 samples/s
|
| 630 |
+
Step 23730/44000 | Loss: 2.9063 | LR: 2.25e-04 | 1785.3ms/step | 11.2 samples/s
|
| 631 |
+
Step 23740/44000 | Loss: 2.9326 | LR: 2.25e-04 | 1783.9ms/step | 11.2 samples/s
|
| 632 |
+
Step 23750/44000 | Loss: 2.9249 | LR: 2.25e-04 | 1786.6ms/step | 11.2 samples/s
|
| 633 |
+
Step 23760/44000 | Loss: 2.9178 | LR: 2.25e-04 | 1786.6ms/step | 11.2 samples/s
|
| 634 |
+
Step 23770/44000 | Loss: 2.9238 | LR: 2.25e-04 | 1787.6ms/step | 11.2 samples/s
|
| 635 |
+
Step 23780/44000 | Loss: 2.9490 | LR: 2.25e-04 | 1787.0ms/step | 11.2 samples/s
|
| 636 |
+
Step 23790/44000 | Loss: 2.9312 | LR: 2.24e-04 | 1787.2ms/step | 11.2 samples/s
|
| 637 |
+
Step 23800/44000 | Loss: 2.9404 | LR: 2.24e-04 | 1786.2ms/step | 11.2 samples/s
|
| 638 |
+
Step 23810/44000 | Loss: 2.9420 | LR: 2.24e-04 | 1786.6ms/step | 11.2 samples/s
|
| 639 |
+
Step 23820/44000 | Loss: 2.9315 | LR: 2.24e-04 | 1785.1ms/step | 11.2 samples/s
|
| 640 |
+
Step 23830/44000 | Loss: 2.9264 | LR: 2.24e-04 | 1784.9ms/step | 11.2 samples/s
|
| 641 |
+
Step 23840/44000 | Loss: 2.9111 | LR: 2.23e-04 | 1784.6ms/step | 11.2 samples/s
|
| 642 |
+
Step 23850/44000 | Loss: 2.9539 | LR: 2.23e-04 | 1785.7ms/step | 11.2 samples/s
|
| 643 |
+
Step 23860/44000 | Loss: 2.9504 | LR: 2.23e-04 | 1786.9ms/step | 11.2 samples/s
|
| 644 |
+
Step 23870/44000 | Loss: 2.9326 | LR: 2.23e-04 | 1786.3ms/step | 11.2 samples/s
|
| 645 |
+
Step 23880/44000 | Loss: 2.9230 | LR: 2.23e-04 | 1786.0ms/step | 11.2 samples/s
|
| 646 |
+
Step 23890/44000 | Loss: 2.9233 | LR: 2.23e-04 | 1786.8ms/step | 11.2 samples/s
|
epfl_final/SparseAtt/wandb/run-20260820_140401-ysa9iyov/files/requirements.txt
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
| 142 |
+
entmax==1.3
|
epfl_final/SparseAtt/wandb/run-20260820_140401-ysa9iyov/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-08-20T14:04:01.962254Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"/mloscratch/homes/navasard/sparse/configs_ropectl/resume_a1p0_ddp2_a100/seed43.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/TokenPrune.git",
|
| 14 |
+
"commit": "5637886f0cc760009989b14e68c6ae06d4827418"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "a100-train-4-gpu-3-0-12",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
|
| 20 |
+
"cpu_count": 64,
|
| 21 |
+
"cpu_count_logical": 128,
|
| 22 |
+
"gpu": "NVIDIA A100-SXM4-80GB",
|
| 23 |
+
"gpu_count": 4,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "7679362727936",
|
| 27 |
+
"used": "4570677764096"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1081884934144"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 36 |
+
"memoryTotal": "85899345920",
|
| 37 |
+
"cudaCores": 6912,
|
| 38 |
+
"architecture": "Ampere",
|
| 39 |
+
"uuid": "GPU-acb0c1f0-24c3-b247-3a34-5a388f299636"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 43 |
+
"memoryTotal": "85899345920",
|
| 44 |
+
"cudaCores": 6912,
|
| 45 |
+
"architecture": "Ampere",
|
| 46 |
+
"uuid": "GPU-e671cf21-d915-5ffb-9a1d-d5b78ed74769"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 50 |
+
"memoryTotal": "85899345920",
|
| 51 |
+
"cudaCores": 6912,
|
| 52 |
+
"architecture": "Ampere",
|
| 53 |
+
"uuid": "GPU-6adf3f18-ae44-1e33-62a1-898a55bd9925"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 57 |
+
"memoryTotal": "85899345920",
|
| 58 |
+
"cudaCores": 6912,
|
| 59 |
+
"architecture": "Ampere",
|
| 60 |
+
"uuid": "GPU-bd9e5b3c-f1d9-ad77-5085-33fdd309a690"
|
| 61 |
+
}
|
| 62 |
+
],
|
| 63 |
+
"cudaVersion": "13.0",
|
| 64 |
+
"writerId": "uhtm4tvvdtgt9s7788pu9uyvcoza6g0i"
|
| 65 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260820_140401-ysa9iyov/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260820_140401-ysa9iyov/logs/debug.log
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-08-20 14:04:01,984 INFO MainThread:377 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-08-20 14:04:01,984 INFO MainThread:377 [wandb_setup.py:_flush():81] Configure stats pid to 377
|
| 3 |
+
2026-08-20 14:04:01,984 INFO MainThread:377 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-08-20 14:04:01,984 INFO MainThread:377 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_140401-ysa9iyov/logs/debug.log
|
| 5 |
+
2026-08-20 14:04:01,984 INFO MainThread:377 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_140401-ysa9iyov/logs/debug-internal.log
|
| 6 |
+
2026-08-20 14:04:01,984 INFO MainThread:377 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-08-20 14:04:01,985 INFO MainThread:377 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-18000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 5, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 1000, 'eval_batches': 50, 'logging_steps': 10, 'seed': 43, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'l8k365m_dense_ropectl_a1p0_seed43', 'wandb_tags': ['qwen3', '365m', '8k', 'dense', 'rope-control', 'ddp2-a100'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': 0.0, 'rope_train_position_scale_set': [1.0, 0.9896339488176222, 0.9692159692159692, 0.9420289855072463, 0.8730158730158729], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-08-20 14:04:01,985 INFO MainThread:377 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-08-20 14:04:02,645 INFO MainThread:377 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-08-20 14:04:02,946 INFO MainThread:377 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-08-20 14:04:02,950 INFO MainThread:377 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-08-20 14:04:02,975 INFO MainThread:377 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-08-20 14:04:03,636 INFO MainThread:377 [wandb_init.py:init():1051] run resumed
|
| 15 |
+
2026-08-20 14:04:03,638 INFO MainThread:377 [wandb_init.py:init():1056] starting run threads in backend
|
| 16 |
+
2026-08-20 14:04:03,830 INFO MainThread:377 [wandb_run.py:_console_start():2554] atexit reg
|
| 17 |
+
2026-08-20 14:04:03,830 INFO MainThread:377 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 18 |
+
2026-08-20 14:04:03,830 INFO MainThread:377 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 19 |
+
2026-08-20 14:04:03,830 INFO MainThread:377 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 20 |
+
2026-08-20 14:04:03,837 INFO MainThread:377 [wandb_init.py:init():1094] run started, returning control to user process
|
epfl_final/SparseAtt/wandb/run-20260820_140401-ysa9iyov/run-ysa9iyov.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0394873ce4e8be55a80545af2c3afd7c0406c6d4257b609e22eb7e85bfe5dc7a
|
| 3 |
+
size 2228224
|
epfl_final/SparseAtt/wandb/run-20260820_140811-xfnxdn24/files/config.yaml
ADDED
|
@@ -0,0 +1,200 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.26.1
|
| 4 |
+
e:
|
| 5 |
+
2i8ld990jrgpbs4ijmg08s8bl505xjkz:
|
| 6 |
+
args:
|
| 7 |
+
- --config
|
| 8 |
+
- /mloscratch/homes/navasard/sparse/configs_ropectl/resume_a1p0_ddp2_a10040/seed45.json
|
| 9 |
+
codePath: train.py
|
| 10 |
+
codePathLocal: train.py
|
| 11 |
+
cpu_count: 32
|
| 12 |
+
cpu_count_logical: 64
|
| 13 |
+
cudaVersion: "13.0"
|
| 14 |
+
disk:
|
| 15 |
+
/:
|
| 16 |
+
total: "7679361679360"
|
| 17 |
+
used: "2889477971968"
|
| 18 |
+
email: ruben.navasardyan@mail.utoronto.ca
|
| 19 |
+
executable: /mloscratch/homes/navasard/conda/bin/python3.13
|
| 20 |
+
git:
|
| 21 |
+
commit: 5637886f0cc760009989b14e68c6ae06d4827418
|
| 22 |
+
remote: https://github.com/rubentium/TokenPrune.git
|
| 23 |
+
gpu: NVIDIA A100-SXM4-40GB
|
| 24 |
+
gpu_count: 4
|
| 25 |
+
gpu_nvidia:
|
| 26 |
+
- architecture: Ampere
|
| 27 |
+
cudaCores: 6912
|
| 28 |
+
memoryTotal: "42949672960"
|
| 29 |
+
name: NVIDIA A100-SXM4-40GB
|
| 30 |
+
uuid: GPU-fec533de-5bf5-cbb1-59be-0eacf3047129
|
| 31 |
+
- architecture: Ampere
|
| 32 |
+
cudaCores: 6912
|
| 33 |
+
memoryTotal: "42949672960"
|
| 34 |
+
name: NVIDIA A100-SXM4-40GB
|
| 35 |
+
uuid: GPU-95e16bd3-71b4-4352-501e-2fddd6eec868
|
| 36 |
+
- architecture: Ampere
|
| 37 |
+
cudaCores: 6912
|
| 38 |
+
memoryTotal: "42949672960"
|
| 39 |
+
name: NVIDIA A100-SXM4-40GB
|
| 40 |
+
uuid: GPU-086d5a87-2d0c-ecc2-e858-d39983c380ed
|
| 41 |
+
- architecture: Ampere
|
| 42 |
+
cudaCores: 6912
|
| 43 |
+
memoryTotal: "42949672960"
|
| 44 |
+
name: NVIDIA A100-SXM4-40GB
|
| 45 |
+
uuid: GPU-6b0e8c05-a91f-b234-b772-05ea54f42725
|
| 46 |
+
host: a100-40gb-train-4-gpu-2-0-6
|
| 47 |
+
memory:
|
| 48 |
+
total: "540741947392"
|
| 49 |
+
os: Linux-6.8.0-124-generic-x86_64-with-glibc2.35
|
| 50 |
+
program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
|
| 51 |
+
python: CPython 3.13.13
|
| 52 |
+
root: /mloscratch/homes/navasard/sparse/SparseAtt
|
| 53 |
+
startedAt: "2026-08-20T14:08:11.259091Z"
|
| 54 |
+
writerId: 2i8ld990jrgpbs4ijmg08s8bl505xjkz
|
| 55 |
+
m: []
|
| 56 |
+
python_version: 3.13.13
|
| 57 |
+
t:
|
| 58 |
+
"1":
|
| 59 |
+
- 1
|
| 60 |
+
- 5
|
| 61 |
+
- 11
|
| 62 |
+
- 49
|
| 63 |
+
- 51
|
| 64 |
+
- 53
|
| 65 |
+
- 71
|
| 66 |
+
"2":
|
| 67 |
+
- 1
|
| 68 |
+
- 5
|
| 69 |
+
- 11
|
| 70 |
+
- 49
|
| 71 |
+
- 51
|
| 72 |
+
- 53
|
| 73 |
+
- 71
|
| 74 |
+
"3":
|
| 75 |
+
- 2
|
| 76 |
+
- 5
|
| 77 |
+
- 13
|
| 78 |
+
- 14
|
| 79 |
+
- 15
|
| 80 |
+
- 16
|
| 81 |
+
- 61
|
| 82 |
+
- 62
|
| 83 |
+
"4": 3.13.13
|
| 84 |
+
"5": 0.26.1
|
| 85 |
+
"6": 5.7.0
|
| 86 |
+
"12": 0.26.1
|
| 87 |
+
"13": linux-x86_64
|
| 88 |
+
attention_dropout:
|
| 89 |
+
value: 0
|
| 90 |
+
attention_type:
|
| 91 |
+
value: sparse_stochastic
|
| 92 |
+
checkpoint_activations:
|
| 93 |
+
value: false
|
| 94 |
+
dense_mode:
|
| 95 |
+
value: true
|
| 96 |
+
dense_step_intervals:
|
| 97 |
+
value: null
|
| 98 |
+
eval_batches:
|
| 99 |
+
value: 50
|
| 100 |
+
eval_steps:
|
| 101 |
+
value: 1000
|
| 102 |
+
force_new_run:
|
| 103 |
+
value: false
|
| 104 |
+
gradient_accumulation_steps:
|
| 105 |
+
value: 5
|
| 106 |
+
head_dim:
|
| 107 |
+
value: null
|
| 108 |
+
hidden_size:
|
| 109 |
+
value: 768
|
| 110 |
+
init_from_scratch:
|
| 111 |
+
value: true
|
| 112 |
+
intermediate_size:
|
| 113 |
+
value: 3072
|
| 114 |
+
learning_rate:
|
| 115 |
+
value: 0.0005
|
| 116 |
+
logging_steps:
|
| 117 |
+
value: 10
|
| 118 |
+
lr_scheduler:
|
| 119 |
+
value: cosine
|
| 120 |
+
max_grad_norm:
|
| 121 |
+
value: 1
|
| 122 |
+
max_seq_length:
|
| 123 |
+
value: 8192
|
| 124 |
+
max_steps:
|
| 125 |
+
value: 44000
|
| 126 |
+
mixed_precision:
|
| 127 |
+
value: bf16
|
| 128 |
+
model_dtype:
|
| 129 |
+
value: bfloat16
|
| 130 |
+
model_path:
|
| 131 |
+
value: /mloscratch/homes/navasard/qwen3_06b_base
|
| 132 |
+
num_attention_heads:
|
| 133 |
+
value: 6
|
| 134 |
+
num_epochs:
|
| 135 |
+
value: 1
|
| 136 |
+
num_hidden_layers:
|
| 137 |
+
value: 28
|
| 138 |
+
num_key_value_heads:
|
| 139 |
+
value: 2
|
| 140 |
+
num_workers:
|
| 141 |
+
value: 8
|
| 142 |
+
output_dir:
|
| 143 |
+
value: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735
|
| 144 |
+
per_device_batch_size:
|
| 145 |
+
value: 2
|
| 146 |
+
resume_from_checkpoint:
|
| 147 |
+
value: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-17000
|
| 148 |
+
rope_train_position_scale_set:
|
| 149 |
+
value:
|
| 150 |
+
- 1
|
| 151 |
+
- 0.9896339488176222
|
| 152 |
+
- 0.9692159692159692
|
| 153 |
+
- 0.9420289855072463
|
| 154 |
+
- 0.8730158730158729
|
| 155 |
+
save_steps:
|
| 156 |
+
value: 1000
|
| 157 |
+
seed:
|
| 158 |
+
value: 45
|
| 159 |
+
sparse_block_size:
|
| 160 |
+
value: 128
|
| 161 |
+
sparse_layer_indices:
|
| 162 |
+
value: null
|
| 163 |
+
sparse_mask_cache_dir:
|
| 164 |
+
value: null
|
| 165 |
+
sparse_mask_pool_size:
|
| 166 |
+
value: 1
|
| 167 |
+
sparse_mask_seed:
|
| 168 |
+
value: 0
|
| 169 |
+
sparse_mixed_nrb_set:
|
| 170 |
+
value: null
|
| 171 |
+
sparse_num_random_blocks:
|
| 172 |
+
value: 31
|
| 173 |
+
sparse_val_dense_compare:
|
| 174 |
+
value: false
|
| 175 |
+
strategy:
|
| 176 |
+
value: ddp
|
| 177 |
+
train_path:
|
| 178 |
+
value: /mloscratch/homes/navasard/data/fineweb_large/tokenized
|
| 179 |
+
val_path:
|
| 180 |
+
value: null
|
| 181 |
+
verbose:
|
| 182 |
+
value: true
|
| 183 |
+
wandb_enabled:
|
| 184 |
+
value: true
|
| 185 |
+
wandb_name:
|
| 186 |
+
value: l8k365m_dense_ropectl_a1p0_seed45
|
| 187 |
+
wandb_project:
|
| 188 |
+
value: stochastic_sparsity
|
| 189 |
+
wandb_tags:
|
| 190 |
+
value:
|
| 191 |
+
- qwen3
|
| 192 |
+
- 365m
|
| 193 |
+
- 8k
|
| 194 |
+
- dense
|
| 195 |
+
- rope-control
|
| 196 |
+
- ddp2-a100-40gb
|
| 197 |
+
warmup_steps:
|
| 198 |
+
value: 750
|
| 199 |
+
weight_decay:
|
| 200 |
+
value: 0.01
|
epfl_final/SparseAtt/wandb/run-20260820_140811-xfnxdn24/files/output.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260820_140811-xfnxdn24/files/requirements.txt
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
| 142 |
+
entmax==1.3
|
epfl_final/SparseAtt/wandb/run-20260820_140811-xfnxdn24/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-08-20T14:08:11.259091Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"/mloscratch/homes/navasard/sparse/configs_ropectl/resume_a1p0_ddp2_a10040/seed45.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/TokenPrune.git",
|
| 14 |
+
"commit": "5637886f0cc760009989b14e68c6ae06d4827418"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "a100-40gb-train-4-gpu-2-0-6",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
|
| 20 |
+
"cpu_count": 32,
|
| 21 |
+
"cpu_count_logical": 64,
|
| 22 |
+
"gpu": "NVIDIA A100-SXM4-40GB",
|
| 23 |
+
"gpu_count": 4,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "7679361679360",
|
| 27 |
+
"used": "2889477971968"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "540741947392"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA A100-SXM4-40GB",
|
| 36 |
+
"memoryTotal": "42949672960",
|
| 37 |
+
"cudaCores": 6912,
|
| 38 |
+
"architecture": "Ampere",
|
| 39 |
+
"uuid": "GPU-fec533de-5bf5-cbb1-59be-0eacf3047129"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA A100-SXM4-40GB",
|
| 43 |
+
"memoryTotal": "42949672960",
|
| 44 |
+
"cudaCores": 6912,
|
| 45 |
+
"architecture": "Ampere",
|
| 46 |
+
"uuid": "GPU-95e16bd3-71b4-4352-501e-2fddd6eec868"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"name": "NVIDIA A100-SXM4-40GB",
|
| 50 |
+
"memoryTotal": "42949672960",
|
| 51 |
+
"cudaCores": 6912,
|
| 52 |
+
"architecture": "Ampere",
|
| 53 |
+
"uuid": "GPU-086d5a87-2d0c-ecc2-e858-d39983c380ed"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "NVIDIA A100-SXM4-40GB",
|
| 57 |
+
"memoryTotal": "42949672960",
|
| 58 |
+
"cudaCores": 6912,
|
| 59 |
+
"architecture": "Ampere",
|
| 60 |
+
"uuid": "GPU-6b0e8c05-a91f-b234-b772-05ea54f42725"
|
| 61 |
+
}
|
| 62 |
+
],
|
| 63 |
+
"cudaVersion": "13.0",
|
| 64 |
+
"writerId": "2i8ld990jrgpbs4ijmg08s8bl505xjkz"
|
| 65 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260820_140811-xfnxdn24/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"eval/loss":2.8730159530639647,"eval/uses_dense_attention":1,"eval/dense_interval_end":-1,"_wandb.runtime":10163,"train/global_step":44000,"eval/dense_interval_active":0,"speed/steps_per_sec":0.4287300740699672,"_timestamp":1.7872973784204266e+09,"train/epoch":0.2906439083679023,"_wandb":{"runtime":79297},"speed/ms_per_step":2332.469916343689,"_step":44000,"train/loss":2.873705039024353,"eval/perplexity":17.690290912536277,"speed/samples_per_sec":8.574601481399345,"eval/uses_sparse_attention":0,"_runtime":79297.653695322,"eval/dense_interval_start":-1,"train/learning_rate":0}
|
epfl_final/SparseAtt/wandb/run-20260820_140811-xfnxdn24/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260820_140811-xfnxdn24/logs/debug.log
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-08-20 14:08:11,281 INFO MainThread:364 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-08-20 14:08:11,281 INFO MainThread:364 [wandb_setup.py:_flush():81] Configure stats pid to 364
|
| 3 |
+
2026-08-20 14:08:11,281 INFO MainThread:364 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-08-20 14:08:11,281 INFO MainThread:364 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_140811-xfnxdn24/logs/debug.log
|
| 5 |
+
2026-08-20 14:08:11,281 INFO MainThread:364 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_140811-xfnxdn24/logs/debug-internal.log
|
| 6 |
+
2026-08-20 14:08:11,281 INFO MainThread:364 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-08-20 14:08:11,281 INFO MainThread:364 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-17000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 5, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 1000, 'eval_batches': 50, 'logging_steps': 10, 'seed': 45, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'l8k365m_dense_ropectl_a1p0_seed45', 'wandb_tags': ['qwen3', '365m', '8k', 'dense', 'rope-control', 'ddp2-a100-40gb'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': 0.0, 'rope_train_position_scale_set': [1.0, 0.9896339488176222, 0.9692159692159692, 0.9420289855072463, 0.8730158730158729], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-08-20 14:08:11,281 INFO MainThread:364 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-08-20 14:08:12,485 INFO MainThread:364 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-08-20 14:08:13,128 INFO MainThread:364 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-08-20 14:08:13,130 INFO MainThread:364 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-08-20 14:08:13,187 INFO MainThread:364 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-08-20 14:08:13,863 INFO MainThread:364 [wandb_init.py:init():1051] run resumed
|
| 15 |
+
2026-08-20 14:08:13,864 INFO MainThread:364 [wandb_init.py:init():1056] starting run threads in backend
|
| 16 |
+
2026-08-20 14:08:14,172 INFO MainThread:364 [wandb_run.py:_console_start():2554] atexit reg
|
| 17 |
+
2026-08-20 14:08:14,172 INFO MainThread:364 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 18 |
+
2026-08-20 14:08:14,172 INFO MainThread:364 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 19 |
+
2026-08-20 14:08:14,172 INFO MainThread:364 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 20 |
+
2026-08-20 14:08:14,182 INFO MainThread:364 [wandb_init.py:init():1094] run started, returning control to user process
|
| 21 |
+
2026-08-21 07:29:48,511 INFO MainThread:364 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/xfnxdn24
|
| 22 |
+
2026-08-21 07:29:48,512 INFO MainThread:364 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
|
| 23 |
+
2026-08-21 07:29:48,513 INFO MainThread:364 [wandb_run.py:_restore():2502] restore
|
| 24 |
+
2026-08-21 07:29:48,513 INFO MainThread:364 [wandb_run.py:_restore():2508] restore done
|
| 25 |
+
2026-08-21 07:29:50,112 INFO MainThread:364 [wandb_run.py:_footer_sync_info():3889] logging synced files
|
epfl_final/SparseAtt/wandb/run-20260820_140811-xfnxdn24/run-xfnxdn24.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9cb89b83928337ec859d50670d7ad812a605a4b5d1e14d715fa32c1f80eef347
|
| 3 |
+
size 12320679
|
epfl_final/SparseAtt/wandb/run-20260820_140814-bajm6b15/files/config.yaml
ADDED
|
@@ -0,0 +1,200 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.26.1
|
| 4 |
+
e:
|
| 5 |
+
vmd5xzzcppyu853jiiavwydvytup5hpw:
|
| 6 |
+
args:
|
| 7 |
+
- --config
|
| 8 |
+
- /mloscratch/homes/navasard/sparse/configs_ropectl/resume_a1p0_ddp2_a10040/seed44.json
|
| 9 |
+
codePath: train.py
|
| 10 |
+
codePathLocal: train.py
|
| 11 |
+
cpu_count: 32
|
| 12 |
+
cpu_count_logical: 64
|
| 13 |
+
cudaVersion: "13.0"
|
| 14 |
+
disk:
|
| 15 |
+
/:
|
| 16 |
+
total: "7679361679360"
|
| 17 |
+
used: "2889477996544"
|
| 18 |
+
email: ruben.navasardyan@mail.utoronto.ca
|
| 19 |
+
executable: /mloscratch/homes/navasard/conda/bin/python3.13
|
| 20 |
+
git:
|
| 21 |
+
commit: 5637886f0cc760009989b14e68c6ae06d4827418
|
| 22 |
+
remote: https://github.com/rubentium/TokenPrune.git
|
| 23 |
+
gpu: NVIDIA A100-SXM4-40GB
|
| 24 |
+
gpu_count: 4
|
| 25 |
+
gpu_nvidia:
|
| 26 |
+
- architecture: Ampere
|
| 27 |
+
cudaCores: 6912
|
| 28 |
+
memoryTotal: "42949672960"
|
| 29 |
+
name: NVIDIA A100-SXM4-40GB
|
| 30 |
+
uuid: GPU-fec533de-5bf5-cbb1-59be-0eacf3047129
|
| 31 |
+
- architecture: Ampere
|
| 32 |
+
cudaCores: 6912
|
| 33 |
+
memoryTotal: "42949672960"
|
| 34 |
+
name: NVIDIA A100-SXM4-40GB
|
| 35 |
+
uuid: GPU-95e16bd3-71b4-4352-501e-2fddd6eec868
|
| 36 |
+
- architecture: Ampere
|
| 37 |
+
cudaCores: 6912
|
| 38 |
+
memoryTotal: "42949672960"
|
| 39 |
+
name: NVIDIA A100-SXM4-40GB
|
| 40 |
+
uuid: GPU-086d5a87-2d0c-ecc2-e858-d39983c380ed
|
| 41 |
+
- architecture: Ampere
|
| 42 |
+
cudaCores: 6912
|
| 43 |
+
memoryTotal: "42949672960"
|
| 44 |
+
name: NVIDIA A100-SXM4-40GB
|
| 45 |
+
uuid: GPU-6b0e8c05-a91f-b234-b772-05ea54f42725
|
| 46 |
+
host: a100-40gb-train-4-gpu-2-0-6
|
| 47 |
+
memory:
|
| 48 |
+
total: "540741947392"
|
| 49 |
+
os: Linux-6.8.0-124-generic-x86_64-with-glibc2.35
|
| 50 |
+
program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
|
| 51 |
+
python: CPython 3.13.13
|
| 52 |
+
root: /mloscratch/homes/navasard/sparse/SparseAtt
|
| 53 |
+
startedAt: "2026-08-20T14:08:14.266122Z"
|
| 54 |
+
writerId: vmd5xzzcppyu853jiiavwydvytup5hpw
|
| 55 |
+
m: []
|
| 56 |
+
python_version: 3.13.13
|
| 57 |
+
t:
|
| 58 |
+
"1":
|
| 59 |
+
- 1
|
| 60 |
+
- 5
|
| 61 |
+
- 11
|
| 62 |
+
- 49
|
| 63 |
+
- 51
|
| 64 |
+
- 53
|
| 65 |
+
- 71
|
| 66 |
+
"2":
|
| 67 |
+
- 1
|
| 68 |
+
- 5
|
| 69 |
+
- 11
|
| 70 |
+
- 49
|
| 71 |
+
- 51
|
| 72 |
+
- 53
|
| 73 |
+
- 71
|
| 74 |
+
"3":
|
| 75 |
+
- 2
|
| 76 |
+
- 5
|
| 77 |
+
- 13
|
| 78 |
+
- 14
|
| 79 |
+
- 15
|
| 80 |
+
- 16
|
| 81 |
+
- 61
|
| 82 |
+
- 62
|
| 83 |
+
"4": 3.13.13
|
| 84 |
+
"5": 0.26.1
|
| 85 |
+
"6": 5.7.0
|
| 86 |
+
"12": 0.26.1
|
| 87 |
+
"13": linux-x86_64
|
| 88 |
+
attention_dropout:
|
| 89 |
+
value: 0
|
| 90 |
+
attention_type:
|
| 91 |
+
value: sparse_stochastic
|
| 92 |
+
checkpoint_activations:
|
| 93 |
+
value: false
|
| 94 |
+
dense_mode:
|
| 95 |
+
value: true
|
| 96 |
+
dense_step_intervals:
|
| 97 |
+
value: null
|
| 98 |
+
eval_batches:
|
| 99 |
+
value: 50
|
| 100 |
+
eval_steps:
|
| 101 |
+
value: 1000
|
| 102 |
+
force_new_run:
|
| 103 |
+
value: false
|
| 104 |
+
gradient_accumulation_steps:
|
| 105 |
+
value: 5
|
| 106 |
+
head_dim:
|
| 107 |
+
value: null
|
| 108 |
+
hidden_size:
|
| 109 |
+
value: 768
|
| 110 |
+
init_from_scratch:
|
| 111 |
+
value: true
|
| 112 |
+
intermediate_size:
|
| 113 |
+
value: 3072
|
| 114 |
+
learning_rate:
|
| 115 |
+
value: 0.0005
|
| 116 |
+
logging_steps:
|
| 117 |
+
value: 10
|
| 118 |
+
lr_scheduler:
|
| 119 |
+
value: cosine
|
| 120 |
+
max_grad_norm:
|
| 121 |
+
value: 1
|
| 122 |
+
max_seq_length:
|
| 123 |
+
value: 8192
|
| 124 |
+
max_steps:
|
| 125 |
+
value: 44000
|
| 126 |
+
mixed_precision:
|
| 127 |
+
value: bf16
|
| 128 |
+
model_dtype:
|
| 129 |
+
value: bfloat16
|
| 130 |
+
model_path:
|
| 131 |
+
value: /mloscratch/homes/navasard/qwen3_06b_base
|
| 132 |
+
num_attention_heads:
|
| 133 |
+
value: 6
|
| 134 |
+
num_epochs:
|
| 135 |
+
value: 1
|
| 136 |
+
num_hidden_layers:
|
| 137 |
+
value: 28
|
| 138 |
+
num_key_value_heads:
|
| 139 |
+
value: 2
|
| 140 |
+
num_workers:
|
| 141 |
+
value: 8
|
| 142 |
+
output_dir:
|
| 143 |
+
value: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858
|
| 144 |
+
per_device_batch_size:
|
| 145 |
+
value: 2
|
| 146 |
+
resume_from_checkpoint:
|
| 147 |
+
value: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-17000
|
| 148 |
+
rope_train_position_scale_set:
|
| 149 |
+
value:
|
| 150 |
+
- 1
|
| 151 |
+
- 0.9896339488176222
|
| 152 |
+
- 0.9692159692159692
|
| 153 |
+
- 0.9420289855072463
|
| 154 |
+
- 0.8730158730158729
|
| 155 |
+
save_steps:
|
| 156 |
+
value: 1000
|
| 157 |
+
seed:
|
| 158 |
+
value: 44
|
| 159 |
+
sparse_block_size:
|
| 160 |
+
value: 128
|
| 161 |
+
sparse_layer_indices:
|
| 162 |
+
value: null
|
| 163 |
+
sparse_mask_cache_dir:
|
| 164 |
+
value: null
|
| 165 |
+
sparse_mask_pool_size:
|
| 166 |
+
value: 1
|
| 167 |
+
sparse_mask_seed:
|
| 168 |
+
value: 0
|
| 169 |
+
sparse_mixed_nrb_set:
|
| 170 |
+
value: null
|
| 171 |
+
sparse_num_random_blocks:
|
| 172 |
+
value: 31
|
| 173 |
+
sparse_val_dense_compare:
|
| 174 |
+
value: false
|
| 175 |
+
strategy:
|
| 176 |
+
value: ddp
|
| 177 |
+
train_path:
|
| 178 |
+
value: /mloscratch/homes/navasard/data/fineweb_large/tokenized
|
| 179 |
+
val_path:
|
| 180 |
+
value: null
|
| 181 |
+
verbose:
|
| 182 |
+
value: true
|
| 183 |
+
wandb_enabled:
|
| 184 |
+
value: true
|
| 185 |
+
wandb_name:
|
| 186 |
+
value: l8k365m_dense_ropectl_a1p0_seed44
|
| 187 |
+
wandb_project:
|
| 188 |
+
value: stochastic_sparsity
|
| 189 |
+
wandb_tags:
|
| 190 |
+
value:
|
| 191 |
+
- qwen3
|
| 192 |
+
- 365m
|
| 193 |
+
- 8k
|
| 194 |
+
- dense
|
| 195 |
+
- rope-control
|
| 196 |
+
- ddp2-a100-40gb
|
| 197 |
+
warmup_steps:
|
| 198 |
+
value: 750
|
| 199 |
+
weight_decay:
|
| 200 |
+
value: 0.01
|
epfl_final/SparseAtt/wandb/run-20260820_140814-bajm6b15/files/output.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260820_140814-bajm6b15/files/requirements.txt
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
| 142 |
+
entmax==1.3
|
epfl_final/SparseAtt/wandb/run-20260820_140814-bajm6b15/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-08-20T14:08:14.266122Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"/mloscratch/homes/navasard/sparse/configs_ropectl/resume_a1p0_ddp2_a10040/seed44.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/TokenPrune.git",
|
| 14 |
+
"commit": "5637886f0cc760009989b14e68c6ae06d4827418"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "a100-40gb-train-4-gpu-2-0-6",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
|
| 20 |
+
"cpu_count": 32,
|
| 21 |
+
"cpu_count_logical": 64,
|
| 22 |
+
"gpu": "NVIDIA A100-SXM4-40GB",
|
| 23 |
+
"gpu_count": 4,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "7679361679360",
|
| 27 |
+
"used": "2889477996544"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "540741947392"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA A100-SXM4-40GB",
|
| 36 |
+
"memoryTotal": "42949672960",
|
| 37 |
+
"cudaCores": 6912,
|
| 38 |
+
"architecture": "Ampere",
|
| 39 |
+
"uuid": "GPU-fec533de-5bf5-cbb1-59be-0eacf3047129"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA A100-SXM4-40GB",
|
| 43 |
+
"memoryTotal": "42949672960",
|
| 44 |
+
"cudaCores": 6912,
|
| 45 |
+
"architecture": "Ampere",
|
| 46 |
+
"uuid": "GPU-95e16bd3-71b4-4352-501e-2fddd6eec868"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"name": "NVIDIA A100-SXM4-40GB",
|
| 50 |
+
"memoryTotal": "42949672960",
|
| 51 |
+
"cudaCores": 6912,
|
| 52 |
+
"architecture": "Ampere",
|
| 53 |
+
"uuid": "GPU-086d5a87-2d0c-ecc2-e858-d39983c380ed"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "NVIDIA A100-SXM4-40GB",
|
| 57 |
+
"memoryTotal": "42949672960",
|
| 58 |
+
"cudaCores": 6912,
|
| 59 |
+
"architecture": "Ampere",
|
| 60 |
+
"uuid": "GPU-6b0e8c05-a91f-b234-b772-05ea54f42725"
|
| 61 |
+
}
|
| 62 |
+
],
|
| 63 |
+
"cudaVersion": "13.0",
|
| 64 |
+
"writerId": "vmd5xzzcppyu853jiiavwydvytup5hpw"
|
| 65 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260820_140814-bajm6b15/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"eval/dense_interval_start":-1,"train/learning_rate":0,"eval/dense_interval_end":-1,"_wandb.runtime":10159,"train/epoch":0.2906439083679023,"eval/uses_sparse_attention":0,"_step":44000,"eval/perplexity":18.075437185913934,"train/loss":2.857089877128601,"eval/dense_interval_active":0,"speed/ms_per_step":1914.4633769989014,"train/global_step":44000,"speed/samples_per_sec":10.446791638998,"speed/steps_per_sec":0.5223395819499,"_timestamp":1.787297462039496e+09,"_wandb":{"runtime":79366},"_runtime":79366.530813584,"eval/uses_dense_attention":1,"eval/loss":2.894553955078125}
|
epfl_final/SparseAtt/wandb/run-20260820_140814-bajm6b15/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260820_140814-bajm6b15/logs/debug.log
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-08-20 14:08:14,290 INFO MainThread:360 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-08-20 14:08:14,290 INFO MainThread:360 [wandb_setup.py:_flush():81] Configure stats pid to 360
|
| 3 |
+
2026-08-20 14:08:14,290 INFO MainThread:360 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-08-20 14:08:14,290 INFO MainThread:360 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_140814-bajm6b15/logs/debug.log
|
| 5 |
+
2026-08-20 14:08:14,290 INFO MainThread:360 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_140814-bajm6b15/logs/debug-internal.log
|
| 6 |
+
2026-08-20 14:08:14,290 INFO MainThread:360 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-08-20 14:08:14,291 INFO MainThread:360 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-17000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 5, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 1000, 'eval_batches': 50, 'logging_steps': 10, 'seed': 44, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'l8k365m_dense_ropectl_a1p0_seed44', 'wandb_tags': ['qwen3', '365m', '8k', 'dense', 'rope-control', 'ddp2-a100-40gb'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': 0.0, 'rope_train_position_scale_set': [1.0, 0.9896339488176222, 0.9692159692159692, 0.9420289855072463, 0.8730158730158729], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-08-20 14:08:14,291 INFO MainThread:360 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-08-20 14:08:14,569 INFO MainThread:360 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-08-20 14:08:14,873 INFO MainThread:360 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-08-20 14:08:14,876 INFO MainThread:360 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-08-20 14:08:14,919 INFO MainThread:360 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-08-20 14:08:15,698 INFO MainThread:360 [wandb_init.py:init():1051] run resumed
|
| 15 |
+
2026-08-20 14:08:15,699 INFO MainThread:360 [wandb_init.py:init():1056] starting run threads in backend
|
| 16 |
+
2026-08-20 14:08:15,982 INFO MainThread:360 [wandb_run.py:_console_start():2554] atexit reg
|
| 17 |
+
2026-08-20 14:08:15,983 INFO MainThread:360 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 18 |
+
2026-08-20 14:08:15,983 INFO MainThread:360 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 19 |
+
2026-08-20 14:08:15,983 INFO MainThread:360 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 20 |
+
2026-08-20 14:08:16,001 INFO MainThread:360 [wandb_init.py:init():1094] run started, returning control to user process
|
| 21 |
+
2026-08-21 07:31:11,228 INFO MainThread:360 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/bajm6b15
|
| 22 |
+
2026-08-21 07:31:11,229 INFO MainThread:360 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
|
| 23 |
+
2026-08-21 07:31:11,230 INFO MainThread:360 [wandb_run.py:_restore():2502] restore
|
| 24 |
+
2026-08-21 07:31:11,230 INFO MainThread:360 [wandb_run.py:_restore():2508] restore done
|
| 25 |
+
2026-08-21 07:31:13,245 INFO MainThread:360 [wandb_run.py:_footer_sync_info():3889] logging synced files
|
epfl_final/SparseAtt/wandb/run-20260820_140814-bajm6b15/run-bajm6b15.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f739930e3b4543a564b89f8ff6e72d7d07495836281322e481c17f1a162e314e
|
| 3 |
+
size 12324377
|
epfl_final/SparseAtt/wandb/run-20260820_144758-zovy63wq/files/config.yaml
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.26.1
|
| 4 |
+
e:
|
| 5 |
+
mkv4jtz4fzkoz1u7bnodz4fizbdn0rho:
|
| 6 |
+
args:
|
| 7 |
+
- --config
|
| 8 |
+
- /mloscratch/homes/navasard/sparse/configs_ropectl/resume_a1p0_wandb/seed46.json
|
| 9 |
+
codePath: train.py
|
| 10 |
+
codePathLocal: train.py
|
| 11 |
+
cpu_count: 96
|
| 12 |
+
cpu_count_logical: 192
|
| 13 |
+
cudaVersion: "13.0"
|
| 14 |
+
disk:
|
| 15 |
+
/:
|
| 16 |
+
total: "9374931091456"
|
| 17 |
+
used: "5355710636032"
|
| 18 |
+
email: ruben.navasardyan@mail.utoronto.ca
|
| 19 |
+
executable: /mloscratch/homes/navasard/conda/bin/python3.13
|
| 20 |
+
git:
|
| 21 |
+
commit: 5637886f0cc760009989b14e68c6ae06d4827418
|
| 22 |
+
remote: https://github.com/rubentium/TokenPrune.git
|
| 23 |
+
gpu: NVIDIA H100 80GB HBM3
|
| 24 |
+
gpu_count: 1
|
| 25 |
+
gpu_nvidia:
|
| 26 |
+
- architecture: Hopper
|
| 27 |
+
cudaCores: 16896
|
| 28 |
+
memoryTotal: "85520809984"
|
| 29 |
+
name: NVIDIA H100 80GB HBM3
|
| 30 |
+
uuid: GPU-32209e2e-7887-e730-7611-251622a8e78f
|
| 31 |
+
host: h100-workspace-0-0
|
| 32 |
+
memory:
|
| 33 |
+
total: "1623005626368"
|
| 34 |
+
os: Linux-6.8.0-124-generic-x86_64-with-glibc2.35
|
| 35 |
+
program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
|
| 36 |
+
python: CPython 3.13.13
|
| 37 |
+
root: /mloscratch/homes/navasard/sparse/SparseAtt
|
| 38 |
+
startedAt: "2026-08-20T14:47:58.454100Z"
|
| 39 |
+
writerId: mkv4jtz4fzkoz1u7bnodz4fizbdn0rho
|
| 40 |
+
m: []
|
| 41 |
+
python_version: 3.13.13
|
| 42 |
+
t:
|
| 43 |
+
"1":
|
| 44 |
+
- 1
|
| 45 |
+
- 5
|
| 46 |
+
- 11
|
| 47 |
+
- 49
|
| 48 |
+
- 51
|
| 49 |
+
- 53
|
| 50 |
+
- 71
|
| 51 |
+
"2":
|
| 52 |
+
- 1
|
| 53 |
+
- 5
|
| 54 |
+
- 11
|
| 55 |
+
- 49
|
| 56 |
+
- 51
|
| 57 |
+
- 53
|
| 58 |
+
- 71
|
| 59 |
+
"3":
|
| 60 |
+
- 2
|
| 61 |
+
- 5
|
| 62 |
+
- 13
|
| 63 |
+
- 14
|
| 64 |
+
- 15
|
| 65 |
+
- 16
|
| 66 |
+
- 61
|
| 67 |
+
- 62
|
| 68 |
+
"4": 3.13.13
|
| 69 |
+
"5": 0.26.1
|
| 70 |
+
"6": 5.7.0
|
| 71 |
+
"12": 0.26.1
|
| 72 |
+
"13": linux-x86_64
|
| 73 |
+
attention_dropout:
|
| 74 |
+
value: 0
|
| 75 |
+
attention_type:
|
| 76 |
+
value: sparse_stochastic
|
| 77 |
+
checkpoint_activations:
|
| 78 |
+
value: false
|
| 79 |
+
dense_mode:
|
| 80 |
+
value: true
|
| 81 |
+
dense_step_intervals:
|
| 82 |
+
value: null
|
| 83 |
+
eval_batches:
|
| 84 |
+
value: 50
|
| 85 |
+
eval_steps:
|
| 86 |
+
value: 1000
|
| 87 |
+
force_new_run:
|
| 88 |
+
value: false
|
| 89 |
+
gradient_accumulation_steps:
|
| 90 |
+
value: 10
|
| 91 |
+
head_dim:
|
| 92 |
+
value: null
|
| 93 |
+
hidden_size:
|
| 94 |
+
value: 768
|
| 95 |
+
init_from_scratch:
|
| 96 |
+
value: true
|
| 97 |
+
intermediate_size:
|
| 98 |
+
value: 3072
|
| 99 |
+
learning_rate:
|
| 100 |
+
value: 0.0005
|
| 101 |
+
logging_steps:
|
| 102 |
+
value: 10
|
| 103 |
+
lr_scheduler:
|
| 104 |
+
value: cosine
|
| 105 |
+
max_grad_norm:
|
| 106 |
+
value: 1
|
| 107 |
+
max_seq_length:
|
| 108 |
+
value: 8192
|
| 109 |
+
max_steps:
|
| 110 |
+
value: 44000
|
| 111 |
+
mixed_precision:
|
| 112 |
+
value: bf16
|
| 113 |
+
model_dtype:
|
| 114 |
+
value: bfloat16
|
| 115 |
+
model_path:
|
| 116 |
+
value: /mloscratch/homes/navasard/qwen3_06b_base
|
| 117 |
+
num_attention_heads:
|
| 118 |
+
value: 6
|
| 119 |
+
num_epochs:
|
| 120 |
+
value: 1
|
| 121 |
+
num_hidden_layers:
|
| 122 |
+
value: 28
|
| 123 |
+
num_key_value_heads:
|
| 124 |
+
value: 2
|
| 125 |
+
num_workers:
|
| 126 |
+
value: 8
|
| 127 |
+
output_dir:
|
| 128 |
+
value: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735
|
| 129 |
+
per_device_batch_size:
|
| 130 |
+
value: 2
|
| 131 |
+
resume_from_checkpoint:
|
| 132 |
+
value: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-17000
|
| 133 |
+
rope_train_position_scale_set:
|
| 134 |
+
value:
|
| 135 |
+
- 1
|
| 136 |
+
- 0.9896339488176222
|
| 137 |
+
- 0.9692159692159692
|
| 138 |
+
- 0.9420289855072463
|
| 139 |
+
- 0.8730158730158729
|
| 140 |
+
save_steps:
|
| 141 |
+
value: 1000
|
| 142 |
+
seed:
|
| 143 |
+
value: 46
|
| 144 |
+
sparse_block_size:
|
| 145 |
+
value: 128
|
| 146 |
+
sparse_layer_indices:
|
| 147 |
+
value: null
|
| 148 |
+
sparse_mask_cache_dir:
|
| 149 |
+
value: null
|
| 150 |
+
sparse_mask_pool_size:
|
| 151 |
+
value: 1
|
| 152 |
+
sparse_mask_seed:
|
| 153 |
+
value: 0
|
| 154 |
+
sparse_mixed_nrb_set:
|
| 155 |
+
value: null
|
| 156 |
+
sparse_num_random_blocks:
|
| 157 |
+
value: 31
|
| 158 |
+
sparse_val_dense_compare:
|
| 159 |
+
value: false
|
| 160 |
+
strategy:
|
| 161 |
+
value: ddp
|
| 162 |
+
train_path:
|
| 163 |
+
value: /mloscratch/homes/navasard/data/fineweb_large/tokenized
|
| 164 |
+
val_path:
|
| 165 |
+
value: null
|
| 166 |
+
verbose:
|
| 167 |
+
value: true
|
| 168 |
+
wandb_enabled:
|
| 169 |
+
value: true
|
| 170 |
+
wandb_name:
|
| 171 |
+
value: l8k365m_dense_ropectl_a1p0_seed46
|
| 172 |
+
wandb_project:
|
| 173 |
+
value: stochastic_sparsity
|
| 174 |
+
wandb_tags:
|
| 175 |
+
value:
|
| 176 |
+
- qwen3
|
| 177 |
+
- 365m
|
| 178 |
+
- 8k
|
| 179 |
+
- dense
|
| 180 |
+
- rope-control
|
| 181 |
+
warmup_steps:
|
| 182 |
+
value: 750
|
| 183 |
+
weight_decay:
|
| 184 |
+
value: 0.01
|
epfl_final/SparseAtt/wandb/run-20260820_144758-zovy63wq/files/output.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260820_144758-zovy63wq/files/requirements.txt
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
| 142 |
+
entmax==1.3
|
epfl_final/SparseAtt/wandb/run-20260820_144758-zovy63wq/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-08-20T14:47:58.454100Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"/mloscratch/homes/navasard/sparse/configs_ropectl/resume_a1p0_wandb/seed46.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/TokenPrune.git",
|
| 14 |
+
"commit": "5637886f0cc760009989b14e68c6ae06d4827418"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "h100-workspace-0-0",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 23 |
+
"gpu_count": 1,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "9374931091456",
|
| 27 |
+
"used": "5355710636032"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1623005626368"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-32209e2e-7887-e730-7611-251622a8e78f"
|
| 40 |
+
}
|
| 41 |
+
],
|
| 42 |
+
"cudaVersion": "13.0",
|
| 43 |
+
"writerId": "mkv4jtz4fzkoz1u7bnodz4fizbdn0rho"
|
| 44 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260820_144758-zovy63wq/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"eval/dense_interval_end":-1,"speed/ms_per_step":1385.7184886932373,"eval/uses_sparse_attention":0,"_timestamp":1.787275670406833e+09,"_wandb":{"runtime":55207},"_runtime":55207.494384632,"speed/steps_per_sec":0.7216472957238391,"train/learning_rate":0,"train/loss":2.8717809176445006,"eval/uses_dense_attention":1,"eval/dense_interval_active":0,"eval/dense_interval_start":-1,"eval/loss":2.884106925010681,"eval/perplexity":17.88758550474596,"train/global_step":44000,"_step":44000,"train/epoch":0.2906439083679023,"_wandb.runtime":10167,"speed/samples_per_sec":14.432945914476782}
|
epfl_final/SparseAtt/wandb/run-20260820_144758-zovy63wq/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260820_144758-zovy63wq/logs/debug.log
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-08-20 14:47:58,471 INFO MainThread:165 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-08-20 14:47:58,471 INFO MainThread:165 [wandb_setup.py:_flush():81] Configure stats pid to 165
|
| 3 |
+
2026-08-20 14:47:58,471 INFO MainThread:165 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-08-20 14:47:58,471 INFO MainThread:165 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_144758-zovy63wq/logs/debug.log
|
| 5 |
+
2026-08-20 14:47:58,471 INFO MainThread:165 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_144758-zovy63wq/logs/debug-internal.log
|
| 6 |
+
2026-08-20 14:47:58,471 INFO MainThread:165 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-08-20 14:47:58,471 INFO MainThread:165 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-17000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 1000, 'eval_batches': 50, 'logging_steps': 10, 'seed': 46, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'l8k365m_dense_ropectl_a1p0_seed46', 'wandb_tags': ['qwen3', '365m', '8k', 'dense', 'rope-control'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': 0.0, 'rope_train_position_scale_set': [1.0, 0.9896339488176222, 0.9692159692159692, 0.9420289855072463, 0.8730158730158729], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-08-20 14:47:58,471 INFO MainThread:165 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-08-20 14:47:59,167 INFO MainThread:165 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-08-20 14:47:59,578 INFO MainThread:165 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-08-20 14:47:59,580 INFO MainThread:165 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-08-20 14:47:59,604 INFO MainThread:165 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-08-20 14:48:00,071 INFO MainThread:165 [wandb_init.py:init():1051] run resumed
|
| 15 |
+
2026-08-20 14:48:00,072 INFO MainThread:165 [wandb_init.py:init():1056] starting run threads in backend
|
| 16 |
+
2026-08-20 14:48:00,334 INFO MainThread:165 [wandb_run.py:_console_start():2554] atexit reg
|
| 17 |
+
2026-08-20 14:48:00,335 INFO MainThread:165 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 18 |
+
2026-08-20 14:48:00,335 INFO MainThread:165 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 19 |
+
2026-08-20 14:48:00,335 INFO MainThread:165 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 20 |
+
2026-08-20 14:48:00,342 INFO MainThread:165 [wandb_init.py:init():1094] run started, returning control to user process
|
| 21 |
+
2026-08-21 01:27:56,565 INFO MainThread:165 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/zovy63wq
|
| 22 |
+
2026-08-21 01:27:56,566 INFO MainThread:165 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
|
| 23 |
+
2026-08-21 01:27:56,567 INFO MainThread:165 [wandb_run.py:_restore():2502] restore
|
| 24 |
+
2026-08-21 01:27:56,567 INFO MainThread:165 [wandb_run.py:_restore():2508] restore done
|
| 25 |
+
2026-08-21 01:27:58,506 INFO MainThread:165 [wandb_run.py:_footer_sync_info():3889] logging synced files
|
epfl_final/SparseAtt/wandb/run-20260820_144758-zovy63wq/run-zovy63wq.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bda9f321764b2017ab0e00974cbac2f7afbee08e407111860e90225d7cf18bcc
|
| 3 |
+
size 7630182
|
epfl_final/SparseAtt/wandb/run-20260820_194209-jqstnvej/files/config.yaml
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.26.1
|
| 4 |
+
e:
|
| 5 |
+
8l9cf1l8zaolhsew8f38inve9wj109an:
|
| 6 |
+
args:
|
| 7 |
+
- --config
|
| 8 |
+
- /mloscratch/homes/navasard/sparse/configs_ropectl/resume_a1p0_wandb/seed42.json
|
| 9 |
+
codePath: train.py
|
| 10 |
+
codePathLocal: train.py
|
| 11 |
+
cpu_count: 96
|
| 12 |
+
cpu_count_logical: 192
|
| 13 |
+
cudaVersion: "13.0"
|
| 14 |
+
disk:
|
| 15 |
+
/:
|
| 16 |
+
total: "9374931091456"
|
| 17 |
+
used: "7170272956416"
|
| 18 |
+
email: ruben.navasardyan@mail.utoronto.ca
|
| 19 |
+
executable: /mloscratch/homes/navasard/conda/bin/python3.13
|
| 20 |
+
git:
|
| 21 |
+
commit: 5637886f0cc760009989b14e68c6ae06d4827418
|
| 22 |
+
remote: https://github.com/rubentium/TokenPrune.git
|
| 23 |
+
gpu: NVIDIA H100 80GB HBM3
|
| 24 |
+
gpu_count: 4
|
| 25 |
+
gpu_nvidia:
|
| 26 |
+
- architecture: Hopper
|
| 27 |
+
cudaCores: 16896
|
| 28 |
+
memoryTotal: "85520809984"
|
| 29 |
+
name: NVIDIA H100 80GB HBM3
|
| 30 |
+
uuid: GPU-b817d241-acce-e5dd-a749-3447239c5207
|
| 31 |
+
- architecture: Hopper
|
| 32 |
+
cudaCores: 16896
|
| 33 |
+
memoryTotal: "85520809984"
|
| 34 |
+
name: NVIDIA H100 80GB HBM3
|
| 35 |
+
uuid: GPU-165543e5-e049-5219-e717-e1216ac960de
|
| 36 |
+
- architecture: Hopper
|
| 37 |
+
cudaCores: 16896
|
| 38 |
+
memoryTotal: "85520809984"
|
| 39 |
+
name: NVIDIA H100 80GB HBM3
|
| 40 |
+
uuid: GPU-47444fc7-ea8c-cef6-aac8-ef408dc8047b
|
| 41 |
+
- architecture: Hopper
|
| 42 |
+
cudaCores: 16896
|
| 43 |
+
memoryTotal: "85520809984"
|
| 44 |
+
name: NVIDIA H100 80GB HBM3
|
| 45 |
+
uuid: GPU-37dc97ad-46ef-50d9-3619-3a6eaea1b6aa
|
| 46 |
+
host: h100-workspace-4-gpu-3-0-6
|
| 47 |
+
memory:
|
| 48 |
+
total: "1623005732864"
|
| 49 |
+
os: Linux-6.8.0-124-generic-x86_64-with-glibc2.35
|
| 50 |
+
program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
|
| 51 |
+
python: CPython 3.13.13
|
| 52 |
+
root: /mloscratch/homes/navasard/sparse/SparseAtt
|
| 53 |
+
startedAt: "2026-08-20T19:42:09.596893Z"
|
| 54 |
+
writerId: 8l9cf1l8zaolhsew8f38inve9wj109an
|
| 55 |
+
m: []
|
| 56 |
+
python_version: 3.13.13
|
| 57 |
+
t:
|
| 58 |
+
"1":
|
| 59 |
+
- 1
|
| 60 |
+
- 5
|
| 61 |
+
- 11
|
| 62 |
+
- 49
|
| 63 |
+
- 51
|
| 64 |
+
- 53
|
| 65 |
+
- 71
|
| 66 |
+
"2":
|
| 67 |
+
- 1
|
| 68 |
+
- 5
|
| 69 |
+
- 11
|
| 70 |
+
- 49
|
| 71 |
+
- 51
|
| 72 |
+
- 53
|
| 73 |
+
- 71
|
| 74 |
+
"3":
|
| 75 |
+
- 2
|
| 76 |
+
- 5
|
| 77 |
+
- 13
|
| 78 |
+
- 14
|
| 79 |
+
- 15
|
| 80 |
+
- 16
|
| 81 |
+
- 61
|
| 82 |
+
- 62
|
| 83 |
+
"4": 3.13.13
|
| 84 |
+
"5": 0.26.1
|
| 85 |
+
"6": 5.7.0
|
| 86 |
+
"12": 0.26.1
|
| 87 |
+
"13": linux-x86_64
|
| 88 |
+
attention_dropout:
|
| 89 |
+
value: 0
|
| 90 |
+
attention_type:
|
| 91 |
+
value: sparse_stochastic
|
| 92 |
+
checkpoint_activations:
|
| 93 |
+
value: false
|
| 94 |
+
dense_mode:
|
| 95 |
+
value: true
|
| 96 |
+
dense_step_intervals:
|
| 97 |
+
value: null
|
| 98 |
+
eval_batches:
|
| 99 |
+
value: 50
|
| 100 |
+
eval_steps:
|
| 101 |
+
value: 1000
|
| 102 |
+
force_new_run:
|
| 103 |
+
value: false
|
| 104 |
+
gradient_accumulation_steps:
|
| 105 |
+
value: 10
|
| 106 |
+
head_dim:
|
| 107 |
+
value: null
|
| 108 |
+
hidden_size:
|
| 109 |
+
value: 768
|
| 110 |
+
init_from_scratch:
|
| 111 |
+
value: true
|
| 112 |
+
intermediate_size:
|
| 113 |
+
value: 3072
|
| 114 |
+
learning_rate:
|
| 115 |
+
value: 0.0005
|
| 116 |
+
logging_steps:
|
| 117 |
+
value: 10
|
| 118 |
+
lr_scheduler:
|
| 119 |
+
value: cosine
|
| 120 |
+
max_grad_norm:
|
| 121 |
+
value: 1
|
| 122 |
+
max_seq_length:
|
| 123 |
+
value: 8192
|
| 124 |
+
max_steps:
|
| 125 |
+
value: 44000
|
| 126 |
+
mixed_precision:
|
| 127 |
+
value: bf16
|
| 128 |
+
model_dtype:
|
| 129 |
+
value: bfloat16
|
| 130 |
+
model_path:
|
| 131 |
+
value: /mloscratch/homes/navasard/qwen3_06b_base
|
| 132 |
+
num_attention_heads:
|
| 133 |
+
value: 6
|
| 134 |
+
num_epochs:
|
| 135 |
+
value: 1
|
| 136 |
+
num_hidden_layers:
|
| 137 |
+
value: 28
|
| 138 |
+
num_key_value_heads:
|
| 139 |
+
value: 2
|
| 140 |
+
num_workers:
|
| 141 |
+
value: 8
|
| 142 |
+
output_dir:
|
| 143 |
+
value: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735
|
| 144 |
+
per_device_batch_size:
|
| 145 |
+
value: 2
|
| 146 |
+
resume_from_checkpoint:
|
| 147 |
+
value: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-22000
|
| 148 |
+
rope_train_position_scale_set:
|
| 149 |
+
value:
|
| 150 |
+
- 1
|
| 151 |
+
- 0.9896339488176222
|
| 152 |
+
- 0.9692159692159692
|
| 153 |
+
- 0.9420289855072463
|
| 154 |
+
- 0.8730158730158729
|
| 155 |
+
save_steps:
|
| 156 |
+
value: 1000
|
| 157 |
+
seed:
|
| 158 |
+
value: 42
|
| 159 |
+
sparse_block_size:
|
| 160 |
+
value: 128
|
| 161 |
+
sparse_layer_indices:
|
| 162 |
+
value: null
|
| 163 |
+
sparse_mask_cache_dir:
|
| 164 |
+
value: null
|
| 165 |
+
sparse_mask_pool_size:
|
| 166 |
+
value: 1
|
| 167 |
+
sparse_mask_seed:
|
| 168 |
+
value: 0
|
| 169 |
+
sparse_mixed_nrb_set:
|
| 170 |
+
value: null
|
| 171 |
+
sparse_num_random_blocks:
|
| 172 |
+
value: 31
|
| 173 |
+
sparse_val_dense_compare:
|
| 174 |
+
value: false
|
| 175 |
+
strategy:
|
| 176 |
+
value: ddp
|
| 177 |
+
train_path:
|
| 178 |
+
value: /mloscratch/homes/navasard/data/fineweb_large/tokenized
|
| 179 |
+
val_path:
|
| 180 |
+
value: null
|
| 181 |
+
verbose:
|
| 182 |
+
value: true
|
| 183 |
+
wandb_enabled:
|
| 184 |
+
value: true
|
| 185 |
+
wandb_name:
|
| 186 |
+
value: l8k365m_dense_ropectl_a1p0_seed42
|
| 187 |
+
wandb_project:
|
| 188 |
+
value: stochastic_sparsity
|
| 189 |
+
wandb_tags:
|
| 190 |
+
value:
|
| 191 |
+
- qwen3
|
| 192 |
+
- 365m
|
| 193 |
+
- 8k
|
| 194 |
+
- dense
|
| 195 |
+
- rope-control
|
| 196 |
+
warmup_steps:
|
| 197 |
+
value: 750
|
| 198 |
+
weight_decay:
|
| 199 |
+
value: 0.01
|
epfl_final/SparseAtt/wandb/run-20260820_194209-jqstnvej/files/output.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260820_194209-jqstnvej/files/requirements.txt
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
| 142 |
+
entmax==1.3
|
epfl_final/SparseAtt/wandb/run-20260820_194209-jqstnvej/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-08-20T19:42:09.596893Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"/mloscratch/homes/navasard/sparse/configs_ropectl/resume_a1p0_wandb/seed42.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/TokenPrune.git",
|
| 14 |
+
"commit": "5637886f0cc760009989b14e68c6ae06d4827418"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "h100-workspace-4-gpu-3-0-6",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 23 |
+
"gpu_count": 4,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "9374931091456",
|
| 27 |
+
"used": "7170272956416"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1623005732864"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-b817d241-acce-e5dd-a749-3447239c5207"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 43 |
+
"memoryTotal": "85520809984",
|
| 44 |
+
"cudaCores": 16896,
|
| 45 |
+
"architecture": "Hopper",
|
| 46 |
+
"uuid": "GPU-165543e5-e049-5219-e717-e1216ac960de"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 50 |
+
"memoryTotal": "85520809984",
|
| 51 |
+
"cudaCores": 16896,
|
| 52 |
+
"architecture": "Hopper",
|
| 53 |
+
"uuid": "GPU-47444fc7-ea8c-cef6-aac8-ef408dc8047b"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 57 |
+
"memoryTotal": "85520809984",
|
| 58 |
+
"cudaCores": 16896,
|
| 59 |
+
"architecture": "Hopper",
|
| 60 |
+
"uuid": "GPU-37dc97ad-46ef-50d9-3619-3a6eaea1b6aa"
|
| 61 |
+
}
|
| 62 |
+
],
|
| 63 |
+
"cudaVersion": "13.0",
|
| 64 |
+
"writerId": "8l9cf1l8zaolhsew8f38inve9wj109an"
|
| 65 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260820_194209-jqstnvej/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"_runtime":59340.189901118,"_timestamp":1.787286475768016e+09,"train/global_step":44000,"train/loss":2.8999343729019165,"train/epoch":0.2906439083679023,"_wandb.runtime":10161,"speed/steps_per_sec":0.7301364448469594,"train/learning_rate":0,"eval/uses_sparse_attention":0,"eval/loss":2.893159435749054,"eval/uses_dense_attention":1,"eval/dense_interval_active":0,"eval/dense_interval_start":-1,"eval/dense_interval_end":-1,"eval/perplexity":18.050248206718,"speed/samples_per_sec":14.602728896939189,"_step":44000,"_wandb":{"runtime":59340},"speed/ms_per_step":1369.6070194244385}
|
epfl_final/SparseAtt/wandb/run-20260820_194209-jqstnvej/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260820_194209-jqstnvej/logs/debug.log
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-08-20 19:42:09,619 INFO MainThread:146 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-08-20 19:42:09,620 INFO MainThread:146 [wandb_setup.py:_flush():81] Configure stats pid to 146
|
| 3 |
+
2026-08-20 19:42:09,620 INFO MainThread:146 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-08-20 19:42:09,620 INFO MainThread:146 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_194209-jqstnvej/logs/debug.log
|
| 5 |
+
2026-08-20 19:42:09,620 INFO MainThread:146 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_194209-jqstnvej/logs/debug-internal.log
|
| 6 |
+
2026-08-20 19:42:09,620 INFO MainThread:146 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-08-20 19:42:09,620 INFO MainThread:146 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-22000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 1000, 'eval_batches': 50, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'l8k365m_dense_ropectl_a1p0_seed42', 'wandb_tags': ['qwen3', '365m', '8k', 'dense', 'rope-control'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': 0.0, 'rope_train_position_scale_set': [1.0, 0.9896339488176222, 0.9692159692159692, 0.9420289855072463, 0.8730158730158729], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-08-20 19:42:09,620 INFO MainThread:146 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-08-20 19:42:10,593 INFO MainThread:146 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-08-20 19:42:11,267 INFO MainThread:146 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-08-20 19:42:11,273 INFO MainThread:146 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-08-20 19:42:11,361 INFO MainThread:146 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-08-20 19:42:12,026 INFO MainThread:146 [wandb_init.py:init():1051] run resumed
|
| 15 |
+
2026-08-20 19:42:12,028 INFO MainThread:146 [wandb_init.py:init():1056] starting run threads in backend
|
| 16 |
+
2026-08-20 19:42:12,302 INFO MainThread:146 [wandb_run.py:_console_start():2554] atexit reg
|
| 17 |
+
2026-08-20 19:42:12,302 INFO MainThread:146 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 18 |
+
2026-08-20 19:42:12,302 INFO MainThread:146 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 19 |
+
2026-08-20 19:42:12,302 INFO MainThread:146 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 20 |
+
2026-08-20 19:42:12,312 INFO MainThread:146 [wandb_init.py:init():1094] run started, returning control to user process
|
| 21 |
+
2026-08-21 04:28:02,217 INFO MainThread:146 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/jqstnvej
|
| 22 |
+
2026-08-21 04:28:02,218 INFO MainThread:146 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
|
| 23 |
+
2026-08-21 04:28:02,219 INFO MainThread:146 [wandb_run.py:_restore():2502] restore
|
| 24 |
+
2026-08-21 04:28:02,219 INFO MainThread:146 [wandb_run.py:_restore():2508] restore done
|
| 25 |
+
2026-08-21 04:28:04,111 INFO MainThread:146 [wandb_run.py:_footer_sync_info():3889] logging synced files
|
epfl_final/SparseAtt/wandb/run-20260820_194209-jqstnvej/run-jqstnvej.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ca1976048e29ef909ae58697656daaefe6a22446e6be10e87e25c841380aa326
|
| 3 |
+
size 11532765
|