EPFL final checkpoints and W&B history 1001-1050 of 1463
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +8 -0
- epfl_final/SparseAtt/wandb/run-20260823_140341-qctui00g/files/output.log +285 -0
- epfl_final/SparseAtt/wandb/run-20260823_140341-qctui00g/files/requirements.txt +142 -0
- epfl_final/SparseAtt/wandb/run-20260823_140341-qctui00g/run-qctui00g.wandb +3 -0
- epfl_final/SparseAtt/wandb/run-20260823_142101-rgmkfpnl/files/output.log +613 -0
- epfl_final/SparseAtt/wandb/run-20260823_142101-rgmkfpnl/files/requirements.txt +142 -0
- epfl_final/SparseAtt/wandb/run-20260823_142101-rgmkfpnl/files/wandb-metadata.json +72 -0
- epfl_final/SparseAtt/wandb/run-20260823_142101-rgmkfpnl/logs/debug-internal.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260823_142101-rgmkfpnl/logs/debug.log +20 -0
- epfl_final/SparseAtt/wandb/run-20260823_142101-rgmkfpnl/run-rgmkfpnl.wandb +3 -0
- epfl_final/SparseAtt/wandb/run-20260823_150341-i8ewyy95/files/output.log +814 -0
- epfl_final/SparseAtt/wandb/run-20260823_150341-i8ewyy95/files/requirements.txt +142 -0
- epfl_final/SparseAtt/wandb/run-20260823_150341-i8ewyy95/files/wandb-metadata.json +79 -0
- epfl_final/SparseAtt/wandb/run-20260823_150341-i8ewyy95/logs/debug-internal.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260823_150341-i8ewyy95/logs/debug.log +20 -0
- epfl_final/SparseAtt/wandb/run-20260823_150341-i8ewyy95/run-i8ewyy95.wandb +3 -0
- epfl_final/SparseAtt/wandb/run-20260824_034449-rgmkfpnl/files/output.log +367 -0
- epfl_final/SparseAtt/wandb/run-20260824_034449-rgmkfpnl/files/requirements.txt +142 -0
- epfl_final/SparseAtt/wandb/run-20260824_034449-rgmkfpnl/files/wandb-metadata.json +72 -0
- epfl_final/SparseAtt/wandb/run-20260824_034449-rgmkfpnl/logs/debug-internal.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260824_034449-rgmkfpnl/logs/debug.log +20 -0
- epfl_final/SparseAtt/wandb/run-20260824_034449-rgmkfpnl/run-rgmkfpnl.wandb +3 -0
- epfl_final/SparseAtt/wandb/run-20260824_034710-i8ewyy95/files/output.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260824_034710-i8ewyy95/files/requirements.txt +142 -0
- epfl_final/SparseAtt/wandb/run-20260824_034710-i8ewyy95/files/wandb-metadata.json +79 -0
- epfl_final/SparseAtt/wandb/run-20260824_034710-i8ewyy95/logs/debug-internal.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260824_034710-i8ewyy95/logs/debug.log +20 -0
- epfl_final/SparseAtt/wandb/run-20260824_034710-i8ewyy95/run-i8ewyy95.wandb +3 -0
- epfl_final/SparseAtt/wandb/run-20260824_132232-rgmkfpnl/files/output.log +111 -0
- epfl_final/SparseAtt/wandb/run-20260824_132232-rgmkfpnl/files/requirements.txt +142 -0
- epfl_final/SparseAtt/wandb/run-20260824_132232-rgmkfpnl/files/wandb-metadata.json +51 -0
- epfl_final/SparseAtt/wandb/run-20260824_132232-rgmkfpnl/logs/debug-internal.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260824_132232-rgmkfpnl/logs/debug.log +20 -0
- epfl_final/SparseAtt/wandb/run-20260824_132232-rgmkfpnl/run-rgmkfpnl.wandb +3 -0
- epfl_final/SparseAtt/wandb/run-20260824_181605-i8ewyy95/files/config.yaml +210 -0
- epfl_final/SparseAtt/wandb/run-20260824_181605-i8ewyy95/files/output.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260824_181605-i8ewyy95/files/requirements.txt +142 -0
- epfl_final/SparseAtt/wandb/run-20260824_181605-i8ewyy95/files/wandb-metadata.json +79 -0
- epfl_final/SparseAtt/wandb/run-20260824_181605-i8ewyy95/files/wandb-summary.json +1 -0
- epfl_final/SparseAtt/wandb/run-20260824_181605-i8ewyy95/logs/debug-internal.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260824_181605-i8ewyy95/logs/debug.log +25 -0
- epfl_final/SparseAtt/wandb/run-20260824_181605-i8ewyy95/run-i8ewyy95.wandb +3 -0
- epfl_final/SparseAtt/wandb/run-20260824_181749-rgmkfpnl/files/output.log +708 -0
- epfl_final/SparseAtt/wandb/run-20260824_181749-rgmkfpnl/files/requirements.txt +142 -0
- epfl_final/SparseAtt/wandb/run-20260824_181749-rgmkfpnl/files/wandb-metadata.json +51 -0
- epfl_final/SparseAtt/wandb/run-20260824_181749-rgmkfpnl/logs/debug-internal.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260824_181749-rgmkfpnl/logs/debug.log +20 -0
- epfl_final/SparseAtt/wandb/run-20260824_181749-rgmkfpnl/run-rgmkfpnl.wandb +3 -0
- epfl_final/SparseAtt/wandb/run-20260825_031743-qctui00g/files/wandb-metadata.json +79 -0
- epfl_final/SparseAtt/wandb/run-20260825_031743-qctui00g/logs/debug-internal.log +0 -0
.gitattributes
CHANGED
|
@@ -131,3 +131,11 @@ epfl_final/SparseAtt/wandb/run-20260822_161907-9nppzrru/run-9nppzrru.wandb filte
|
|
| 131 |
epfl_final/SparseAtt/wandb/run-20260822_210348-i8ewyy95/run-i8ewyy95.wandb filter=lfs diff=lfs merge=lfs -text
|
| 132 |
epfl_final/SparseAtt/wandb/run-20260823_041148-i8ewyy95/run-i8ewyy95.wandb filter=lfs diff=lfs merge=lfs -text
|
| 133 |
epfl_final/SparseAtt/wandb/run-20260823_043841-rgmkfpnl/run-rgmkfpnl.wandb filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 131 |
epfl_final/SparseAtt/wandb/run-20260822_210348-i8ewyy95/run-i8ewyy95.wandb filter=lfs diff=lfs merge=lfs -text
|
| 132 |
epfl_final/SparseAtt/wandb/run-20260823_041148-i8ewyy95/run-i8ewyy95.wandb filter=lfs diff=lfs merge=lfs -text
|
| 133 |
epfl_final/SparseAtt/wandb/run-20260823_043841-rgmkfpnl/run-rgmkfpnl.wandb filter=lfs diff=lfs merge=lfs -text
|
| 134 |
+
epfl_final/SparseAtt/wandb/run-20260823_140341-qctui00g/run-qctui00g.wandb filter=lfs diff=lfs merge=lfs -text
|
| 135 |
+
epfl_final/SparseAtt/wandb/run-20260823_142101-rgmkfpnl/run-rgmkfpnl.wandb filter=lfs diff=lfs merge=lfs -text
|
| 136 |
+
epfl_final/SparseAtt/wandb/run-20260823_150341-i8ewyy95/run-i8ewyy95.wandb filter=lfs diff=lfs merge=lfs -text
|
| 137 |
+
epfl_final/SparseAtt/wandb/run-20260824_034449-rgmkfpnl/run-rgmkfpnl.wandb filter=lfs diff=lfs merge=lfs -text
|
| 138 |
+
epfl_final/SparseAtt/wandb/run-20260824_034710-i8ewyy95/run-i8ewyy95.wandb filter=lfs diff=lfs merge=lfs -text
|
| 139 |
+
epfl_final/SparseAtt/wandb/run-20260824_132232-rgmkfpnl/run-rgmkfpnl.wandb filter=lfs diff=lfs merge=lfs -text
|
| 140 |
+
epfl_final/SparseAtt/wandb/run-20260824_181605-i8ewyy95/run-i8ewyy95.wandb filter=lfs diff=lfs merge=lfs -text
|
| 141 |
+
epfl_final/SparseAtt/wandb/run-20260824_181749-rgmkfpnl/run-rgmkfpnl.wandb filter=lfs diff=lfs merge=lfs -text
|
epfl_final/SparseAtt/wandb/run-20260823_140341-qctui00g/files/output.log
ADDED
|
@@ -0,0 +1,285 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Starting training...
|
| 2 |
+
Switching attention mode for step 1: dense (dense step interval)
|
| 3 |
+
Step 10/50462 | Loss: 12.1742 | LR: 6.67e-06 | 11731.3ms/step | 5.1 samples/s
|
| 4 |
+
Step 20/50462 | Loss: 11.8320 | LR: 1.33e-05 | 4686.4ms/step | 12.8 samples/s
|
| 5 |
+
Step 30/50462 | Loss: 10.8631 | LR: 2.00e-05 | 4685.8ms/step | 12.8 samples/s
|
| 6 |
+
Step 40/50462 | Loss: 10.0487 | LR: 2.67e-05 | 4683.0ms/step | 12.8 samples/s
|
| 7 |
+
Step 50/50462 | Loss: 9.6692 | LR: 3.33e-05 | 4685.5ms/step | 12.8 samples/s
|
| 8 |
+
Step 60/50462 | Loss: 9.2179 | LR: 4.00e-05 | 4687.5ms/step | 12.8 samples/s
|
| 9 |
+
Step 70/50462 | Loss: 8.6882 | LR: 4.67e-05 | 4694.4ms/step | 12.8 samples/s
|
| 10 |
+
Step 80/50462 | Loss: 8.2368 | LR: 5.33e-05 | 4694.2ms/step | 12.8 samples/s
|
| 11 |
+
Step 90/50462 | Loss: 7.8637 | LR: 6.00e-05 | 4696.7ms/step | 12.8 samples/s
|
| 12 |
+
Step 100/50462 | Loss: 7.5250 | LR: 6.67e-05 | 4698.3ms/step | 12.8 samples/s
|
| 13 |
+
Step 110/50462 | Loss: 7.2680 | LR: 7.33e-05 | 4700.0ms/step | 12.8 samples/s
|
| 14 |
+
Step 120/50462 | Loss: 7.1073 | LR: 8.00e-05 | 4701.1ms/step | 12.8 samples/s
|
| 15 |
+
Step 130/50462 | Loss: 6.9883 | LR: 8.67e-05 | 4704.2ms/step | 12.8 samples/s
|
| 16 |
+
Step 140/50462 | Loss: 6.9082 | LR: 9.33e-05 | 4706.6ms/step | 12.7 samples/s
|
| 17 |
+
Step 150/50462 | Loss: 6.7566 | LR: 1.00e-04 | 4705.5ms/step | 12.8 samples/s
|
| 18 |
+
Step 160/50462 | Loss: 6.7090 | LR: 1.07e-04 | 4707.2ms/step | 12.7 samples/s
|
| 19 |
+
Step 170/50462 | Loss: 6.5772 | LR: 1.13e-04 | 4706.8ms/step | 12.7 samples/s
|
| 20 |
+
Step 180/50462 | Loss: 6.4874 | LR: 1.20e-04 | 4709.3ms/step | 12.7 samples/s
|
| 21 |
+
Step 190/50462 | Loss: 6.3780 | LR: 1.27e-04 | 4710.0ms/step | 12.7 samples/s
|
| 22 |
+
Step 200/50462 | Loss: 6.3171 | LR: 1.33e-04 | 4710.6ms/step | 12.7 samples/s
|
| 23 |
+
Step 210/50462 | Loss: 6.2308 | LR: 1.40e-04 | 4711.4ms/step | 12.7 samples/s
|
| 24 |
+
Step 220/50462 | Loss: 6.2443 | LR: 1.47e-04 | 4711.4ms/step | 12.7 samples/s
|
| 25 |
+
Step 230/50462 | Loss: 6.1113 | LR: 1.53e-04 | 4711.8ms/step | 12.7 samples/s
|
| 26 |
+
Step 240/50462 | Loss: 6.0704 | LR: 1.60e-04 | 4713.0ms/step | 12.7 samples/s
|
| 27 |
+
Step 250/50462 | Loss: 6.0070 | LR: 1.67e-04 | 4713.5ms/step | 12.7 samples/s
|
| 28 |
+
Validation at step 250: Loss = 5.9502, Perplexity = 383.82
|
| 29 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-250
|
| 30 |
+
Step 260/50462 | Loss: 5.9161 | LR: 1.73e-04 | 10552.8ms/step | 5.7 samples/s
|
| 31 |
+
Step 270/50462 | Loss: 5.9269 | LR: 1.80e-04 | 4709.3ms/step | 12.7 samples/s
|
| 32 |
+
Step 280/50462 | Loss: 5.8761 | LR: 1.87e-04 | 4718.1ms/step | 12.7 samples/s
|
| 33 |
+
Step 290/50462 | Loss: 5.7764 | LR: 1.93e-04 | 4721.0ms/step | 12.7 samples/s
|
| 34 |
+
Step 300/50462 | Loss: 5.7620 | LR: 2.00e-04 | 4720.5ms/step | 12.7 samples/s
|
| 35 |
+
Step 310/50462 | Loss: 5.7145 | LR: 2.07e-04 | 4721.2ms/step | 12.7 samples/s
|
| 36 |
+
Step 320/50462 | Loss: 5.6582 | LR: 2.13e-04 | 4721.3ms/step | 12.7 samples/s
|
| 37 |
+
Step 330/50462 | Loss: 5.5893 | LR: 2.20e-04 | 4722.5ms/step | 12.7 samples/s
|
| 38 |
+
Step 340/50462 | Loss: 5.5291 | LR: 2.27e-04 | 4724.2ms/step | 12.7 samples/s
|
| 39 |
+
Step 350/50462 | Loss: 5.4462 | LR: 2.33e-04 | 4726.4ms/step | 12.7 samples/s
|
| 40 |
+
Step 360/50462 | Loss: 5.4114 | LR: 2.40e-04 | 4727.0ms/step | 12.7 samples/s
|
| 41 |
+
Step 370/50462 | Loss: 5.3658 | LR: 2.47e-04 | 4723.6ms/step | 12.7 samples/s
|
| 42 |
+
Step 380/50462 | Loss: 5.3126 | LR: 2.53e-04 | 4727.8ms/step | 12.7 samples/s
|
| 43 |
+
Step 390/50462 | Loss: 5.3079 | LR: 2.60e-04 | 4723.7ms/step | 12.7 samples/s
|
| 44 |
+
Step 400/50462 | Loss: 5.2380 | LR: 2.67e-04 | 4722.7ms/step | 12.7 samples/s
|
| 45 |
+
Step 410/50462 | Loss: 5.2100 | LR: 2.73e-04 | 4722.8ms/step | 12.7 samples/s
|
| 46 |
+
Step 420/50462 | Loss: 5.1419 | LR: 2.80e-04 | 4725.1ms/step | 12.7 samples/s
|
| 47 |
+
Step 430/50462 | Loss: 5.0929 | LR: 2.87e-04 | 4723.0ms/step | 12.7 samples/s
|
| 48 |
+
Step 440/50462 | Loss: 5.0683 | LR: 2.93e-04 | 4723.7ms/step | 12.7 samples/s
|
| 49 |
+
Step 450/50462 | Loss: 5.0881 | LR: 3.00e-04 | 4722.1ms/step | 12.7 samples/s
|
| 50 |
+
Step 460/50462 | Loss: 5.0190 | LR: 3.07e-04 | 4725.6ms/step | 12.7 samples/s
|
| 51 |
+
Step 470/50462 | Loss: 4.8955 | LR: 3.13e-04 | 4728.7ms/step | 12.7 samples/s
|
| 52 |
+
Step 480/50462 | Loss: 4.9059 | LR: 3.20e-04 | 4731.7ms/step | 12.7 samples/s
|
| 53 |
+
Step 490/50462 | Loss: 4.9201 | LR: 3.27e-04 | 4732.2ms/step | 12.7 samples/s
|
| 54 |
+
Step 500/50462 | Loss: 4.8885 | LR: 3.33e-04 | 4724.2ms/step | 12.7 samples/s
|
| 55 |
+
Validation at step 500: Loss = 4.8038, Perplexity = 121.97
|
| 56 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-500
|
| 57 |
+
Step 510/50462 | Loss: 4.7781 | LR: 3.40e-04 | 6965.7ms/step | 8.6 samples/s
|
| 58 |
+
Step 520/50462 | Loss: 4.8151 | LR: 3.47e-04 | 4710.8ms/step | 12.7 samples/s
|
| 59 |
+
Step 530/50462 | Loss: 4.8086 | LR: 3.53e-04 | 4716.2ms/step | 12.7 samples/s
|
| 60 |
+
Step 540/50462 | Loss: 4.7796 | LR: 3.60e-04 | 4721.8ms/step | 12.7 samples/s
|
| 61 |
+
Step 550/50462 | Loss: 4.7198 | LR: 3.67e-04 | 4726.1ms/step | 12.7 samples/s
|
| 62 |
+
Step 560/50462 | Loss: 4.6588 | LR: 3.73e-04 | 4725.2ms/step | 12.7 samples/s
|
| 63 |
+
Step 570/50462 | Loss: 4.6077 | LR: 3.80e-04 | 4726.1ms/step | 12.7 samples/s
|
| 64 |
+
Step 580/50462 | Loss: 4.6800 | LR: 3.87e-04 | 4727.3ms/step | 12.7 samples/s
|
| 65 |
+
Step 590/50462 | Loss: 4.5749 | LR: 3.93e-04 | 4726.6ms/step | 12.7 samples/s
|
| 66 |
+
Step 600/50462 | Loss: 4.5416 | LR: 4.00e-04 | 4725.3ms/step | 12.7 samples/s
|
| 67 |
+
Step 610/50462 | Loss: 4.5025 | LR: 4.07e-04 | 4725.2ms/step | 12.7 samples/s
|
| 68 |
+
Step 620/50462 | Loss: 4.5348 | LR: 4.13e-04 | 4723.7ms/step | 12.7 samples/s
|
| 69 |
+
Step 630/50462 | Loss: 4.5014 | LR: 4.20e-04 | 4724.4ms/step | 12.7 samples/s
|
| 70 |
+
Step 640/50462 | Loss: 4.3765 | LR: 4.27e-04 | 4724.0ms/step | 12.7 samples/s
|
| 71 |
+
Step 650/50462 | Loss: 4.3733 | LR: 4.33e-04 | 4728.0ms/step | 12.7 samples/s
|
| 72 |
+
Step 660/50462 | Loss: 4.4487 | LR: 4.40e-04 | 4729.0ms/step | 12.7 samples/s
|
| 73 |
+
Step 670/50462 | Loss: 4.3735 | LR: 4.47e-04 | 4724.5ms/step | 12.7 samples/s
|
| 74 |
+
Step 680/50462 | Loss: 4.2940 | LR: 4.53e-04 | 4724.8ms/step | 12.7 samples/s
|
| 75 |
+
Step 690/50462 | Loss: 4.2584 | LR: 4.60e-04 | 4725.7ms/step | 12.7 samples/s
|
| 76 |
+
Step 700/50462 | Loss: 4.2832 | LR: 4.67e-04 | 4725.6ms/step | 12.7 samples/s
|
| 77 |
+
Step 710/50462 | Loss: 4.2303 | LR: 4.73e-04 | 4724.7ms/step | 12.7 samples/s
|
| 78 |
+
Step 720/50462 | Loss: 4.2239 | LR: 4.80e-04 | 4724.2ms/step | 12.7 samples/s
|
| 79 |
+
Step 730/50462 | Loss: 4.2088 | LR: 4.87e-04 | 4725.5ms/step | 12.7 samples/s
|
| 80 |
+
Step 740/50462 | Loss: 4.1720 | LR: 4.93e-04 | 4725.0ms/step | 12.7 samples/s
|
| 81 |
+
Step 750/50462 | Loss: 4.0841 | LR: 5.00e-04 | 4727.0ms/step | 12.7 samples/s
|
| 82 |
+
Validation at step 750: Loss = 4.0713, Perplexity = 58.63
|
| 83 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-750
|
| 84 |
+
Step 760/50462 | Loss: 4.1001 | LR: 5.00e-04 | 6971.9ms/step | 8.6 samples/s
|
| 85 |
+
Step 770/50462 | Loss: 4.1039 | LR: 5.00e-04 | 4704.8ms/step | 12.8 samples/s
|
| 86 |
+
Step 780/50462 | Loss: 4.0593 | LR: 5.00e-04 | 4713.6ms/step | 12.7 samples/s
|
| 87 |
+
Step 790/50462 | Loss: 4.0365 | LR: 5.00e-04 | 4716.9ms/step | 12.7 samples/s
|
| 88 |
+
Step 800/50462 | Loss: 4.0207 | LR: 5.00e-04 | 4720.8ms/step | 12.7 samples/s
|
| 89 |
+
Step 810/50462 | Loss: 3.9969 | LR: 5.00e-04 | 4721.3ms/step | 12.7 samples/s
|
| 90 |
+
Step 820/50462 | Loss: 3.9676 | LR: 5.00e-04 | 4720.4ms/step | 12.7 samples/s
|
| 91 |
+
Step 830/50462 | Loss: 3.9502 | LR: 5.00e-04 | 4721.6ms/step | 12.7 samples/s
|
| 92 |
+
Step 840/50462 | Loss: 3.8992 | LR: 5.00e-04 | 4719.3ms/step | 12.7 samples/s
|
| 93 |
+
Step 850/50462 | Loss: 3.8830 | LR: 5.00e-04 | 4719.8ms/step | 12.7 samples/s
|
| 94 |
+
Step 860/50462 | Loss: 3.8926 | LR: 5.00e-04 | 4720.0ms/step | 12.7 samples/s
|
| 95 |
+
Step 870/50462 | Loss: 3.8647 | LR: 5.00e-04 | 4721.3ms/step | 12.7 samples/s
|
| 96 |
+
Step 880/50462 | Loss: 3.8662 | LR: 5.00e-04 | 4720.4ms/step | 12.7 samples/s
|
| 97 |
+
Step 890/50462 | Loss: 3.8305 | LR: 5.00e-04 | 4720.5ms/step | 12.7 samples/s
|
| 98 |
+
Step 900/50462 | Loss: 3.8499 | LR: 5.00e-04 | 4721.2ms/step | 12.7 samples/s
|
| 99 |
+
Step 910/50462 | Loss: 3.8032 | LR: 5.00e-04 | 4720.0ms/step | 12.7 samples/s
|
| 100 |
+
Step 920/50462 | Loss: 3.7715 | LR: 5.00e-04 | 4722.8ms/step | 12.7 samples/s
|
| 101 |
+
Step 930/50462 | Loss: 3.7964 | LR: 5.00e-04 | 4722.1ms/step | 12.7 samples/s
|
| 102 |
+
Step 940/50462 | Loss: 3.7777 | LR: 5.00e-04 | 4721.3ms/step | 12.7 samples/s
|
| 103 |
+
Step 950/50462 | Loss: 3.7519 | LR: 5.00e-04 | 4719.6ms/step | 12.7 samples/s
|
| 104 |
+
Step 960/50462 | Loss: 3.7624 | LR: 5.00e-04 | 4720.2ms/step | 12.7 samples/s
|
| 105 |
+
Step 970/50462 | Loss: 3.7137 | LR: 5.00e-04 | 4721.8ms/step | 12.7 samples/s
|
| 106 |
+
Step 980/50462 | Loss: 3.7370 | LR: 5.00e-04 | 4721.2ms/step | 12.7 samples/s
|
| 107 |
+
Step 990/50462 | Loss: 3.7066 | LR: 5.00e-04 | 4720.1ms/step | 12.7 samples/s
|
| 108 |
+
Step 1000/50462 | Loss: 3.6887 | LR: 5.00e-04 | 4720.9ms/step | 12.7 samples/s
|
| 109 |
+
Validation at step 1000: Loss = 3.6835, Perplexity = 39.78
|
| 110 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-1000
|
| 111 |
+
Step 1010/50462 | Loss: 3.6802 | LR: 5.00e-04 | 6982.7ms/step | 8.6 samples/s
|
| 112 |
+
Step 1020/50462 | Loss: 3.6538 | LR: 5.00e-04 | 4709.8ms/step | 12.7 samples/s
|
| 113 |
+
Step 1030/50462 | Loss: 3.6368 | LR: 5.00e-04 | 4720.9ms/step | 12.7 samples/s
|
| 114 |
+
Step 1040/50462 | Loss: 3.6837 | LR: 5.00e-04 | 4723.9ms/step | 12.7 samples/s
|
| 115 |
+
Step 1050/50462 | Loss: 3.6485 | LR: 5.00e-04 | 4723.9ms/step | 12.7 samples/s
|
| 116 |
+
Step 1060/50462 | Loss: 3.6665 | LR: 5.00e-04 | 4725.5ms/step | 12.7 samples/s
|
| 117 |
+
Step 1070/50462 | Loss: 3.6531 | LR: 5.00e-04 | 4728.3ms/step | 12.7 samples/s
|
| 118 |
+
Step 1080/50462 | Loss: 3.6400 | LR: 5.00e-04 | 4728.7ms/step | 12.7 samples/s
|
| 119 |
+
Step 1090/50462 | Loss: 3.5922 | LR: 5.00e-04 | 4725.9ms/step | 12.7 samples/s
|
| 120 |
+
Step 1100/50462 | Loss: 3.6015 | LR: 5.00e-04 | 4725.1ms/step | 12.7 samples/s
|
| 121 |
+
Step 1110/50462 | Loss: 3.5827 | LR: 5.00e-04 | 4725.1ms/step | 12.7 samples/s
|
| 122 |
+
Step 1120/50462 | Loss: 3.6256 | LR: 5.00e-04 | 4724.4ms/step | 12.7 samples/s
|
| 123 |
+
Step 1130/50462 | Loss: 3.5688 | LR: 5.00e-04 | 4726.0ms/step | 12.7 samples/s
|
| 124 |
+
Step 1140/50462 | Loss: 3.5873 | LR: 5.00e-04 | 4725.2ms/step | 12.7 samples/s
|
| 125 |
+
Step 1150/50462 | Loss: 3.5207 | LR: 5.00e-04 | 4724.6ms/step | 12.7 samples/s
|
| 126 |
+
Step 1160/50462 | Loss: 3.5826 | LR: 5.00e-04 | 4726.0ms/step | 12.7 samples/s
|
| 127 |
+
Step 1170/50462 | Loss: 3.5051 | LR: 5.00e-04 | 4728.4ms/step | 12.7 samples/s
|
| 128 |
+
Step 1180/50462 | Loss: 3.5318 | LR: 5.00e-04 | 4727.1ms/step | 12.7 samples/s
|
| 129 |
+
Step 1190/50462 | Loss: 3.5266 | LR: 5.00e-04 | 4726.0ms/step | 12.7 samples/s
|
| 130 |
+
Step 1200/50462 | Loss: 3.5324 | LR: 5.00e-04 | 4726.7ms/step | 12.7 samples/s
|
| 131 |
+
Step 1210/50462 | Loss: 3.5045 | LR: 5.00e-04 | 4726.1ms/step | 12.7 samples/s
|
| 132 |
+
Step 1220/50462 | Loss: 3.5279 | LR: 5.00e-04 | 4726.7ms/step | 12.7 samples/s
|
| 133 |
+
Step 1230/50462 | Loss: 3.5277 | LR: 5.00e-04 | 4727.0ms/step | 12.7 samples/s
|
| 134 |
+
Step 1240/50462 | Loss: 3.5033 | LR: 5.00e-04 | 4727.3ms/step | 12.7 samples/s
|
| 135 |
+
Step 1250/50462 | Loss: 3.4781 | LR: 5.00e-04 | 4725.7ms/step | 12.7 samples/s
|
| 136 |
+
Validation at step 1250: Loss = 3.4788, Perplexity = 32.42
|
| 137 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-1250
|
| 138 |
+
Step 1260/50462 | Loss: 3.4758 | LR: 5.00e-04 | 6962.5ms/step | 8.6 samples/s
|
| 139 |
+
Step 1270/50462 | Loss: 3.4578 | LR: 5.00e-04 | 4710.7ms/step | 12.7 samples/s
|
| 140 |
+
Step 1280/50462 | Loss: 3.4830 | LR: 5.00e-04 | 4718.4ms/step | 12.7 samples/s
|
| 141 |
+
Step 1290/50462 | Loss: 3.5056 | LR: 5.00e-04 | 4725.4ms/step | 12.7 samples/s
|
| 142 |
+
Step 1300/50462 | Loss: 3.4777 | LR: 5.00e-04 | 4725.3ms/step | 12.7 samples/s
|
| 143 |
+
Step 1310/50462 | Loss: 3.4519 | LR: 5.00e-04 | 4725.8ms/step | 12.7 samples/s
|
| 144 |
+
Step 1320/50462 | Loss: 3.4839 | LR: 5.00e-04 | 4726.3ms/step | 12.7 samples/s
|
| 145 |
+
Step 1330/50462 | Loss: 3.4750 | LR: 5.00e-04 | 4728.4ms/step | 12.7 samples/s
|
| 146 |
+
Step 1340/50462 | Loss: 3.4846 | LR: 5.00e-04 | 4726.6ms/step | 12.7 samples/s
|
| 147 |
+
Step 1350/50462 | Loss: 3.4618 | LR: 5.00e-04 | 4725.3ms/step | 12.7 samples/s
|
| 148 |
+
Step 1360/50462 | Loss: 3.4365 | LR: 5.00e-04 | 4727.4ms/step | 12.7 samples/s
|
| 149 |
+
Step 1370/50462 | Loss: 3.4206 | LR: 5.00e-04 | 4726.1ms/step | 12.7 samples/s
|
| 150 |
+
Step 1380/50462 | Loss: 3.4555 | LR: 5.00e-04 | 4726.2ms/step | 12.7 samples/s
|
| 151 |
+
Step 1390/50462 | Loss: 3.4204 | LR: 5.00e-04 | 4727.3ms/step | 12.7 samples/s
|
| 152 |
+
Step 1400/50462 | Loss: 3.4113 | LR: 5.00e-04 | 4726.5ms/step | 12.7 samples/s
|
| 153 |
+
Step 1410/50462 | Loss: 3.3973 | LR: 5.00e-04 | 4726.2ms/step | 12.7 samples/s
|
| 154 |
+
Step 1420/50462 | Loss: 3.3653 | LR: 5.00e-04 | 4725.4ms/step | 12.7 samples/s
|
| 155 |
+
Step 1430/50462 | Loss: 3.3901 | LR: 5.00e-04 | 4727.1ms/step | 12.7 samples/s
|
| 156 |
+
Step 1440/50462 | Loss: 3.3916 | LR: 5.00e-04 | 4727.2ms/step | 12.7 samples/s
|
| 157 |
+
Step 1450/50462 | Loss: 3.4192 | LR: 5.00e-04 | 4728.3ms/step | 12.7 samples/s
|
| 158 |
+
Step 1460/50462 | Loss: 3.3773 | LR: 5.00e-04 | 4726.5ms/step | 12.7 samples/s
|
| 159 |
+
Step 1470/50462 | Loss: 3.4135 | LR: 5.00e-04 | 4726.3ms/step | 12.7 samples/s
|
| 160 |
+
Step 1480/50462 | Loss: 3.3473 | LR: 5.00e-04 | 4728.3ms/step | 12.7 samples/s
|
| 161 |
+
Step 1490/50462 | Loss: 3.3635 | LR: 5.00e-04 | 4726.4ms/step | 12.7 samples/s
|
| 162 |
+
Step 1500/50462 | Loss: 3.3885 | LR: 5.00e-04 | 4726.5ms/step | 12.7 samples/s
|
| 163 |
+
Validation at step 1500: Loss = 3.3587, Perplexity = 28.75
|
| 164 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-1500
|
| 165 |
+
Step 1510/50462 | Loss: 3.3174 | LR: 5.00e-04 | 6964.7ms/step | 8.6 samples/s
|
| 166 |
+
Step 1520/50462 | Loss: 3.3599 | LR: 5.00e-04 | 4705.3ms/step | 12.8 samples/s
|
| 167 |
+
Step 1530/50462 | Loss: 3.3518 | LR: 5.00e-04 | 4713.3ms/step | 12.7 samples/s
|
| 168 |
+
Step 1540/50462 | Loss: 3.3621 | LR: 5.00e-04 | 4718.4ms/step | 12.7 samples/s
|
| 169 |
+
Step 1550/50462 | Loss: 3.3820 | LR: 5.00e-04 | 4722.1ms/step | 12.7 samples/s
|
| 170 |
+
Step 1560/50462 | Loss: 3.3465 | LR: 5.00e-04 | 4721.6ms/step | 12.7 samples/s
|
| 171 |
+
Step 1570/50462 | Loss: 3.2939 | LR: 5.00e-04 | 4727.5ms/step | 12.7 samples/s
|
| 172 |
+
Step 1580/50462 | Loss: 3.3606 | LR: 5.00e-04 | 4725.3ms/step | 12.7 samples/s
|
| 173 |
+
Step 1590/50462 | Loss: 3.3297 | LR: 5.00e-04 | 4723.0ms/step | 12.7 samples/s
|
| 174 |
+
Step 1600/50462 | Loss: 3.3123 | LR: 5.00e-04 | 4726.4ms/step | 12.7 samples/s
|
| 175 |
+
Step 1610/50462 | Loss: 3.3400 | LR: 5.00e-04 | 4725.7ms/step | 12.7 samples/s
|
| 176 |
+
Step 1620/50462 | Loss: 3.3135 | LR: 5.00e-04 | 4724.1ms/step | 12.7 samples/s
|
| 177 |
+
Step 1630/50462 | Loss: 3.3332 | LR: 5.00e-04 | 4723.3ms/step | 12.7 samples/s
|
| 178 |
+
Step 1640/50462 | Loss: 3.3286 | LR: 5.00e-04 | 4723.5ms/step | 12.7 samples/s
|
| 179 |
+
Step 1650/50462 | Loss: 3.3389 | LR: 5.00e-04 | 4723.6ms/step | 12.7 samples/s
|
| 180 |
+
Step 1660/50462 | Loss: 3.2935 | LR: 5.00e-04 | 4723.7ms/step | 12.7 samples/s
|
| 181 |
+
Step 1670/50462 | Loss: 3.2991 | LR: 5.00e-04 | 4725.6ms/step | 12.7 samples/s
|
| 182 |
+
Step 1680/50462 | Loss: 3.2835 | LR: 5.00e-04 | 4723.9ms/step | 12.7 samples/s
|
| 183 |
+
Step 1690/50462 | Loss: 3.2968 | LR: 5.00e-04 | 4723.8ms/step | 12.7 samples/s
|
| 184 |
+
Step 1700/50462 | Loss: 3.2718 | LR: 5.00e-04 | 4725.1ms/step | 12.7 samples/s
|
| 185 |
+
Step 1710/50462 | Loss: 3.3012 | LR: 5.00e-04 | 4726.8ms/step | 12.7 samples/s
|
| 186 |
+
Step 1720/50462 | Loss: 3.2430 | LR: 5.00e-04 | 4728.1ms/step | 12.7 samples/s
|
| 187 |
+
Step 1730/50462 | Loss: 3.2939 | LR: 5.00e-04 | 4729.9ms/step | 12.7 samples/s
|
| 188 |
+
Step 1740/50462 | Loss: 3.2910 | LR: 5.00e-04 | 4728.6ms/step | 12.7 samples/s
|
| 189 |
+
Step 1750/50462 | Loss: 3.2874 | LR: 5.00e-04 | 4726.8ms/step | 12.7 samples/s
|
| 190 |
+
Validation at step 1750: Loss = 3.2720, Perplexity = 26.36
|
| 191 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-1750
|
| 192 |
+
Step 1760/50462 | Loss: 3.3130 | LR: 4.99e-04 | 7002.9ms/step | 8.6 samples/s
|
| 193 |
+
Step 1770/50462 | Loss: 3.2957 | LR: 4.99e-04 | 4715.3ms/step | 12.7 samples/s
|
| 194 |
+
Step 1780/50462 | Loss: 3.2438 | LR: 4.99e-04 | 4724.5ms/step | 12.7 samples/s
|
| 195 |
+
Step 1790/50462 | Loss: 3.2295 | LR: 4.99e-04 | 4729.4ms/step | 12.7 samples/s
|
| 196 |
+
Step 1800/50462 | Loss: 3.2176 | LR: 4.99e-04 | 4730.0ms/step | 12.7 samples/s
|
| 197 |
+
Step 1810/50462 | Loss: 3.2666 | LR: 4.99e-04 | 4729.2ms/step | 12.7 samples/s
|
| 198 |
+
Step 1820/50462 | Loss: 3.2809 | LR: 4.99e-04 | 4729.6ms/step | 12.7 samples/s
|
| 199 |
+
Step 1830/50462 | Loss: 3.2583 | LR: 4.99e-04 | 4733.9ms/step | 12.7 samples/s
|
| 200 |
+
Step 1840/50462 | Loss: 3.2340 | LR: 4.99e-04 | 4730.8ms/step | 12.7 samples/s
|
| 201 |
+
Step 1850/50462 | Loss: 3.1894 | LR: 4.99e-04 | 4730.2ms/step | 12.7 samples/s
|
| 202 |
+
Step 1860/50462 | Loss: 3.2571 | LR: 4.99e-04 | 4730.5ms/step | 12.7 samples/s
|
| 203 |
+
Step 1870/50462 | Loss: 3.2353 | LR: 4.99e-04 | 4727.9ms/step | 12.7 samples/s
|
| 204 |
+
Step 1880/50462 | Loss: 3.2044 | LR: 4.99e-04 | 4728.8ms/step | 12.7 samples/s
|
| 205 |
+
Step 1890/50462 | Loss: 3.2278 | LR: 4.99e-04 | 4729.9ms/step | 12.7 samples/s
|
| 206 |
+
Step 1900/50462 | Loss: 3.3148 | LR: 4.99e-04 | 4730.5ms/step | 12.7 samples/s
|
| 207 |
+
Step 1910/50462 | Loss: 3.2301 | LR: 4.99e-04 | 4729.8ms/step | 12.7 samples/s
|
| 208 |
+
Step 1920/50462 | Loss: 3.2385 | LR: 4.99e-04 | 4731.7ms/step | 12.7 samples/s
|
| 209 |
+
Step 1930/50462 | Loss: 3.2242 | LR: 4.99e-04 | 4730.1ms/step | 12.7 samples/s
|
| 210 |
+
Step 1940/50462 | Loss: 3.2372 | LR: 4.99e-04 | 4729.3ms/step | 12.7 samples/s
|
| 211 |
+
Step 1950/50462 | Loss: 3.2175 | LR: 4.99e-04 | 4730.5ms/step | 12.7 samples/s
|
| 212 |
+
Step 1960/50462 | Loss: 3.2014 | LR: 4.99e-04 | 4729.8ms/step | 12.7 samples/s
|
| 213 |
+
Step 1970/50462 | Loss: 3.2542 | LR: 4.99e-04 | 4730.2ms/step | 12.7 samples/s
|
| 214 |
+
Step 1980/50462 | Loss: 3.2235 | LR: 4.99e-04 | 4732.0ms/step | 12.7 samples/s
|
| 215 |
+
Step 1990/50462 | Loss: 3.2450 | LR: 4.99e-04 | 4731.8ms/step | 12.7 samples/s
|
| 216 |
+
Step 2000/50462 | Loss: 3.2124 | LR: 4.99e-04 | 4733.8ms/step | 12.7 samples/s
|
| 217 |
+
Validation at step 2000: Loss = 3.2086, Perplexity = 24.74
|
| 218 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-2000
|
| 219 |
+
Step 2010/50462 | Loss: 3.2572 | LR: 4.99e-04 | 6993.8ms/step | 8.6 samples/s
|
| 220 |
+
Step 2020/50462 | Loss: 3.2106 | LR: 4.99e-04 | 4715.2ms/step | 12.7 samples/s
|
| 221 |
+
Step 2030/50462 | Loss: 3.2275 | LR: 4.99e-04 | 4719.9ms/step | 12.7 samples/s
|
| 222 |
+
Step 2040/50462 | Loss: 3.2156 | LR: 4.99e-04 | 4724.1ms/step | 12.7 samples/s
|
| 223 |
+
Step 2050/50462 | Loss: 3.2348 | LR: 4.99e-04 | 4724.6ms/step | 12.7 samples/s
|
| 224 |
+
Step 2060/50462 | Loss: 3.2143 | LR: 4.99e-04 | 4726.6ms/step | 12.7 samples/s
|
| 225 |
+
Step 2070/50462 | Loss: 3.2341 | LR: 4.99e-04 | 4726.1ms/step | 12.7 samples/s
|
| 226 |
+
Step 2080/50462 | Loss: 3.2059 | LR: 4.99e-04 | 4726.9ms/step | 12.7 samples/s
|
| 227 |
+
Step 2090/50462 | Loss: 3.1874 | LR: 4.99e-04 | 4725.4ms/step | 12.7 samples/s
|
| 228 |
+
Step 2100/50462 | Loss: 3.1995 | LR: 4.99e-04 | 4728.1ms/step | 12.7 samples/s
|
| 229 |
+
Step 2110/50462 | Loss: 3.1995 | LR: 4.99e-04 | 4725.9ms/step | 12.7 samples/s
|
| 230 |
+
Step 2120/50462 | Loss: 3.1980 | LR: 4.99e-04 | 4725.7ms/step | 12.7 samples/s
|
| 231 |
+
Step 2130/50462 | Loss: 3.1753 | LR: 4.99e-04 | 4725.2ms/step | 12.7 samples/s
|
| 232 |
+
Step 2140/50462 | Loss: 3.1705 | LR: 4.99e-04 | 4727.7ms/step | 12.7 samples/s
|
| 233 |
+
Step 2150/50462 | Loss: 3.1975 | LR: 4.99e-04 | 4727.7ms/step | 12.7 samples/s
|
| 234 |
+
Step 2160/50462 | Loss: 3.2377 | LR: 4.99e-04 | 4725.5ms/step | 12.7 samples/s
|
| 235 |
+
Step 2170/50462 | Loss: 3.1811 | LR: 4.99e-04 | 4725.1ms/step | 12.7 samples/s
|
| 236 |
+
Step 2180/50462 | Loss: 3.1681 | LR: 4.99e-04 | 4726.8ms/step | 12.7 samples/s
|
| 237 |
+
Step 2190/50462 | Loss: 3.1791 | LR: 4.99e-04 | 4727.1ms/step | 12.7 samples/s
|
| 238 |
+
Step 2200/50462 | Loss: 3.1861 | LR: 4.99e-04 | 4727.6ms/step | 12.7 samples/s
|
| 239 |
+
Step 2210/50462 | Loss: 3.1742 | LR: 4.99e-04 | 4726.3ms/step | 12.7 samples/s
|
| 240 |
+
Step 2220/50462 | Loss: 3.1546 | LR: 4.99e-04 | 4727.0ms/step | 12.7 samples/s
|
| 241 |
+
Step 2230/50462 | Loss: 3.1727 | LR: 4.99e-04 | 4727.6ms/step | 12.7 samples/s
|
| 242 |
+
Step 2240/50462 | Loss: 3.1488 | LR: 4.99e-04 | 4725.7ms/step | 12.7 samples/s
|
| 243 |
+
Step 2250/50462 | Loss: 3.1842 | LR: 4.99e-04 | 4726.7ms/step | 12.7 samples/s
|
| 244 |
+
Validation at step 2250: Loss = 3.1563, Perplexity = 23.48
|
| 245 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-2250
|
| 246 |
+
Step 2260/50462 | Loss: 3.1741 | LR: 4.99e-04 | 6990.2ms/step | 8.6 samples/s
|
| 247 |
+
Step 2270/50462 | Loss: 3.1603 | LR: 4.99e-04 | 4711.9ms/step | 12.7 samples/s
|
| 248 |
+
Step 2280/50462 | Loss: 3.1767 | LR: 4.99e-04 | 4720.4ms/step | 12.7 samples/s
|
| 249 |
+
Step 2290/50462 | Loss: 3.1850 | LR: 4.99e-04 | 4725.3ms/step | 12.7 samples/s
|
| 250 |
+
Step 2300/50462 | Loss: 3.1567 | LR: 4.99e-04 | 4727.0ms/step | 12.7 samples/s
|
| 251 |
+
Step 2310/50462 | Loss: 3.1578 | LR: 4.99e-04 | 4726.9ms/step | 12.7 samples/s
|
| 252 |
+
Step 2320/50462 | Loss: 3.1718 | LR: 4.99e-04 | 4727.6ms/step | 12.7 samples/s
|
| 253 |
+
Step 2330/50462 | Loss: 3.1383 | LR: 4.99e-04 | 4727.4ms/step | 12.7 samples/s
|
| 254 |
+
Step 2340/50462 | Loss: 3.1241 | LR: 4.99e-04 | 4727.4ms/step | 12.7 samples/s
|
| 255 |
+
Step 2350/50462 | Loss: 3.0744 | LR: 4.99e-04 | 4728.8ms/step | 12.7 samples/s
|
| 256 |
+
Step 2360/50462 | Loss: 3.1817 | LR: 4.99e-04 | 4728.2ms/step | 12.7 samples/s
|
| 257 |
+
Step 2370/50462 | Loss: 3.1758 | LR: 4.99e-04 | 4729.2ms/step | 12.7 samples/s
|
| 258 |
+
Step 2380/50462 | Loss: 3.1503 | LR: 4.99e-04 | 4728.7ms/step | 12.7 samples/s
|
| 259 |
+
Step 2390/50462 | Loss: 3.0959 | LR: 4.99e-04 | 4728.8ms/step | 12.7 samples/s
|
| 260 |
+
Step 2400/50462 | Loss: 3.1322 | LR: 4.99e-04 | 4730.2ms/step | 12.7 samples/s
|
| 261 |
+
Step 2410/50462 | Loss: 3.1433 | LR: 4.99e-04 | 4728.3ms/step | 12.7 samples/s
|
| 262 |
+
Step 2420/50462 | Loss: 3.1539 | LR: 4.99e-04 | 4730.5ms/step | 12.7 samples/s
|
| 263 |
+
Step 2430/50462 | Loss: 3.1273 | LR: 4.99e-04 | 4728.7ms/step | 12.7 samples/s
|
| 264 |
+
Step 2440/50462 | Loss: 3.1468 | LR: 4.99e-04 | 4731.2ms/step | 12.7 samples/s
|
| 265 |
+
Step 2450/50462 | Loss: 3.1141 | LR: 4.99e-04 | 4730.5ms/step | 12.7 samples/s
|
| 266 |
+
Step 2460/50462 | Loss: 3.0949 | LR: 4.99e-04 | 4729.4ms/step | 12.7 samples/s
|
| 267 |
+
Step 2470/50462 | Loss: 3.1297 | LR: 4.99e-04 | 4730.3ms/step | 12.7 samples/s
|
| 268 |
+
Step 2480/50462 | Loss: 3.0989 | LR: 4.99e-04 | 4729.9ms/step | 12.7 samples/s
|
| 269 |
+
Step 2490/50462 | Loss: 3.1122 | LR: 4.98e-04 | 4729.0ms/step | 12.7 samples/s
|
| 270 |
+
Step 2500/50462 | Loss: 3.1597 | LR: 4.98e-04 | 4728.5ms/step | 12.7 samples/s
|
| 271 |
+
Validation at step 2500: Loss = 3.1156, Perplexity = 22.55
|
| 272 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-2500
|
| 273 |
+
Step 2510/50462 | Loss: 3.0855 | LR: 4.98e-04 | 6997.4ms/step | 8.6 samples/s
|
| 274 |
+
Step 2520/50462 | Loss: 3.1177 | LR: 4.98e-04 | 4715.5ms/step | 12.7 samples/s
|
| 275 |
+
Step 2530/50462 | Loss: 3.1661 | LR: 4.98e-04 | 4723.3ms/step | 12.7 samples/s
|
| 276 |
+
Step 2540/50462 | Loss: 3.1501 | LR: 4.98e-04 | 4724.6ms/step | 12.7 samples/s
|
| 277 |
+
Step 2550/50462 | Loss: 3.1158 | LR: 4.98e-04 | 4724.0ms/step | 12.7 samples/s
|
| 278 |
+
Step 2560/50462 | Loss: 3.0982 | LR: 4.98e-04 | 4725.0ms/step | 12.7 samples/s
|
| 279 |
+
Step 2570/50462 | Loss: 3.1328 | LR: 4.98e-04 | 4722.0ms/step | 12.7 samples/s
|
| 280 |
+
Step 2580/50462 | Loss: 3.0830 | LR: 4.98e-04 | 4723.9ms/step | 12.7 samples/s
|
| 281 |
+
Step 2590/50462 | Loss: 3.1025 | LR: 4.98e-04 | 4723.8ms/step | 12.7 samples/s
|
| 282 |
+
Step 2600/50462 | Loss: 3.1344 | LR: 4.98e-04 | 4723.3ms/step | 12.7 samples/s
|
| 283 |
+
Step 2610/50462 | Loss: 3.0735 | LR: 4.98e-04 | 4722.8ms/step | 12.7 samples/s
|
| 284 |
+
Step 2620/50462 | Loss: 3.1237 | LR: 4.98e-04 | 4723.9ms/step | 12.7 samples/s
|
| 285 |
+
Step 2630/50462 | Loss: 3.1402 | LR: 4.98e-04 | 4725.5ms/step | 12.7 samples/s
|
epfl_final/SparseAtt/wandb/run-20260823_140341-qctui00g/files/requirements.txt
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
| 142 |
+
entmax==1.3
|
epfl_final/SparseAtt/wandb/run-20260823_140341-qctui00g/run-qctui00g.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1231928bb0c6c295392d32fff234fbadfab7cfa29e416b2b51b40d94c00dc4ee
|
| 3 |
+
size 2850816
|
epfl_final/SparseAtt/wandb/run-20260823_142101-rgmkfpnl/files/output.log
ADDED
|
@@ -0,0 +1,613 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Starting training...
|
| 2 |
+
Validation at step 4750: Loss = 2.9765, Perplexity = 19.62
|
| 3 |
+
wandb: WARNING Tried to log to step 4750 that is less than the current step 4851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 4 |
+
Switching attention mode for step 4751: dense (dense step interval)
|
| 5 |
+
Step 4760/50462 | Loss: 2.7898 | LR: 4.92e-04 | 25557.5ms/step | 2.3 samples/s
|
| 6 |
+
wandb: WARNING Tried to log to step 4760 that is less than the current step 4851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 7 |
+
Step 4770/50462 | Loss: 2.9189 | LR: 4.92e-04 | 5606.2ms/step | 10.7 samples/s
|
| 8 |
+
wandb: WARNING Tried to log to step 4770 that is less than the current step 4851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 9 |
+
Step 4780/50462 | Loss: 2.8885 | LR: 4.92e-04 | 5613.3ms/step | 10.7 samples/s
|
| 10 |
+
wandb: WARNING Tried to log to step 4780 that is less than the current step 4851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 11 |
+
Step 4790/50462 | Loss: 2.9275 | LR: 4.92e-04 | 5611.0ms/step | 10.7 samples/s
|
| 12 |
+
wandb: WARNING Tried to log to step 4790 that is less than the current step 4851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 13 |
+
Step 4800/50462 | Loss: 2.9016 | LR: 4.92e-04 | 5616.0ms/step | 10.7 samples/s
|
| 14 |
+
wandb: WARNING Tried to log to step 4800 that is less than the current step 4851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 15 |
+
Step 4810/50462 | Loss: 2.9204 | LR: 4.92e-04 | 5617.1ms/step | 10.7 samples/s
|
| 16 |
+
wandb: WARNING Tried to log to step 4810 that is less than the current step 4851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 17 |
+
Step 4820/50462 | Loss: 2.9325 | LR: 4.92e-04 | 5620.1ms/step | 10.7 samples/s
|
| 18 |
+
wandb: WARNING Tried to log to step 4820 that is less than the current step 4851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 19 |
+
Step 4830/50462 | Loss: 2.8947 | LR: 4.92e-04 | 5620.8ms/step | 10.7 samples/s
|
| 20 |
+
wandb: WARNING Tried to log to step 4830 that is less than the current step 4851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 21 |
+
Step 4840/50462 | Loss: 2.9317 | LR: 4.92e-04 | 5617.2ms/step | 10.7 samples/s
|
| 22 |
+
wandb: WARNING Tried to log to step 4840 that is less than the current step 4851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 23 |
+
Step 4850/50462 | Loss: 2.9170 | LR: 4.92e-04 | 5620.6ms/step | 10.7 samples/s
|
| 24 |
+
wandb: WARNING Tried to log to step 4850 that is less than the current step 4851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 25 |
+
Step 4860/50462 | Loss: 2.8961 | LR: 4.92e-04 | 5621.4ms/step | 10.7 samples/s
|
| 26 |
+
Step 4870/50462 | Loss: 2.9214 | LR: 4.92e-04 | 5624.5ms/step | 10.7 samples/s
|
| 27 |
+
Step 4880/50462 | Loss: 2.9081 | LR: 4.92e-04 | 5627.3ms/step | 10.7 samples/s
|
| 28 |
+
Step 4890/50462 | Loss: 2.8927 | LR: 4.91e-04 | 5624.2ms/step | 10.7 samples/s
|
| 29 |
+
Step 4900/50462 | Loss: 2.9027 | LR: 4.91e-04 | 5619.8ms/step | 10.7 samples/s
|
| 30 |
+
Step 4910/50462 | Loss: 2.9412 | LR: 4.91e-04 | 5622.9ms/step | 10.7 samples/s
|
| 31 |
+
Step 4920/50462 | Loss: 2.9107 | LR: 4.91e-04 | 5620.5ms/step | 10.7 samples/s
|
| 32 |
+
Step 4930/50462 | Loss: 2.8734 | LR: 4.91e-04 | 5614.3ms/step | 10.7 samples/s
|
| 33 |
+
Step 4940/50462 | Loss: 2.9371 | LR: 4.91e-04 | 5594.0ms/step | 10.7 samples/s
|
| 34 |
+
Step 4950/50462 | Loss: 2.9249 | LR: 4.91e-04 | 5587.1ms/step | 10.7 samples/s
|
| 35 |
+
Step 4960/50462 | Loss: 2.9282 | LR: 4.91e-04 | 5588.0ms/step | 10.7 samples/s
|
| 36 |
+
Step 4970/50462 | Loss: 2.9214 | LR: 4.91e-04 | 5588.2ms/step | 10.7 samples/s
|
| 37 |
+
Step 4980/50462 | Loss: 2.9222 | LR: 4.91e-04 | 5600.1ms/step | 10.7 samples/s
|
| 38 |
+
Step 4990/50462 | Loss: 2.9054 | LR: 4.91e-04 | 5609.8ms/step | 10.7 samples/s
|
| 39 |
+
Step 5000/50462 | Loss: 2.9347 | LR: 4.91e-04 | 5621.1ms/step | 10.7 samples/s
|
| 40 |
+
Validation at step 5000: Loss = 2.9632, Perplexity = 19.36
|
| 41 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-5000
|
| 42 |
+
Step 5010/50462 | Loss: 2.9167 | LR: 4.91e-04 | 8408.3ms/step | 7.1 samples/s
|
| 43 |
+
Step 5020/50462 | Loss: 2.9162 | LR: 4.91e-04 | 5628.8ms/step | 10.7 samples/s
|
| 44 |
+
Step 5030/50462 | Loss: 2.9067 | LR: 4.91e-04 | 5630.2ms/step | 10.7 samples/s
|
| 45 |
+
Step 5040/50462 | Loss: 2.8993 | LR: 4.91e-04 | 5630.2ms/step | 10.7 samples/s
|
| 46 |
+
Step 5050/50462 | Loss: 2.9362 | LR: 4.91e-04 | 5629.8ms/step | 10.7 samples/s
|
| 47 |
+
Step 5060/50462 | Loss: 2.9193 | LR: 4.91e-04 | 5630.6ms/step | 10.7 samples/s
|
| 48 |
+
Step 5070/50462 | Loss: 2.9416 | LR: 4.91e-04 | 5631.0ms/step | 10.7 samples/s
|
| 49 |
+
Step 5080/50462 | Loss: 2.8938 | LR: 4.91e-04 | 5627.7ms/step | 10.7 samples/s
|
| 50 |
+
Step 5090/50462 | Loss: 2.8894 | LR: 4.91e-04 | 5629.1ms/step | 10.7 samples/s
|
| 51 |
+
Step 5100/50462 | Loss: 2.9151 | LR: 4.91e-04 | 5624.0ms/step | 10.7 samples/s
|
| 52 |
+
Step 5110/50462 | Loss: 2.9174 | LR: 4.91e-04 | 5626.5ms/step | 10.7 samples/s
|
| 53 |
+
Step 5120/50462 | Loss: 2.8755 | LR: 4.91e-04 | 5629.6ms/step | 10.7 samples/s
|
| 54 |
+
Step 5130/50462 | Loss: 2.9104 | LR: 4.90e-04 | 5624.6ms/step | 10.7 samples/s
|
| 55 |
+
Step 5140/50462 | Loss: 2.8762 | LR: 4.90e-04 | 5625.8ms/step | 10.7 samples/s
|
| 56 |
+
Step 5150/50462 | Loss: 2.8867 | LR: 4.90e-04 | 5629.5ms/step | 10.7 samples/s
|
| 57 |
+
Step 5160/50462 | Loss: 2.8901 | LR: 4.90e-04 | 5631.5ms/step | 10.7 samples/s
|
| 58 |
+
Step 5170/50462 | Loss: 2.9315 | LR: 4.90e-04 | 5628.2ms/step | 10.7 samples/s
|
| 59 |
+
Step 5180/50462 | Loss: 2.9286 | LR: 4.90e-04 | 5628.2ms/step | 10.7 samples/s
|
| 60 |
+
Step 5190/50462 | Loss: 2.8828 | LR: 4.90e-04 | 5626.5ms/step | 10.7 samples/s
|
| 61 |
+
Step 5200/50462 | Loss: 2.9138 | LR: 4.90e-04 | 5628.6ms/step | 10.7 samples/s
|
| 62 |
+
Step 5210/50462 | Loss: 2.8465 | LR: 4.90e-04 | 5630.6ms/step | 10.7 samples/s
|
| 63 |
+
Step 5220/50462 | Loss: 2.8790 | LR: 4.90e-04 | 5629.3ms/step | 10.7 samples/s
|
| 64 |
+
Step 5230/50462 | Loss: 2.9057 | LR: 4.90e-04 | 5631.1ms/step | 10.7 samples/s
|
| 65 |
+
Step 5240/50462 | Loss: 2.8852 | LR: 4.90e-04 | 5630.2ms/step | 10.7 samples/s
|
| 66 |
+
Step 5250/50462 | Loss: 2.9079 | LR: 4.90e-04 | 5625.9ms/step | 10.7 samples/s
|
| 67 |
+
Validation at step 5250: Loss = 2.9504, Perplexity = 19.11
|
| 68 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-5250
|
| 69 |
+
Step 5260/50462 | Loss: 2.8622 | LR: 4.90e-04 | 8411.9ms/step | 7.1 samples/s
|
| 70 |
+
Step 5270/50462 | Loss: 2.8674 | LR: 4.90e-04 | 5626.8ms/step | 10.7 samples/s
|
| 71 |
+
Step 5280/50462 | Loss: 2.8786 | LR: 4.90e-04 | 5629.7ms/step | 10.7 samples/s
|
| 72 |
+
Step 5290/50462 | Loss: 2.9092 | LR: 4.90e-04 | 5632.1ms/step | 10.7 samples/s
|
| 73 |
+
Step 5300/50462 | Loss: 2.8912 | LR: 4.90e-04 | 5630.1ms/step | 10.7 samples/s
|
| 74 |
+
Step 5310/50462 | Loss: 2.8852 | LR: 4.90e-04 | 5632.0ms/step | 10.7 samples/s
|
| 75 |
+
Step 5320/50462 | Loss: 2.8587 | LR: 4.90e-04 | 5634.3ms/step | 10.6 samples/s
|
| 76 |
+
Step 5330/50462 | Loss: 2.8758 | LR: 4.90e-04 | 5630.3ms/step | 10.7 samples/s
|
| 77 |
+
Step 5340/50462 | Loss: 2.9077 | LR: 4.90e-04 | 5628.4ms/step | 10.7 samples/s
|
| 78 |
+
Step 5350/50462 | Loss: 2.8832 | LR: 4.90e-04 | 5631.1ms/step | 10.7 samples/s
|
| 79 |
+
Step 5360/50462 | Loss: 2.9022 | LR: 4.89e-04 | 5630.8ms/step | 10.7 samples/s
|
| 80 |
+
Step 5370/50462 | Loss: 2.8662 | LR: 4.89e-04 | 5632.8ms/step | 10.7 samples/s
|
| 81 |
+
Step 5380/50462 | Loss: 2.8646 | LR: 4.89e-04 | 5631.1ms/step | 10.7 samples/s
|
| 82 |
+
Step 5390/50462 | Loss: 2.9104 | LR: 4.89e-04 | 5630.2ms/step | 10.7 samples/s
|
| 83 |
+
Step 5400/50462 | Loss: 2.9088 | LR: 4.89e-04 | 5629.6ms/step | 10.7 samples/s
|
| 84 |
+
Step 5410/50462 | Loss: 2.9090 | LR: 4.89e-04 | 5629.7ms/step | 10.7 samples/s
|
| 85 |
+
Step 5420/50462 | Loss: 2.8929 | LR: 4.89e-04 | 5627.7ms/step | 10.7 samples/s
|
| 86 |
+
Step 5430/50462 | Loss: 2.9173 | LR: 4.89e-04 | 5631.1ms/step | 10.7 samples/s
|
| 87 |
+
Step 5440/50462 | Loss: 2.9053 | LR: 4.89e-04 | 5632.1ms/step | 10.7 samples/s
|
| 88 |
+
Step 5450/50462 | Loss: 2.9171 | LR: 4.89e-04 | 5629.6ms/step | 10.7 samples/s
|
| 89 |
+
Step 5460/50462 | Loss: 2.8715 | LR: 4.89e-04 | 5631.5ms/step | 10.7 samples/s
|
| 90 |
+
Step 5470/50462 | Loss: 2.8659 | LR: 4.89e-04 | 5632.2ms/step | 10.7 samples/s
|
| 91 |
+
Step 5480/50462 | Loss: 2.9038 | LR: 4.89e-04 | 5631.0ms/step | 10.7 samples/s
|
| 92 |
+
Step 5490/50462 | Loss: 2.8241 | LR: 4.89e-04 | 5632.1ms/step | 10.7 samples/s
|
| 93 |
+
Step 5500/50462 | Loss: 2.8918 | LR: 4.89e-04 | 5631.3ms/step | 10.7 samples/s
|
| 94 |
+
Validation at step 5500: Loss = 2.9392, Perplexity = 18.90
|
| 95 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-5500
|
| 96 |
+
Step 5510/50462 | Loss: 2.8884 | LR: 4.89e-04 | 8372.1ms/step | 7.2 samples/s
|
| 97 |
+
Step 5520/50462 | Loss: 2.8531 | LR: 4.89e-04 | 5630.6ms/step | 10.7 samples/s
|
| 98 |
+
Step 5530/50462 | Loss: 2.8670 | LR: 4.89e-04 | 5631.0ms/step | 10.7 samples/s
|
| 99 |
+
Step 5540/50462 | Loss: 2.8558 | LR: 4.89e-04 | 5632.1ms/step | 10.7 samples/s
|
| 100 |
+
Step 5550/50462 | Loss: 2.8815 | LR: 4.89e-04 | 5628.9ms/step | 10.7 samples/s
|
| 101 |
+
Step 5560/50462 | Loss: 2.8765 | LR: 4.89e-04 | 5630.0ms/step | 10.7 samples/s
|
| 102 |
+
Step 5570/50462 | Loss: 2.8838 | LR: 4.88e-04 | 5630.8ms/step | 10.7 samples/s
|
| 103 |
+
Step 5580/50462 | Loss: 2.9248 | LR: 4.88e-04 | 5632.9ms/step | 10.7 samples/s
|
| 104 |
+
Step 5590/50462 | Loss: 2.8654 | LR: 4.88e-04 | 5632.0ms/step | 10.7 samples/s
|
| 105 |
+
Step 5600/50462 | Loss: 2.9087 | LR: 4.88e-04 | 5630.2ms/step | 10.7 samples/s
|
| 106 |
+
Step 5610/50462 | Loss: 2.8404 | LR: 4.88e-04 | 5628.7ms/step | 10.7 samples/s
|
| 107 |
+
Step 5620/50462 | Loss: 2.8812 | LR: 4.88e-04 | 5629.5ms/step | 10.7 samples/s
|
| 108 |
+
Step 5630/50462 | Loss: 2.8908 | LR: 4.88e-04 | 5631.6ms/step | 10.7 samples/s
|
| 109 |
+
Step 5640/50462 | Loss: 2.8975 | LR: 4.88e-04 | 5631.9ms/step | 10.7 samples/s
|
| 110 |
+
Step 5650/50462 | Loss: 2.9143 | LR: 4.88e-04 | 5631.8ms/step | 10.7 samples/s
|
| 111 |
+
Step 5660/50462 | Loss: 2.8830 | LR: 4.88e-04 | 5631.2ms/step | 10.7 samples/s
|
| 112 |
+
Step 5670/50462 | Loss: 2.8973 | LR: 4.88e-04 | 5630.7ms/step | 10.7 samples/s
|
| 113 |
+
Step 5680/50462 | Loss: 2.8657 | LR: 4.88e-04 | 5632.2ms/step | 10.7 samples/s
|
| 114 |
+
Step 5690/50462 | Loss: 2.8848 | LR: 4.88e-04 | 5632.3ms/step | 10.7 samples/s
|
| 115 |
+
Step 5700/50462 | Loss: 2.8677 | LR: 4.88e-04 | 5631.7ms/step | 10.7 samples/s
|
| 116 |
+
Step 5710/50462 | Loss: 2.8588 | LR: 4.88e-04 | 5630.7ms/step | 10.7 samples/s
|
| 117 |
+
Step 5720/50462 | Loss: 2.8786 | LR: 4.88e-04 | 5631.9ms/step | 10.7 samples/s
|
| 118 |
+
Step 5730/50462 | Loss: 2.8848 | LR: 4.88e-04 | 5631.5ms/step | 10.7 samples/s
|
| 119 |
+
Step 5740/50462 | Loss: 2.9005 | LR: 4.88e-04 | 5630.7ms/step | 10.7 samples/s
|
| 120 |
+
Step 5750/50462 | Loss: 2.8510 | LR: 4.88e-04 | 5632.8ms/step | 10.7 samples/s
|
| 121 |
+
Validation at step 5750: Loss = 2.9269, Perplexity = 18.67
|
| 122 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-5750
|
| 123 |
+
Step 5760/50462 | Loss: 2.8559 | LR: 4.88e-04 | 8387.7ms/step | 7.2 samples/s
|
| 124 |
+
Step 5770/50462 | Loss: 2.8836 | LR: 4.88e-04 | 5627.9ms/step | 10.7 samples/s
|
| 125 |
+
Step 5780/50462 | Loss: 2.8623 | LR: 4.87e-04 | 5627.9ms/step | 10.7 samples/s
|
| 126 |
+
Step 5790/50462 | Loss: 2.8753 | LR: 4.87e-04 | 5628.3ms/step | 10.7 samples/s
|
| 127 |
+
Step 5800/50462 | Loss: 2.8568 | LR: 4.87e-04 | 5627.9ms/step | 10.7 samples/s
|
| 128 |
+
Step 5810/50462 | Loss: 2.8833 | LR: 4.87e-04 | 5628.6ms/step | 10.7 samples/s
|
| 129 |
+
Step 5820/50462 | Loss: 2.9242 | LR: 4.87e-04 | 5628.5ms/step | 10.7 samples/s
|
| 130 |
+
Step 5830/50462 | Loss: 2.8731 | LR: 4.87e-04 | 5629.8ms/step | 10.7 samples/s
|
| 131 |
+
Step 5840/50462 | Loss: 2.8296 | LR: 4.87e-04 | 5630.7ms/step | 10.7 samples/s
|
| 132 |
+
Step 5850/50462 | Loss: 2.8613 | LR: 4.87e-04 | 5629.5ms/step | 10.7 samples/s
|
| 133 |
+
Step 5860/50462 | Loss: 2.8531 | LR: 4.87e-04 | 5630.3ms/step | 10.7 samples/s
|
| 134 |
+
Step 5870/50462 | Loss: 2.8372 | LR: 4.87e-04 | 5630.5ms/step | 10.7 samples/s
|
| 135 |
+
Step 5880/50462 | Loss: 2.8499 | LR: 4.87e-04 | 5633.4ms/step | 10.7 samples/s
|
| 136 |
+
Step 5890/50462 | Loss: 2.9092 | LR: 4.87e-04 | 5630.0ms/step | 10.7 samples/s
|
| 137 |
+
Step 5900/50462 | Loss: 2.8151 | LR: 4.87e-04 | 5630.1ms/step | 10.7 samples/s
|
| 138 |
+
Step 5910/50462 | Loss: 2.8835 | LR: 4.87e-04 | 5630.8ms/step | 10.7 samples/s
|
| 139 |
+
Step 5920/50462 | Loss: 2.8911 | LR: 4.87e-04 | 5631.8ms/step | 10.7 samples/s
|
| 140 |
+
Step 5930/50462 | Loss: 2.8540 | LR: 4.87e-04 | 5629.2ms/step | 10.7 samples/s
|
| 141 |
+
Step 5940/50462 | Loss: 2.8611 | LR: 4.87e-04 | 5631.2ms/step | 10.7 samples/s
|
| 142 |
+
Step 5950/50462 | Loss: 2.8628 | LR: 4.87e-04 | 5628.4ms/step | 10.7 samples/s
|
| 143 |
+
Step 5960/50462 | Loss: 2.8701 | LR: 4.87e-04 | 5626.0ms/step | 10.7 samples/s
|
| 144 |
+
Step 5970/50462 | Loss: 2.8578 | LR: 4.87e-04 | 5627.0ms/step | 10.7 samples/s
|
| 145 |
+
Step 5980/50462 | Loss: 2.9127 | LR: 4.86e-04 | 5626.9ms/step | 10.7 samples/s
|
| 146 |
+
Step 5990/50462 | Loss: 2.8624 | LR: 4.86e-04 | 5628.2ms/step | 10.7 samples/s
|
| 147 |
+
Step 6000/50462 | Loss: 2.8871 | LR: 4.86e-04 | 5628.3ms/step | 10.7 samples/s
|
| 148 |
+
Validation at step 6000: Loss = 2.9159, Perplexity = 18.47
|
| 149 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-6000
|
| 150 |
+
Step 6010/50462 | Loss: 2.8736 | LR: 4.86e-04 | 8467.4ms/step | 7.1 samples/s
|
| 151 |
+
Step 6020/50462 | Loss: 2.8459 | LR: 4.86e-04 | 5625.4ms/step | 10.7 samples/s
|
| 152 |
+
Step 6030/50462 | Loss: 2.8760 | LR: 4.86e-04 | 5626.1ms/step | 10.7 samples/s
|
| 153 |
+
Step 6040/50462 | Loss: 2.8854 | LR: 4.86e-04 | 5625.5ms/step | 10.7 samples/s
|
| 154 |
+
Step 6050/50462 | Loss: 2.8499 | LR: 4.86e-04 | 5627.0ms/step | 10.7 samples/s
|
| 155 |
+
Step 6060/50462 | Loss: 2.8394 | LR: 4.86e-04 | 5626.7ms/step | 10.7 samples/s
|
| 156 |
+
Step 6070/50462 | Loss: 2.8636 | LR: 4.86e-04 | 5627.3ms/step | 10.7 samples/s
|
| 157 |
+
Step 6080/50462 | Loss: 2.8463 | LR: 4.86e-04 | 5626.4ms/step | 10.7 samples/s
|
| 158 |
+
Step 6090/50462 | Loss: 2.8518 | LR: 4.86e-04 | 5626.2ms/step | 10.7 samples/s
|
| 159 |
+
Step 6100/50462 | Loss: 2.8666 | LR: 4.86e-04 | 5625.7ms/step | 10.7 samples/s
|
| 160 |
+
Step 6110/50462 | Loss: 2.8935 | LR: 4.86e-04 | 5626.4ms/step | 10.7 samples/s
|
| 161 |
+
Step 6120/50462 | Loss: 2.8419 | LR: 4.86e-04 | 5625.8ms/step | 10.7 samples/s
|
| 162 |
+
Step 6130/50462 | Loss: 2.8559 | LR: 4.86e-04 | 5625.9ms/step | 10.7 samples/s
|
| 163 |
+
Step 6140/50462 | Loss: 2.8538 | LR: 4.86e-04 | 5627.4ms/step | 10.7 samples/s
|
| 164 |
+
Step 6150/50462 | Loss: 2.8704 | LR: 4.86e-04 | 5625.8ms/step | 10.7 samples/s
|
| 165 |
+
Step 6160/50462 | Loss: 2.8512 | LR: 4.86e-04 | 5624.5ms/step | 10.7 samples/s
|
| 166 |
+
Step 6170/50462 | Loss: 2.8844 | LR: 4.85e-04 | 5625.2ms/step | 10.7 samples/s
|
| 167 |
+
Step 6180/50462 | Loss: 2.8691 | LR: 4.85e-04 | 5626.8ms/step | 10.7 samples/s
|
| 168 |
+
Step 6190/50462 | Loss: 2.8418 | LR: 4.85e-04 | 5628.1ms/step | 10.7 samples/s
|
| 169 |
+
Step 6200/50462 | Loss: 2.8738 | LR: 4.85e-04 | 5629.4ms/step | 10.7 samples/s
|
| 170 |
+
Step 6210/50462 | Loss: 2.8575 | LR: 4.85e-04 | 5628.7ms/step | 10.7 samples/s
|
| 171 |
+
Step 6220/50462 | Loss: 2.8286 | LR: 4.85e-04 | 5629.6ms/step | 10.7 samples/s
|
| 172 |
+
Step 6230/50462 | Loss: 2.8197 | LR: 4.85e-04 | 5627.0ms/step | 10.7 samples/s
|
| 173 |
+
Step 6240/50462 | Loss: 2.8540 | LR: 4.85e-04 | 5626.2ms/step | 10.7 samples/s
|
| 174 |
+
Step 6250/50462 | Loss: 2.8531 | LR: 4.85e-04 | 5625.4ms/step | 10.7 samples/s
|
| 175 |
+
Validation at step 6250: Loss = 2.9014, Perplexity = 18.20
|
| 176 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-6250
|
| 177 |
+
Step 6260/50462 | Loss: 2.8687 | LR: 4.85e-04 | 8514.2ms/step | 7.0 samples/s
|
| 178 |
+
Step 6270/50462 | Loss: 2.8768 | LR: 4.85e-04 | 5625.1ms/step | 10.7 samples/s
|
| 179 |
+
Step 6280/50462 | Loss: 2.8565 | LR: 4.85e-04 | 5625.4ms/step | 10.7 samples/s
|
| 180 |
+
Step 6290/50462 | Loss: 2.8830 | LR: 4.85e-04 | 5622.4ms/step | 10.7 samples/s
|
| 181 |
+
Step 6300/50462 | Loss: 2.8696 | LR: 4.85e-04 | 5608.7ms/step | 10.7 samples/s
|
| 182 |
+
Step 6310/50462 | Loss: 2.8627 | LR: 4.85e-04 | 5606.0ms/step | 10.7 samples/s
|
| 183 |
+
Step 6320/50462 | Loss: 2.8211 | LR: 4.85e-04 | 5606.3ms/step | 10.7 samples/s
|
| 184 |
+
Step 6330/50462 | Loss: 2.8506 | LR: 4.85e-04 | 5605.5ms/step | 10.7 samples/s
|
| 185 |
+
Step 6340/50462 | Loss: 2.8594 | LR: 4.85e-04 | 5600.9ms/step | 10.7 samples/s
|
| 186 |
+
Step 6350/50462 | Loss: 2.8198 | LR: 4.85e-04 | 5602.0ms/step | 10.7 samples/s
|
| 187 |
+
Step 6360/50462 | Loss: 2.8180 | LR: 4.84e-04 | 5601.4ms/step | 10.7 samples/s
|
| 188 |
+
Step 6370/50462 | Loss: 2.8644 | LR: 4.84e-04 | 5602.0ms/step | 10.7 samples/s
|
| 189 |
+
Step 6380/50462 | Loss: 2.8690 | LR: 4.84e-04 | 5602.2ms/step | 10.7 samples/s
|
| 190 |
+
Step 6390/50462 | Loss: 2.8321 | LR: 4.84e-04 | 5601.1ms/step | 10.7 samples/s
|
| 191 |
+
Step 6400/50462 | Loss: 2.8484 | LR: 4.84e-04 | 5601.2ms/step | 10.7 samples/s
|
| 192 |
+
Step 6410/50462 | Loss: 2.8332 | LR: 4.84e-04 | 5604.4ms/step | 10.7 samples/s
|
| 193 |
+
Step 6420/50462 | Loss: 2.8245 | LR: 4.84e-04 | 5600.4ms/step | 10.7 samples/s
|
| 194 |
+
Step 6430/50462 | Loss: 2.8550 | LR: 4.84e-04 | 5601.2ms/step | 10.7 samples/s
|
| 195 |
+
Step 6440/50462 | Loss: 2.8666 | LR: 4.84e-04 | 5600.7ms/step | 10.7 samples/s
|
| 196 |
+
Step 6450/50462 | Loss: 2.8331 | LR: 4.84e-04 | 5603.5ms/step | 10.7 samples/s
|
| 197 |
+
Step 6460/50462 | Loss: 2.8651 | LR: 4.84e-04 | 5601.2ms/step | 10.7 samples/s
|
| 198 |
+
Step 6470/50462 | Loss: 2.8439 | LR: 4.84e-04 | 5602.0ms/step | 10.7 samples/s
|
| 199 |
+
Step 6480/50462 | Loss: 2.8011 | LR: 4.84e-04 | 5602.5ms/step | 10.7 samples/s
|
| 200 |
+
Step 6490/50462 | Loss: 2.8101 | LR: 4.84e-04 | 5602.7ms/step | 10.7 samples/s
|
| 201 |
+
Step 6500/50462 | Loss: 2.8176 | LR: 4.84e-04 | 5601.2ms/step | 10.7 samples/s
|
| 202 |
+
Validation at step 6500: Loss = 2.8927, Perplexity = 18.04
|
| 203 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-6500
|
| 204 |
+
Step 6510/50462 | Loss: 2.7805 | LR: 4.84e-04 | 8498.7ms/step | 7.1 samples/s
|
| 205 |
+
Step 6520/50462 | Loss: 2.8407 | LR: 4.84e-04 | 5599.8ms/step | 10.7 samples/s
|
| 206 |
+
Step 6530/50462 | Loss: 2.8762 | LR: 4.84e-04 | 5601.4ms/step | 10.7 samples/s
|
| 207 |
+
Step 6540/50462 | Loss: 2.8507 | LR: 4.83e-04 | 5602.0ms/step | 10.7 samples/s
|
| 208 |
+
Step 6550/50462 | Loss: 2.8411 | LR: 4.83e-04 | 5602.1ms/step | 10.7 samples/s
|
| 209 |
+
Step 6560/50462 | Loss: 2.8222 | LR: 4.83e-04 | 5602.8ms/step | 10.7 samples/s
|
| 210 |
+
Step 6570/50462 | Loss: 2.7923 | LR: 4.83e-04 | 5603.2ms/step | 10.7 samples/s
|
| 211 |
+
Step 6580/50462 | Loss: 2.8730 | LR: 4.83e-04 | 5603.4ms/step | 10.7 samples/s
|
| 212 |
+
Step 6590/50462 | Loss: 2.8510 | LR: 4.83e-04 | 5604.8ms/step | 10.7 samples/s
|
| 213 |
+
Step 6600/50462 | Loss: 2.8475 | LR: 4.83e-04 | 5602.8ms/step | 10.7 samples/s
|
| 214 |
+
Step 6610/50462 | Loss: 2.8278 | LR: 4.83e-04 | 5603.6ms/step | 10.7 samples/s
|
| 215 |
+
Step 6620/50462 | Loss: 2.8478 | LR: 4.83e-04 | 5604.3ms/step | 10.7 samples/s
|
| 216 |
+
Step 6630/50462 | Loss: 2.8333 | LR: 4.83e-04 | 5602.5ms/step | 10.7 samples/s
|
| 217 |
+
Step 6640/50462 | Loss: 2.8981 | LR: 4.83e-04 | 5601.9ms/step | 10.7 samples/s
|
| 218 |
+
Step 6650/50462 | Loss: 2.8616 | LR: 4.83e-04 | 5600.7ms/step | 10.7 samples/s
|
| 219 |
+
Step 6660/50462 | Loss: 2.7948 | LR: 4.83e-04 | 5600.7ms/step | 10.7 samples/s
|
| 220 |
+
Step 6670/50462 | Loss: 2.8245 | LR: 4.83e-04 | 5601.0ms/step | 10.7 samples/s
|
| 221 |
+
Step 6680/50462 | Loss: 2.8394 | LR: 4.83e-04 | 5603.6ms/step | 10.7 samples/s
|
| 222 |
+
Step 6690/50462 | Loss: 2.7885 | LR: 4.83e-04 | 5601.9ms/step | 10.7 samples/s
|
| 223 |
+
Step 6700/50462 | Loss: 2.8298 | LR: 4.83e-04 | 5602.5ms/step | 10.7 samples/s
|
| 224 |
+
Step 6710/50462 | Loss: 2.8079 | LR: 4.82e-04 | 5602.8ms/step | 10.7 samples/s
|
| 225 |
+
Step 6720/50462 | Loss: 2.8333 | LR: 4.82e-04 | 5603.4ms/step | 10.7 samples/s
|
| 226 |
+
Step 6730/50462 | Loss: 2.8491 | LR: 4.82e-04 | 5603.8ms/step | 10.7 samples/s
|
| 227 |
+
Step 6740/50462 | Loss: 2.8368 | LR: 4.82e-04 | 5602.5ms/step | 10.7 samples/s
|
| 228 |
+
Step 6750/50462 | Loss: 2.8223 | LR: 4.82e-04 | 5602.5ms/step | 10.7 samples/s
|
| 229 |
+
Validation at step 6750: Loss = 2.8829, Perplexity = 17.87
|
| 230 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-6750
|
| 231 |
+
Step 6760/50462 | Loss: 2.8407 | LR: 4.82e-04 | 8528.3ms/step | 7.0 samples/s
|
| 232 |
+
Step 6770/50462 | Loss: 2.7726 | LR: 4.82e-04 | 5600.9ms/step | 10.7 samples/s
|
| 233 |
+
Step 6780/50462 | Loss: 2.8296 | LR: 4.82e-04 | 5600.1ms/step | 10.7 samples/s
|
| 234 |
+
Step 6790/50462 | Loss: 2.8262 | LR: 4.82e-04 | 5602.8ms/step | 10.7 samples/s
|
| 235 |
+
Step 6800/50462 | Loss: 2.8336 | LR: 4.82e-04 | 5601.9ms/step | 10.7 samples/s
|
| 236 |
+
Step 6810/50462 | Loss: 2.8418 | LR: 4.82e-04 | 5598.7ms/step | 10.7 samples/s
|
| 237 |
+
Step 6820/50462 | Loss: 2.8416 | LR: 4.82e-04 | 5599.5ms/step | 10.7 samples/s
|
| 238 |
+
Step 6830/50462 | Loss: 2.8223 | LR: 4.82e-04 | 5598.7ms/step | 10.7 samples/s
|
| 239 |
+
Step 6840/50462 | Loss: 2.8191 | LR: 4.82e-04 | 5599.6ms/step | 10.7 samples/s
|
| 240 |
+
Step 6850/50462 | Loss: 2.8408 | LR: 4.82e-04 | 5599.3ms/step | 10.7 samples/s
|
| 241 |
+
Step 6860/50462 | Loss: 2.7948 | LR: 4.82e-04 | 5607.1ms/step | 10.7 samples/s
|
| 242 |
+
Step 6870/50462 | Loss: 2.8232 | LR: 4.82e-04 | 5602.0ms/step | 10.7 samples/s
|
| 243 |
+
Step 6880/50462 | Loss: 2.8308 | LR: 4.81e-04 | 5608.6ms/step | 10.7 samples/s
|
| 244 |
+
Step 6890/50462 | Loss: 2.8011 | LR: 4.81e-04 | 5612.1ms/step | 10.7 samples/s
|
| 245 |
+
Step 6900/50462 | Loss: 2.7990 | LR: 4.81e-04 | 5608.8ms/step | 10.7 samples/s
|
| 246 |
+
Step 6910/50462 | Loss: 2.8602 | LR: 4.81e-04 | 5609.3ms/step | 10.7 samples/s
|
| 247 |
+
Step 6920/50462 | Loss: 2.8411 | LR: 4.81e-04 | 5610.0ms/step | 10.7 samples/s
|
| 248 |
+
Step 6930/50462 | Loss: 2.8294 | LR: 4.81e-04 | 5611.2ms/step | 10.7 samples/s
|
| 249 |
+
Step 6940/50462 | Loss: 2.8712 | LR: 4.81e-04 | 5611.2ms/step | 10.7 samples/s
|
| 250 |
+
Step 6950/50462 | Loss: 2.8333 | LR: 4.81e-04 | 5607.3ms/step | 10.7 samples/s
|
| 251 |
+
Step 6960/50462 | Loss: 2.8152 | LR: 4.81e-04 | 5608.0ms/step | 10.7 samples/s
|
| 252 |
+
Step 6970/50462 | Loss: 2.8187 | LR: 4.81e-04 | 5608.8ms/step | 10.7 samples/s
|
| 253 |
+
Step 6980/50462 | Loss: 2.8442 | LR: 4.81e-04 | 5610.3ms/step | 10.7 samples/s
|
| 254 |
+
Step 6990/50462 | Loss: 2.8114 | LR: 4.81e-04 | 5608.5ms/step | 10.7 samples/s
|
| 255 |
+
Step 7000/50462 | Loss: 2.8674 | LR: 4.81e-04 | 5606.9ms/step | 10.7 samples/s
|
| 256 |
+
Validation at step 7000: Loss = 2.8742, Perplexity = 17.71
|
| 257 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-7000
|
| 258 |
+
Step 7010/50462 | Loss: 2.8103 | LR: 4.81e-04 | 8484.8ms/step | 7.1 samples/s
|
| 259 |
+
Step 7020/50462 | Loss: 2.7829 | LR: 4.81e-04 | 5606.5ms/step | 10.7 samples/s
|
| 260 |
+
Step 7030/50462 | Loss: 2.7681 | LR: 4.81e-04 | 5606.5ms/step | 10.7 samples/s
|
| 261 |
+
Step 7040/50462 | Loss: 2.8460 | LR: 4.81e-04 | 5606.5ms/step | 10.7 samples/s
|
| 262 |
+
Step 7050/50462 | Loss: 2.7953 | LR: 4.80e-04 | 5606.3ms/step | 10.7 samples/s
|
| 263 |
+
Step 7060/50462 | Loss: 2.8414 | LR: 4.80e-04 | 5604.6ms/step | 10.7 samples/s
|
| 264 |
+
Step 7070/50462 | Loss: 2.8295 | LR: 4.80e-04 | 5607.4ms/step | 10.7 samples/s
|
| 265 |
+
Step 7080/50462 | Loss: 2.8229 | LR: 4.80e-04 | 5609.0ms/step | 10.7 samples/s
|
| 266 |
+
Step 7090/50462 | Loss: 2.8258 | LR: 4.80e-04 | 5607.6ms/step | 10.7 samples/s
|
| 267 |
+
Step 7100/50462 | Loss: 2.8283 | LR: 4.80e-04 | 5607.3ms/step | 10.7 samples/s
|
| 268 |
+
Step 7110/50462 | Loss: 2.8068 | LR: 4.80e-04 | 5605.6ms/step | 10.7 samples/s
|
| 269 |
+
Step 7120/50462 | Loss: 2.8171 | LR: 4.80e-04 | 5608.4ms/step | 10.7 samples/s
|
| 270 |
+
Step 7130/50462 | Loss: 2.8284 | LR: 4.80e-04 | 5609.2ms/step | 10.7 samples/s
|
| 271 |
+
Step 7140/50462 | Loss: 2.7921 | LR: 4.80e-04 | 5607.4ms/step | 10.7 samples/s
|
| 272 |
+
Step 7150/50462 | Loss: 2.8284 | LR: 4.80e-04 | 5605.7ms/step | 10.7 samples/s
|
| 273 |
+
Step 7160/50462 | Loss: 2.7888 | LR: 4.80e-04 | 5608.1ms/step | 10.7 samples/s
|
| 274 |
+
Step 7170/50462 | Loss: 2.8640 | LR: 4.80e-04 | 5606.7ms/step | 10.7 samples/s
|
| 275 |
+
Step 7180/50462 | Loss: 2.7964 | LR: 4.80e-04 | 5604.3ms/step | 10.7 samples/s
|
| 276 |
+
Step 7190/50462 | Loss: 2.8355 | LR: 4.80e-04 | 5606.2ms/step | 10.7 samples/s
|
| 277 |
+
Step 7200/50462 | Loss: 2.8251 | LR: 4.80e-04 | 5605.8ms/step | 10.7 samples/s
|
| 278 |
+
Step 7210/50462 | Loss: 2.8124 | LR: 4.79e-04 | 5606.9ms/step | 10.7 samples/s
|
| 279 |
+
Step 7220/50462 | Loss: 2.8326 | LR: 4.79e-04 | 5607.1ms/step | 10.7 samples/s
|
| 280 |
+
Step 7230/50462 | Loss: 2.8360 | LR: 4.79e-04 | 5607.2ms/step | 10.7 samples/s
|
| 281 |
+
Step 7240/50462 | Loss: 2.7862 | LR: 4.79e-04 | 5604.5ms/step | 10.7 samples/s
|
| 282 |
+
Step 7250/50462 | Loss: 2.8325 | LR: 4.79e-04 | 5607.8ms/step | 10.7 samples/s
|
| 283 |
+
Validation at step 7250: Loss = 2.8687, Perplexity = 17.61
|
| 284 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-7250
|
| 285 |
+
Step 7260/50462 | Loss: 2.8578 | LR: 4.79e-04 | 8405.5ms/step | 7.1 samples/s
|
| 286 |
+
Step 7270/50462 | Loss: 2.8250 | LR: 4.79e-04 | 5602.0ms/step | 10.7 samples/s
|
| 287 |
+
Step 7280/50462 | Loss: 2.8292 | LR: 4.79e-04 | 5604.6ms/step | 10.7 samples/s
|
| 288 |
+
Step 7290/50462 | Loss: 2.8172 | LR: 4.79e-04 | 5605.3ms/step | 10.7 samples/s
|
| 289 |
+
Step 7300/50462 | Loss: 2.7778 | LR: 4.79e-04 | 5606.0ms/step | 10.7 samples/s
|
| 290 |
+
Step 7310/50462 | Loss: 2.8234 | LR: 4.79e-04 | 5607.1ms/step | 10.7 samples/s
|
| 291 |
+
Step 7320/50462 | Loss: 2.8029 | LR: 4.79e-04 | 5609.3ms/step | 10.7 samples/s
|
| 292 |
+
Step 7330/50462 | Loss: 2.8036 | LR: 4.79e-04 | 5607.5ms/step | 10.7 samples/s
|
| 293 |
+
Step 7340/50462 | Loss: 2.8200 | LR: 4.79e-04 | 5605.8ms/step | 10.7 samples/s
|
| 294 |
+
Step 7350/50462 | Loss: 2.7972 | LR: 4.79e-04 | 5606.6ms/step | 10.7 samples/s
|
| 295 |
+
Step 7360/50462 | Loss: 2.7684 | LR: 4.79e-04 | 5607.0ms/step | 10.7 samples/s
|
| 296 |
+
Step 7370/50462 | Loss: 2.8106 | LR: 4.78e-04 | 5606.2ms/step | 10.7 samples/s
|
| 297 |
+
Step 7380/50462 | Loss: 2.7749 | LR: 4.78e-04 | 5607.3ms/step | 10.7 samples/s
|
| 298 |
+
Step 7390/50462 | Loss: 2.8407 | LR: 4.78e-04 | 5607.1ms/step | 10.7 samples/s
|
| 299 |
+
Step 7400/50462 | Loss: 2.8236 | LR: 4.78e-04 | 5607.3ms/step | 10.7 samples/s
|
| 300 |
+
Step 7410/50462 | Loss: 2.8280 | LR: 4.78e-04 | 5604.0ms/step | 10.7 samples/s
|
| 301 |
+
Step 7420/50462 | Loss: 2.8239 | LR: 4.78e-04 | 5605.8ms/step | 10.7 samples/s
|
| 302 |
+
Step 7430/50462 | Loss: 2.7875 | LR: 4.78e-04 | 5610.1ms/step | 10.7 samples/s
|
| 303 |
+
Step 7440/50462 | Loss: 2.8026 | LR: 4.78e-04 | 5607.4ms/step | 10.7 samples/s
|
| 304 |
+
Step 7450/50462 | Loss: 2.8018 | LR: 4.78e-04 | 5605.3ms/step | 10.7 samples/s
|
| 305 |
+
Step 7460/50462 | Loss: 2.8252 | LR: 4.78e-04 | 5608.0ms/step | 10.7 samples/s
|
| 306 |
+
Step 7470/50462 | Loss: 2.7949 | LR: 4.78e-04 | 5605.8ms/step | 10.7 samples/s
|
| 307 |
+
Step 7480/50462 | Loss: 2.8329 | LR: 4.78e-04 | 5607.9ms/step | 10.7 samples/s
|
| 308 |
+
Step 7490/50462 | Loss: 2.7891 | LR: 4.78e-04 | 5606.8ms/step | 10.7 samples/s
|
| 309 |
+
Step 7500/50462 | Loss: 2.7779 | LR: 4.78e-04 | 5603.6ms/step | 10.7 samples/s
|
| 310 |
+
Validation at step 7500: Loss = 2.8601, Perplexity = 17.46
|
| 311 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-7500
|
| 312 |
+
Step 7510/50462 | Loss: 2.8206 | LR: 4.78e-04 | 8499.2ms/step | 7.1 samples/s
|
| 313 |
+
Step 7520/50462 | Loss: 2.8255 | LR: 4.77e-04 | 5596.4ms/step | 10.7 samples/s
|
| 314 |
+
Step 7530/50462 | Loss: 2.8260 | LR: 4.77e-04 | 5602.2ms/step | 10.7 samples/s
|
| 315 |
+
Step 7540/50462 | Loss: 2.7634 | LR: 4.77e-04 | 5605.4ms/step | 10.7 samples/s
|
| 316 |
+
Step 7550/50462 | Loss: 2.7544 | LR: 4.77e-04 | 5605.7ms/step | 10.7 samples/s
|
| 317 |
+
Step 7560/50462 | Loss: 2.8360 | LR: 4.77e-04 | 5604.6ms/step | 10.7 samples/s
|
| 318 |
+
Step 7570/50462 | Loss: 2.7680 | LR: 4.77e-04 | 5613.9ms/step | 10.7 samples/s
|
| 319 |
+
Step 7580/50462 | Loss: 2.7691 | LR: 4.77e-04 | 5609.5ms/step | 10.7 samples/s
|
| 320 |
+
Step 7590/50462 | Loss: 2.7912 | LR: 4.77e-04 | 5609.0ms/step | 10.7 samples/s
|
| 321 |
+
Step 7600/50462 | Loss: 2.8269 | LR: 4.77e-04 | 5610.9ms/step | 10.7 samples/s
|
| 322 |
+
Step 7610/50462 | Loss: 2.8439 | LR: 4.77e-04 | 5613.6ms/step | 10.7 samples/s
|
| 323 |
+
Step 7620/50462 | Loss: 2.8021 | LR: 4.77e-04 | 5615.1ms/step | 10.7 samples/s
|
| 324 |
+
Step 7630/50462 | Loss: 2.8128 | LR: 4.77e-04 | 5608.9ms/step | 10.7 samples/s
|
| 325 |
+
Step 7640/50462 | Loss: 2.8043 | LR: 4.77e-04 | 5616.4ms/step | 10.7 samples/s
|
| 326 |
+
Step 7650/50462 | Loss: 2.8083 | LR: 4.77e-04 | 5630.2ms/step | 10.7 samples/s
|
| 327 |
+
Step 7660/50462 | Loss: 2.7890 | LR: 4.77e-04 | 5617.2ms/step | 10.7 samples/s
|
| 328 |
+
Step 7670/50462 | Loss: 2.7772 | LR: 4.76e-04 | 5607.5ms/step | 10.7 samples/s
|
| 329 |
+
Step 7680/50462 | Loss: 2.7793 | LR: 4.76e-04 | 5610.5ms/step | 10.7 samples/s
|
| 330 |
+
Step 7690/50462 | Loss: 2.8036 | LR: 4.76e-04 | 5620.3ms/step | 10.7 samples/s
|
| 331 |
+
Step 7700/50462 | Loss: 2.7944 | LR: 4.76e-04 | 5610.1ms/step | 10.7 samples/s
|
| 332 |
+
Step 7710/50462 | Loss: 2.7520 | LR: 4.76e-04 | 5610.0ms/step | 10.7 samples/s
|
| 333 |
+
Step 7720/50462 | Loss: 2.8115 | LR: 4.76e-04 | 5608.5ms/step | 10.7 samples/s
|
| 334 |
+
Step 7730/50462 | Loss: 2.8230 | LR: 4.76e-04 | 5609.3ms/step | 10.7 samples/s
|
| 335 |
+
Step 7740/50462 | Loss: 2.8089 | LR: 4.76e-04 | 5608.4ms/step | 10.7 samples/s
|
| 336 |
+
Step 7750/50462 | Loss: 2.7953 | LR: 4.76e-04 | 5607.9ms/step | 10.7 samples/s
|
| 337 |
+
Validation at step 7750: Loss = 2.8521, Perplexity = 17.32
|
| 338 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-7750
|
| 339 |
+
Step 7760/50462 | Loss: 2.8306 | LR: 4.76e-04 | 8331.5ms/step | 7.2 samples/s
|
| 340 |
+
Step 7770/50462 | Loss: 2.8193 | LR: 4.76e-04 | 5599.0ms/step | 10.7 samples/s
|
| 341 |
+
Step 7780/50462 | Loss: 2.7795 | LR: 4.76e-04 | 5605.1ms/step | 10.7 samples/s
|
| 342 |
+
Step 7790/50462 | Loss: 2.7692 | LR: 4.76e-04 | 5607.7ms/step | 10.7 samples/s
|
| 343 |
+
Step 7800/50462 | Loss: 2.7805 | LR: 4.76e-04 | 5607.5ms/step | 10.7 samples/s
|
| 344 |
+
Step 7810/50462 | Loss: 2.8043 | LR: 4.76e-04 | 5605.2ms/step | 10.7 samples/s
|
| 345 |
+
Step 7820/50462 | Loss: 2.7824 | LR: 4.75e-04 | 5610.0ms/step | 10.7 samples/s
|
| 346 |
+
Step 7830/50462 | Loss: 2.7889 | LR: 4.75e-04 | 5607.2ms/step | 10.7 samples/s
|
| 347 |
+
Step 7840/50462 | Loss: 2.8430 | LR: 4.75e-04 | 5612.5ms/step | 10.7 samples/s
|
| 348 |
+
Step 7850/50462 | Loss: 2.7800 | LR: 4.75e-04 | 5612.6ms/step | 10.7 samples/s
|
| 349 |
+
Step 7860/50462 | Loss: 2.8191 | LR: 4.75e-04 | 5609.4ms/step | 10.7 samples/s
|
| 350 |
+
Step 7870/50462 | Loss: 2.8351 | LR: 4.75e-04 | 5608.0ms/step | 10.7 samples/s
|
| 351 |
+
Step 7880/50462 | Loss: 2.7958 | LR: 4.75e-04 | 5607.2ms/step | 10.7 samples/s
|
| 352 |
+
Step 7890/50462 | Loss: 2.8373 | LR: 4.75e-04 | 5609.8ms/step | 10.7 samples/s
|
| 353 |
+
Step 7900/50462 | Loss: 2.8238 | LR: 4.75e-04 | 5614.4ms/step | 10.7 samples/s
|
| 354 |
+
Step 7910/50462 | Loss: 2.7748 | LR: 4.75e-04 | 5605.2ms/step | 10.7 samples/s
|
| 355 |
+
Step 7920/50462 | Loss: 2.7886 | LR: 4.75e-04 | 5609.6ms/step | 10.7 samples/s
|
| 356 |
+
Step 7930/50462 | Loss: 2.7858 | LR: 4.75e-04 | 5609.4ms/step | 10.7 samples/s
|
| 357 |
+
Step 7940/50462 | Loss: 2.8201 | LR: 4.75e-04 | 5608.5ms/step | 10.7 samples/s
|
| 358 |
+
Step 7950/50462 | Loss: 2.8178 | LR: 4.75e-04 | 5617.4ms/step | 10.7 samples/s
|
| 359 |
+
Step 7960/50462 | Loss: 2.8199 | LR: 4.74e-04 | 5618.3ms/step | 10.7 samples/s
|
| 360 |
+
Step 7970/50462 | Loss: 2.8284 | LR: 4.74e-04 | 5609.5ms/step | 10.7 samples/s
|
| 361 |
+
Step 7980/50462 | Loss: 2.8000 | LR: 4.74e-04 | 5613.1ms/step | 10.7 samples/s
|
| 362 |
+
Step 7990/50462 | Loss: 2.7719 | LR: 4.74e-04 | 5619.0ms/step | 10.7 samples/s
|
| 363 |
+
Step 8000/50462 | Loss: 2.8038 | LR: 4.74e-04 | 5615.3ms/step | 10.7 samples/s
|
| 364 |
+
Validation at step 8000: Loss = 2.8459, Perplexity = 17.22
|
| 365 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-8000
|
| 366 |
+
Step 8010/50462 | Loss: 2.7816 | LR: 4.74e-04 | 8393.4ms/step | 7.1 samples/s
|
| 367 |
+
Step 8020/50462 | Loss: 2.7759 | LR: 4.74e-04 | 5607.4ms/step | 10.7 samples/s
|
| 368 |
+
Step 8030/50462 | Loss: 2.7861 | LR: 4.74e-04 | 5611.8ms/step | 10.7 samples/s
|
| 369 |
+
Step 8040/50462 | Loss: 2.7721 | LR: 4.74e-04 | 5614.7ms/step | 10.7 samples/s
|
| 370 |
+
Step 8050/50462 | Loss: 2.7873 | LR: 4.74e-04 | 5609.6ms/step | 10.7 samples/s
|
| 371 |
+
Step 8060/50462 | Loss: 2.7801 | LR: 4.74e-04 | 5615.1ms/step | 10.7 samples/s
|
| 372 |
+
Step 8070/50462 | Loss: 2.8520 | LR: 4.74e-04 | 5614.2ms/step | 10.7 samples/s
|
| 373 |
+
Step 8080/50462 | Loss: 2.7732 | LR: 4.74e-04 | 5617.2ms/step | 10.7 samples/s
|
| 374 |
+
Step 8090/50462 | Loss: 2.7643 | LR: 4.74e-04 | 5613.3ms/step | 10.7 samples/s
|
| 375 |
+
Step 8100/50462 | Loss: 2.8193 | LR: 4.74e-04 | 5615.8ms/step | 10.7 samples/s
|
| 376 |
+
Step 8110/50462 | Loss: 2.7653 | LR: 4.73e-04 | 5613.5ms/step | 10.7 samples/s
|
| 377 |
+
Step 8120/50462 | Loss: 2.7890 | LR: 4.73e-04 | 5615.0ms/step | 10.7 samples/s
|
| 378 |
+
Step 8130/50462 | Loss: 2.7724 | LR: 4.73e-04 | 5613.0ms/step | 10.7 samples/s
|
| 379 |
+
Step 8140/50462 | Loss: 2.7874 | LR: 4.73e-04 | 5612.8ms/step | 10.7 samples/s
|
| 380 |
+
Step 8150/50462 | Loss: 2.7921 | LR: 4.73e-04 | 5609.4ms/step | 10.7 samples/s
|
| 381 |
+
Step 8160/50462 | Loss: 2.8130 | LR: 4.73e-04 | 5609.0ms/step | 10.7 samples/s
|
| 382 |
+
Step 8170/50462 | Loss: 2.8034 | LR: 4.73e-04 | 5610.4ms/step | 10.7 samples/s
|
| 383 |
+
Step 8180/50462 | Loss: 2.8194 | LR: 4.73e-04 | 5611.1ms/step | 10.7 samples/s
|
| 384 |
+
Step 8190/50462 | Loss: 2.7971 | LR: 4.73e-04 | 5608.1ms/step | 10.7 samples/s
|
| 385 |
+
Step 8200/50462 | Loss: 2.8226 | LR: 4.73e-04 | 5608.1ms/step | 10.7 samples/s
|
| 386 |
+
Step 8210/50462 | Loss: 2.7842 | LR: 4.73e-04 | 5608.2ms/step | 10.7 samples/s
|
| 387 |
+
Step 8220/50462 | Loss: 2.7515 | LR: 4.73e-04 | 5608.8ms/step | 10.7 samples/s
|
| 388 |
+
Step 8230/50462 | Loss: 2.7711 | LR: 4.73e-04 | 5606.8ms/step | 10.7 samples/s
|
| 389 |
+
Step 8240/50462 | Loss: 2.8089 | LR: 4.73e-04 | 5608.8ms/step | 10.7 samples/s
|
| 390 |
+
Step 8250/50462 | Loss: 2.7797 | LR: 4.72e-04 | 5609.6ms/step | 10.7 samples/s
|
| 391 |
+
Validation at step 8250: Loss = 2.8379, Perplexity = 17.08
|
| 392 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-8250
|
| 393 |
+
Step 8260/50462 | Loss: 2.8130 | LR: 4.72e-04 | 8335.4ms/step | 7.2 samples/s
|
| 394 |
+
Step 8270/50462 | Loss: 2.7933 | LR: 4.72e-04 | 5602.1ms/step | 10.7 samples/s
|
| 395 |
+
Step 8280/50462 | Loss: 2.7686 | LR: 4.72e-04 | 5606.8ms/step | 10.7 samples/s
|
| 396 |
+
Step 8290/50462 | Loss: 2.7837 | LR: 4.72e-04 | 5606.9ms/step | 10.7 samples/s
|
| 397 |
+
Step 8300/50462 | Loss: 2.7548 | LR: 4.72e-04 | 5606.6ms/step | 10.7 samples/s
|
| 398 |
+
Step 8310/50462 | Loss: 2.8076 | LR: 4.72e-04 | 5608.0ms/step | 10.7 samples/s
|
| 399 |
+
Step 8320/50462 | Loss: 2.7677 | LR: 4.72e-04 | 5608.8ms/step | 10.7 samples/s
|
| 400 |
+
Step 8330/50462 | Loss: 2.7699 | LR: 4.72e-04 | 5609.2ms/step | 10.7 samples/s
|
| 401 |
+
Step 8340/50462 | Loss: 2.8124 | LR: 4.72e-04 | 5607.6ms/step | 10.7 samples/s
|
| 402 |
+
Step 8350/50462 | Loss: 2.7834 | LR: 4.72e-04 | 5606.7ms/step | 10.7 samples/s
|
| 403 |
+
Step 8360/50462 | Loss: 2.8255 | LR: 4.72e-04 | 5607.4ms/step | 10.7 samples/s
|
| 404 |
+
Step 8370/50462 | Loss: 2.7844 | LR: 4.72e-04 | 5605.1ms/step | 10.7 samples/s
|
| 405 |
+
Step 8380/50462 | Loss: 2.7884 | LR: 4.71e-04 | 5607.2ms/step | 10.7 samples/s
|
| 406 |
+
Step 8390/50462 | Loss: 2.7772 | LR: 4.71e-04 | 5608.0ms/step | 10.7 samples/s
|
| 407 |
+
Step 8400/50462 | Loss: 2.7743 | LR: 4.71e-04 | 5606.6ms/step | 10.7 samples/s
|
| 408 |
+
Step 8410/50462 | Loss: 2.7925 | LR: 4.71e-04 | 5606.6ms/step | 10.7 samples/s
|
| 409 |
+
Step 8420/50462 | Loss: 2.8248 | LR: 4.71e-04 | 5607.0ms/step | 10.7 samples/s
|
| 410 |
+
Step 8430/50462 | Loss: 2.7775 | LR: 4.71e-04 | 5606.2ms/step | 10.7 samples/s
|
| 411 |
+
Step 8440/50462 | Loss: 2.7937 | LR: 4.71e-04 | 5607.1ms/step | 10.7 samples/s
|
| 412 |
+
Step 8450/50462 | Loss: 2.7623 | LR: 4.71e-04 | 5606.1ms/step | 10.7 samples/s
|
| 413 |
+
Step 8460/50462 | Loss: 2.7462 | LR: 4.71e-04 | 5608.2ms/step | 10.7 samples/s
|
| 414 |
+
Step 8470/50462 | Loss: 2.8054 | LR: 4.71e-04 | 5614.4ms/step | 10.7 samples/s
|
| 415 |
+
Step 8480/50462 | Loss: 2.7772 | LR: 4.71e-04 | 5615.1ms/step | 10.7 samples/s
|
| 416 |
+
Step 8490/50462 | Loss: 2.7954 | LR: 4.71e-04 | 5609.9ms/step | 10.7 samples/s
|
| 417 |
+
Step 8500/50462 | Loss: 2.7854 | LR: 4.71e-04 | 5612.6ms/step | 10.7 samples/s
|
| 418 |
+
Validation at step 8500: Loss = 2.8318, Perplexity = 16.98
|
| 419 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-8500
|
| 420 |
+
Step 8510/50462 | Loss: 2.7415 | LR: 4.71e-04 | 8383.3ms/step | 7.2 samples/s
|
| 421 |
+
Step 8520/50462 | Loss: 2.7724 | LR: 4.70e-04 | 5607.3ms/step | 10.7 samples/s
|
| 422 |
+
Step 8530/50462 | Loss: 2.7970 | LR: 4.70e-04 | 5611.9ms/step | 10.7 samples/s
|
| 423 |
+
Step 8540/50462 | Loss: 2.7206 | LR: 4.70e-04 | 5610.2ms/step | 10.7 samples/s
|
| 424 |
+
Step 8550/50462 | Loss: 2.7360 | LR: 4.70e-04 | 5611.6ms/step | 10.7 samples/s
|
| 425 |
+
Step 8560/50462 | Loss: 2.7586 | LR: 4.70e-04 | 5613.6ms/step | 10.7 samples/s
|
| 426 |
+
Step 8570/50462 | Loss: 2.7953 | LR: 4.70e-04 | 5614.1ms/step | 10.7 samples/s
|
| 427 |
+
Step 8580/50462 | Loss: 2.7844 | LR: 4.70e-04 | 5613.4ms/step | 10.7 samples/s
|
| 428 |
+
Step 8590/50462 | Loss: 2.7743 | LR: 4.70e-04 | 5611.3ms/step | 10.7 samples/s
|
| 429 |
+
Step 8600/50462 | Loss: 2.7710 | LR: 4.70e-04 | 5614.2ms/step | 10.7 samples/s
|
| 430 |
+
Step 8610/50462 | Loss: 2.7338 | LR: 4.70e-04 | 5614.5ms/step | 10.7 samples/s
|
| 431 |
+
Step 8620/50462 | Loss: 2.7743 | LR: 4.70e-04 | 5616.0ms/step | 10.7 samples/s
|
| 432 |
+
Step 8630/50462 | Loss: 2.7633 | LR: 4.70e-04 | 5616.2ms/step | 10.7 samples/s
|
| 433 |
+
Step 8640/50462 | Loss: 2.7525 | LR: 4.70e-04 | 5614.0ms/step | 10.7 samples/s
|
| 434 |
+
Step 8650/50462 | Loss: 2.7807 | LR: 4.69e-04 | 5615.4ms/step | 10.7 samples/s
|
| 435 |
+
Step 8660/50462 | Loss: 2.7697 | LR: 4.69e-04 | 5617.6ms/step | 10.7 samples/s
|
| 436 |
+
Step 8670/50462 | Loss: 2.7909 | LR: 4.69e-04 | 5615.0ms/step | 10.7 samples/s
|
| 437 |
+
Step 8680/50462 | Loss: 2.7759 | LR: 4.69e-04 | 5616.2ms/step | 10.7 samples/s
|
| 438 |
+
Step 8690/50462 | Loss: 2.7675 | LR: 4.69e-04 | 5619.6ms/step | 10.7 samples/s
|
| 439 |
+
Step 8700/50462 | Loss: 2.8104 | LR: 4.69e-04 | 5618.5ms/step | 10.7 samples/s
|
| 440 |
+
Step 8710/50462 | Loss: 2.7951 | LR: 4.69e-04 | 5613.6ms/step | 10.7 samples/s
|
| 441 |
+
Step 8720/50462 | Loss: 2.7897 | LR: 4.69e-04 | 5616.5ms/step | 10.7 samples/s
|
| 442 |
+
Step 8730/50462 | Loss: 2.7354 | LR: 4.69e-04 | 5615.2ms/step | 10.7 samples/s
|
| 443 |
+
Step 8740/50462 | Loss: 2.7694 | LR: 4.69e-04 | 5615.6ms/step | 10.7 samples/s
|
| 444 |
+
Step 8750/50462 | Loss: 2.8096 | LR: 4.69e-04 | 5617.2ms/step | 10.7 samples/s
|
| 445 |
+
Validation at step 8750: Loss = 2.8289, Perplexity = 16.93
|
| 446 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-8750
|
| 447 |
+
Step 8760/50462 | Loss: 2.7627 | LR: 4.69e-04 | 8396.2ms/step | 7.1 samples/s
|
| 448 |
+
Step 8770/50462 | Loss: 2.7783 | LR: 4.69e-04 | 5602.9ms/step | 10.7 samples/s
|
| 449 |
+
Step 8780/50462 | Loss: 2.7793 | LR: 4.68e-04 | 5604.7ms/step | 10.7 samples/s
|
| 450 |
+
Step 8790/50462 | Loss: 2.7471 | LR: 4.68e-04 | 5604.8ms/step | 10.7 samples/s
|
| 451 |
+
Step 8800/50462 | Loss: 2.7721 | LR: 4.68e-04 | 5606.9ms/step | 10.7 samples/s
|
| 452 |
+
Step 8810/50462 | Loss: 2.8129 | LR: 4.68e-04 | 5606.7ms/step | 10.7 samples/s
|
| 453 |
+
Step 8820/50462 | Loss: 2.7578 | LR: 4.68e-04 | 5605.5ms/step | 10.7 samples/s
|
| 454 |
+
Step 8830/50462 | Loss: 2.8022 | LR: 4.68e-04 | 5604.8ms/step | 10.7 samples/s
|
| 455 |
+
Step 8840/50462 | Loss: 2.8083 | LR: 4.68e-04 | 5609.0ms/step | 10.7 samples/s
|
| 456 |
+
Step 8850/50462 | Loss: 2.7074 | LR: 4.68e-04 | 5608.0ms/step | 10.7 samples/s
|
| 457 |
+
Step 8860/50462 | Loss: 2.7886 | LR: 4.68e-04 | 5607.5ms/step | 10.7 samples/s
|
| 458 |
+
Step 8870/50462 | Loss: 2.7390 | LR: 4.68e-04 | 5606.7ms/step | 10.7 samples/s
|
| 459 |
+
Step 8880/50462 | Loss: 2.7719 | LR: 4.68e-04 | 5613.8ms/step | 10.7 samples/s
|
| 460 |
+
Step 8890/50462 | Loss: 2.7811 | LR: 4.68e-04 | 5611.0ms/step | 10.7 samples/s
|
| 461 |
+
Step 8900/50462 | Loss: 2.7505 | LR: 4.68e-04 | 5606.0ms/step | 10.7 samples/s
|
| 462 |
+
Step 8910/50462 | Loss: 2.7700 | LR: 4.67e-04 | 5607.6ms/step | 10.7 samples/s
|
| 463 |
+
Step 8920/50462 | Loss: 2.7748 | LR: 4.67e-04 | 5607.2ms/step | 10.7 samples/s
|
| 464 |
+
Step 8930/50462 | Loss: 2.7463 | LR: 4.67e-04 | 5607.8ms/step | 10.7 samples/s
|
| 465 |
+
Step 8940/50462 | Loss: 2.7942 | LR: 4.67e-04 | 5605.7ms/step | 10.7 samples/s
|
| 466 |
+
Step 8950/50462 | Loss: 2.7552 | LR: 4.67e-04 | 5608.3ms/step | 10.7 samples/s
|
| 467 |
+
Step 8960/50462 | Loss: 2.7496 | LR: 4.67e-04 | 5605.8ms/step | 10.7 samples/s
|
| 468 |
+
Step 8970/50462 | Loss: 2.7428 | LR: 4.67e-04 | 5606.2ms/step | 10.7 samples/s
|
| 469 |
+
Step 8980/50462 | Loss: 2.8126 | LR: 4.67e-04 | 5608.2ms/step | 10.7 samples/s
|
| 470 |
+
Step 8990/50462 | Loss: 2.7853 | LR: 4.67e-04 | 5605.9ms/step | 10.7 samples/s
|
| 471 |
+
Step 9000/50462 | Loss: 2.7523 | LR: 4.67e-04 | 5607.3ms/step | 10.7 samples/s
|
| 472 |
+
Validation at step 9000: Loss = 2.8188, Perplexity = 16.76
|
| 473 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-9000
|
| 474 |
+
Step 9010/50462 | Loss: 2.7694 | LR: 4.67e-04 | 8490.8ms/step | 7.1 samples/s
|
| 475 |
+
Step 9020/50462 | Loss: 2.7579 | LR: 4.67e-04 | 5609.8ms/step | 10.7 samples/s
|
| 476 |
+
Step 9030/50462 | Loss: 2.7669 | LR: 4.67e-04 | 5610.7ms/step | 10.7 samples/s
|
| 477 |
+
Step 9040/50462 | Loss: 2.7845 | LR: 4.66e-04 | 5613.3ms/step | 10.7 samples/s
|
| 478 |
+
Step 9050/50462 | Loss: 2.7730 | LR: 4.66e-04 | 5612.6ms/step | 10.7 samples/s
|
| 479 |
+
Step 9060/50462 | Loss: 2.7563 | LR: 4.66e-04 | 5612.0ms/step | 10.7 samples/s
|
| 480 |
+
Step 9070/50462 | Loss: 2.7574 | LR: 4.66e-04 | 5611.3ms/step | 10.7 samples/s
|
| 481 |
+
Step 9080/50462 | Loss: 2.7771 | LR: 4.66e-04 | 5609.7ms/step | 10.7 samples/s
|
| 482 |
+
Step 9090/50462 | Loss: 2.7305 | LR: 4.66e-04 | 5611.8ms/step | 10.7 samples/s
|
| 483 |
+
Step 9100/50462 | Loss: 2.7742 | LR: 4.66e-04 | 5612.3ms/step | 10.7 samples/s
|
| 484 |
+
Step 9110/50462 | Loss: 2.7700 | LR: 4.66e-04 | 5610.0ms/step | 10.7 samples/s
|
| 485 |
+
Step 9120/50462 | Loss: 2.7834 | LR: 4.66e-04 | 5610.3ms/step | 10.7 samples/s
|
| 486 |
+
Step 9130/50462 | Loss: 2.7510 | LR: 4.66e-04 | 5612.1ms/step | 10.7 samples/s
|
| 487 |
+
Step 9140/50462 | Loss: 2.7705 | LR: 4.66e-04 | 5610.0ms/step | 10.7 samples/s
|
| 488 |
+
Step 9150/50462 | Loss: 2.8000 | LR: 4.66e-04 | 5611.5ms/step | 10.7 samples/s
|
| 489 |
+
Step 9160/50462 | Loss: 2.7886 | LR: 4.66e-04 | 5613.0ms/step | 10.7 samples/s
|
| 490 |
+
Step 9170/50462 | Loss: 2.7411 | LR: 4.65e-04 | 5615.7ms/step | 10.7 samples/s
|
| 491 |
+
Step 9180/50462 | Loss: 2.7651 | LR: 4.65e-04 | 5609.5ms/step | 10.7 samples/s
|
| 492 |
+
Step 9190/50462 | Loss: 2.7721 | LR: 4.65e-04 | 5609.0ms/step | 10.7 samples/s
|
| 493 |
+
Step 9200/50462 | Loss: 2.7539 | LR: 4.65e-04 | 5610.0ms/step | 10.7 samples/s
|
| 494 |
+
Step 9210/50462 | Loss: 2.7694 | LR: 4.65e-04 | 5609.8ms/step | 10.7 samples/s
|
| 495 |
+
Step 9220/50462 | Loss: 2.7531 | LR: 4.65e-04 | 5613.4ms/step | 10.7 samples/s
|
| 496 |
+
Step 9230/50462 | Loss: 2.7699 | LR: 4.65e-04 | 5612.8ms/step | 10.7 samples/s
|
| 497 |
+
Step 9240/50462 | Loss: 2.7466 | LR: 4.65e-04 | 5610.0ms/step | 10.7 samples/s
|
| 498 |
+
Step 9250/50462 | Loss: 2.7779 | LR: 4.65e-04 | 5611.1ms/step | 10.7 samples/s
|
| 499 |
+
Validation at step 9250: Loss = 2.8126, Perplexity = 16.65
|
| 500 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-9250
|
| 501 |
+
Step 9260/50462 | Loss: 2.7286 | LR: 4.65e-04 | 8343.4ms/step | 7.2 samples/s
|
| 502 |
+
Step 9270/50462 | Loss: 2.7800 | LR: 4.65e-04 | 5605.4ms/step | 10.7 samples/s
|
| 503 |
+
Step 9280/50462 | Loss: 2.7675 | LR: 4.65e-04 | 5608.0ms/step | 10.7 samples/s
|
| 504 |
+
Step 9290/50462 | Loss: 2.7647 | LR: 4.64e-04 | 5607.5ms/step | 10.7 samples/s
|
| 505 |
+
Step 9300/50462 | Loss: 2.7586 | LR: 4.64e-04 | 5608.3ms/step | 10.7 samples/s
|
| 506 |
+
Step 9310/50462 | Loss: 2.7486 | LR: 4.64e-04 | 5609.4ms/step | 10.7 samples/s
|
| 507 |
+
Step 9320/50462 | Loss: 2.7386 | LR: 4.64e-04 | 5607.4ms/step | 10.7 samples/s
|
| 508 |
+
Step 9330/50462 | Loss: 2.7595 | LR: 4.64e-04 | 5607.2ms/step | 10.7 samples/s
|
| 509 |
+
Step 9340/50462 | Loss: 2.7449 | LR: 4.64e-04 | 5606.1ms/step | 10.7 samples/s
|
| 510 |
+
Step 9350/50462 | Loss: 2.7588 | LR: 4.64e-04 | 5608.8ms/step | 10.7 samples/s
|
| 511 |
+
Step 9360/50462 | Loss: 2.7406 | LR: 4.64e-04 | 5610.4ms/step | 10.7 samples/s
|
| 512 |
+
Step 9370/50462 | Loss: 2.7582 | LR: 4.64e-04 | 5610.3ms/step | 10.7 samples/s
|
| 513 |
+
Step 9380/50462 | Loss: 2.7559 | LR: 4.64e-04 | 5607.9ms/step | 10.7 samples/s
|
| 514 |
+
Step 9390/50462 | Loss: 2.7721 | LR: 4.64e-04 | 5608.1ms/step | 10.7 samples/s
|
| 515 |
+
Step 9400/50462 | Loss: 2.7702 | LR: 4.64e-04 | 5610.2ms/step | 10.7 samples/s
|
| 516 |
+
Step 9410/50462 | Loss: 2.7491 | LR: 4.63e-04 | 5608.6ms/step | 10.7 samples/s
|
| 517 |
+
Step 9420/50462 | Loss: 2.7359 | LR: 4.63e-04 | 5605.8ms/step | 10.7 samples/s
|
| 518 |
+
Step 9430/50462 | Loss: 2.7381 | LR: 4.63e-04 | 5607.1ms/step | 10.7 samples/s
|
| 519 |
+
Step 9440/50462 | Loss: 2.7312 | LR: 4.63e-04 | 5607.7ms/step | 10.7 samples/s
|
| 520 |
+
Step 9450/50462 | Loss: 2.7743 | LR: 4.63e-04 | 5608.7ms/step | 10.7 samples/s
|
| 521 |
+
Step 9460/50462 | Loss: 2.7416 | LR: 4.63e-04 | 5607.5ms/step | 10.7 samples/s
|
| 522 |
+
Step 9470/50462 | Loss: 2.7670 | LR: 4.63e-04 | 5606.4ms/step | 10.7 samples/s
|
| 523 |
+
Step 9480/50462 | Loss: 2.7492 | LR: 4.63e-04 | 5606.1ms/step | 10.7 samples/s
|
| 524 |
+
Step 9490/50462 | Loss: 2.7678 | LR: 4.63e-04 | 5604.6ms/step | 10.7 samples/s
|
| 525 |
+
Step 9500/50462 | Loss: 2.7475 | LR: 4.63e-04 | 5606.2ms/step | 10.7 samples/s
|
| 526 |
+
Validation at step 9500: Loss = 2.8092, Perplexity = 16.60
|
| 527 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-9500
|
| 528 |
+
Step 9510/50462 | Loss: 2.7561 | LR: 4.63e-04 | 8325.2ms/step | 7.2 samples/s
|
| 529 |
+
Step 9520/50462 | Loss: 2.7911 | LR: 4.63e-04 | 5604.5ms/step | 10.7 samples/s
|
| 530 |
+
Step 9530/50462 | Loss: 2.7658 | LR: 4.62e-04 | 5604.2ms/step | 10.7 samples/s
|
| 531 |
+
Step 9540/50462 | Loss: 2.7188 | LR: 4.62e-04 | 5604.3ms/step | 10.7 samples/s
|
| 532 |
+
Step 9550/50462 | Loss: 2.7615 | LR: 4.62e-04 | 5606.1ms/step | 10.7 samples/s
|
| 533 |
+
Step 9560/50462 | Loss: 2.7964 | LR: 4.62e-04 | 5605.2ms/step | 10.7 samples/s
|
| 534 |
+
Step 9570/50462 | Loss: 2.7694 | LR: 4.62e-04 | 5605.6ms/step | 10.7 samples/s
|
| 535 |
+
Step 9580/50462 | Loss: 2.7394 | LR: 4.62e-04 | 5606.6ms/step | 10.7 samples/s
|
| 536 |
+
Step 9590/50462 | Loss: 2.7903 | LR: 4.62e-04 | 5606.8ms/step | 10.7 samples/s
|
| 537 |
+
Step 9600/50462 | Loss: 2.7657 | LR: 4.62e-04 | 5604.6ms/step | 10.7 samples/s
|
| 538 |
+
Step 9610/50462 | Loss: 2.7484 | LR: 4.62e-04 | 5605.9ms/step | 10.7 samples/s
|
| 539 |
+
Step 9620/50462 | Loss: 2.7443 | LR: 4.62e-04 | 5604.5ms/step | 10.7 samples/s
|
| 540 |
+
Step 9630/50462 | Loss: 2.7518 | LR: 4.62e-04 | 5606.0ms/step | 10.7 samples/s
|
| 541 |
+
Step 9640/50462 | Loss: 2.7772 | LR: 4.62e-04 | 5605.0ms/step | 10.7 samples/s
|
| 542 |
+
Step 9650/50462 | Loss: 2.8076 | LR: 4.61e-04 | 5604.1ms/step | 10.7 samples/s
|
| 543 |
+
Step 9660/50462 | Loss: 2.7280 | LR: 4.61e-04 | 5604.8ms/step | 10.7 samples/s
|
| 544 |
+
Step 9670/50462 | Loss: 2.7233 | LR: 4.61e-04 | 5603.4ms/step | 10.7 samples/s
|
| 545 |
+
Step 9680/50462 | Loss: 2.7231 | LR: 4.61e-04 | 5604.8ms/step | 10.7 samples/s
|
| 546 |
+
Step 9690/50462 | Loss: 2.7473 | LR: 4.61e-04 | 5606.5ms/step | 10.7 samples/s
|
| 547 |
+
Step 9700/50462 | Loss: 2.7192 | LR: 4.61e-04 | 5606.6ms/step | 10.7 samples/s
|
| 548 |
+
Step 9710/50462 | Loss: 2.7704 | LR: 4.61e-04 | 5606.9ms/step | 10.7 samples/s
|
| 549 |
+
Step 9720/50462 | Loss: 2.7421 | LR: 4.61e-04 | 5604.9ms/step | 10.7 samples/s
|
| 550 |
+
Step 9730/50462 | Loss: 2.7645 | LR: 4.61e-04 | 5605.1ms/step | 10.7 samples/s
|
| 551 |
+
Step 9740/50462 | Loss: 2.7574 | LR: 4.61e-04 | 5605.9ms/step | 10.7 samples/s
|
| 552 |
+
Step 9750/50462 | Loss: 2.7395 | LR: 4.61e-04 | 5608.0ms/step | 10.7 samples/s
|
| 553 |
+
Validation at step 9750: Loss = 2.8019, Perplexity = 16.48
|
| 554 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-9750
|
| 555 |
+
Step 9760/50462 | Loss: 2.7212 | LR: 4.61e-04 | 8356.6ms/step | 7.2 samples/s
|
| 556 |
+
Step 9770/50462 | Loss: 2.7606 | LR: 4.60e-04 | 5600.8ms/step | 10.7 samples/s
|
| 557 |
+
Step 9780/50462 | Loss: 2.7806 | LR: 4.60e-04 | 5602.7ms/step | 10.7 samples/s
|
| 558 |
+
Step 9790/50462 | Loss: 2.7184 | LR: 4.60e-04 | 5603.6ms/step | 10.7 samples/s
|
| 559 |
+
Step 9800/50462 | Loss: 2.7798 | LR: 4.60e-04 | 5602.7ms/step | 10.7 samples/s
|
| 560 |
+
Step 9810/50462 | Loss: 2.7105 | LR: 4.60e-04 | 5603.7ms/step | 10.7 samples/s
|
| 561 |
+
Step 9820/50462 | Loss: 2.7196 | LR: 4.60e-04 | 5604.5ms/step | 10.7 samples/s
|
| 562 |
+
Step 9830/50462 | Loss: 2.7237 | LR: 4.60e-04 | 5605.7ms/step | 10.7 samples/s
|
| 563 |
+
Step 9840/50462 | Loss: 2.7693 | LR: 4.60e-04 | 5607.4ms/step | 10.7 samples/s
|
| 564 |
+
Step 9850/50462 | Loss: 2.7523 | LR: 4.60e-04 | 5606.8ms/step | 10.7 samples/s
|
| 565 |
+
Step 9860/50462 | Loss: 2.7584 | LR: 4.60e-04 | 5606.1ms/step | 10.7 samples/s
|
| 566 |
+
Step 9870/50462 | Loss: 2.7052 | LR: 4.60e-04 | 5606.1ms/step | 10.7 samples/s
|
| 567 |
+
Step 9880/50462 | Loss: 2.8008 | LR: 4.60e-04 | 5604.4ms/step | 10.7 samples/s
|
| 568 |
+
Step 9890/50462 | Loss: 2.7258 | LR: 4.59e-04 | 5606.0ms/step | 10.7 samples/s
|
| 569 |
+
Step 9900/50462 | Loss: 2.7258 | LR: 4.59e-04 | 5608.7ms/step | 10.7 samples/s
|
| 570 |
+
Step 9910/50462 | Loss: 2.7649 | LR: 4.59e-04 | 5606.1ms/step | 10.7 samples/s
|
| 571 |
+
Step 9920/50462 | Loss: 2.7208 | LR: 4.59e-04 | 5610.0ms/step | 10.7 samples/s
|
| 572 |
+
Step 9930/50462 | Loss: 2.7479 | LR: 4.59e-04 | 5607.0ms/step | 10.7 samples/s
|
| 573 |
+
Step 9940/50462 | Loss: 2.7224 | LR: 4.59e-04 | 5609.0ms/step | 10.7 samples/s
|
| 574 |
+
Step 9950/50462 | Loss: 2.7372 | LR: 4.59e-04 | 5606.2ms/step | 10.7 samples/s
|
| 575 |
+
Step 9960/50462 | Loss: 2.7000 | LR: 4.59e-04 | 5609.8ms/step | 10.7 samples/s
|
| 576 |
+
Step 9970/50462 | Loss: 2.7467 | LR: 4.59e-04 | 5609.7ms/step | 10.7 samples/s
|
| 577 |
+
Step 9980/50462 | Loss: 2.7468 | LR: 4.59e-04 | 5609.3ms/step | 10.7 samples/s
|
| 578 |
+
Step 9990/50462 | Loss: 2.7452 | LR: 4.59e-04 | 5609.3ms/step | 10.7 samples/s
|
| 579 |
+
Step 10000/50462 | Loss: 2.7438 | LR: 4.58e-04 | 5607.2ms/step | 10.7 samples/s
|
| 580 |
+
Validation at step 10000: Loss = 2.7972, Perplexity = 16.40
|
| 581 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-10000
|
| 582 |
+
Step 10010/50462 | Loss: 2.7701 | LR: 4.58e-04 | 8397.1ms/step | 7.1 samples/s
|
| 583 |
+
Step 10020/50462 | Loss: 2.7533 | LR: 4.58e-04 | 5607.5ms/step | 10.7 samples/s
|
| 584 |
+
Step 10030/50462 | Loss: 2.7500 | LR: 4.58e-04 | 5604.4ms/step | 10.7 samples/s
|
| 585 |
+
Step 10040/50462 | Loss: 2.7883 | LR: 4.58e-04 | 5607.2ms/step | 10.7 samples/s
|
| 586 |
+
Step 10050/50462 | Loss: 2.7378 | LR: 4.58e-04 | 5602.0ms/step | 10.7 samples/s
|
| 587 |
+
Step 10060/50462 | Loss: 2.7471 | LR: 4.58e-04 | 5598.9ms/step | 10.7 samples/s
|
| 588 |
+
Step 10070/50462 | Loss: 2.7227 | LR: 4.58e-04 | 5600.5ms/step | 10.7 samples/s
|
| 589 |
+
Step 10080/50462 | Loss: 2.7259 | LR: 4.58e-04 | 5602.9ms/step | 10.7 samples/s
|
| 590 |
+
Step 10090/50462 | Loss: 2.7601 | LR: 4.58e-04 | 5601.1ms/step | 10.7 samples/s
|
| 591 |
+
Step 10100/50462 | Loss: 2.7453 | LR: 4.58e-04 | 5600.5ms/step | 10.7 samples/s
|
| 592 |
+
Step 10110/50462 | Loss: 2.7469 | LR: 4.58e-04 | 5601.9ms/step | 10.7 samples/s
|
| 593 |
+
Step 10120/50462 | Loss: 2.7661 | LR: 4.57e-04 | 5604.0ms/step | 10.7 samples/s
|
| 594 |
+
Step 10130/50462 | Loss: 2.7740 | LR: 4.57e-04 | 5600.8ms/step | 10.7 samples/s
|
| 595 |
+
Step 10140/50462 | Loss: 2.7727 | LR: 4.57e-04 | 5599.4ms/step | 10.7 samples/s
|
| 596 |
+
Step 10150/50462 | Loss: 2.7383 | LR: 4.57e-04 | 5601.9ms/step | 10.7 samples/s
|
| 597 |
+
Step 10160/50462 | Loss: 2.7311 | LR: 4.57e-04 | 5601.6ms/step | 10.7 samples/s
|
| 598 |
+
Step 10170/50462 | Loss: 2.7111 | LR: 4.57e-04 | 5601.0ms/step | 10.7 samples/s
|
| 599 |
+
Step 10180/50462 | Loss: 2.7212 | LR: 4.57e-04 | 5602.6ms/step | 10.7 samples/s
|
| 600 |
+
Step 10190/50462 | Loss: 2.7325 | LR: 4.57e-04 | 5602.4ms/step | 10.7 samples/s
|
| 601 |
+
Step 10200/50462 | Loss: 2.7312 | LR: 4.57e-04 | 5602.2ms/step | 10.7 samples/s
|
| 602 |
+
Step 10210/50462 | Loss: 2.7310 | LR: 4.57e-04 | 5604.6ms/step | 10.7 samples/s
|
| 603 |
+
Step 10220/50462 | Loss: 2.7002 | LR: 4.57e-04 | 5601.5ms/step | 10.7 samples/s
|
| 604 |
+
Step 10230/50462 | Loss: 2.7561 | LR: 4.56e-04 | 5600.3ms/step | 10.7 samples/s
|
| 605 |
+
Step 10240/50462 | Loss: 2.7453 | LR: 4.56e-04 | 5601.4ms/step | 10.7 samples/s
|
| 606 |
+
Step 10250/50462 | Loss: 2.7798 | LR: 4.56e-04 | 5600.2ms/step | 10.7 samples/s
|
| 607 |
+
Validation at step 10250: Loss = 2.7931, Perplexity = 16.33
|
| 608 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-10250
|
| 609 |
+
Step 10260/50462 | Loss: 2.7353 | LR: 4.56e-04 | 8450.8ms/step | 7.1 samples/s
|
| 610 |
+
Step 10270/50462 | Loss: 2.7236 | LR: 4.56e-04 | 5597.8ms/step | 10.7 samples/s
|
| 611 |
+
Step 10280/50462 | Loss: 2.7148 | LR: 4.56e-04 | 5607.9ms/step | 10.7 samples/s
|
| 612 |
+
Step 10290/50462 | Loss: 2.7074 | LR: 4.56e-04 | 5606.0ms/step | 10.7 samples/s
|
| 613 |
+
Step 10300/50462 | Loss: 2.7144 | LR: 4.56e-04 | 5608.7ms/step | 10.7 samples/s
|
epfl_final/SparseAtt/wandb/run-20260823_142101-rgmkfpnl/files/requirements.txt
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
| 142 |
+
entmax==1.3
|
epfl_final/SparseAtt/wandb/run-20260823_142101-rgmkfpnl/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-08-23T14:21:01.746009Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"configs/p2_1p2b_dense_seed70_5gpu.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/TokenPrune.git",
|
| 14 |
+
"commit": "5637886f0cc760009989b14e68c6ae06d4827418"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "a100-train-5-gpu-0-2",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python",
|
| 20 |
+
"cpu_count": 64,
|
| 21 |
+
"cpu_count_logical": 128,
|
| 22 |
+
"gpu": "NVIDIA A100-SXM4-80GB",
|
| 23 |
+
"gpu_count": 5,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "7679362727936",
|
| 27 |
+
"used": "4658920669184"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1081884971008"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 36 |
+
"memoryTotal": "85899345920",
|
| 37 |
+
"cudaCores": 6912,
|
| 38 |
+
"architecture": "Ampere",
|
| 39 |
+
"uuid": "GPU-da403873-77fd-0400-8874-ee85122e86ce"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 43 |
+
"memoryTotal": "85899345920",
|
| 44 |
+
"cudaCores": 6912,
|
| 45 |
+
"architecture": "Ampere",
|
| 46 |
+
"uuid": "GPU-ac7c2b06-0e54-6c0e-5c5e-bef297b937e2"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 50 |
+
"memoryTotal": "85899345920",
|
| 51 |
+
"cudaCores": 6912,
|
| 52 |
+
"architecture": "Ampere",
|
| 53 |
+
"uuid": "GPU-2f2cf903-6615-a5f8-353b-076dccd64125"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 57 |
+
"memoryTotal": "85899345920",
|
| 58 |
+
"cudaCores": 6912,
|
| 59 |
+
"architecture": "Ampere",
|
| 60 |
+
"uuid": "GPU-75d03f5a-9fc2-0173-5629-3ef21c2f450c"
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 64 |
+
"memoryTotal": "85899345920",
|
| 65 |
+
"cudaCores": 6912,
|
| 66 |
+
"architecture": "Ampere",
|
| 67 |
+
"uuid": "GPU-1d4bb5b9-c680-19af-9659-b21aa13b3c4d"
|
| 68 |
+
}
|
| 69 |
+
],
|
| 70 |
+
"cudaVersion": "13.0",
|
| 71 |
+
"writerId": "nm0kqnffv47zosz16jwa3rz1idddou6r"
|
| 72 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260823_142101-rgmkfpnl/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260823_142101-rgmkfpnl/logs/debug.log
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-08-23 14:21:01,766 INFO MainThread:22335 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-08-23 14:21:01,766 INFO MainThread:22335 [wandb_setup.py:_flush():81] Configure stats pid to 22335
|
| 3 |
+
2026-08-23 14:21:01,766 INFO MainThread:22335 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-08-23 14:21:01,766 INFO MainThread:22335 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260823_142101-rgmkfpnl/logs/debug.log
|
| 5 |
+
2026-08-23 14:21:01,766 INFO MainThread:22335 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260823_142101-rgmkfpnl/logs/debug-internal.log
|
| 6 |
+
2026-08-23 14:21:01,766 INFO MainThread:22335 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-08-23 14:21:01,766 INFO MainThread:22335 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-4750', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 12, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 70, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b dense 8k seed70 5gpu resume', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'dense', 'seed70'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-08-23 14:21:01,766 INFO MainThread:22335 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-08-23 14:21:02,071 INFO MainThread:22335 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-08-23 14:21:02,380 INFO MainThread:22335 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-08-23 14:21:02,390 INFO MainThread:22335 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-08-23 14:21:02,411 INFO MainThread:22335 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-08-23 14:21:02,903 INFO MainThread:22335 [wandb_init.py:init():1051] run resumed
|
| 15 |
+
2026-08-23 14:21:02,904 INFO MainThread:22335 [wandb_init.py:init():1056] starting run threads in backend
|
| 16 |
+
2026-08-23 14:21:03,109 INFO MainThread:22335 [wandb_run.py:_console_start():2554] atexit reg
|
| 17 |
+
2026-08-23 14:21:03,109 INFO MainThread:22335 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 18 |
+
2026-08-23 14:21:03,109 INFO MainThread:22335 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 19 |
+
2026-08-23 14:21:03,109 INFO MainThread:22335 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 20 |
+
2026-08-23 14:21:03,117 INFO MainThread:22335 [wandb_init.py:init():1094] run started, returning control to user process
|
epfl_final/SparseAtt/wandb/run-20260823_142101-rgmkfpnl/run-rgmkfpnl.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:13bd52e44387ee38c7efa86a4009e492ccbfbf16bde168e3226ca6d7c7a7f275
|
| 3 |
+
size 6258688
|
epfl_final/SparseAtt/wandb/run-20260823_150341-i8ewyy95/files/output.log
ADDED
|
@@ -0,0 +1,814 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Starting training...
|
| 2 |
+
Validation at step 16750: Loss = 2.6770, Perplexity = 14.54
|
| 3 |
+
wandb: WARNING Tried to log to step 16750 that is less than the current step 16811. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 4 |
+
Switching attention mode for step 16751: sparse_stochastic (configured sparse attention)
|
| 5 |
+
Step 16760/50462 | Loss: 2.6845 | LR: 3.83e-04 | 42321.5ms/step | 1.4 samples/s
|
| 6 |
+
wandb: WARNING Tried to log to step 16760 that is less than the current step 16811. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 7 |
+
Step 16770/50462 | Loss: 2.6890 | LR: 3.82e-04 | 2741.6ms/step | 21.9 samples/s
|
| 8 |
+
wandb: WARNING Tried to log to step 16770 that is less than the current step 16811. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 9 |
+
Step 16780/50462 | Loss: 2.6879 | LR: 3.82e-04 | 2748.5ms/step | 21.8 samples/s
|
| 10 |
+
wandb: WARNING Tried to log to step 16780 that is less than the current step 16811. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 11 |
+
Step 16790/50462 | Loss: 2.6737 | LR: 3.82e-04 | 2739.1ms/step | 21.9 samples/s
|
| 12 |
+
wandb: WARNING Tried to log to step 16790 that is less than the current step 16811. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 13 |
+
Step 16800/50462 | Loss: 2.6492 | LR: 3.82e-04 | 2736.2ms/step | 21.9 samples/s
|
| 14 |
+
wandb: WARNING Tried to log to step 16800 that is less than the current step 16811. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 15 |
+
Step 16810/50462 | Loss: 2.6735 | LR: 3.82e-04 | 2744.5ms/step | 21.9 samples/s
|
| 16 |
+
wandb: WARNING Tried to log to step 16810 that is less than the current step 16811. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 17 |
+
Step 16820/50462 | Loss: 2.6886 | LR: 3.82e-04 | 2740.7ms/step | 21.9 samples/s
|
| 18 |
+
Step 16830/50462 | Loss: 2.6647 | LR: 3.82e-04 | 2738.4ms/step | 21.9 samples/s
|
| 19 |
+
Step 16840/50462 | Loss: 2.6860 | LR: 3.82e-04 | 2741.9ms/step | 21.9 samples/s
|
| 20 |
+
Step 16850/50462 | Loss: 2.6593 | LR: 3.81e-04 | 2734.1ms/step | 21.9 samples/s
|
| 21 |
+
Step 16860/50462 | Loss: 2.6914 | LR: 3.81e-04 | 2751.8ms/step | 21.8 samples/s
|
| 22 |
+
Step 16870/50462 | Loss: 2.7246 | LR: 3.81e-04 | 2746.3ms/step | 21.8 samples/s
|
| 23 |
+
Step 16880/50462 | Loss: 2.6777 | LR: 3.81e-04 | 2743.7ms/step | 21.9 samples/s
|
| 24 |
+
Step 16890/50462 | Loss: 2.6550 | LR: 3.81e-04 | 2740.4ms/step | 21.9 samples/s
|
| 25 |
+
Step 16900/50462 | Loss: 2.6746 | LR: 3.81e-04 | 2747.2ms/step | 21.8 samples/s
|
| 26 |
+
Step 16910/50462 | Loss: 2.7249 | LR: 3.81e-04 | 2734.0ms/step | 21.9 samples/s
|
| 27 |
+
Step 16920/50462 | Loss: 2.6715 | LR: 3.80e-04 | 2737.6ms/step | 21.9 samples/s
|
| 28 |
+
Step 16930/50462 | Loss: 2.6982 | LR: 3.80e-04 | 2739.3ms/step | 21.9 samples/s
|
| 29 |
+
Step 16940/50462 | Loss: 2.6699 | LR: 3.80e-04 | 2741.7ms/step | 21.9 samples/s
|
| 30 |
+
Step 16950/50462 | Loss: 2.6797 | LR: 3.80e-04 | 2745.6ms/step | 21.9 samples/s
|
| 31 |
+
Step 16960/50462 | Loss: 2.6754 | LR: 3.80e-04 | 2747.9ms/step | 21.8 samples/s
|
| 32 |
+
Step 16970/50462 | Loss: 2.6537 | LR: 3.80e-04 | 2748.1ms/step | 21.8 samples/s
|
| 33 |
+
Step 16980/50462 | Loss: 2.6712 | LR: 3.80e-04 | 2738.0ms/step | 21.9 samples/s
|
| 34 |
+
Step 16990/50462 | Loss: 2.6581 | LR: 3.79e-04 | 2754.7ms/step | 21.8 samples/s
|
| 35 |
+
Step 17000/50462 | Loss: 2.7017 | LR: 3.79e-04 | 2724.3ms/step | 22.0 samples/s
|
| 36 |
+
Validation at step 17000: Loss = 2.6745, Perplexity = 14.51
|
| 37 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-17000
|
| 38 |
+
Step 17010/50462 | Loss: 2.6479 | LR: 3.79e-04 | 4095.8ms/step | 14.6 samples/s
|
| 39 |
+
Step 17020/50462 | Loss: 2.7154 | LR: 3.79e-04 | 2732.7ms/step | 22.0 samples/s
|
| 40 |
+
Step 17030/50462 | Loss: 2.6598 | LR: 3.79e-04 | 2744.0ms/step | 21.9 samples/s
|
| 41 |
+
Step 17040/50462 | Loss: 2.7000 | LR: 3.79e-04 | 2741.9ms/step | 21.9 samples/s
|
| 42 |
+
Step 17050/50462 | Loss: 2.7051 | LR: 3.79e-04 | 2742.7ms/step | 21.9 samples/s
|
| 43 |
+
Step 17060/50462 | Loss: 2.6419 | LR: 3.79e-04 | 2741.8ms/step | 21.9 samples/s
|
| 44 |
+
Step 17070/50462 | Loss: 2.6524 | LR: 3.78e-04 | 2744.3ms/step | 21.9 samples/s
|
| 45 |
+
Step 17080/50462 | Loss: 2.7199 | LR: 3.78e-04 | 2750.9ms/step | 21.8 samples/s
|
| 46 |
+
Step 17090/50462 | Loss: 2.6643 | LR: 3.78e-04 | 2733.8ms/step | 21.9 samples/s
|
| 47 |
+
Step 17100/50462 | Loss: 2.6472 | LR: 3.78e-04 | 2747.4ms/step | 21.8 samples/s
|
| 48 |
+
Step 17110/50462 | Loss: 2.6479 | LR: 3.78e-04 | 2728.8ms/step | 22.0 samples/s
|
| 49 |
+
Step 17120/50462 | Loss: 2.6519 | LR: 3.78e-04 | 2737.4ms/step | 21.9 samples/s
|
| 50 |
+
Step 17130/50462 | Loss: 2.7193 | LR: 3.78e-04 | 2741.4ms/step | 21.9 samples/s
|
| 51 |
+
Step 17140/50462 | Loss: 2.6912 | LR: 3.77e-04 | 2745.5ms/step | 21.9 samples/s
|
| 52 |
+
Step 17150/50462 | Loss: 2.6687 | LR: 3.77e-04 | 2743.4ms/step | 21.9 samples/s
|
| 53 |
+
Step 17160/50462 | Loss: 2.6548 | LR: 3.77e-04 | 2739.5ms/step | 21.9 samples/s
|
| 54 |
+
Step 17170/50462 | Loss: 2.6578 | LR: 3.77e-04 | 2745.8ms/step | 21.9 samples/s
|
| 55 |
+
Step 17180/50462 | Loss: 2.6645 | LR: 3.77e-04 | 2744.1ms/step | 21.9 samples/s
|
| 56 |
+
Step 17190/50462 | Loss: 2.6589 | LR: 3.77e-04 | 2742.1ms/step | 21.9 samples/s
|
| 57 |
+
Step 17200/50462 | Loss: 2.6407 | LR: 3.77e-04 | 2738.5ms/step | 21.9 samples/s
|
| 58 |
+
Step 17210/50462 | Loss: 2.6723 | LR: 3.77e-04 | 2736.7ms/step | 21.9 samples/s
|
| 59 |
+
Step 17220/50462 | Loss: 2.6787 | LR: 3.76e-04 | 2737.8ms/step | 21.9 samples/s
|
| 60 |
+
Step 17230/50462 | Loss: 2.6773 | LR: 3.76e-04 | 2748.0ms/step | 21.8 samples/s
|
| 61 |
+
Step 17240/50462 | Loss: 2.7250 | LR: 3.76e-04 | 2745.2ms/step | 21.9 samples/s
|
| 62 |
+
Step 17250/50462 | Loss: 2.6901 | LR: 3.76e-04 | 2740.2ms/step | 21.9 samples/s
|
| 63 |
+
Validation at step 17250: Loss = 2.6714, Perplexity = 14.46
|
| 64 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-17250
|
| 65 |
+
Step 17260/50462 | Loss: 2.6524 | LR: 3.76e-04 | 4167.8ms/step | 14.4 samples/s
|
| 66 |
+
Step 17270/50462 | Loss: 2.6861 | LR: 3.76e-04 | 2739.5ms/step | 21.9 samples/s
|
| 67 |
+
Step 17280/50462 | Loss: 2.6878 | LR: 3.76e-04 | 2746.9ms/step | 21.8 samples/s
|
| 68 |
+
Step 17290/50462 | Loss: 2.6853 | LR: 3.75e-04 | 2733.2ms/step | 22.0 samples/s
|
| 69 |
+
Step 17300/50462 | Loss: 2.6508 | LR: 3.75e-04 | 2745.2ms/step | 21.9 samples/s
|
| 70 |
+
Step 17310/50462 | Loss: 2.6582 | LR: 3.75e-04 | 2740.2ms/step | 21.9 samples/s
|
| 71 |
+
Step 17320/50462 | Loss: 2.6454 | LR: 3.75e-04 | 2732.3ms/step | 22.0 samples/s
|
| 72 |
+
Step 17330/50462 | Loss: 2.7101 | LR: 3.75e-04 | 2735.3ms/step | 21.9 samples/s
|
| 73 |
+
Step 17340/50462 | Loss: 2.6979 | LR: 3.75e-04 | 2744.4ms/step | 21.9 samples/s
|
| 74 |
+
Step 17350/50462 | Loss: 2.7007 | LR: 3.75e-04 | 2739.7ms/step | 21.9 samples/s
|
| 75 |
+
Step 17360/50462 | Loss: 2.6736 | LR: 3.74e-04 | 2740.2ms/step | 21.9 samples/s
|
| 76 |
+
Step 17370/50462 | Loss: 2.6425 | LR: 3.74e-04 | 2736.9ms/step | 21.9 samples/s
|
| 77 |
+
Step 17380/50462 | Loss: 2.7250 | LR: 3.74e-04 | 2738.6ms/step | 21.9 samples/s
|
| 78 |
+
Step 17390/50462 | Loss: 2.7061 | LR: 3.74e-04 | 2744.2ms/step | 21.9 samples/s
|
| 79 |
+
Step 17400/50462 | Loss: 2.6572 | LR: 3.74e-04 | 2752.7ms/step | 21.8 samples/s
|
| 80 |
+
Step 17410/50462 | Loss: 2.6793 | LR: 3.74e-04 | 2731.4ms/step | 22.0 samples/s
|
| 81 |
+
Step 17420/50462 | Loss: 2.6298 | LR: 3.74e-04 | 2729.2ms/step | 22.0 samples/s
|
| 82 |
+
Step 17430/50462 | Loss: 2.6423 | LR: 3.74e-04 | 2735.2ms/step | 21.9 samples/s
|
| 83 |
+
Step 17440/50462 | Loss: 2.6059 | LR: 3.73e-04 | 2741.7ms/step | 21.9 samples/s
|
| 84 |
+
Step 17450/50462 | Loss: 2.6906 | LR: 3.73e-04 | 2738.0ms/step | 21.9 samples/s
|
| 85 |
+
Step 17460/50462 | Loss: 2.6816 | LR: 3.73e-04 | 2735.4ms/step | 21.9 samples/s
|
| 86 |
+
Step 17470/50462 | Loss: 2.6713 | LR: 3.73e-04 | 2733.6ms/step | 21.9 samples/s
|
| 87 |
+
Step 17480/50462 | Loss: 2.6729 | LR: 3.73e-04 | 2742.8ms/step | 21.9 samples/s
|
| 88 |
+
Step 17490/50462 | Loss: 2.7170 | LR: 3.73e-04 | 2733.3ms/step | 22.0 samples/s
|
| 89 |
+
Step 17500/50462 | Loss: 2.6628 | LR: 3.73e-04 | 2738.7ms/step | 21.9 samples/s
|
| 90 |
+
Validation at step 17500: Loss = 2.6724, Perplexity = 14.47
|
| 91 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-17500
|
| 92 |
+
Step 17510/50462 | Loss: 2.6499 | LR: 3.72e-04 | 4209.1ms/step | 14.3 samples/s
|
| 93 |
+
Step 17520/50462 | Loss: 2.6490 | LR: 3.72e-04 | 2745.2ms/step | 21.9 samples/s
|
| 94 |
+
Step 17530/50462 | Loss: 2.6474 | LR: 3.72e-04 | 2733.7ms/step | 21.9 samples/s
|
| 95 |
+
Step 17540/50462 | Loss: 2.6584 | LR: 3.72e-04 | 2741.5ms/step | 21.9 samples/s
|
| 96 |
+
Step 17550/50462 | Loss: 2.6596 | LR: 3.72e-04 | 2747.0ms/step | 21.8 samples/s
|
| 97 |
+
Step 17560/50462 | Loss: 2.6277 | LR: 3.72e-04 | 2733.8ms/step | 21.9 samples/s
|
| 98 |
+
Step 17570/50462 | Loss: 2.6458 | LR: 3.72e-04 | 2731.4ms/step | 22.0 samples/s
|
| 99 |
+
Step 17580/50462 | Loss: 2.6486 | LR: 3.71e-04 | 2731.8ms/step | 22.0 samples/s
|
| 100 |
+
Step 17590/50462 | Loss: 2.6779 | LR: 3.71e-04 | 2735.2ms/step | 21.9 samples/s
|
| 101 |
+
Step 17600/50462 | Loss: 2.6830 | LR: 3.71e-04 | 2733.2ms/step | 22.0 samples/s
|
| 102 |
+
Step 17610/50462 | Loss: 2.7174 | LR: 3.71e-04 | 2732.7ms/step | 22.0 samples/s
|
| 103 |
+
Step 17620/50462 | Loss: 2.6315 | LR: 3.71e-04 | 2742.0ms/step | 21.9 samples/s
|
| 104 |
+
Step 17630/50462 | Loss: 2.7254 | LR: 3.71e-04 | 2733.7ms/step | 21.9 samples/s
|
| 105 |
+
Step 17640/50462 | Loss: 2.6422 | LR: 3.71e-04 | 2737.1ms/step | 21.9 samples/s
|
| 106 |
+
Step 17650/50462 | Loss: 2.6546 | LR: 3.70e-04 | 2736.9ms/step | 21.9 samples/s
|
| 107 |
+
Step 17660/50462 | Loss: 2.6475 | LR: 3.70e-04 | 2735.7ms/step | 21.9 samples/s
|
| 108 |
+
Step 17670/50462 | Loss: 2.6282 | LR: 3.70e-04 | 2746.5ms/step | 21.8 samples/s
|
| 109 |
+
Step 17680/50462 | Loss: 2.6354 | LR: 3.70e-04 | 2731.6ms/step | 22.0 samples/s
|
| 110 |
+
Step 17690/50462 | Loss: 2.6998 | LR: 3.70e-04 | 2736.1ms/step | 21.9 samples/s
|
| 111 |
+
Step 17700/50462 | Loss: 2.6663 | LR: 3.70e-04 | 2734.1ms/step | 21.9 samples/s
|
| 112 |
+
Step 17710/50462 | Loss: 2.6304 | LR: 3.70e-04 | 2739.0ms/step | 21.9 samples/s
|
| 113 |
+
Step 17720/50462 | Loss: 2.6309 | LR: 3.69e-04 | 2734.9ms/step | 21.9 samples/s
|
| 114 |
+
Step 17730/50462 | Loss: 2.6613 | LR: 3.69e-04 | 2736.2ms/step | 21.9 samples/s
|
| 115 |
+
Step 17740/50462 | Loss: 2.6881 | LR: 3.69e-04 | 2732.6ms/step | 22.0 samples/s
|
| 116 |
+
Step 17750/50462 | Loss: 2.6692 | LR: 3.69e-04 | 2738.2ms/step | 21.9 samples/s
|
| 117 |
+
Validation at step 17750: Loss = 2.6653, Perplexity = 14.37
|
| 118 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-17750
|
| 119 |
+
Step 17760/50462 | Loss: 2.6379 | LR: 3.69e-04 | 4110.4ms/step | 14.6 samples/s
|
| 120 |
+
Step 17770/50462 | Loss: 2.6693 | LR: 3.69e-04 | 2728.9ms/step | 22.0 samples/s
|
| 121 |
+
Step 17780/50462 | Loss: 2.6336 | LR: 3.69e-04 | 2729.8ms/step | 22.0 samples/s
|
| 122 |
+
Step 17790/50462 | Loss: 2.6621 | LR: 3.69e-04 | 2726.5ms/step | 22.0 samples/s
|
| 123 |
+
Step 17800/50462 | Loss: 2.6762 | LR: 3.68e-04 | 2735.8ms/step | 21.9 samples/s
|
| 124 |
+
Step 17810/50462 | Loss: 2.6745 | LR: 3.68e-04 | 2736.1ms/step | 21.9 samples/s
|
| 125 |
+
Step 17820/50462 | Loss: 2.6650 | LR: 3.68e-04 | 2736.9ms/step | 21.9 samples/s
|
| 126 |
+
Step 17830/50462 | Loss: 2.6496 | LR: 3.68e-04 | 2739.1ms/step | 21.9 samples/s
|
| 127 |
+
Step 17840/50462 | Loss: 2.7096 | LR: 3.68e-04 | 2735.9ms/step | 21.9 samples/s
|
| 128 |
+
Step 17850/50462 | Loss: 2.6736 | LR: 3.68e-04 | 2736.1ms/step | 21.9 samples/s
|
| 129 |
+
Step 17860/50462 | Loss: 2.6645 | LR: 3.68e-04 | 2750.8ms/step | 21.8 samples/s
|
| 130 |
+
Step 17870/50462 | Loss: 2.7052 | LR: 3.67e-04 | 2742.4ms/step | 21.9 samples/s
|
| 131 |
+
Step 17880/50462 | Loss: 2.6519 | LR: 3.67e-04 | 2735.3ms/step | 21.9 samples/s
|
| 132 |
+
Step 17890/50462 | Loss: 2.6666 | LR: 3.67e-04 | 2732.2ms/step | 22.0 samples/s
|
| 133 |
+
Step 17900/50462 | Loss: 2.6343 | LR: 3.67e-04 | 2742.8ms/step | 21.9 samples/s
|
| 134 |
+
Step 17910/50462 | Loss: 2.6359 | LR: 3.67e-04 | 2736.5ms/step | 21.9 samples/s
|
| 135 |
+
Step 17920/50462 | Loss: 2.6724 | LR: 3.67e-04 | 2743.3ms/step | 21.9 samples/s
|
| 136 |
+
Step 17930/50462 | Loss: 2.6395 | LR: 3.67e-04 | 2729.1ms/step | 22.0 samples/s
|
| 137 |
+
Step 17940/50462 | Loss: 2.6774 | LR: 3.66e-04 | 2744.6ms/step | 21.9 samples/s
|
| 138 |
+
Step 17950/50462 | Loss: 2.6698 | LR: 3.66e-04 | 2737.7ms/step | 21.9 samples/s
|
| 139 |
+
Step 17960/50462 | Loss: 2.6940 | LR: 3.66e-04 | 2738.6ms/step | 21.9 samples/s
|
| 140 |
+
Step 17970/50462 | Loss: 2.6282 | LR: 3.66e-04 | 2738.9ms/step | 21.9 samples/s
|
| 141 |
+
Step 17980/50462 | Loss: 2.6692 | LR: 3.66e-04 | 2734.5ms/step | 21.9 samples/s
|
| 142 |
+
Step 17990/50462 | Loss: 2.6628 | LR: 3.66e-04 | 2729.8ms/step | 22.0 samples/s
|
| 143 |
+
Step 18000/50462 | Loss: 2.6642 | LR: 3.66e-04 | 2731.5ms/step | 22.0 samples/s
|
| 144 |
+
Validation at step 18000: Loss = 2.6625, Perplexity = 14.33
|
| 145 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-18000
|
| 146 |
+
Step 18010/50462 | Loss: 2.6936 | LR: 3.65e-04 | 4074.4ms/step | 14.7 samples/s
|
| 147 |
+
Step 18020/50462 | Loss: 2.6692 | LR: 3.65e-04 | 2730.7ms/step | 22.0 samples/s
|
| 148 |
+
Step 18030/50462 | Loss: 2.6487 | LR: 3.65e-04 | 2737.9ms/step | 21.9 samples/s
|
| 149 |
+
Step 18040/50462 | Loss: 2.6453 | LR: 3.65e-04 | 2734.5ms/step | 21.9 samples/s
|
| 150 |
+
Step 18050/50462 | Loss: 2.6541 | LR: 3.65e-04 | 2733.8ms/step | 21.9 samples/s
|
| 151 |
+
Step 18060/50462 | Loss: 2.6618 | LR: 3.65e-04 | 2734.5ms/step | 21.9 samples/s
|
| 152 |
+
Step 18070/50462 | Loss: 2.7331 | LR: 3.65e-04 | 2734.5ms/step | 21.9 samples/s
|
| 153 |
+
Step 18080/50462 | Loss: 2.6736 | LR: 3.64e-04 | 2735.7ms/step | 21.9 samples/s
|
| 154 |
+
Step 18090/50462 | Loss: 2.6733 | LR: 3.64e-04 | 2739.5ms/step | 21.9 samples/s
|
| 155 |
+
Step 18100/50462 | Loss: 2.6680 | LR: 3.64e-04 | 2733.1ms/step | 22.0 samples/s
|
| 156 |
+
Step 18110/50462 | Loss: 2.6850 | LR: 3.64e-04 | 2740.1ms/step | 21.9 samples/s
|
| 157 |
+
Step 18120/50462 | Loss: 2.6658 | LR: 3.64e-04 | 2737.0ms/step | 21.9 samples/s
|
| 158 |
+
Step 18130/50462 | Loss: 2.6800 | LR: 3.64e-04 | 2732.1ms/step | 22.0 samples/s
|
| 159 |
+
Step 18140/50462 | Loss: 2.6739 | LR: 3.64e-04 | 2726.7ms/step | 22.0 samples/s
|
| 160 |
+
Step 18150/50462 | Loss: 2.6608 | LR: 3.63e-04 | 2730.1ms/step | 22.0 samples/s
|
| 161 |
+
Step 18160/50462 | Loss: 2.6525 | LR: 3.63e-04 | 2727.5ms/step | 22.0 samples/s
|
| 162 |
+
Step 18170/50462 | Loss: 2.6415 | LR: 3.63e-04 | 2727.3ms/step | 22.0 samples/s
|
| 163 |
+
Step 18180/50462 | Loss: 2.6567 | LR: 3.63e-04 | 2729.1ms/step | 22.0 samples/s
|
| 164 |
+
Step 18190/50462 | Loss: 2.6893 | LR: 3.63e-04 | 2735.0ms/step | 21.9 samples/s
|
| 165 |
+
Step 18200/50462 | Loss: 2.6453 | LR: 3.63e-04 | 2731.5ms/step | 22.0 samples/s
|
| 166 |
+
Step 18210/50462 | Loss: 2.6536 | LR: 3.63e-04 | 2731.3ms/step | 22.0 samples/s
|
| 167 |
+
Step 18220/50462 | Loss: 2.6949 | LR: 3.62e-04 | 2728.5ms/step | 22.0 samples/s
|
| 168 |
+
Step 18230/50462 | Loss: 2.6505 | LR: 3.62e-04 | 2728.7ms/step | 22.0 samples/s
|
| 169 |
+
Step 18240/50462 | Loss: 2.7005 | LR: 3.62e-04 | 2738.0ms/step | 21.9 samples/s
|
| 170 |
+
Step 18250/50462 | Loss: 2.6361 | LR: 3.62e-04 | 2730.0ms/step | 22.0 samples/s
|
| 171 |
+
Validation at step 18250: Loss = 2.6568, Perplexity = 14.25
|
| 172 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-18250
|
| 173 |
+
Step 18260/50462 | Loss: 2.6532 | LR: 3.62e-04 | 4068.8ms/step | 14.7 samples/s
|
| 174 |
+
Step 18270/50462 | Loss: 2.6540 | LR: 3.62e-04 | 2725.4ms/step | 22.0 samples/s
|
| 175 |
+
Step 18280/50462 | Loss: 2.6575 | LR: 3.62e-04 | 2728.6ms/step | 22.0 samples/s
|
| 176 |
+
Step 18290/50462 | Loss: 2.6261 | LR: 3.62e-04 | 2726.9ms/step | 22.0 samples/s
|
| 177 |
+
Step 18300/50462 | Loss: 2.6784 | LR: 3.61e-04 | 2733.5ms/step | 22.0 samples/s
|
| 178 |
+
Step 18310/50462 | Loss: 2.6237 | LR: 3.61e-04 | 2732.6ms/step | 22.0 samples/s
|
| 179 |
+
Step 18320/50462 | Loss: 2.6791 | LR: 3.61e-04 | 2739.1ms/step | 21.9 samples/s
|
| 180 |
+
Step 18330/50462 | Loss: 2.6377 | LR: 3.61e-04 | 2738.0ms/step | 21.9 samples/s
|
| 181 |
+
Step 18340/50462 | Loss: 2.6544 | LR: 3.61e-04 | 2721.9ms/step | 22.0 samples/s
|
| 182 |
+
Step 18350/50462 | Loss: 2.6899 | LR: 3.61e-04 | 2735.6ms/step | 21.9 samples/s
|
| 183 |
+
Step 18360/50462 | Loss: 2.6950 | LR: 3.61e-04 | 2738.1ms/step | 21.9 samples/s
|
| 184 |
+
Step 18370/50462 | Loss: 2.6819 | LR: 3.60e-04 | 2731.8ms/step | 22.0 samples/s
|
| 185 |
+
Step 18380/50462 | Loss: 2.6574 | LR: 3.60e-04 | 2736.8ms/step | 21.9 samples/s
|
| 186 |
+
Step 18390/50462 | Loss: 2.6187 | LR: 3.60e-04 | 2730.4ms/step | 22.0 samples/s
|
| 187 |
+
Step 18400/50462 | Loss: 2.6540 | LR: 3.60e-04 | 2734.9ms/step | 21.9 samples/s
|
| 188 |
+
Step 18410/50462 | Loss: 2.6629 | LR: 3.60e-04 | 2728.9ms/step | 22.0 samples/s
|
| 189 |
+
Step 18420/50462 | Loss: 2.6746 | LR: 3.60e-04 | 2729.4ms/step | 22.0 samples/s
|
| 190 |
+
Step 18430/50462 | Loss: 2.6708 | LR: 3.60e-04 | 2730.2ms/step | 22.0 samples/s
|
| 191 |
+
Step 18440/50462 | Loss: 2.6403 | LR: 3.59e-04 | 2731.1ms/step | 22.0 samples/s
|
| 192 |
+
Step 18450/50462 | Loss: 2.6549 | LR: 3.59e-04 | 2732.7ms/step | 22.0 samples/s
|
| 193 |
+
Step 18460/50462 | Loss: 2.6867 | LR: 3.59e-04 | 2734.0ms/step | 21.9 samples/s
|
| 194 |
+
Step 18470/50462 | Loss: 2.6399 | LR: 3.59e-04 | 2734.2ms/step | 21.9 samples/s
|
| 195 |
+
Step 18480/50462 | Loss: 2.6712 | LR: 3.59e-04 | 2729.2ms/step | 22.0 samples/s
|
| 196 |
+
Step 18490/50462 | Loss: 2.6583 | LR: 3.59e-04 | 2743.3ms/step | 21.9 samples/s
|
| 197 |
+
Step 18500/50462 | Loss: 2.6141 | LR: 3.59e-04 | 2730.3ms/step | 22.0 samples/s
|
| 198 |
+
Validation at step 18500: Loss = 2.6571, Perplexity = 14.25
|
| 199 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-18500
|
| 200 |
+
Step 18510/50462 | Loss: 2.6709 | LR: 3.58e-04 | 4073.5ms/step | 14.7 samples/s
|
| 201 |
+
Step 18520/50462 | Loss: 2.6600 | LR: 3.58e-04 | 2727.3ms/step | 22.0 samples/s
|
| 202 |
+
Step 18530/50462 | Loss: 2.6412 | LR: 3.58e-04 | 2730.3ms/step | 22.0 samples/s
|
| 203 |
+
Step 18540/50462 | Loss: 2.6809 | LR: 3.58e-04 | 2737.8ms/step | 21.9 samples/s
|
| 204 |
+
Step 18550/50462 | Loss: 2.6322 | LR: 3.58e-04 | 2733.1ms/step | 22.0 samples/s
|
| 205 |
+
Step 18560/50462 | Loss: 2.6070 | LR: 3.58e-04 | 2732.7ms/step | 22.0 samples/s
|
| 206 |
+
Step 18570/50462 | Loss: 2.6590 | LR: 3.58e-04 | 2731.0ms/step | 22.0 samples/s
|
| 207 |
+
Step 18580/50462 | Loss: 2.6038 | LR: 3.57e-04 | 2731.8ms/step | 22.0 samples/s
|
| 208 |
+
Step 18590/50462 | Loss: 2.6436 | LR: 3.57e-04 | 2734.1ms/step | 21.9 samples/s
|
| 209 |
+
Step 18600/50462 | Loss: 2.6454 | LR: 3.57e-04 | 2733.4ms/step | 22.0 samples/s
|
| 210 |
+
Step 18610/50462 | Loss: 2.6729 | LR: 3.57e-04 | 2734.3ms/step | 21.9 samples/s
|
| 211 |
+
Step 18620/50462 | Loss: 2.6297 | LR: 3.57e-04 | 2729.1ms/step | 22.0 samples/s
|
| 212 |
+
Step 18630/50462 | Loss: 2.6217 | LR: 3.57e-04 | 2732.8ms/step | 22.0 samples/s
|
| 213 |
+
Step 18640/50462 | Loss: 2.6318 | LR: 3.57e-04 | 2730.3ms/step | 22.0 samples/s
|
| 214 |
+
Step 18650/50462 | Loss: 2.6130 | LR: 3.56e-04 | 2727.3ms/step | 22.0 samples/s
|
| 215 |
+
Step 18660/50462 | Loss: 2.6246 | LR: 3.56e-04 | 2746.8ms/step | 21.8 samples/s
|
| 216 |
+
Step 18670/50462 | Loss: 2.6687 | LR: 3.56e-04 | 2741.2ms/step | 21.9 samples/s
|
| 217 |
+
Step 18680/50462 | Loss: 2.6482 | LR: 3.56e-04 | 2740.6ms/step | 21.9 samples/s
|
| 218 |
+
Step 18690/50462 | Loss: 2.6537 | LR: 3.56e-04 | 2742.4ms/step | 21.9 samples/s
|
| 219 |
+
Step 18700/50462 | Loss: 2.6520 | LR: 3.56e-04 | 2742.5ms/step | 21.9 samples/s
|
| 220 |
+
Step 18710/50462 | Loss: 2.6375 | LR: 3.56e-04 | 2741.4ms/step | 21.9 samples/s
|
| 221 |
+
Step 18720/50462 | Loss: 2.6420 | LR: 3.55e-04 | 2735.4ms/step | 21.9 samples/s
|
| 222 |
+
Step 18730/50462 | Loss: 2.6709 | LR: 3.55e-04 | 2723.8ms/step | 22.0 samples/s
|
| 223 |
+
Step 18740/50462 | Loss: 2.6446 | LR: 3.55e-04 | 2746.4ms/step | 21.8 samples/s
|
| 224 |
+
Step 18750/50462 | Loss: 2.6592 | LR: 3.55e-04 | 2746.0ms/step | 21.8 samples/s
|
| 225 |
+
Validation at step 18750: Loss = 2.6571, Perplexity = 14.26
|
| 226 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-18750
|
| 227 |
+
Step 18760/50462 | Loss: 2.6690 | LR: 3.55e-04 | 4123.2ms/step | 14.6 samples/s
|
| 228 |
+
Step 18770/50462 | Loss: 2.6084 | LR: 3.55e-04 | 2746.6ms/step | 21.8 samples/s
|
| 229 |
+
Step 18780/50462 | Loss: 2.6240 | LR: 3.55e-04 | 2734.4ms/step | 21.9 samples/s
|
| 230 |
+
Step 18790/50462 | Loss: 2.7283 | LR: 3.54e-04 | 2724.6ms/step | 22.0 samples/s
|
| 231 |
+
Step 18800/50462 | Loss: 2.6264 | LR: 3.54e-04 | 2734.5ms/step | 21.9 samples/s
|
| 232 |
+
Step 18810/50462 | Loss: 2.6524 | LR: 3.54e-04 | 2736.2ms/step | 21.9 samples/s
|
| 233 |
+
Step 18820/50462 | Loss: 2.6710 | LR: 3.54e-04 | 2727.1ms/step | 22.0 samples/s
|
| 234 |
+
Step 18830/50462 | Loss: 2.6730 | LR: 3.54e-04 | 2725.1ms/step | 22.0 samples/s
|
| 235 |
+
Step 18840/50462 | Loss: 2.6750 | LR: 3.54e-04 | 2727.7ms/step | 22.0 samples/s
|
| 236 |
+
Step 18850/50462 | Loss: 2.6786 | LR: 3.54e-04 | 2733.9ms/step | 21.9 samples/s
|
| 237 |
+
Step 18860/50462 | Loss: 2.6556 | LR: 3.53e-04 | 2733.4ms/step | 22.0 samples/s
|
| 238 |
+
Step 18870/50462 | Loss: 2.7275 | LR: 3.53e-04 | 2725.6ms/step | 22.0 samples/s
|
| 239 |
+
Step 18880/50462 | Loss: 2.6388 | LR: 3.53e-04 | 2732.3ms/step | 22.0 samples/s
|
| 240 |
+
Step 18890/50462 | Loss: 2.6753 | LR: 3.53e-04 | 2734.6ms/step | 21.9 samples/s
|
| 241 |
+
Step 18900/50462 | Loss: 2.6620 | LR: 3.53e-04 | 2731.9ms/step | 22.0 samples/s
|
| 242 |
+
Step 18910/50462 | Loss: 2.6348 | LR: 3.53e-04 | 2732.1ms/step | 22.0 samples/s
|
| 243 |
+
Step 18920/50462 | Loss: 2.6709 | LR: 3.53e-04 | 2728.0ms/step | 22.0 samples/s
|
| 244 |
+
Step 18930/50462 | Loss: 2.6344 | LR: 3.52e-04 | 2719.1ms/step | 22.1 samples/s
|
| 245 |
+
Step 18940/50462 | Loss: 2.6501 | LR: 3.52e-04 | 2723.8ms/step | 22.0 samples/s
|
| 246 |
+
Step 18950/50462 | Loss: 2.6530 | LR: 3.52e-04 | 2725.3ms/step | 22.0 samples/s
|
| 247 |
+
Step 18960/50462 | Loss: 2.6482 | LR: 3.52e-04 | 2730.9ms/step | 22.0 samples/s
|
| 248 |
+
Step 18970/50462 | Loss: 2.6611 | LR: 3.52e-04 | 2720.8ms/step | 22.1 samples/s
|
| 249 |
+
Step 18980/50462 | Loss: 2.6298 | LR: 3.52e-04 | 2734.1ms/step | 21.9 samples/s
|
| 250 |
+
Step 18990/50462 | Loss: 2.6984 | LR: 3.52e-04 | 2741.2ms/step | 21.9 samples/s
|
| 251 |
+
Step 19000/50462 | Loss: 2.6279 | LR: 3.51e-04 | 2743.1ms/step | 21.9 samples/s
|
| 252 |
+
Validation at step 19000: Loss = 2.6523, Perplexity = 14.19
|
| 253 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-19000
|
| 254 |
+
Step 19010/50462 | Loss: 2.6076 | LR: 3.51e-04 | 4144.8ms/step | 14.5 samples/s
|
| 255 |
+
Step 19020/50462 | Loss: 2.6465 | LR: 3.51e-04 | 2733.1ms/step | 22.0 samples/s
|
| 256 |
+
Step 19030/50462 | Loss: 2.6569 | LR: 3.51e-04 | 2734.2ms/step | 21.9 samples/s
|
| 257 |
+
Step 19040/50462 | Loss: 2.6762 | LR: 3.51e-04 | 2737.5ms/step | 21.9 samples/s
|
| 258 |
+
Step 19050/50462 | Loss: 2.6975 | LR: 3.51e-04 | 2735.8ms/step | 21.9 samples/s
|
| 259 |
+
Step 19060/50462 | Loss: 2.6794 | LR: 3.50e-04 | 2743.4ms/step | 21.9 samples/s
|
| 260 |
+
Step 19070/50462 | Loss: 2.6772 | LR: 3.50e-04 | 2725.8ms/step | 22.0 samples/s
|
| 261 |
+
Step 19080/50462 | Loss: 2.6643 | LR: 3.50e-04 | 2726.9ms/step | 22.0 samples/s
|
| 262 |
+
Step 19090/50462 | Loss: 2.6438 | LR: 3.50e-04 | 2734.7ms/step | 21.9 samples/s
|
| 263 |
+
Step 19100/50462 | Loss: 2.6369 | LR: 3.50e-04 | 2726.4ms/step | 22.0 samples/s
|
| 264 |
+
Step 19110/50462 | Loss: 2.6664 | LR: 3.50e-04 | 2731.8ms/step | 22.0 samples/s
|
| 265 |
+
Step 19120/50462 | Loss: 2.6536 | LR: 3.50e-04 | 2730.6ms/step | 22.0 samples/s
|
| 266 |
+
Step 19130/50462 | Loss: 2.6321 | LR: 3.49e-04 | 2724.8ms/step | 22.0 samples/s
|
| 267 |
+
Step 19140/50462 | Loss: 2.6259 | LR: 3.49e-04 | 2732.3ms/step | 22.0 samples/s
|
| 268 |
+
Step 19150/50462 | Loss: 2.6396 | LR: 3.49e-04 | 2728.0ms/step | 22.0 samples/s
|
| 269 |
+
Step 19160/50462 | Loss: 2.6879 | LR: 3.49e-04 | 2732.7ms/step | 22.0 samples/s
|
| 270 |
+
Step 19170/50462 | Loss: 2.6140 | LR: 3.49e-04 | 2744.2ms/step | 21.9 samples/s
|
| 271 |
+
Step 19180/50462 | Loss: 2.6137 | LR: 3.49e-04 | 2735.8ms/step | 21.9 samples/s
|
| 272 |
+
Step 19190/50462 | Loss: 2.6189 | LR: 3.49e-04 | 2736.9ms/step | 21.9 samples/s
|
| 273 |
+
Step 19200/50462 | Loss: 2.6316 | LR: 3.48e-04 | 2732.4ms/step | 22.0 samples/s
|
| 274 |
+
Step 19210/50462 | Loss: 2.6419 | LR: 3.48e-04 | 2730.9ms/step | 22.0 samples/s
|
| 275 |
+
Step 19220/50462 | Loss: 2.6276 | LR: 3.48e-04 | 2728.5ms/step | 22.0 samples/s
|
| 276 |
+
Step 19230/50462 | Loss: 2.6202 | LR: 3.48e-04 | 2734.4ms/step | 21.9 samples/s
|
| 277 |
+
Step 19240/50462 | Loss: 2.6284 | LR: 3.48e-04 | 2732.9ms/step | 22.0 samples/s
|
| 278 |
+
Step 19250/50462 | Loss: 2.6117 | LR: 3.48e-04 | 2739.4ms/step | 21.9 samples/s
|
| 279 |
+
Validation at step 19250: Loss = 2.6510, Perplexity = 14.17
|
| 280 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-19250
|
| 281 |
+
Step 19260/50462 | Loss: 2.6737 | LR: 3.48e-04 | 4093.0ms/step | 14.7 samples/s
|
| 282 |
+
Step 19270/50462 | Loss: 2.6222 | LR: 3.47e-04 | 2733.5ms/step | 21.9 samples/s
|
| 283 |
+
Step 19280/50462 | Loss: 2.6719 | LR: 3.47e-04 | 2726.4ms/step | 22.0 samples/s
|
| 284 |
+
Step 19290/50462 | Loss: 2.6361 | LR: 3.47e-04 | 2728.0ms/step | 22.0 samples/s
|
| 285 |
+
Step 19300/50462 | Loss: 2.6214 | LR: 3.47e-04 | 2740.0ms/step | 21.9 samples/s
|
| 286 |
+
Step 19310/50462 | Loss: 2.6255 | LR: 3.47e-04 | 2736.7ms/step | 21.9 samples/s
|
| 287 |
+
Step 19320/50462 | Loss: 2.6366 | LR: 3.47e-04 | 2734.8ms/step | 21.9 samples/s
|
| 288 |
+
Step 19330/50462 | Loss: 2.6558 | LR: 3.47e-04 | 2733.3ms/step | 22.0 samples/s
|
| 289 |
+
Step 19340/50462 | Loss: 2.6690 | LR: 3.46e-04 | 2745.0ms/step | 21.9 samples/s
|
| 290 |
+
Step 19350/50462 | Loss: 2.6327 | LR: 3.46e-04 | 2739.8ms/step | 21.9 samples/s
|
| 291 |
+
Step 19360/50462 | Loss: 2.6336 | LR: 3.46e-04 | 2737.0ms/step | 21.9 samples/s
|
| 292 |
+
Step 19370/50462 | Loss: 2.7082 | LR: 3.46e-04 | 2740.3ms/step | 21.9 samples/s
|
| 293 |
+
Step 19380/50462 | Loss: 2.6353 | LR: 3.46e-04 | 2768.5ms/step | 21.7 samples/s
|
| 294 |
+
Step 19390/50462 | Loss: 2.6185 | LR: 3.46e-04 | 2757.6ms/step | 21.8 samples/s
|
| 295 |
+
Step 19400/50462 | Loss: 2.6284 | LR: 3.46e-04 | 2755.9ms/step | 21.8 samples/s
|
| 296 |
+
Step 19410/50462 | Loss: 2.6367 | LR: 3.45e-04 | 2747.6ms/step | 21.8 samples/s
|
| 297 |
+
Step 19420/50462 | Loss: 2.6211 | LR: 3.45e-04 | 2759.2ms/step | 21.7 samples/s
|
| 298 |
+
Step 19430/50462 | Loss: 2.6264 | LR: 3.45e-04 | 2751.1ms/step | 21.8 samples/s
|
| 299 |
+
Step 19440/50462 | Loss: 2.6321 | LR: 3.45e-04 | 2747.0ms/step | 21.8 samples/s
|
| 300 |
+
Step 19450/50462 | Loss: 2.6392 | LR: 3.45e-04 | 2747.6ms/step | 21.8 samples/s
|
| 301 |
+
Step 19460/50462 | Loss: 2.6357 | LR: 3.45e-04 | 2753.7ms/step | 21.8 samples/s
|
| 302 |
+
Step 19470/50462 | Loss: 2.6179 | LR: 3.45e-04 | 2758.6ms/step | 21.8 samples/s
|
| 303 |
+
Step 19480/50462 | Loss: 2.6813 | LR: 3.44e-04 | 2744.7ms/step | 21.9 samples/s
|
| 304 |
+
Step 19490/50462 | Loss: 2.6897 | LR: 3.44e-04 | 2745.4ms/step | 21.9 samples/s
|
| 305 |
+
Step 19500/50462 | Loss: 2.6153 | LR: 3.44e-04 | 2750.4ms/step | 21.8 samples/s
|
| 306 |
+
Validation at step 19500: Loss = 2.6464, Perplexity = 14.10
|
| 307 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-19500
|
| 308 |
+
Step 19510/50462 | Loss: 2.6549 | LR: 3.44e-04 | 4187.4ms/step | 14.3 samples/s
|
| 309 |
+
Step 19520/50462 | Loss: 2.5752 | LR: 3.44e-04 | 2740.7ms/step | 21.9 samples/s
|
| 310 |
+
Step 19530/50462 | Loss: 2.6645 | LR: 3.44e-04 | 2757.0ms/step | 21.8 samples/s
|
| 311 |
+
Step 19540/50462 | Loss: 2.6547 | LR: 3.44e-04 | 2750.1ms/step | 21.8 samples/s
|
| 312 |
+
Step 19550/50462 | Loss: 2.6796 | LR: 3.43e-04 | 2741.8ms/step | 21.9 samples/s
|
| 313 |
+
Step 19560/50462 | Loss: 2.6689 | LR: 3.43e-04 | 2744.6ms/step | 21.9 samples/s
|
| 314 |
+
Step 19570/50462 | Loss: 2.6225 | LR: 3.43e-04 | 2740.4ms/step | 21.9 samples/s
|
| 315 |
+
Step 19580/50462 | Loss: 2.6871 | LR: 3.43e-04 | 2736.8ms/step | 21.9 samples/s
|
| 316 |
+
Step 19590/50462 | Loss: 2.6407 | LR: 3.43e-04 | 2736.0ms/step | 21.9 samples/s
|
| 317 |
+
Step 19600/50462 | Loss: 2.6722 | LR: 3.43e-04 | 2743.9ms/step | 21.9 samples/s
|
| 318 |
+
Step 19610/50462 | Loss: 2.6735 | LR: 3.42e-04 | 2737.5ms/step | 21.9 samples/s
|
| 319 |
+
Step 19620/50462 | Loss: 2.6257 | LR: 3.42e-04 | 2740.9ms/step | 21.9 samples/s
|
| 320 |
+
Step 19630/50462 | Loss: 2.6467 | LR: 3.42e-04 | 2740.4ms/step | 21.9 samples/s
|
| 321 |
+
Step 19640/50462 | Loss: 2.6254 | LR: 3.42e-04 | 2749.0ms/step | 21.8 samples/s
|
| 322 |
+
Step 19650/50462 | Loss: 2.6664 | LR: 3.42e-04 | 2745.2ms/step | 21.9 samples/s
|
| 323 |
+
Step 19660/50462 | Loss: 2.6260 | LR: 3.42e-04 | 2744.2ms/step | 21.9 samples/s
|
| 324 |
+
Step 19670/50462 | Loss: 2.6469 | LR: 3.42e-04 | 2729.6ms/step | 22.0 samples/s
|
| 325 |
+
Step 19680/50462 | Loss: 2.6301 | LR: 3.41e-04 | 2741.3ms/step | 21.9 samples/s
|
| 326 |
+
Step 19690/50462 | Loss: 2.6725 | LR: 3.41e-04 | 2744.2ms/step | 21.9 samples/s
|
| 327 |
+
Step 19700/50462 | Loss: 2.6481 | LR: 3.41e-04 | 2750.0ms/step | 21.8 samples/s
|
| 328 |
+
Step 19710/50462 | Loss: 2.6438 | LR: 3.41e-04 | 2753.0ms/step | 21.8 samples/s
|
| 329 |
+
Step 19720/50462 | Loss: 2.6878 | LR: 3.41e-04 | 2743.8ms/step | 21.9 samples/s
|
| 330 |
+
Step 19730/50462 | Loss: 2.6475 | LR: 3.41e-04 | 2734.0ms/step | 21.9 samples/s
|
| 331 |
+
Step 19740/50462 | Loss: 2.6744 | LR: 3.41e-04 | 2743.0ms/step | 21.9 samples/s
|
| 332 |
+
Step 19750/50462 | Loss: 2.6847 | LR: 3.40e-04 | 2740.9ms/step | 21.9 samples/s
|
| 333 |
+
Validation at step 19750: Loss = 2.6442, Perplexity = 14.07
|
| 334 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-19750
|
| 335 |
+
Step 19760/50462 | Loss: 2.6104 | LR: 3.40e-04 | 4156.0ms/step | 14.4 samples/s
|
| 336 |
+
Step 19770/50462 | Loss: 2.6588 | LR: 3.40e-04 | 2748.4ms/step | 21.8 samples/s
|
| 337 |
+
Step 19780/50462 | Loss: 2.6679 | LR: 3.40e-04 | 2751.8ms/step | 21.8 samples/s
|
| 338 |
+
Step 19790/50462 | Loss: 2.6127 | LR: 3.40e-04 | 2738.3ms/step | 21.9 samples/s
|
| 339 |
+
Step 19800/50462 | Loss: 2.6541 | LR: 3.40e-04 | 2738.5ms/step | 21.9 samples/s
|
| 340 |
+
Step 19810/50462 | Loss: 2.6461 | LR: 3.40e-04 | 2746.5ms/step | 21.8 samples/s
|
| 341 |
+
Step 19820/50462 | Loss: 2.6442 | LR: 3.39e-04 | 2741.2ms/step | 21.9 samples/s
|
| 342 |
+
Step 19830/50462 | Loss: 2.6599 | LR: 3.39e-04 | 2740.8ms/step | 21.9 samples/s
|
| 343 |
+
Step 19840/50462 | Loss: 2.6507 | LR: 3.39e-04 | 2744.8ms/step | 21.9 samples/s
|
| 344 |
+
Step 19850/50462 | Loss: 2.6332 | LR: 3.39e-04 | 2748.7ms/step | 21.8 samples/s
|
| 345 |
+
Step 19860/50462 | Loss: 2.6273 | LR: 3.39e-04 | 2744.8ms/step | 21.9 samples/s
|
| 346 |
+
Step 19870/50462 | Loss: 2.6322 | LR: 3.39e-04 | 2738.2ms/step | 21.9 samples/s
|
| 347 |
+
Step 19880/50462 | Loss: 2.6419 | LR: 3.39e-04 | 2746.5ms/step | 21.8 samples/s
|
| 348 |
+
Step 19890/50462 | Loss: 2.6561 | LR: 3.38e-04 | 2745.8ms/step | 21.9 samples/s
|
| 349 |
+
Step 19900/50462 | Loss: 2.6247 | LR: 3.38e-04 | 2737.8ms/step | 21.9 samples/s
|
| 350 |
+
Step 19910/50462 | Loss: 2.6649 | LR: 3.38e-04 | 2739.0ms/step | 21.9 samples/s
|
| 351 |
+
Step 19920/50462 | Loss: 2.6307 | LR: 3.38e-04 | 2744.8ms/step | 21.9 samples/s
|
| 352 |
+
Step 19930/50462 | Loss: 2.6360 | LR: 3.38e-04 | 2752.0ms/step | 21.8 samples/s
|
| 353 |
+
Step 19940/50462 | Loss: 2.6153 | LR: 3.38e-04 | 2755.8ms/step | 21.8 samples/s
|
| 354 |
+
Step 19950/50462 | Loss: 2.6410 | LR: 3.37e-04 | 2743.0ms/step | 21.9 samples/s
|
| 355 |
+
Step 19960/50462 | Loss: 2.6531 | LR: 3.37e-04 | 2737.4ms/step | 21.9 samples/s
|
| 356 |
+
Step 19970/50462 | Loss: 2.6188 | LR: 3.37e-04 | 2748.7ms/step | 21.8 samples/s
|
| 357 |
+
Step 19980/50462 | Loss: 2.6537 | LR: 3.37e-04 | 2748.8ms/step | 21.8 samples/s
|
| 358 |
+
Step 19990/50462 | Loss: 2.6197 | LR: 3.37e-04 | 2742.3ms/step | 21.9 samples/s
|
| 359 |
+
Step 20000/50462 | Loss: 2.6187 | LR: 3.37e-04 | 2735.7ms/step | 21.9 samples/s
|
| 360 |
+
Validation at step 20000: Loss = 2.6416, Perplexity = 14.04
|
| 361 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-20000
|
| 362 |
+
Step 20010/50462 | Loss: 2.6349 | LR: 3.37e-04 | 4124.5ms/step | 14.5 samples/s
|
| 363 |
+
Step 20020/50462 | Loss: 2.6578 | LR: 3.36e-04 | 2741.0ms/step | 21.9 samples/s
|
| 364 |
+
Step 20030/50462 | Loss: 2.6743 | LR: 3.36e-04 | 2747.1ms/step | 21.8 samples/s
|
| 365 |
+
Step 20040/50462 | Loss: 2.6234 | LR: 3.36e-04 | 2745.5ms/step | 21.9 samples/s
|
| 366 |
+
Step 20050/50462 | Loss: 2.6694 | LR: 3.36e-04 | 2738.8ms/step | 21.9 samples/s
|
| 367 |
+
Step 20060/50462 | Loss: 2.6619 | LR: 3.36e-04 | 2744.1ms/step | 21.9 samples/s
|
| 368 |
+
Step 20070/50462 | Loss: 2.6483 | LR: 3.36e-04 | 2747.5ms/step | 21.8 samples/s
|
| 369 |
+
Step 20080/50462 | Loss: 2.6292 | LR: 3.36e-04 | 2745.9ms/step | 21.9 samples/s
|
| 370 |
+
Step 20090/50462 | Loss: 2.6646 | LR: 3.35e-04 | 2745.6ms/step | 21.9 samples/s
|
| 371 |
+
Step 20100/50462 | Loss: 2.6342 | LR: 3.35e-04 | 2744.1ms/step | 21.9 samples/s
|
| 372 |
+
Step 20110/50462 | Loss: 2.6467 | LR: 3.35e-04 | 2735.0ms/step | 21.9 samples/s
|
| 373 |
+
Step 20120/50462 | Loss: 2.6245 | LR: 3.35e-04 | 2748.6ms/step | 21.8 samples/s
|
| 374 |
+
Step 20130/50462 | Loss: 2.6186 | LR: 3.35e-04 | 2743.1ms/step | 21.9 samples/s
|
| 375 |
+
Step 20140/50462 | Loss: 2.6173 | LR: 3.35e-04 | 2749.7ms/step | 21.8 samples/s
|
| 376 |
+
Step 20150/50462 | Loss: 2.6647 | LR: 3.35e-04 | 2752.9ms/step | 21.8 samples/s
|
| 377 |
+
Step 20160/50462 | Loss: 2.5865 | LR: 3.34e-04 | 2742.4ms/step | 21.9 samples/s
|
| 378 |
+
Step 20170/50462 | Loss: 2.6214 | LR: 3.34e-04 | 2744.4ms/step | 21.9 samples/s
|
| 379 |
+
Step 20180/50462 | Loss: 2.5806 | LR: 3.34e-04 | 2749.6ms/step | 21.8 samples/s
|
| 380 |
+
Step 20190/50462 | Loss: 2.6364 | LR: 3.34e-04 | 2743.0ms/step | 21.9 samples/s
|
| 381 |
+
Step 20200/50462 | Loss: 2.6392 | LR: 3.34e-04 | 2746.1ms/step | 21.8 samples/s
|
| 382 |
+
Step 20210/50462 | Loss: 2.6154 | LR: 3.34e-04 | 2756.0ms/step | 21.8 samples/s
|
| 383 |
+
Step 20220/50462 | Loss: 2.6140 | LR: 3.33e-04 | 2739.5ms/step | 21.9 samples/s
|
| 384 |
+
Step 20230/50462 | Loss: 2.6760 | LR: 3.33e-04 | 2743.0ms/step | 21.9 samples/s
|
| 385 |
+
Step 20240/50462 | Loss: 2.6766 | LR: 3.33e-04 | 2734.0ms/step | 21.9 samples/s
|
| 386 |
+
Step 20250/50462 | Loss: 2.6327 | LR: 3.33e-04 | 2746.3ms/step | 21.8 samples/s
|
| 387 |
+
Validation at step 20250: Loss = 2.6395, Perplexity = 14.01
|
| 388 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-20250
|
| 389 |
+
Step 20260/50462 | Loss: 2.6432 | LR: 3.33e-04 | 4112.8ms/step | 14.6 samples/s
|
| 390 |
+
Step 20270/50462 | Loss: 2.6299 | LR: 3.33e-04 | 2748.3ms/step | 21.8 samples/s
|
| 391 |
+
Step 20280/50462 | Loss: 2.5800 | LR: 3.33e-04 | 2746.0ms/step | 21.9 samples/s
|
| 392 |
+
Step 20290/50462 | Loss: 2.6297 | LR: 3.32e-04 | 2745.5ms/step | 21.9 samples/s
|
| 393 |
+
Step 20300/50462 | Loss: 2.6427 | LR: 3.32e-04 | 2740.1ms/step | 21.9 samples/s
|
| 394 |
+
Step 20310/50462 | Loss: 2.6623 | LR: 3.32e-04 | 2744.4ms/step | 21.9 samples/s
|
| 395 |
+
Step 20320/50462 | Loss: 2.6565 | LR: 3.32e-04 | 2750.3ms/step | 21.8 samples/s
|
| 396 |
+
Step 20330/50462 | Loss: 2.6644 | LR: 3.32e-04 | 2749.8ms/step | 21.8 samples/s
|
| 397 |
+
Step 20340/50462 | Loss: 2.6502 | LR: 3.32e-04 | 2748.5ms/step | 21.8 samples/s
|
| 398 |
+
Step 20350/50462 | Loss: 2.6573 | LR: 3.32e-04 | 2749.2ms/step | 21.8 samples/s
|
| 399 |
+
Step 20360/50462 | Loss: 2.6312 | LR: 3.31e-04 | 2743.9ms/step | 21.9 samples/s
|
| 400 |
+
Step 20370/50462 | Loss: 2.6345 | LR: 3.31e-04 | 2747.3ms/step | 21.8 samples/s
|
| 401 |
+
Step 20380/50462 | Loss: 2.6714 | LR: 3.31e-04 | 2755.9ms/step | 21.8 samples/s
|
| 402 |
+
Step 20390/50462 | Loss: 2.6111 | LR: 3.31e-04 | 2746.9ms/step | 21.8 samples/s
|
| 403 |
+
Step 20400/50462 | Loss: 2.6295 | LR: 3.31e-04 | 2745.4ms/step | 21.9 samples/s
|
| 404 |
+
Step 20410/50462 | Loss: 2.6130 | LR: 3.31e-04 | 2750.8ms/step | 21.8 samples/s
|
| 405 |
+
Step 20420/50462 | Loss: 2.6510 | LR: 3.30e-04 | 2745.5ms/step | 21.9 samples/s
|
| 406 |
+
Step 20430/50462 | Loss: 2.6519 | LR: 3.30e-04 | 2739.8ms/step | 21.9 samples/s
|
| 407 |
+
Step 20440/50462 | Loss: 2.5872 | LR: 3.30e-04 | 2750.8ms/step | 21.8 samples/s
|
| 408 |
+
Step 20450/50462 | Loss: 2.6059 | LR: 3.30e-04 | 2745.1ms/step | 21.9 samples/s
|
| 409 |
+
Step 20460/50462 | Loss: 2.6290 | LR: 3.30e-04 | 2737.4ms/step | 21.9 samples/s
|
| 410 |
+
Step 20470/50462 | Loss: 2.6615 | LR: 3.30e-04 | 2747.6ms/step | 21.8 samples/s
|
| 411 |
+
Step 20480/50462 | Loss: 2.6529 | LR: 3.30e-04 | 2751.9ms/step | 21.8 samples/s
|
| 412 |
+
Step 20490/50462 | Loss: 2.6639 | LR: 3.29e-04 | 2751.0ms/step | 21.8 samples/s
|
| 413 |
+
Step 20500/50462 | Loss: 2.6275 | LR: 3.29e-04 | 2752.1ms/step | 21.8 samples/s
|
| 414 |
+
Validation at step 20500: Loss = 2.6379, Perplexity = 13.98
|
| 415 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-20500
|
| 416 |
+
Step 20510/50462 | Loss: 2.6431 | LR: 3.29e-04 | 4139.4ms/step | 14.5 samples/s
|
| 417 |
+
Step 20520/50462 | Loss: 2.6414 | LR: 3.29e-04 | 2752.5ms/step | 21.8 samples/s
|
| 418 |
+
Step 20530/50462 | Loss: 2.6104 | LR: 3.29e-04 | 2748.6ms/step | 21.8 samples/s
|
| 419 |
+
Step 20540/50462 | Loss: 2.6352 | LR: 3.29e-04 | 2750.5ms/step | 21.8 samples/s
|
| 420 |
+
Step 20550/50462 | Loss: 2.6229 | LR: 3.29e-04 | 2747.2ms/step | 21.8 samples/s
|
| 421 |
+
Step 20560/50462 | Loss: 2.6329 | LR: 3.28e-04 | 2743.5ms/step | 21.9 samples/s
|
| 422 |
+
Step 20570/50462 | Loss: 2.6224 | LR: 3.28e-04 | 2747.9ms/step | 21.8 samples/s
|
| 423 |
+
Step 20580/50462 | Loss: 2.6400 | LR: 3.28e-04 | 2743.1ms/step | 21.9 samples/s
|
| 424 |
+
Step 20590/50462 | Loss: 2.6448 | LR: 3.28e-04 | 2746.8ms/step | 21.8 samples/s
|
| 425 |
+
Step 20600/50462 | Loss: 2.6504 | LR: 3.28e-04 | 2743.1ms/step | 21.9 samples/s
|
| 426 |
+
Step 20610/50462 | Loss: 2.6678 | LR: 3.28e-04 | 2753.0ms/step | 21.8 samples/s
|
| 427 |
+
Step 20620/50462 | Loss: 2.6032 | LR: 3.27e-04 | 2749.1ms/step | 21.8 samples/s
|
| 428 |
+
Step 20630/50462 | Loss: 2.6442 | LR: 3.27e-04 | 2745.8ms/step | 21.9 samples/s
|
| 429 |
+
Step 20640/50462 | Loss: 2.6657 | LR: 3.27e-04 | 2739.0ms/step | 21.9 samples/s
|
| 430 |
+
Step 20650/50462 | Loss: 2.6376 | LR: 3.27e-04 | 2746.1ms/step | 21.8 samples/s
|
| 431 |
+
Step 20660/50462 | Loss: 2.6566 | LR: 3.27e-04 | 2743.4ms/step | 21.9 samples/s
|
| 432 |
+
Step 20670/50462 | Loss: 2.6494 | LR: 3.27e-04 | 2756.7ms/step | 21.8 samples/s
|
| 433 |
+
Step 20680/50462 | Loss: 2.6050 | LR: 3.27e-04 | 2744.9ms/step | 21.9 samples/s
|
| 434 |
+
Step 20690/50462 | Loss: 2.6466 | LR: 3.26e-04 | 2746.5ms/step | 21.8 samples/s
|
| 435 |
+
Step 20700/50462 | Loss: 2.6591 | LR: 3.26e-04 | 2745.4ms/step | 21.9 samples/s
|
| 436 |
+
Step 20710/50462 | Loss: 2.6571 | LR: 3.26e-04 | 2742.9ms/step | 21.9 samples/s
|
| 437 |
+
Step 20720/50462 | Loss: 2.6123 | LR: 3.26e-04 | 2742.4ms/step | 21.9 samples/s
|
| 438 |
+
Step 20730/50462 | Loss: 2.6722 | LR: 3.26e-04 | 2739.9ms/step | 21.9 samples/s
|
| 439 |
+
Step 20740/50462 | Loss: 2.6118 | LR: 3.26e-04 | 2741.5ms/step | 21.9 samples/s
|
| 440 |
+
Step 20750/50462 | Loss: 2.6206 | LR: 3.26e-04 | 2759.7ms/step | 21.7 samples/s
|
| 441 |
+
Validation at step 20750: Loss = 2.6356, Perplexity = 13.95
|
| 442 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-20750
|
| 443 |
+
Step 20760/50462 | Loss: 2.6315 | LR: 3.25e-04 | 4129.2ms/step | 14.5 samples/s
|
| 444 |
+
Step 20770/50462 | Loss: 2.6317 | LR: 3.25e-04 | 2741.4ms/step | 21.9 samples/s
|
| 445 |
+
Step 20780/50462 | Loss: 2.6625 | LR: 3.25e-04 | 2736.2ms/step | 21.9 samples/s
|
| 446 |
+
Step 20790/50462 | Loss: 2.6452 | LR: 3.25e-04 | 2745.4ms/step | 21.9 samples/s
|
| 447 |
+
Step 20800/50462 | Loss: 2.6462 | LR: 3.25e-04 | 2740.5ms/step | 21.9 samples/s
|
| 448 |
+
Step 20810/50462 | Loss: 2.5724 | LR: 3.25e-04 | 2747.5ms/step | 21.8 samples/s
|
| 449 |
+
Step 20820/50462 | Loss: 2.6278 | LR: 3.24e-04 | 2741.8ms/step | 21.9 samples/s
|
| 450 |
+
Step 20830/50462 | Loss: 2.6719 | LR: 3.24e-04 | 2741.1ms/step | 21.9 samples/s
|
| 451 |
+
Step 20840/50462 | Loss: 2.6066 | LR: 3.24e-04 | 2747.6ms/step | 21.8 samples/s
|
| 452 |
+
Step 20850/50462 | Loss: 2.6700 | LR: 3.24e-04 | 2743.0ms/step | 21.9 samples/s
|
| 453 |
+
Step 20860/50462 | Loss: 2.6401 | LR: 3.24e-04 | 2748.8ms/step | 21.8 samples/s
|
| 454 |
+
Step 20870/50462 | Loss: 2.6319 | LR: 3.24e-04 | 2751.7ms/step | 21.8 samples/s
|
| 455 |
+
Step 20880/50462 | Loss: 2.6221 | LR: 3.24e-04 | 2737.9ms/step | 21.9 samples/s
|
| 456 |
+
Step 20890/50462 | Loss: 2.6450 | LR: 3.23e-04 | 2744.3ms/step | 21.9 samples/s
|
| 457 |
+
Step 20900/50462 | Loss: 2.6214 | LR: 3.23e-04 | 2746.5ms/step | 21.8 samples/s
|
| 458 |
+
Step 20910/50462 | Loss: 2.6581 | LR: 3.23e-04 | 2747.4ms/step | 21.8 samples/s
|
| 459 |
+
Step 20920/50462 | Loss: 2.6554 | LR: 3.23e-04 | 2748.7ms/step | 21.8 samples/s
|
| 460 |
+
Step 20930/50462 | Loss: 2.6590 | LR: 3.23e-04 | 2746.5ms/step | 21.8 samples/s
|
| 461 |
+
Step 20940/50462 | Loss: 2.5726 | LR: 3.23e-04 | 2745.9ms/step | 21.9 samples/s
|
| 462 |
+
Step 20950/50462 | Loss: 2.6213 | LR: 3.23e-04 | 2745.3ms/step | 21.9 samples/s
|
| 463 |
+
Step 20960/50462 | Loss: 2.6527 | LR: 3.22e-04 | 2737.3ms/step | 21.9 samples/s
|
| 464 |
+
Step 20970/50462 | Loss: 2.6566 | LR: 3.22e-04 | 2747.2ms/step | 21.8 samples/s
|
| 465 |
+
Step 20980/50462 | Loss: 2.6400 | LR: 3.22e-04 | 2753.4ms/step | 21.8 samples/s
|
| 466 |
+
Step 20990/50462 | Loss: 2.6241 | LR: 3.22e-04 | 2746.4ms/step | 21.8 samples/s
|
| 467 |
+
Step 21000/50462 | Loss: 2.6749 | LR: 3.22e-04 | 2741.1ms/step | 21.9 samples/s
|
| 468 |
+
Validation at step 21000: Loss = 2.6333, Perplexity = 13.92
|
| 469 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-21000
|
| 470 |
+
Step 21010/50462 | Loss: 2.6459 | LR: 3.22e-04 | 4151.9ms/step | 14.5 samples/s
|
| 471 |
+
Step 21020/50462 | Loss: 2.6339 | LR: 3.21e-04 | 2739.5ms/step | 21.9 samples/s
|
| 472 |
+
Step 21030/50462 | Loss: 2.6527 | LR: 3.21e-04 | 2753.2ms/step | 21.8 samples/s
|
| 473 |
+
Step 21040/50462 | Loss: 2.6239 | LR: 3.21e-04 | 2744.8ms/step | 21.9 samples/s
|
| 474 |
+
Step 21050/50462 | Loss: 2.6492 | LR: 3.21e-04 | 2746.8ms/step | 21.8 samples/s
|
| 475 |
+
Step 21060/50462 | Loss: 2.6417 | LR: 3.21e-04 | 2747.8ms/step | 21.8 samples/s
|
| 476 |
+
Step 21070/50462 | Loss: 2.6265 | LR: 3.21e-04 | 2742.7ms/step | 21.9 samples/s
|
| 477 |
+
Step 21080/50462 | Loss: 2.6315 | LR: 3.21e-04 | 2741.2ms/step | 21.9 samples/s
|
| 478 |
+
Step 21090/50462 | Loss: 2.6467 | LR: 3.20e-04 | 2742.3ms/step | 21.9 samples/s
|
| 479 |
+
Step 21100/50462 | Loss: 2.6455 | LR: 3.20e-04 | 2754.3ms/step | 21.8 samples/s
|
| 480 |
+
Step 21110/50462 | Loss: 2.5656 | LR: 3.20e-04 | 2750.3ms/step | 21.8 samples/s
|
| 481 |
+
Step 21120/50462 | Loss: 2.6083 | LR: 3.20e-04 | 2738.9ms/step | 21.9 samples/s
|
| 482 |
+
Step 21130/50462 | Loss: 2.6556 | LR: 3.20e-04 | 2745.0ms/step | 21.9 samples/s
|
| 483 |
+
Step 21140/50462 | Loss: 2.6538 | LR: 3.20e-04 | 2741.8ms/step | 21.9 samples/s
|
| 484 |
+
Step 21150/50462 | Loss: 2.6317 | LR: 3.19e-04 | 2735.7ms/step | 21.9 samples/s
|
| 485 |
+
Step 21160/50462 | Loss: 2.5922 | LR: 3.19e-04 | 2750.2ms/step | 21.8 samples/s
|
| 486 |
+
Step 21170/50462 | Loss: 2.6330 | LR: 3.19e-04 | 2740.3ms/step | 21.9 samples/s
|
| 487 |
+
Step 21180/50462 | Loss: 2.6665 | LR: 3.19e-04 | 2741.7ms/step | 21.9 samples/s
|
| 488 |
+
Step 21190/50462 | Loss: 2.6266 | LR: 3.19e-04 | 2745.6ms/step | 21.9 samples/s
|
| 489 |
+
Step 21200/50462 | Loss: 2.6091 | LR: 3.19e-04 | 2740.1ms/step | 21.9 samples/s
|
| 490 |
+
Step 21210/50462 | Loss: 2.6392 | LR: 3.19e-04 | 2742.6ms/step | 21.9 samples/s
|
| 491 |
+
Step 21220/50462 | Loss: 2.6085 | LR: 3.18e-04 | 2745.5ms/step | 21.9 samples/s
|
| 492 |
+
Step 21230/50462 | Loss: 2.6369 | LR: 3.18e-04 | 2735.1ms/step | 21.9 samples/s
|
| 493 |
+
Step 21240/50462 | Loss: 2.6343 | LR: 3.18e-04 | 2739.6ms/step | 21.9 samples/s
|
| 494 |
+
Step 21250/50462 | Loss: 2.6423 | LR: 3.18e-04 | 2740.1ms/step | 21.9 samples/s
|
| 495 |
+
Validation at step 21250: Loss = 2.6301, Perplexity = 13.87
|
| 496 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-21250
|
| 497 |
+
Step 21260/50462 | Loss: 2.6793 | LR: 3.18e-04 | 4130.8ms/step | 14.5 samples/s
|
| 498 |
+
Step 21270/50462 | Loss: 2.6500 | LR: 3.18e-04 | 2739.7ms/step | 21.9 samples/s
|
| 499 |
+
Step 21280/50462 | Loss: 2.6139 | LR: 3.17e-04 | 2746.8ms/step | 21.8 samples/s
|
| 500 |
+
Step 21290/50462 | Loss: 2.6811 | LR: 3.17e-04 | 2747.2ms/step | 21.8 samples/s
|
| 501 |
+
Step 21300/50462 | Loss: 2.6074 | LR: 3.17e-04 | 2734.3ms/step | 21.9 samples/s
|
| 502 |
+
Step 21310/50462 | Loss: 2.6130 | LR: 3.17e-04 | 2748.2ms/step | 21.8 samples/s
|
| 503 |
+
Step 21320/50462 | Loss: 2.6454 | LR: 3.17e-04 | 2735.1ms/step | 21.9 samples/s
|
| 504 |
+
Step 21330/50462 | Loss: 2.6593 | LR: 3.17e-04 | 2749.4ms/step | 21.8 samples/s
|
| 505 |
+
Step 21340/50462 | Loss: 2.6823 | LR: 3.17e-04 | 2747.6ms/step | 21.8 samples/s
|
| 506 |
+
Step 21350/50462 | Loss: 2.6067 | LR: 3.16e-04 | 2750.6ms/step | 21.8 samples/s
|
| 507 |
+
Step 21360/50462 | Loss: 2.5933 | LR: 3.16e-04 | 2735.8ms/step | 21.9 samples/s
|
| 508 |
+
Step 21370/50462 | Loss: 2.6192 | LR: 3.16e-04 | 2744.1ms/step | 21.9 samples/s
|
| 509 |
+
Step 21380/50462 | Loss: 2.6514 | LR: 3.16e-04 | 2742.7ms/step | 21.9 samples/s
|
| 510 |
+
Step 21390/50462 | Loss: 2.6398 | LR: 3.16e-04 | 2744.1ms/step | 21.9 samples/s
|
| 511 |
+
Step 21400/50462 | Loss: 2.6140 | LR: 3.16e-04 | 2742.3ms/step | 21.9 samples/s
|
| 512 |
+
Step 21410/50462 | Loss: 2.6531 | LR: 3.16e-04 | 2741.5ms/step | 21.9 samples/s
|
| 513 |
+
Step 21420/50462 | Loss: 2.5842 | LR: 3.15e-04 | 2750.8ms/step | 21.8 samples/s
|
| 514 |
+
Step 21430/50462 | Loss: 2.5973 | LR: 3.15e-04 | 2740.2ms/step | 21.9 samples/s
|
| 515 |
+
Step 21440/50462 | Loss: 2.6398 | LR: 3.15e-04 | 2737.9ms/step | 21.9 samples/s
|
| 516 |
+
Step 21450/50462 | Loss: 2.5989 | LR: 3.15e-04 | 2747.9ms/step | 21.8 samples/s
|
| 517 |
+
Step 21460/50462 | Loss: 2.6326 | LR: 3.15e-04 | 2747.4ms/step | 21.8 samples/s
|
| 518 |
+
Step 21470/50462 | Loss: 2.6256 | LR: 3.15e-04 | 2754.6ms/step | 21.8 samples/s
|
| 519 |
+
Step 21480/50462 | Loss: 2.6065 | LR: 3.14e-04 | 2745.8ms/step | 21.9 samples/s
|
| 520 |
+
Step 21490/50462 | Loss: 2.6349 | LR: 3.14e-04 | 2745.8ms/step | 21.9 samples/s
|
| 521 |
+
Step 21500/50462 | Loss: 2.6193 | LR: 3.14e-04 | 2742.2ms/step | 21.9 samples/s
|
| 522 |
+
Validation at step 21500: Loss = 2.6269, Perplexity = 13.83
|
| 523 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-21500
|
| 524 |
+
Step 21510/50462 | Loss: 2.6076 | LR: 3.14e-04 | 4137.9ms/step | 14.5 samples/s
|
| 525 |
+
Step 21520/50462 | Loss: 2.6034 | LR: 3.14e-04 | 2732.0ms/step | 22.0 samples/s
|
| 526 |
+
Step 21530/50462 | Loss: 2.6098 | LR: 3.14e-04 | 2732.0ms/step | 22.0 samples/s
|
| 527 |
+
Step 21540/50462 | Loss: 2.6340 | LR: 3.14e-04 | 2752.3ms/step | 21.8 samples/s
|
| 528 |
+
Step 21550/50462 | Loss: 2.5621 | LR: 3.13e-04 | 2740.7ms/step | 21.9 samples/s
|
| 529 |
+
Step 21560/50462 | Loss: 2.6294 | LR: 3.13e-04 | 2738.9ms/step | 21.9 samples/s
|
| 530 |
+
Step 21570/50462 | Loss: 2.6834 | LR: 3.13e-04 | 2749.0ms/step | 21.8 samples/s
|
| 531 |
+
Step 21580/50462 | Loss: 2.6112 | LR: 3.13e-04 | 2739.6ms/step | 21.9 samples/s
|
| 532 |
+
Step 21590/50462 | Loss: 2.6126 | LR: 3.13e-04 | 2743.7ms/step | 21.9 samples/s
|
| 533 |
+
Step 21600/50462 | Loss: 2.6056 | LR: 3.13e-04 | 2752.2ms/step | 21.8 samples/s
|
| 534 |
+
Step 21610/50462 | Loss: 2.6146 | LR: 3.12e-04 | 2750.3ms/step | 21.8 samples/s
|
| 535 |
+
Step 21620/50462 | Loss: 2.6243 | LR: 3.12e-04 | 2745.7ms/step | 21.9 samples/s
|
| 536 |
+
Step 21630/50462 | Loss: 2.6064 | LR: 3.12e-04 | 2737.9ms/step | 21.9 samples/s
|
| 537 |
+
Step 21640/50462 | Loss: 2.6573 | LR: 3.12e-04 | 2746.1ms/step | 21.8 samples/s
|
| 538 |
+
Step 21650/50462 | Loss: 2.6310 | LR: 3.12e-04 | 2747.0ms/step | 21.8 samples/s
|
| 539 |
+
Step 21660/50462 | Loss: 2.6704 | LR: 3.12e-04 | 2741.1ms/step | 21.9 samples/s
|
| 540 |
+
Step 21670/50462 | Loss: 2.5794 | LR: 3.12e-04 | 2739.7ms/step | 21.9 samples/s
|
| 541 |
+
Step 21680/50462 | Loss: 2.7129 | LR: 3.11e-04 | 2737.4ms/step | 21.9 samples/s
|
| 542 |
+
Step 21690/50462 | Loss: 2.6355 | LR: 3.11e-04 | 2743.4ms/step | 21.9 samples/s
|
| 543 |
+
Step 21700/50462 | Loss: 2.6279 | LR: 3.11e-04 | 2741.5ms/step | 21.9 samples/s
|
| 544 |
+
Step 21710/50462 | Loss: 2.6621 | LR: 3.11e-04 | 2745.5ms/step | 21.9 samples/s
|
| 545 |
+
Step 21720/50462 | Loss: 2.6081 | LR: 3.11e-04 | 2746.8ms/step | 21.8 samples/s
|
| 546 |
+
Step 21730/50462 | Loss: 2.6302 | LR: 3.11e-04 | 2740.8ms/step | 21.9 samples/s
|
| 547 |
+
Step 21740/50462 | Loss: 2.5968 | LR: 3.10e-04 | 2748.9ms/step | 21.8 samples/s
|
| 548 |
+
Step 21750/50462 | Loss: 2.6490 | LR: 3.10e-04 | 2748.5ms/step | 21.8 samples/s
|
| 549 |
+
Validation at step 21750: Loss = 2.6235, Perplexity = 13.78
|
| 550 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-21750
|
| 551 |
+
Step 21760/50462 | Loss: 2.6053 | LR: 3.10e-04 | 4136.8ms/step | 14.5 samples/s
|
| 552 |
+
Step 21770/50462 | Loss: 2.6212 | LR: 3.10e-04 | 2740.0ms/step | 21.9 samples/s
|
| 553 |
+
Step 21780/50462 | Loss: 2.6365 | LR: 3.10e-04 | 2739.4ms/step | 21.9 samples/s
|
| 554 |
+
Step 21790/50462 | Loss: 2.6033 | LR: 3.10e-04 | 2747.3ms/step | 21.8 samples/s
|
| 555 |
+
Step 21800/50462 | Loss: 2.6104 | LR: 3.10e-04 | 2741.3ms/step | 21.9 samples/s
|
| 556 |
+
Step 21810/50462 | Loss: 2.6218 | LR: 3.09e-04 | 2744.7ms/step | 21.9 samples/s
|
| 557 |
+
Step 21820/50462 | Loss: 2.5914 | LR: 3.09e-04 | 2739.3ms/step | 21.9 samples/s
|
| 558 |
+
Step 21830/50462 | Loss: 2.5989 | LR: 3.09e-04 | 2748.1ms/step | 21.8 samples/s
|
| 559 |
+
Step 21840/50462 | Loss: 2.6581 | LR: 3.09e-04 | 2746.0ms/step | 21.9 samples/s
|
| 560 |
+
Step 21850/50462 | Loss: 2.5708 | LR: 3.09e-04 | 2734.1ms/step | 21.9 samples/s
|
| 561 |
+
Step 21860/50462 | Loss: 2.6033 | LR: 3.09e-04 | 2744.0ms/step | 21.9 samples/s
|
| 562 |
+
Step 21870/50462 | Loss: 2.6187 | LR: 3.08e-04 | 2742.4ms/step | 21.9 samples/s
|
| 563 |
+
Step 21880/50462 | Loss: 2.6382 | LR: 3.08e-04 | 2740.4ms/step | 21.9 samples/s
|
| 564 |
+
Step 21890/50462 | Loss: 2.6377 | LR: 3.08e-04 | 2742.5ms/step | 21.9 samples/s
|
| 565 |
+
Step 21900/50462 | Loss: 2.6030 | LR: 3.08e-04 | 2747.2ms/step | 21.8 samples/s
|
| 566 |
+
Step 21910/50462 | Loss: 2.5579 | LR: 3.08e-04 | 2740.1ms/step | 21.9 samples/s
|
| 567 |
+
Step 21920/50462 | Loss: 2.6545 | LR: 3.08e-04 | 2746.3ms/step | 21.8 samples/s
|
| 568 |
+
Step 21930/50462 | Loss: 2.6085 | LR: 3.08e-04 | 2751.5ms/step | 21.8 samples/s
|
| 569 |
+
Step 21940/50462 | Loss: 2.6217 | LR: 3.07e-04 | 2741.7ms/step | 21.9 samples/s
|
| 570 |
+
Step 21950/50462 | Loss: 2.6008 | LR: 3.07e-04 | 2737.2ms/step | 21.9 samples/s
|
| 571 |
+
Step 21960/50462 | Loss: 2.5927 | LR: 3.07e-04 | 2739.9ms/step | 21.9 samples/s
|
| 572 |
+
Step 21970/50462 | Loss: 2.6327 | LR: 3.07e-04 | 2742.3ms/step | 21.9 samples/s
|
| 573 |
+
Step 21980/50462 | Loss: 2.6524 | LR: 3.07e-04 | 2745.1ms/step | 21.9 samples/s
|
| 574 |
+
Step 21990/50462 | Loss: 2.6330 | LR: 3.07e-04 | 2737.4ms/step | 21.9 samples/s
|
| 575 |
+
Step 22000/50462 | Loss: 2.6035 | LR: 3.06e-04 | 2738.6ms/step | 21.9 samples/s
|
| 576 |
+
Validation at step 22000: Loss = 2.6218, Perplexity = 13.76
|
| 577 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-22000
|
| 578 |
+
Step 22010/50462 | Loss: 2.6054 | LR: 3.06e-04 | 4142.0ms/step | 14.5 samples/s
|
| 579 |
+
Step 22020/50462 | Loss: 2.6393 | LR: 3.06e-04 | 2744.8ms/step | 21.9 samples/s
|
| 580 |
+
Step 22030/50462 | Loss: 2.5999 | LR: 3.06e-04 | 2739.8ms/step | 21.9 samples/s
|
| 581 |
+
Step 22040/50462 | Loss: 2.6181 | LR: 3.06e-04 | 2748.9ms/step | 21.8 samples/s
|
| 582 |
+
Step 22050/50462 | Loss: 2.6324 | LR: 3.06e-04 | 2740.4ms/step | 21.9 samples/s
|
| 583 |
+
Step 22060/50462 | Loss: 2.5881 | LR: 3.06e-04 | 2744.3ms/step | 21.9 samples/s
|
| 584 |
+
Step 22070/50462 | Loss: 2.6568 | LR: 3.05e-04 | 2740.0ms/step | 21.9 samples/s
|
| 585 |
+
Step 22080/50462 | Loss: 2.6082 | LR: 3.05e-04 | 2740.1ms/step | 21.9 samples/s
|
| 586 |
+
Step 22090/50462 | Loss: 2.6628 | LR: 3.05e-04 | 2741.3ms/step | 21.9 samples/s
|
| 587 |
+
Step 22100/50462 | Loss: 2.5891 | LR: 3.05e-04 | 2744.4ms/step | 21.9 samples/s
|
| 588 |
+
Step 22110/50462 | Loss: 2.6443 | LR: 3.05e-04 | 2750.0ms/step | 21.8 samples/s
|
| 589 |
+
Step 22120/50462 | Loss: 2.6316 | LR: 3.05e-04 | 2744.3ms/step | 21.9 samples/s
|
| 590 |
+
Step 22130/50462 | Loss: 2.5886 | LR: 3.04e-04 | 2745.0ms/step | 21.9 samples/s
|
| 591 |
+
Step 22140/50462 | Loss: 2.6418 | LR: 3.04e-04 | 2746.4ms/step | 21.8 samples/s
|
| 592 |
+
Step 22150/50462 | Loss: 2.6301 | LR: 3.04e-04 | 2750.8ms/step | 21.8 samples/s
|
| 593 |
+
Step 22160/50462 | Loss: 2.6622 | LR: 3.04e-04 | 2744.5ms/step | 21.9 samples/s
|
| 594 |
+
Step 22170/50462 | Loss: 2.6287 | LR: 3.04e-04 | 2741.3ms/step | 21.9 samples/s
|
| 595 |
+
Step 22180/50462 | Loss: 2.6496 | LR: 3.04e-04 | 2737.3ms/step | 21.9 samples/s
|
| 596 |
+
Step 22190/50462 | Loss: 2.6109 | LR: 3.04e-04 | 2748.6ms/step | 21.8 samples/s
|
| 597 |
+
Step 22200/50462 | Loss: 2.5839 | LR: 3.03e-04 | 2738.7ms/step | 21.9 samples/s
|
| 598 |
+
Step 22210/50462 | Loss: 2.5803 | LR: 3.03e-04 | 2738.7ms/step | 21.9 samples/s
|
| 599 |
+
Step 22220/50462 | Loss: 2.6208 | LR: 3.03e-04 | 2746.7ms/step | 21.8 samples/s
|
| 600 |
+
Step 22230/50462 | Loss: 2.6143 | LR: 3.03e-04 | 2751.6ms/step | 21.8 samples/s
|
| 601 |
+
Step 22240/50462 | Loss: 2.6381 | LR: 3.03e-04 | 2741.2ms/step | 21.9 samples/s
|
| 602 |
+
Step 22250/50462 | Loss: 2.6007 | LR: 3.03e-04 | 2743.7ms/step | 21.9 samples/s
|
| 603 |
+
Validation at step 22250: Loss = 2.6194, Perplexity = 13.73
|
| 604 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-22250
|
| 605 |
+
Step 22260/50462 | Loss: 2.6005 | LR: 3.02e-04 | 4120.1ms/step | 14.6 samples/s
|
| 606 |
+
Step 22270/50462 | Loss: 2.6306 | LR: 3.02e-04 | 2740.8ms/step | 21.9 samples/s
|
| 607 |
+
Step 22280/50462 | Loss: 2.6487 | LR: 3.02e-04 | 2745.1ms/step | 21.9 samples/s
|
| 608 |
+
Step 22290/50462 | Loss: 2.5856 | LR: 3.02e-04 | 2744.5ms/step | 21.9 samples/s
|
| 609 |
+
Step 22300/50462 | Loss: 2.6223 | LR: 3.02e-04 | 2745.7ms/step | 21.9 samples/s
|
| 610 |
+
Step 22310/50462 | Loss: 2.6192 | LR: 3.02e-04 | 2742.7ms/step | 21.9 samples/s
|
| 611 |
+
Step 22320/50462 | Loss: 2.6331 | LR: 3.02e-04 | 2750.3ms/step | 21.8 samples/s
|
| 612 |
+
Step 22330/50462 | Loss: 2.6105 | LR: 3.01e-04 | 2744.3ms/step | 21.9 samples/s
|
| 613 |
+
Step 22340/50462 | Loss: 2.6420 | LR: 3.01e-04 | 2743.0ms/step | 21.9 samples/s
|
| 614 |
+
Step 22350/50462 | Loss: 2.6249 | LR: 3.01e-04 | 2745.6ms/step | 21.9 samples/s
|
| 615 |
+
Step 22360/50462 | Loss: 2.6020 | LR: 3.01e-04 | 2742.2ms/step | 21.9 samples/s
|
| 616 |
+
Step 22370/50462 | Loss: 2.6161 | LR: 3.01e-04 | 2737.7ms/step | 21.9 samples/s
|
| 617 |
+
Step 22380/50462 | Loss: 2.6322 | LR: 3.01e-04 | 2737.1ms/step | 21.9 samples/s
|
| 618 |
+
Step 22390/50462 | Loss: 2.6000 | LR: 3.00e-04 | 2740.0ms/step | 21.9 samples/s
|
| 619 |
+
Step 22400/50462 | Loss: 2.6294 | LR: 3.00e-04 | 2738.9ms/step | 21.9 samples/s
|
| 620 |
+
Step 22410/50462 | Loss: 2.6255 | LR: 3.00e-04 | 2741.3ms/step | 21.9 samples/s
|
| 621 |
+
Step 22420/50462 | Loss: 2.6182 | LR: 3.00e-04 | 2741.0ms/step | 21.9 samples/s
|
| 622 |
+
Step 22430/50462 | Loss: 2.6425 | LR: 3.00e-04 | 2737.8ms/step | 21.9 samples/s
|
| 623 |
+
Step 22440/50462 | Loss: 2.6083 | LR: 3.00e-04 | 2734.7ms/step | 21.9 samples/s
|
| 624 |
+
Step 22450/50462 | Loss: 2.6350 | LR: 3.00e-04 | 2747.0ms/step | 21.8 samples/s
|
| 625 |
+
Step 22460/50462 | Loss: 2.6553 | LR: 2.99e-04 | 2746.0ms/step | 21.9 samples/s
|
| 626 |
+
Step 22470/50462 | Loss: 2.6262 | LR: 2.99e-04 | 2739.4ms/step | 21.9 samples/s
|
| 627 |
+
Step 22480/50462 | Loss: 2.6213 | LR: 2.99e-04 | 2737.5ms/step | 21.9 samples/s
|
| 628 |
+
Step 22490/50462 | Loss: 2.6499 | LR: 2.99e-04 | 2742.3ms/step | 21.9 samples/s
|
| 629 |
+
Step 22500/50462 | Loss: 2.6349 | LR: 2.99e-04 | 2742.0ms/step | 21.9 samples/s
|
| 630 |
+
Validation at step 22500: Loss = 2.6177, Perplexity = 13.70
|
| 631 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-22500
|
| 632 |
+
Step 22510/50462 | Loss: 2.5860 | LR: 2.99e-04 | 4134.6ms/step | 14.5 samples/s
|
| 633 |
+
Step 22520/50462 | Loss: 2.6526 | LR: 2.98e-04 | 2738.8ms/step | 21.9 samples/s
|
| 634 |
+
Step 22530/50462 | Loss: 2.6159 | LR: 2.98e-04 | 2743.0ms/step | 21.9 samples/s
|
| 635 |
+
Step 22540/50462 | Loss: 2.5670 | LR: 2.98e-04 | 2739.3ms/step | 21.9 samples/s
|
| 636 |
+
Step 22550/50462 | Loss: 2.6087 | LR: 2.98e-04 | 2735.9ms/step | 21.9 samples/s
|
| 637 |
+
Step 22560/50462 | Loss: 2.6041 | LR: 2.98e-04 | 2737.4ms/step | 21.9 samples/s
|
| 638 |
+
Step 22570/50462 | Loss: 2.5891 | LR: 2.98e-04 | 2735.9ms/step | 21.9 samples/s
|
| 639 |
+
Step 22580/50462 | Loss: 2.6012 | LR: 2.98e-04 | 2742.9ms/step | 21.9 samples/s
|
| 640 |
+
Step 22590/50462 | Loss: 2.6363 | LR: 2.97e-04 | 2754.0ms/step | 21.8 samples/s
|
| 641 |
+
Step 22600/50462 | Loss: 2.6119 | LR: 2.97e-04 | 2734.2ms/step | 21.9 samples/s
|
| 642 |
+
Step 22610/50462 | Loss: 2.6029 | LR: 2.97e-04 | 2739.0ms/step | 21.9 samples/s
|
| 643 |
+
Step 22620/50462 | Loss: 2.6065 | LR: 2.97e-04 | 2740.4ms/step | 21.9 samples/s
|
| 644 |
+
Step 22630/50462 | Loss: 2.6299 | LR: 2.97e-04 | 2740.9ms/step | 21.9 samples/s
|
| 645 |
+
Step 22640/50462 | Loss: 2.6079 | LR: 2.97e-04 | 2744.1ms/step | 21.9 samples/s
|
| 646 |
+
Step 22650/50462 | Loss: 2.5897 | LR: 2.96e-04 | 2744.9ms/step | 21.9 samples/s
|
| 647 |
+
Step 22660/50462 | Loss: 2.5729 | LR: 2.96e-04 | 2740.9ms/step | 21.9 samples/s
|
| 648 |
+
Step 22670/50462 | Loss: 2.6158 | LR: 2.96e-04 | 2742.3ms/step | 21.9 samples/s
|
| 649 |
+
Step 22680/50462 | Loss: 2.6035 | LR: 2.96e-04 | 2740.4ms/step | 21.9 samples/s
|
| 650 |
+
Step 22690/50462 | Loss: 2.6226 | LR: 2.96e-04 | 2734.1ms/step | 21.9 samples/s
|
| 651 |
+
Step 22700/50462 | Loss: 2.6194 | LR: 2.96e-04 | 2736.3ms/step | 21.9 samples/s
|
| 652 |
+
Step 22710/50462 | Loss: 2.5782 | LR: 2.95e-04 | 2746.3ms/step | 21.8 samples/s
|
| 653 |
+
Step 22720/50462 | Loss: 2.6135 | LR: 2.95e-04 | 2741.0ms/step | 21.9 samples/s
|
| 654 |
+
Step 22730/50462 | Loss: 2.6448 | LR: 2.95e-04 | 2743.8ms/step | 21.9 samples/s
|
| 655 |
+
Step 22740/50462 | Loss: 2.6200 | LR: 2.95e-04 | 2746.7ms/step | 21.8 samples/s
|
| 656 |
+
Step 22750/50462 | Loss: 2.5823 | LR: 2.95e-04 | 2747.4ms/step | 21.8 samples/s
|
| 657 |
+
Validation at step 22750: Loss = 2.6145, Perplexity = 13.66
|
| 658 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-22750
|
| 659 |
+
Step 22760/50462 | Loss: 2.6007 | LR: 2.95e-04 | 4112.2ms/step | 14.6 samples/s
|
| 660 |
+
Step 22770/50462 | Loss: 2.6264 | LR: 2.95e-04 | 2741.5ms/step | 21.9 samples/s
|
| 661 |
+
Step 22780/50462 | Loss: 2.5848 | LR: 2.94e-04 | 2745.3ms/step | 21.9 samples/s
|
| 662 |
+
Step 22790/50462 | Loss: 2.6549 | LR: 2.94e-04 | 2744.8ms/step | 21.9 samples/s
|
| 663 |
+
Step 22800/50462 | Loss: 2.6285 | LR: 2.94e-04 | 2745.7ms/step | 21.9 samples/s
|
| 664 |
+
Step 22810/50462 | Loss: 2.6283 | LR: 2.94e-04 | 2750.0ms/step | 21.8 samples/s
|
| 665 |
+
Step 22820/50462 | Loss: 2.6155 | LR: 2.94e-04 | 2740.1ms/step | 21.9 samples/s
|
| 666 |
+
Step 22830/50462 | Loss: 2.5889 | LR: 2.94e-04 | 2736.3ms/step | 21.9 samples/s
|
| 667 |
+
Step 22840/50462 | Loss: 2.5858 | LR: 2.93e-04 | 2737.7ms/step | 21.9 samples/s
|
| 668 |
+
Step 22850/50462 | Loss: 2.6092 | LR: 2.93e-04 | 2742.9ms/step | 21.9 samples/s
|
| 669 |
+
Step 22860/50462 | Loss: 2.6140 | LR: 2.93e-04 | 2754.7ms/step | 21.8 samples/s
|
| 670 |
+
Step 22870/50462 | Loss: 2.6294 | LR: 2.93e-04 | 2745.6ms/step | 21.9 samples/s
|
| 671 |
+
Step 22880/50462 | Loss: 2.6232 | LR: 2.93e-04 | 2750.4ms/step | 21.8 samples/s
|
| 672 |
+
Step 22890/50462 | Loss: 2.6337 | LR: 2.93e-04 | 2744.0ms/step | 21.9 samples/s
|
| 673 |
+
Step 22900/50462 | Loss: 2.6094 | LR: 2.93e-04 | 2730.9ms/step | 22.0 samples/s
|
| 674 |
+
Step 22910/50462 | Loss: 2.5728 | LR: 2.92e-04 | 2739.8ms/step | 21.9 samples/s
|
| 675 |
+
Step 22920/50462 | Loss: 2.6224 | LR: 2.92e-04 | 2739.6ms/step | 21.9 samples/s
|
| 676 |
+
Step 22930/50462 | Loss: 2.5790 | LR: 2.92e-04 | 2739.6ms/step | 21.9 samples/s
|
| 677 |
+
Step 22940/50462 | Loss: 2.5862 | LR: 2.92e-04 | 2739.8ms/step | 21.9 samples/s
|
| 678 |
+
Step 22950/50462 | Loss: 2.5841 | LR: 2.92e-04 | 2740.8ms/step | 21.9 samples/s
|
| 679 |
+
Step 22960/50462 | Loss: 2.6266 | LR: 2.92e-04 | 2745.7ms/step | 21.9 samples/s
|
| 680 |
+
Step 22970/50462 | Loss: 2.6343 | LR: 2.91e-04 | 2742.9ms/step | 21.9 samples/s
|
| 681 |
+
Step 22980/50462 | Loss: 2.5959 | LR: 2.91e-04 | 2740.1ms/step | 21.9 samples/s
|
| 682 |
+
Step 22990/50462 | Loss: 2.6089 | LR: 2.91e-04 | 2746.6ms/step | 21.8 samples/s
|
| 683 |
+
Step 23000/50462 | Loss: 2.6103 | LR: 2.91e-04 | 2744.9ms/step | 21.9 samples/s
|
| 684 |
+
Validation at step 23000: Loss = 2.6121, Perplexity = 13.63
|
| 685 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-23000
|
| 686 |
+
Step 23010/50462 | Loss: 2.6212 | LR: 2.91e-04 | 4115.5ms/step | 14.6 samples/s
|
| 687 |
+
Step 23020/50462 | Loss: 2.6052 | LR: 2.91e-04 | 2737.7ms/step | 21.9 samples/s
|
| 688 |
+
Step 23030/50462 | Loss: 2.6464 | LR: 2.91e-04 | 2741.6ms/step | 21.9 samples/s
|
| 689 |
+
Step 23040/50462 | Loss: 2.6411 | LR: 2.90e-04 | 2741.9ms/step | 21.9 samples/s
|
| 690 |
+
Step 23050/50462 | Loss: 2.6562 | LR: 2.90e-04 | 2742.7ms/step | 21.9 samples/s
|
| 691 |
+
Step 23060/50462 | Loss: 2.6161 | LR: 2.90e-04 | 2741.7ms/step | 21.9 samples/s
|
| 692 |
+
Step 23070/50462 | Loss: 2.6619 | LR: 2.90e-04 | 2743.1ms/step | 21.9 samples/s
|
| 693 |
+
Step 23080/50462 | Loss: 2.5846 | LR: 2.90e-04 | 2736.7ms/step | 21.9 samples/s
|
| 694 |
+
Step 23090/50462 | Loss: 2.6543 | LR: 2.90e-04 | 2733.5ms/step | 21.9 samples/s
|
| 695 |
+
Step 23100/50462 | Loss: 2.6152 | LR: 2.89e-04 | 2744.6ms/step | 21.9 samples/s
|
| 696 |
+
Step 23110/50462 | Loss: 2.6465 | LR: 2.89e-04 | 2739.2ms/step | 21.9 samples/s
|
| 697 |
+
Step 23120/50462 | Loss: 2.6165 | LR: 2.89e-04 | 2741.0ms/step | 21.9 samples/s
|
| 698 |
+
Step 23130/50462 | Loss: 2.6331 | LR: 2.89e-04 | 2737.0ms/step | 21.9 samples/s
|
| 699 |
+
Step 23140/50462 | Loss: 2.5907 | LR: 2.89e-04 | 2743.8ms/step | 21.9 samples/s
|
| 700 |
+
Step 23150/50462 | Loss: 2.6287 | LR: 2.89e-04 | 2732.1ms/step | 22.0 samples/s
|
| 701 |
+
Step 23160/50462 | Loss: 2.5863 | LR: 2.88e-04 | 2736.8ms/step | 21.9 samples/s
|
| 702 |
+
Step 23170/50462 | Loss: 2.6287 | LR: 2.88e-04 | 2745.9ms/step | 21.9 samples/s
|
| 703 |
+
Step 23180/50462 | Loss: 2.6383 | LR: 2.88e-04 | 2743.9ms/step | 21.9 samples/s
|
| 704 |
+
Step 23190/50462 | Loss: 2.6273 | LR: 2.88e-04 | 2742.0ms/step | 21.9 samples/s
|
| 705 |
+
Step 23200/50462 | Loss: 2.6217 | LR: 2.88e-04 | 2740.5ms/step | 21.9 samples/s
|
| 706 |
+
Step 23210/50462 | Loss: 2.6581 | LR: 2.88e-04 | 2742.1ms/step | 21.9 samples/s
|
| 707 |
+
Step 23220/50462 | Loss: 2.5811 | LR: 2.88e-04 | 2742.0ms/step | 21.9 samples/s
|
| 708 |
+
Step 23230/50462 | Loss: 2.6327 | LR: 2.87e-04 | 2743.3ms/step | 21.9 samples/s
|
| 709 |
+
Step 23240/50462 | Loss: 2.6386 | LR: 2.87e-04 | 2735.2ms/step | 21.9 samples/s
|
| 710 |
+
Step 23250/50462 | Loss: 2.6031 | LR: 2.87e-04 | 2741.1ms/step | 21.9 samples/s
|
| 711 |
+
Validation at step 23250: Loss = 2.6092, Perplexity = 13.59
|
| 712 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-23250
|
| 713 |
+
Step 23260/50462 | Loss: 2.5776 | LR: 2.87e-04 | 4169.9ms/step | 14.4 samples/s
|
| 714 |
+
Step 23270/50462 | Loss: 2.5957 | LR: 2.87e-04 | 2740.0ms/step | 21.9 samples/s
|
| 715 |
+
Step 23280/50462 | Loss: 2.6194 | LR: 2.87e-04 | 2731.7ms/step | 22.0 samples/s
|
| 716 |
+
Step 23290/50462 | Loss: 2.5862 | LR: 2.86e-04 | 2756.4ms/step | 21.8 samples/s
|
| 717 |
+
Step 23300/50462 | Loss: 2.6336 | LR: 2.86e-04 | 2749.0ms/step | 21.8 samples/s
|
| 718 |
+
Step 23310/50462 | Loss: 2.5884 | LR: 2.86e-04 | 2746.6ms/step | 21.8 samples/s
|
| 719 |
+
Step 23320/50462 | Loss: 2.6186 | LR: 2.86e-04 | 2738.6ms/step | 21.9 samples/s
|
| 720 |
+
Step 23330/50462 | Loss: 2.6024 | LR: 2.86e-04 | 2736.4ms/step | 21.9 samples/s
|
| 721 |
+
Step 23340/50462 | Loss: 2.6323 | LR: 2.86e-04 | 2738.8ms/step | 21.9 samples/s
|
| 722 |
+
Step 23350/50462 | Loss: 2.6365 | LR: 2.86e-04 | 2737.1ms/step | 21.9 samples/s
|
| 723 |
+
Step 23360/50462 | Loss: 2.5711 | LR: 2.85e-04 | 2738.9ms/step | 21.9 samples/s
|
| 724 |
+
Step 23370/50462 | Loss: 2.6149 | LR: 2.85e-04 | 2742.8ms/step | 21.9 samples/s
|
| 725 |
+
Step 23380/50462 | Loss: 2.6502 | LR: 2.85e-04 | 2746.9ms/step | 21.8 samples/s
|
| 726 |
+
Step 23390/50462 | Loss: 2.6158 | LR: 2.85e-04 | 2737.3ms/step | 21.9 samples/s
|
| 727 |
+
Step 23400/50462 | Loss: 2.6128 | LR: 2.85e-04 | 2737.3ms/step | 21.9 samples/s
|
| 728 |
+
Step 23410/50462 | Loss: 2.6459 | LR: 2.85e-04 | 2733.5ms/step | 22.0 samples/s
|
| 729 |
+
Step 23420/50462 | Loss: 2.5980 | LR: 2.84e-04 | 2741.8ms/step | 21.9 samples/s
|
| 730 |
+
Step 23430/50462 | Loss: 2.5731 | LR: 2.84e-04 | 2742.1ms/step | 21.9 samples/s
|
| 731 |
+
Step 23440/50462 | Loss: 2.6093 | LR: 2.84e-04 | 2738.6ms/step | 21.9 samples/s
|
| 732 |
+
Step 23450/50462 | Loss: 2.5845 | LR: 2.84e-04 | 2743.2ms/step | 21.9 samples/s
|
| 733 |
+
Step 23460/50462 | Loss: 2.6291 | LR: 2.84e-04 | 2751.2ms/step | 21.8 samples/s
|
| 734 |
+
Step 23470/50462 | Loss: 2.6139 | LR: 2.84e-04 | 2746.3ms/step | 21.8 samples/s
|
| 735 |
+
Step 23480/50462 | Loss: 2.6379 | LR: 2.83e-04 | 2751.6ms/step | 21.8 samples/s
|
| 736 |
+
Step 23490/50462 | Loss: 2.5938 | LR: 2.83e-04 | 2735.7ms/step | 21.9 samples/s
|
| 737 |
+
Step 23500/50462 | Loss: 2.6277 | LR: 2.83e-04 | 2738.0ms/step | 21.9 samples/s
|
| 738 |
+
Validation at step 23500: Loss = 2.6078, Perplexity = 13.57
|
| 739 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-23500
|
| 740 |
+
Step 23510/50462 | Loss: 2.5748 | LR: 2.83e-04 | 4114.6ms/step | 14.6 samples/s
|
| 741 |
+
Step 23520/50462 | Loss: 2.6445 | LR: 2.83e-04 | 2739.0ms/step | 21.9 samples/s
|
| 742 |
+
Step 23530/50462 | Loss: 2.5860 | LR: 2.83e-04 | 2738.7ms/step | 21.9 samples/s
|
| 743 |
+
Step 23540/50462 | Loss: 2.6137 | LR: 2.83e-04 | 2742.8ms/step | 21.9 samples/s
|
| 744 |
+
Step 23550/50462 | Loss: 2.5810 | LR: 2.82e-04 | 2749.6ms/step | 21.8 samples/s
|
| 745 |
+
Step 23560/50462 | Loss: 2.6400 | LR: 2.82e-04 | 2738.9ms/step | 21.9 samples/s
|
| 746 |
+
Step 23570/50462 | Loss: 2.6111 | LR: 2.82e-04 | 2737.1ms/step | 21.9 samples/s
|
| 747 |
+
Step 23580/50462 | Loss: 2.5847 | LR: 2.82e-04 | 2730.3ms/step | 22.0 samples/s
|
| 748 |
+
Step 23590/50462 | Loss: 2.5888 | LR: 2.82e-04 | 2743.6ms/step | 21.9 samples/s
|
| 749 |
+
Step 23600/50462 | Loss: 2.6120 | LR: 2.82e-04 | 2737.6ms/step | 21.9 samples/s
|
| 750 |
+
Step 23610/50462 | Loss: 2.5768 | LR: 2.81e-04 | 2734.3ms/step | 21.9 samples/s
|
| 751 |
+
Step 23620/50462 | Loss: 2.5928 | LR: 2.81e-04 | 2737.0ms/step | 21.9 samples/s
|
| 752 |
+
Step 23630/50462 | Loss: 2.6110 | LR: 2.81e-04 | 2743.6ms/step | 21.9 samples/s
|
| 753 |
+
Step 23640/50462 | Loss: 2.5981 | LR: 2.81e-04 | 2749.6ms/step | 21.8 samples/s
|
| 754 |
+
Step 23650/50462 | Loss: 2.6090 | LR: 2.81e-04 | 2739.7ms/step | 21.9 samples/s
|
| 755 |
+
Step 23660/50462 | Loss: 2.6489 | LR: 2.81e-04 | 2743.8ms/step | 21.9 samples/s
|
| 756 |
+
Step 23670/50462 | Loss: 2.6170 | LR: 2.81e-04 | 2742.0ms/step | 21.9 samples/s
|
| 757 |
+
Step 23680/50462 | Loss: 2.6174 | LR: 2.80e-04 | 2741.0ms/step | 21.9 samples/s
|
| 758 |
+
Step 23690/50462 | Loss: 2.5832 | LR: 2.80e-04 | 2738.0ms/step | 21.9 samples/s
|
| 759 |
+
Step 23700/50462 | Loss: 2.5772 | LR: 2.80e-04 | 2742.5ms/step | 21.9 samples/s
|
| 760 |
+
Step 23710/50462 | Loss: 2.6330 | LR: 2.80e-04 | 2732.0ms/step | 22.0 samples/s
|
| 761 |
+
Step 23720/50462 | Loss: 2.6028 | LR: 2.80e-04 | 2739.2ms/step | 21.9 samples/s
|
| 762 |
+
Step 23730/50462 | Loss: 2.5954 | LR: 2.80e-04 | 2742.8ms/step | 21.9 samples/s
|
| 763 |
+
Step 23740/50462 | Loss: 2.6088 | LR: 2.79e-04 | 2734.0ms/step | 21.9 samples/s
|
| 764 |
+
Step 23750/50462 | Loss: 2.5849 | LR: 2.79e-04 | 2728.4ms/step | 22.0 samples/s
|
| 765 |
+
Validation at step 23750: Loss = 2.6059, Perplexity = 13.54
|
| 766 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-23750
|
| 767 |
+
Step 23760/50462 | Loss: 2.6031 | LR: 2.79e-04 | 4113.1ms/step | 14.6 samples/s
|
| 768 |
+
Step 23770/50462 | Loss: 2.5934 | LR: 2.79e-04 | 2745.8ms/step | 21.9 samples/s
|
| 769 |
+
Step 23780/50462 | Loss: 2.5997 | LR: 2.79e-04 | 2731.7ms/step | 22.0 samples/s
|
| 770 |
+
Step 23790/50462 | Loss: 2.5823 | LR: 2.79e-04 | 2736.9ms/step | 21.9 samples/s
|
| 771 |
+
Step 23800/50462 | Loss: 2.5863 | LR: 2.78e-04 | 2741.3ms/step | 21.9 samples/s
|
| 772 |
+
Step 23810/50462 | Loss: 2.6289 | LR: 2.78e-04 | 2740.9ms/step | 21.9 samples/s
|
| 773 |
+
Step 23820/50462 | Loss: 2.5654 | LR: 2.78e-04 | 2735.9ms/step | 21.9 samples/s
|
| 774 |
+
Step 23830/50462 | Loss: 2.6132 | LR: 2.78e-04 | 2734.7ms/step | 21.9 samples/s
|
| 775 |
+
Step 23840/50462 | Loss: 2.6484 | LR: 2.78e-04 | 2735.8ms/step | 21.9 samples/s
|
| 776 |
+
Step 23850/50462 | Loss: 2.6028 | LR: 2.78e-04 | 2733.5ms/step | 22.0 samples/s
|
| 777 |
+
Step 23860/50462 | Loss: 2.5838 | LR: 2.78e-04 | 2740.0ms/step | 21.9 samples/s
|
| 778 |
+
Step 23870/50462 | Loss: 2.6412 | LR: 2.77e-04 | 2742.1ms/step | 21.9 samples/s
|
| 779 |
+
Step 23880/50462 | Loss: 2.5837 | LR: 2.77e-04 | 2736.8ms/step | 21.9 samples/s
|
| 780 |
+
Step 23890/50462 | Loss: 2.6055 | LR: 2.77e-04 | 2750.5ms/step | 21.8 samples/s
|
| 781 |
+
Step 23900/50462 | Loss: 2.6182 | LR: 2.77e-04 | 2735.5ms/step | 21.9 samples/s
|
| 782 |
+
Step 23910/50462 | Loss: 2.6253 | LR: 2.77e-04 | 2740.5ms/step | 21.9 samples/s
|
| 783 |
+
Step 23920/50462 | Loss: 2.5954 | LR: 2.77e-04 | 2740.6ms/step | 21.9 samples/s
|
| 784 |
+
Step 23930/50462 | Loss: 2.6062 | LR: 2.76e-04 | 2731.2ms/step | 22.0 samples/s
|
| 785 |
+
Step 23940/50462 | Loss: 2.5865 | LR: 2.76e-04 | 2730.1ms/step | 22.0 samples/s
|
| 786 |
+
Step 23950/50462 | Loss: 2.6170 | LR: 2.76e-04 | 2733.4ms/step | 22.0 samples/s
|
| 787 |
+
Step 23960/50462 | Loss: 2.5965 | LR: 2.76e-04 | 2742.2ms/step | 21.9 samples/s
|
| 788 |
+
Step 23970/50462 | Loss: 2.6080 | LR: 2.76e-04 | 2735.2ms/step | 21.9 samples/s
|
| 789 |
+
Step 23980/50462 | Loss: 2.6029 | LR: 2.76e-04 | 2743.7ms/step | 21.9 samples/s
|
| 790 |
+
Step 23990/50462 | Loss: 2.6038 | LR: 2.75e-04 | 2740.5ms/step | 21.9 samples/s
|
| 791 |
+
Step 24000/50462 | Loss: 2.5570 | LR: 2.75e-04 | 2746.2ms/step | 21.8 samples/s
|
| 792 |
+
Validation at step 24000: Loss = 2.6042, Perplexity = 13.52
|
| 793 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-24000
|
| 794 |
+
Step 24010/50462 | Loss: 2.6207 | LR: 2.75e-04 | 4173.8ms/step | 14.4 samples/s
|
| 795 |
+
Step 24020/50462 | Loss: 2.6265 | LR: 2.75e-04 | 2746.3ms/step | 21.8 samples/s
|
| 796 |
+
Step 24030/50462 | Loss: 2.6250 | LR: 2.75e-04 | 2738.6ms/step | 21.9 samples/s
|
| 797 |
+
Step 24040/50462 | Loss: 2.6039 | LR: 2.75e-04 | 2744.7ms/step | 21.9 samples/s
|
| 798 |
+
Step 24050/50462 | Loss: 2.5788 | LR: 2.75e-04 | 2731.6ms/step | 22.0 samples/s
|
| 799 |
+
Step 24060/50462 | Loss: 2.6087 | LR: 2.74e-04 | 2742.3ms/step | 21.9 samples/s
|
| 800 |
+
Step 24070/50462 | Loss: 2.6095 | LR: 2.74e-04 | 2747.0ms/step | 21.8 samples/s
|
| 801 |
+
Step 24080/50462 | Loss: 2.5971 | LR: 2.74e-04 | 2742.9ms/step | 21.9 samples/s
|
| 802 |
+
Step 24090/50462 | Loss: 2.5383 | LR: 2.74e-04 | 2733.9ms/step | 21.9 samples/s
|
| 803 |
+
Step 24100/50462 | Loss: 2.5782 | LR: 2.74e-04 | 2731.5ms/step | 22.0 samples/s
|
| 804 |
+
Step 24110/50462 | Loss: 2.6012 | LR: 2.74e-04 | 2732.9ms/step | 22.0 samples/s
|
| 805 |
+
Step 24120/50462 | Loss: 2.6129 | LR: 2.73e-04 | 2734.5ms/step | 21.9 samples/s
|
| 806 |
+
Step 24130/50462 | Loss: 2.5750 | LR: 2.73e-04 | 2726.4ms/step | 22.0 samples/s
|
| 807 |
+
Step 24140/50462 | Loss: 2.5965 | LR: 2.73e-04 | 2749.8ms/step | 21.8 samples/s
|
| 808 |
+
Step 24150/50462 | Loss: 2.5781 | LR: 2.73e-04 | 2740.7ms/step | 21.9 samples/s
|
| 809 |
+
Step 24160/50462 | Loss: 2.5982 | LR: 2.73e-04 | 2744.8ms/step | 21.9 samples/s
|
| 810 |
+
Step 24170/50462 | Loss: 2.6245 | LR: 2.73e-04 | 2745.4ms/step | 21.9 samples/s
|
| 811 |
+
Step 24180/50462 | Loss: 2.6082 | LR: 2.72e-04 | 2736.0ms/step | 21.9 samples/s
|
| 812 |
+
Step 24190/50462 | Loss: 2.6170 | LR: 2.72e-04 | 2744.4ms/step | 21.9 samples/s
|
| 813 |
+
Step 24200/50462 | Loss: 2.5987 | LR: 2.72e-04 | 2736.5ms/step | 21.9 samples/s
|
| 814 |
+
Step 24210/50462 | Loss: 2.5777 | LR: 2.72e-04 | 2747.2ms/step | 21.8 samples/s
|
epfl_final/SparseAtt/wandb/run-20260823_150341-i8ewyy95/files/requirements.txt
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
| 142 |
+
entmax==1.3
|
epfl_final/SparseAtt/wandb/run-20260823_150341-i8ewyy95/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-08-23T15:03:41.406897Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"configs/p2_1p2b_mixed_seed70_6gpu.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/TokenPrune.git",
|
| 14 |
+
"commit": "5637886f0cc760009989b14e68c6ae06d4827418"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "h100-train-6-gpu-0-32",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 23 |
+
"gpu_count": 6,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "9374931091456",
|
| 27 |
+
"used": "6063984922624"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1623005732864"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-85844f64-d144-1c84-306d-f88b5f232a77"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 43 |
+
"memoryTotal": "85520809984",
|
| 44 |
+
"cudaCores": 16896,
|
| 45 |
+
"architecture": "Hopper",
|
| 46 |
+
"uuid": "GPU-c65885f7-52dc-6f23-a3dd-6addf8b518a2"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 50 |
+
"memoryTotal": "85520809984",
|
| 51 |
+
"cudaCores": 16896,
|
| 52 |
+
"architecture": "Hopper",
|
| 53 |
+
"uuid": "GPU-7e349d73-ad76-d62d-635c-266997fedcc1"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 57 |
+
"memoryTotal": "85520809984",
|
| 58 |
+
"cudaCores": 16896,
|
| 59 |
+
"architecture": "Hopper",
|
| 60 |
+
"uuid": "GPU-4e4c2343-4f5b-a2ff-312e-03d402a27803"
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 64 |
+
"memoryTotal": "85520809984",
|
| 65 |
+
"cudaCores": 16896,
|
| 66 |
+
"architecture": "Hopper",
|
| 67 |
+
"uuid": "GPU-fbf0ccd1-8bd4-5f57-6e74-6c0cf7eb70c9"
|
| 68 |
+
},
|
| 69 |
+
{
|
| 70 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 71 |
+
"memoryTotal": "85520809984",
|
| 72 |
+
"cudaCores": 16896,
|
| 73 |
+
"architecture": "Hopper",
|
| 74 |
+
"uuid": "GPU-3a3f0ac7-1596-1797-bf0d-31f3418789cd"
|
| 75 |
+
}
|
| 76 |
+
],
|
| 77 |
+
"cudaVersion": "13.0",
|
| 78 |
+
"writerId": "bp1308ig5kfdsq63v514zd7z5jibb8ct"
|
| 79 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260823_150341-i8ewyy95/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260823_150341-i8ewyy95/logs/debug.log
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-08-23 15:03:41,428 INFO MainThread:222 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-08-23 15:03:41,429 INFO MainThread:222 [wandb_setup.py:_flush():81] Configure stats pid to 222
|
| 3 |
+
2026-08-23 15:03:41,429 INFO MainThread:222 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-08-23 15:03:41,429 INFO MainThread:222 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260823_150341-i8ewyy95/logs/debug.log
|
| 5 |
+
2026-08-23 15:03:41,429 INFO MainThread:222 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260823_150341-i8ewyy95/logs/debug-internal.log
|
| 6 |
+
2026-08-23 15:03:41,429 INFO MainThread:222 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-08-23 15:03:41,432 INFO MainThread:222 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-16750', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 70, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b mixed 8k seed70 6gpu resume', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'mixed', 'seed70'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed', 'sparse_mask_seed': 70, 'sparse_mixed_nrb_set': [63, 47, 31, 21, 11], 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-08-23 15:03:41,432 INFO MainThread:222 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-08-23 15:03:42,226 INFO MainThread:222 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-08-23 15:03:42,707 INFO MainThread:222 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-08-23 15:03:42,710 INFO MainThread:222 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-08-23 15:03:42,752 INFO MainThread:222 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-08-23 15:03:43,251 INFO MainThread:222 [wandb_init.py:init():1051] run resumed
|
| 15 |
+
2026-08-23 15:03:43,254 INFO MainThread:222 [wandb_init.py:init():1056] starting run threads in backend
|
| 16 |
+
2026-08-23 15:03:43,608 INFO MainThread:222 [wandb_run.py:_console_start():2554] atexit reg
|
| 17 |
+
2026-08-23 15:03:43,608 INFO MainThread:222 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 18 |
+
2026-08-23 15:03:43,608 INFO MainThread:222 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 19 |
+
2026-08-23 15:03:43,608 INFO MainThread:222 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 20 |
+
2026-08-23 15:03:43,632 INFO MainThread:222 [wandb_init.py:init():1094] run started, returning control to user process
|
epfl_final/SparseAtt/wandb/run-20260823_150341-i8ewyy95/run-i8ewyy95.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5cdaabffff6132231682266d6acc62286cd6fc888737c5fe1efb55ec19a12993
|
| 3 |
+
size 9732096
|
epfl_final/SparseAtt/wandb/run-20260824_034449-rgmkfpnl/files/output.log
ADDED
|
@@ -0,0 +1,367 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Starting training...
|
| 2 |
+
Validation at step 10250: Loss = 2.7931, Perplexity = 16.33
|
| 3 |
+
wandb: WARNING Tried to log to step 10250 that is less than the current step 10291. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 4 |
+
Switching attention mode for step 10251: dense (dense step interval)
|
| 5 |
+
Step 10260/50462 | Loss: 2.7354 | LR: 4.56e-04 | 37721.5ms/step | 1.6 samples/s
|
| 6 |
+
wandb: WARNING Tried to log to step 10260 that is less than the current step 10291. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 7 |
+
Step 10270/50462 | Loss: 2.7236 | LR: 4.56e-04 | 5615.1ms/step | 10.7 samples/s
|
| 8 |
+
wandb: WARNING Tried to log to step 10270 that is less than the current step 10291. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 9 |
+
Step 10280/50462 | Loss: 2.7148 | LR: 4.56e-04 | 5625.1ms/step | 10.7 samples/s
|
| 10 |
+
wandb: WARNING Tried to log to step 10280 that is less than the current step 10291. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 11 |
+
Step 10290/50462 | Loss: 2.7075 | LR: 4.56e-04 | 5629.4ms/step | 10.7 samples/s
|
| 12 |
+
wandb: WARNING Tried to log to step 10290 that is less than the current step 10291. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 13 |
+
Step 10300/50462 | Loss: 2.7144 | LR: 4.56e-04 | 5632.6ms/step | 10.7 samples/s
|
| 14 |
+
Step 10310/50462 | Loss: 2.7603 | LR: 4.56e-04 | 5633.8ms/step | 10.6 samples/s
|
| 15 |
+
Step 10320/50462 | Loss: 2.7507 | LR: 4.56e-04 | 5636.9ms/step | 10.6 samples/s
|
| 16 |
+
Step 10330/50462 | Loss: 2.7233 | LR: 4.56e-04 | 5638.8ms/step | 10.6 samples/s
|
| 17 |
+
Step 10340/50462 | Loss: 2.7692 | LR: 4.55e-04 | 5640.9ms/step | 10.6 samples/s
|
| 18 |
+
Step 10350/50462 | Loss: 2.7838 | LR: 4.55e-04 | 5641.0ms/step | 10.6 samples/s
|
| 19 |
+
Step 10360/50462 | Loss: 2.7354 | LR: 4.55e-04 | 5640.1ms/step | 10.6 samples/s
|
| 20 |
+
Step 10370/50462 | Loss: 2.7318 | LR: 4.55e-04 | 5641.2ms/step | 10.6 samples/s
|
| 21 |
+
Step 10380/50462 | Loss: 2.6996 | LR: 4.55e-04 | 5640.3ms/step | 10.6 samples/s
|
| 22 |
+
Step 10390/50462 | Loss: 2.7131 | LR: 4.55e-04 | 5641.3ms/step | 10.6 samples/s
|
| 23 |
+
Step 10400/50462 | Loss: 2.7327 | LR: 4.55e-04 | 5642.0ms/step | 10.6 samples/s
|
| 24 |
+
Step 10410/50462 | Loss: 2.7648 | LR: 4.55e-04 | 5640.8ms/step | 10.6 samples/s
|
| 25 |
+
Step 10420/50462 | Loss: 2.7247 | LR: 4.55e-04 | 5640.0ms/step | 10.6 samples/s
|
| 26 |
+
Step 10430/50462 | Loss: 2.7628 | LR: 4.55e-04 | 5641.7ms/step | 10.6 samples/s
|
| 27 |
+
Step 10440/50462 | Loss: 2.7624 | LR: 4.55e-04 | 5639.9ms/step | 10.6 samples/s
|
| 28 |
+
Step 10450/50462 | Loss: 2.7321 | LR: 4.54e-04 | 5641.0ms/step | 10.6 samples/s
|
| 29 |
+
Step 10460/50462 | Loss: 2.7171 | LR: 4.54e-04 | 5640.7ms/step | 10.6 samples/s
|
| 30 |
+
Step 10470/50462 | Loss: 2.7126 | LR: 4.54e-04 | 5641.5ms/step | 10.6 samples/s
|
| 31 |
+
Step 10480/50462 | Loss: 2.7171 | LR: 4.54e-04 | 5642.1ms/step | 10.6 samples/s
|
| 32 |
+
Step 10490/50462 | Loss: 2.7600 | LR: 4.54e-04 | 5641.4ms/step | 10.6 samples/s
|
| 33 |
+
Step 10500/50462 | Loss: 2.7462 | LR: 4.54e-04 | 5641.0ms/step | 10.6 samples/s
|
| 34 |
+
Validation at step 10500: Loss = 2.7867, Perplexity = 16.23
|
| 35 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-10500
|
| 36 |
+
Step 10510/50462 | Loss: 2.7378 | LR: 4.54e-04 | 8481.1ms/step | 7.1 samples/s
|
| 37 |
+
Step 10520/50462 | Loss: 2.6880 | LR: 4.54e-04 | 5627.9ms/step | 10.7 samples/s
|
| 38 |
+
Step 10530/50462 | Loss: 2.7058 | LR: 4.54e-04 | 5633.0ms/step | 10.7 samples/s
|
| 39 |
+
Step 10540/50462 | Loss: 2.7011 | LR: 4.54e-04 | 5635.7ms/step | 10.6 samples/s
|
| 40 |
+
Step 10550/50462 | Loss: 2.7068 | LR: 4.54e-04 | 5635.9ms/step | 10.6 samples/s
|
| 41 |
+
Step 10560/50462 | Loss: 2.7737 | LR: 4.53e-04 | 5635.8ms/step | 10.6 samples/s
|
| 42 |
+
Step 10570/50462 | Loss: 2.7271 | LR: 4.53e-04 | 5637.4ms/step | 10.6 samples/s
|
| 43 |
+
Step 10580/50462 | Loss: 2.7520 | LR: 4.53e-04 | 5637.5ms/step | 10.6 samples/s
|
| 44 |
+
Step 10590/50462 | Loss: 2.7440 | LR: 4.53e-04 | 5642.9ms/step | 10.6 samples/s
|
| 45 |
+
Step 10600/50462 | Loss: 2.7655 | LR: 4.53e-04 | 5636.8ms/step | 10.6 samples/s
|
| 46 |
+
Step 10610/50462 | Loss: 2.7466 | LR: 4.53e-04 | 5643.1ms/step | 10.6 samples/s
|
| 47 |
+
Step 10620/50462 | Loss: 2.7075 | LR: 4.53e-04 | 5643.2ms/step | 10.6 samples/s
|
| 48 |
+
Step 10630/50462 | Loss: 2.7480 | LR: 4.53e-04 | 5641.5ms/step | 10.6 samples/s
|
| 49 |
+
Step 10640/50462 | Loss: 2.7565 | LR: 4.53e-04 | 5641.9ms/step | 10.6 samples/s
|
| 50 |
+
Step 10650/50462 | Loss: 2.7419 | LR: 4.53e-04 | 5640.4ms/step | 10.6 samples/s
|
| 51 |
+
Step 10660/50462 | Loss: 2.7346 | LR: 4.53e-04 | 5643.2ms/step | 10.6 samples/s
|
| 52 |
+
Step 10670/50462 | Loss: 2.7581 | LR: 4.52e-04 | 5638.9ms/step | 10.6 samples/s
|
| 53 |
+
Step 10680/50462 | Loss: 2.7524 | LR: 4.52e-04 | 5640.1ms/step | 10.6 samples/s
|
| 54 |
+
Step 10690/50462 | Loss: 2.7355 | LR: 4.52e-04 | 5640.8ms/step | 10.6 samples/s
|
| 55 |
+
Step 10700/50462 | Loss: 2.7895 | LR: 4.52e-04 | 5641.6ms/step | 10.6 samples/s
|
| 56 |
+
Step 10710/50462 | Loss: 2.7083 | LR: 4.52e-04 | 5641.3ms/step | 10.6 samples/s
|
| 57 |
+
Step 10720/50462 | Loss: 2.7228 | LR: 4.52e-04 | 5641.2ms/step | 10.6 samples/s
|
| 58 |
+
Step 10730/50462 | Loss: 2.7448 | LR: 4.52e-04 | 5639.3ms/step | 10.6 samples/s
|
| 59 |
+
Step 10740/50462 | Loss: 2.7277 | LR: 4.52e-04 | 5640.0ms/step | 10.6 samples/s
|
| 60 |
+
Step 10750/50462 | Loss: 2.7237 | LR: 4.52e-04 | 5641.2ms/step | 10.6 samples/s
|
| 61 |
+
Validation at step 10750: Loss = 2.7808, Perplexity = 16.13
|
| 62 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-10750
|
| 63 |
+
Step 10760/50462 | Loss: 2.7602 | LR: 4.52e-04 | 8396.0ms/step | 7.1 samples/s
|
| 64 |
+
Step 10770/50462 | Loss: 2.7259 | LR: 4.52e-04 | 5637.2ms/step | 10.6 samples/s
|
| 65 |
+
Step 10780/50462 | Loss: 2.7359 | LR: 4.51e-04 | 5644.2ms/step | 10.6 samples/s
|
| 66 |
+
Step 10790/50462 | Loss: 2.7434 | LR: 4.51e-04 | 5645.6ms/step | 10.6 samples/s
|
| 67 |
+
Step 10800/50462 | Loss: 2.6968 | LR: 4.51e-04 | 5644.3ms/step | 10.6 samples/s
|
| 68 |
+
Step 10810/50462 | Loss: 2.7757 | LR: 4.51e-04 | 5645.7ms/step | 10.6 samples/s
|
| 69 |
+
Step 10820/50462 | Loss: 2.7331 | LR: 4.51e-04 | 5642.8ms/step | 10.6 samples/s
|
| 70 |
+
Step 10830/50462 | Loss: 2.7447 | LR: 4.51e-04 | 5646.1ms/step | 10.6 samples/s
|
| 71 |
+
Step 10840/50462 | Loss: 2.7243 | LR: 4.51e-04 | 5648.7ms/step | 10.6 samples/s
|
| 72 |
+
Step 10850/50462 | Loss: 2.7122 | LR: 4.51e-04 | 5647.4ms/step | 10.6 samples/s
|
| 73 |
+
Step 10860/50462 | Loss: 2.7296 | LR: 4.51e-04 | 5647.2ms/step | 10.6 samples/s
|
| 74 |
+
Step 10870/50462 | Loss: 2.7203 | LR: 4.51e-04 | 5647.8ms/step | 10.6 samples/s
|
| 75 |
+
Step 10880/50462 | Loss: 2.7106 | LR: 4.50e-04 | 5646.7ms/step | 10.6 samples/s
|
| 76 |
+
Step 10890/50462 | Loss: 2.7227 | LR: 4.50e-04 | 5645.7ms/step | 10.6 samples/s
|
| 77 |
+
Step 10900/50462 | Loss: 2.7020 | LR: 4.50e-04 | 5647.6ms/step | 10.6 samples/s
|
| 78 |
+
Step 10910/50462 | Loss: 2.6824 | LR: 4.50e-04 | 5647.9ms/step | 10.6 samples/s
|
| 79 |
+
Step 10920/50462 | Loss: 2.7373 | LR: 4.50e-04 | 5647.2ms/step | 10.6 samples/s
|
| 80 |
+
Step 10930/50462 | Loss: 2.6812 | LR: 4.50e-04 | 5647.2ms/step | 10.6 samples/s
|
| 81 |
+
Step 10940/50462 | Loss: 2.7121 | LR: 4.50e-04 | 5645.4ms/step | 10.6 samples/s
|
| 82 |
+
Step 10950/50462 | Loss: 2.7512 | LR: 4.50e-04 | 5644.8ms/step | 10.6 samples/s
|
| 83 |
+
Step 10960/50462 | Loss: 2.6994 | LR: 4.50e-04 | 5645.8ms/step | 10.6 samples/s
|
| 84 |
+
Step 10970/50462 | Loss: 2.7474 | LR: 4.50e-04 | 5646.3ms/step | 10.6 samples/s
|
| 85 |
+
Step 10980/50462 | Loss: 2.7477 | LR: 4.50e-04 | 5645.8ms/step | 10.6 samples/s
|
| 86 |
+
Step 10990/50462 | Loss: 2.7079 | LR: 4.49e-04 | 5645.8ms/step | 10.6 samples/s
|
| 87 |
+
Step 11000/50462 | Loss: 2.7096 | LR: 4.49e-04 | 5647.5ms/step | 10.6 samples/s
|
| 88 |
+
Validation at step 11000: Loss = 2.7744, Perplexity = 16.03
|
| 89 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-11000
|
| 90 |
+
Step 11010/50462 | Loss: 2.7294 | LR: 4.49e-04 | 8370.3ms/step | 7.2 samples/s
|
| 91 |
+
Step 11020/50462 | Loss: 2.7313 | LR: 4.49e-04 | 5642.0ms/step | 10.6 samples/s
|
| 92 |
+
Step 11030/50462 | Loss: 2.7250 | LR: 4.49e-04 | 5642.9ms/step | 10.6 samples/s
|
| 93 |
+
Step 11040/50462 | Loss: 2.7164 | LR: 4.49e-04 | 5644.1ms/step | 10.6 samples/s
|
| 94 |
+
Step 11050/50462 | Loss: 2.7280 | LR: 4.49e-04 | 5643.8ms/step | 10.6 samples/s
|
| 95 |
+
Step 11060/50462 | Loss: 2.7219 | LR: 4.49e-04 | 5643.6ms/step | 10.6 samples/s
|
| 96 |
+
Step 11070/50462 | Loss: 2.7605 | LR: 4.49e-04 | 5643.7ms/step | 10.6 samples/s
|
| 97 |
+
Step 11080/50462 | Loss: 2.6966 | LR: 4.49e-04 | 5645.3ms/step | 10.6 samples/s
|
| 98 |
+
Step 11090/50462 | Loss: 2.7335 | LR: 4.48e-04 | 5645.1ms/step | 10.6 samples/s
|
| 99 |
+
Step 11100/50462 | Loss: 2.7150 | LR: 4.48e-04 | 5644.6ms/step | 10.6 samples/s
|
| 100 |
+
Step 11110/50462 | Loss: 2.7271 | LR: 4.48e-04 | 5647.0ms/step | 10.6 samples/s
|
| 101 |
+
Step 11120/50462 | Loss: 2.7073 | LR: 4.48e-04 | 5644.3ms/step | 10.6 samples/s
|
| 102 |
+
Step 11130/50462 | Loss: 2.7497 | LR: 4.48e-04 | 5644.7ms/step | 10.6 samples/s
|
| 103 |
+
Step 11140/50462 | Loss: 2.7122 | LR: 4.48e-04 | 5644.3ms/step | 10.6 samples/s
|
| 104 |
+
Step 11150/50462 | Loss: 2.7437 | LR: 4.48e-04 | 5645.6ms/step | 10.6 samples/s
|
| 105 |
+
Step 11160/50462 | Loss: 2.7316 | LR: 4.48e-04 | 5645.0ms/step | 10.6 samples/s
|
| 106 |
+
Step 11170/50462 | Loss: 2.7044 | LR: 4.48e-04 | 5644.8ms/step | 10.6 samples/s
|
| 107 |
+
Step 11180/50462 | Loss: 2.7250 | LR: 4.48e-04 | 5644.5ms/step | 10.6 samples/s
|
| 108 |
+
Step 11190/50462 | Loss: 2.7188 | LR: 4.48e-04 | 5646.4ms/step | 10.6 samples/s
|
| 109 |
+
Step 11200/50462 | Loss: 2.7258 | LR: 4.47e-04 | 5645.4ms/step | 10.6 samples/s
|
| 110 |
+
Step 11210/50462 | Loss: 2.6904 | LR: 4.47e-04 | 5644.7ms/step | 10.6 samples/s
|
| 111 |
+
Step 11220/50462 | Loss: 2.7061 | LR: 4.47e-04 | 5645.8ms/step | 10.6 samples/s
|
| 112 |
+
Step 11230/50462 | Loss: 2.7366 | LR: 4.47e-04 | 5646.0ms/step | 10.6 samples/s
|
| 113 |
+
Step 11240/50462 | Loss: 2.7077 | LR: 4.47e-04 | 5644.8ms/step | 10.6 samples/s
|
| 114 |
+
Step 11250/50462 | Loss: 2.7370 | LR: 4.47e-04 | 5645.0ms/step | 10.6 samples/s
|
| 115 |
+
Validation at step 11250: Loss = 2.7708, Perplexity = 15.97
|
| 116 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-11250
|
| 117 |
+
Step 11260/50462 | Loss: 2.7051 | LR: 4.47e-04 | 8360.7ms/step | 7.2 samples/s
|
| 118 |
+
Step 11270/50462 | Loss: 2.7480 | LR: 4.47e-04 | 5641.4ms/step | 10.6 samples/s
|
| 119 |
+
Step 11280/50462 | Loss: 2.7145 | LR: 4.47e-04 | 5649.5ms/step | 10.6 samples/s
|
| 120 |
+
Step 11290/50462 | Loss: 2.7190 | LR: 4.47e-04 | 5650.9ms/step | 10.6 samples/s
|
| 121 |
+
Step 11300/50462 | Loss: 2.7363 | LR: 4.46e-04 | 5651.7ms/step | 10.6 samples/s
|
| 122 |
+
Step 11310/50462 | Loss: 2.7248 | LR: 4.46e-04 | 5651.0ms/step | 10.6 samples/s
|
| 123 |
+
Step 11320/50462 | Loss: 2.6909 | LR: 4.46e-04 | 5651.5ms/step | 10.6 samples/s
|
| 124 |
+
Step 11330/50462 | Loss: 2.7174 | LR: 4.46e-04 | 5653.4ms/step | 10.6 samples/s
|
| 125 |
+
Step 11340/50462 | Loss: 2.7418 | LR: 4.46e-04 | 5651.7ms/step | 10.6 samples/s
|
| 126 |
+
Step 11350/50462 | Loss: 2.7546 | LR: 4.46e-04 | 5651.5ms/step | 10.6 samples/s
|
| 127 |
+
Step 11360/50462 | Loss: 2.7314 | LR: 4.46e-04 | 5653.3ms/step | 10.6 samples/s
|
| 128 |
+
Step 11370/50462 | Loss: 2.7261 | LR: 4.46e-04 | 5652.7ms/step | 10.6 samples/s
|
| 129 |
+
Step 11380/50462 | Loss: 2.7549 | LR: 4.46e-04 | 5650.7ms/step | 10.6 samples/s
|
| 130 |
+
Step 11390/50462 | Loss: 2.7228 | LR: 4.46e-04 | 5652.2ms/step | 10.6 samples/s
|
| 131 |
+
Step 11400/50462 | Loss: 2.6983 | LR: 4.45e-04 | 5652.0ms/step | 10.6 samples/s
|
| 132 |
+
Step 11410/50462 | Loss: 2.7173 | LR: 4.45e-04 | 5651.5ms/step | 10.6 samples/s
|
| 133 |
+
Step 11420/50462 | Loss: 2.7709 | LR: 4.45e-04 | 5651.8ms/step | 10.6 samples/s
|
| 134 |
+
Step 11430/50462 | Loss: 2.6770 | LR: 4.45e-04 | 5653.2ms/step | 10.6 samples/s
|
| 135 |
+
Step 11440/50462 | Loss: 2.6962 | LR: 4.45e-04 | 5651.9ms/step | 10.6 samples/s
|
| 136 |
+
Step 11450/50462 | Loss: 2.6749 | LR: 4.45e-04 | 5651.1ms/step | 10.6 samples/s
|
| 137 |
+
Step 11460/50462 | Loss: 2.7105 | LR: 4.45e-04 | 5652.0ms/step | 10.6 samples/s
|
| 138 |
+
Step 11470/50462 | Loss: 2.7485 | LR: 4.45e-04 | 5650.9ms/step | 10.6 samples/s
|
| 139 |
+
Step 11480/50462 | Loss: 2.7518 | LR: 4.45e-04 | 5653.7ms/step | 10.6 samples/s
|
| 140 |
+
Step 11490/50462 | Loss: 2.6540 | LR: 4.45e-04 | 5658.7ms/step | 10.6 samples/s
|
| 141 |
+
Step 11500/50462 | Loss: 2.7453 | LR: 4.44e-04 | 5661.7ms/step | 10.6 samples/s
|
| 142 |
+
Validation at step 11500: Loss = 2.7652, Perplexity = 15.88
|
| 143 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-11500
|
| 144 |
+
Step 11510/50462 | Loss: 2.6906 | LR: 4.44e-04 | 8349.0ms/step | 7.2 samples/s
|
| 145 |
+
Step 11520/50462 | Loss: 2.6969 | LR: 4.44e-04 | 5640.6ms/step | 10.6 samples/s
|
| 146 |
+
Step 11530/50462 | Loss: 2.6865 | LR: 4.44e-04 | 5642.2ms/step | 10.6 samples/s
|
| 147 |
+
Step 11540/50462 | Loss: 2.7038 | LR: 4.44e-04 | 5641.9ms/step | 10.6 samples/s
|
| 148 |
+
Step 11550/50462 | Loss: 2.7799 | LR: 4.44e-04 | 5642.0ms/step | 10.6 samples/s
|
| 149 |
+
Step 11560/50462 | Loss: 2.7034 | LR: 4.44e-04 | 5642.2ms/step | 10.6 samples/s
|
| 150 |
+
Step 11570/50462 | Loss: 2.7148 | LR: 4.44e-04 | 5642.7ms/step | 10.6 samples/s
|
| 151 |
+
Step 11580/50462 | Loss: 2.7276 | LR: 4.44e-04 | 5643.5ms/step | 10.6 samples/s
|
| 152 |
+
Step 11590/50462 | Loss: 2.7168 | LR: 4.44e-04 | 5644.1ms/step | 10.6 samples/s
|
| 153 |
+
Step 11600/50462 | Loss: 2.7288 | LR: 4.43e-04 | 5644.8ms/step | 10.6 samples/s
|
| 154 |
+
Step 11610/50462 | Loss: 2.7339 | LR: 4.43e-04 | 5645.5ms/step | 10.6 samples/s
|
| 155 |
+
Step 11620/50462 | Loss: 2.6936 | LR: 4.43e-04 | 5644.0ms/step | 10.6 samples/s
|
| 156 |
+
Step 11630/50462 | Loss: 2.7077 | LR: 4.43e-04 | 5643.1ms/step | 10.6 samples/s
|
| 157 |
+
Step 11640/50462 | Loss: 2.7321 | LR: 4.43e-04 | 5642.4ms/step | 10.6 samples/s
|
| 158 |
+
Step 11650/50462 | Loss: 2.7254 | LR: 4.43e-04 | 5640.5ms/step | 10.6 samples/s
|
| 159 |
+
Step 11660/50462 | Loss: 2.7044 | LR: 4.43e-04 | 5643.9ms/step | 10.6 samples/s
|
| 160 |
+
Step 11670/50462 | Loss: 2.7206 | LR: 4.43e-04 | 5642.4ms/step | 10.6 samples/s
|
| 161 |
+
Step 11680/50462 | Loss: 2.6768 | LR: 4.43e-04 | 5643.5ms/step | 10.6 samples/s
|
| 162 |
+
Step 11690/50462 | Loss: 2.6504 | LR: 4.43e-04 | 5644.8ms/step | 10.6 samples/s
|
| 163 |
+
Step 11700/50462 | Loss: 2.7350 | LR: 4.42e-04 | 5644.3ms/step | 10.6 samples/s
|
| 164 |
+
Step 11710/50462 | Loss: 2.6976 | LR: 4.42e-04 | 5643.4ms/step | 10.6 samples/s
|
| 165 |
+
Step 11720/50462 | Loss: 2.6999 | LR: 4.42e-04 | 5643.1ms/step | 10.6 samples/s
|
| 166 |
+
Step 11730/50462 | Loss: 2.6743 | LR: 4.42e-04 | 5639.8ms/step | 10.6 samples/s
|
| 167 |
+
Step 11740/50462 | Loss: 2.6739 | LR: 4.42e-04 | 5650.4ms/step | 10.6 samples/s
|
| 168 |
+
Step 11750/50462 | Loss: 2.6973 | LR: 4.42e-04 | 5647.1ms/step | 10.6 samples/s
|
| 169 |
+
Validation at step 11750: Loss = 2.7606, Perplexity = 15.81
|
| 170 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-11750
|
| 171 |
+
Step 11760/50462 | Loss: 2.6288 | LR: 4.42e-04 | 8366.0ms/step | 7.2 samples/s
|
| 172 |
+
Step 11770/50462 | Loss: 2.7083 | LR: 4.42e-04 | 5632.9ms/step | 10.7 samples/s
|
| 173 |
+
Step 11780/50462 | Loss: 2.7527 | LR: 4.42e-04 | 5635.5ms/step | 10.6 samples/s
|
| 174 |
+
Step 11790/50462 | Loss: 2.6870 | LR: 4.42e-04 | 5637.8ms/step | 10.6 samples/s
|
| 175 |
+
Step 11800/50462 | Loss: 2.6897 | LR: 4.41e-04 | 5636.1ms/step | 10.6 samples/s
|
| 176 |
+
Step 11810/50462 | Loss: 2.7512 | LR: 4.41e-04 | 5638.0ms/step | 10.6 samples/s
|
| 177 |
+
Step 11820/50462 | Loss: 2.7440 | LR: 4.41e-04 | 5637.2ms/step | 10.6 samples/s
|
| 178 |
+
Step 11830/50462 | Loss: 2.7021 | LR: 4.41e-04 | 5638.2ms/step | 10.6 samples/s
|
| 179 |
+
Step 11840/50462 | Loss: 2.6978 | LR: 4.41e-04 | 5637.4ms/step | 10.6 samples/s
|
| 180 |
+
Step 11850/50462 | Loss: 2.7295 | LR: 4.41e-04 | 5637.6ms/step | 10.6 samples/s
|
| 181 |
+
Step 11860/50462 | Loss: 2.6906 | LR: 4.41e-04 | 5636.7ms/step | 10.6 samples/s
|
| 182 |
+
Step 11870/50462 | Loss: 2.6766 | LR: 4.41e-04 | 5635.8ms/step | 10.6 samples/s
|
| 183 |
+
Step 11880/50462 | Loss: 2.7231 | LR: 4.41e-04 | 5637.7ms/step | 10.6 samples/s
|
| 184 |
+
Step 11890/50462 | Loss: 2.7020 | LR: 4.41e-04 | 5636.8ms/step | 10.6 samples/s
|
| 185 |
+
Step 11900/50462 | Loss: 2.7333 | LR: 4.40e-04 | 5639.0ms/step | 10.6 samples/s
|
| 186 |
+
Step 11910/50462 | Loss: 2.7219 | LR: 4.40e-04 | 5641.7ms/step | 10.6 samples/s
|
| 187 |
+
Step 11920/50462 | Loss: 2.7254 | LR: 4.40e-04 | 5640.0ms/step | 10.6 samples/s
|
| 188 |
+
Step 11930/50462 | Loss: 2.6999 | LR: 4.40e-04 | 5638.8ms/step | 10.6 samples/s
|
| 189 |
+
Step 11940/50462 | Loss: 2.6967 | LR: 4.40e-04 | 5639.3ms/step | 10.6 samples/s
|
| 190 |
+
Step 11950/50462 | Loss: 2.6480 | LR: 4.40e-04 | 5637.0ms/step | 10.6 samples/s
|
| 191 |
+
Step 11960/50462 | Loss: 2.7282 | LR: 4.40e-04 | 5638.3ms/step | 10.6 samples/s
|
| 192 |
+
Step 11970/50462 | Loss: 2.7551 | LR: 4.40e-04 | 5638.1ms/step | 10.6 samples/s
|
| 193 |
+
Step 11980/50462 | Loss: 2.7355 | LR: 4.40e-04 | 5638.3ms/step | 10.6 samples/s
|
| 194 |
+
Step 11990/50462 | Loss: 2.6881 | LR: 4.40e-04 | 5638.4ms/step | 10.6 samples/s
|
| 195 |
+
Step 12000/50462 | Loss: 2.6719 | LR: 4.39e-04 | 5639.1ms/step | 10.6 samples/s
|
| 196 |
+
Validation at step 12000: Loss = 2.7572, Perplexity = 15.76
|
| 197 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-12000
|
| 198 |
+
Step 12010/50462 | Loss: 2.7396 | LR: 4.39e-04 | 8379.4ms/step | 7.2 samples/s
|
| 199 |
+
Step 12020/50462 | Loss: 2.7468 | LR: 4.39e-04 | 5626.9ms/step | 10.7 samples/s
|
| 200 |
+
Step 12030/50462 | Loss: 2.6915 | LR: 4.39e-04 | 5633.8ms/step | 10.7 samples/s
|
| 201 |
+
Step 12040/50462 | Loss: 2.6746 | LR: 4.39e-04 | 5637.0ms/step | 10.6 samples/s
|
| 202 |
+
Step 12050/50462 | Loss: 2.6716 | LR: 4.39e-04 | 5640.0ms/step | 10.6 samples/s
|
| 203 |
+
Step 12060/50462 | Loss: 2.7175 | LR: 4.39e-04 | 5638.6ms/step | 10.6 samples/s
|
| 204 |
+
Step 12070/50462 | Loss: 2.6792 | LR: 4.39e-04 | 5639.1ms/step | 10.6 samples/s
|
| 205 |
+
Step 12080/50462 | Loss: 2.7227 | LR: 4.39e-04 | 5639.2ms/step | 10.6 samples/s
|
| 206 |
+
Step 12090/50462 | Loss: 2.7076 | LR: 4.39e-04 | 5640.5ms/step | 10.6 samples/s
|
| 207 |
+
Step 12100/50462 | Loss: 2.6879 | LR: 4.38e-04 | 5638.8ms/step | 10.6 samples/s
|
| 208 |
+
Step 12110/50462 | Loss: 2.7421 | LR: 4.38e-04 | 5638.7ms/step | 10.6 samples/s
|
| 209 |
+
Step 12120/50462 | Loss: 2.7086 | LR: 4.38e-04 | 5639.5ms/step | 10.6 samples/s
|
| 210 |
+
Step 12130/50462 | Loss: 2.7030 | LR: 4.38e-04 | 5638.9ms/step | 10.6 samples/s
|
| 211 |
+
Step 12140/50462 | Loss: 2.7004 | LR: 4.38e-04 | 5640.9ms/step | 10.6 samples/s
|
| 212 |
+
Step 12150/50462 | Loss: 2.7036 | LR: 4.38e-04 | 5640.5ms/step | 10.6 samples/s
|
| 213 |
+
Step 12160/50462 | Loss: 2.7185 | LR: 4.38e-04 | 5638.6ms/step | 10.6 samples/s
|
| 214 |
+
Step 12170/50462 | Loss: 2.7159 | LR: 4.38e-04 | 5638.6ms/step | 10.6 samples/s
|
| 215 |
+
Step 12180/50462 | Loss: 2.7379 | LR: 4.38e-04 | 5638.2ms/step | 10.6 samples/s
|
| 216 |
+
Step 12190/50462 | Loss: 2.7616 | LR: 4.37e-04 | 5637.4ms/step | 10.6 samples/s
|
| 217 |
+
Step 12200/50462 | Loss: 2.6951 | LR: 4.37e-04 | 5637.9ms/step | 10.6 samples/s
|
| 218 |
+
Step 12210/50462 | Loss: 2.6838 | LR: 4.37e-04 | 5638.2ms/step | 10.6 samples/s
|
| 219 |
+
Step 12220/50462 | Loss: 2.6739 | LR: 4.37e-04 | 5639.0ms/step | 10.6 samples/s
|
| 220 |
+
Step 12230/50462 | Loss: 2.7626 | LR: 4.37e-04 | 5640.1ms/step | 10.6 samples/s
|
| 221 |
+
Step 12240/50462 | Loss: 2.7011 | LR: 4.37e-04 | 5639.5ms/step | 10.6 samples/s
|
| 222 |
+
Step 12250/50462 | Loss: 2.6758 | LR: 4.37e-04 | 5639.7ms/step | 10.6 samples/s
|
| 223 |
+
Validation at step 12250: Loss = 2.7546, Perplexity = 15.72
|
| 224 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-12250
|
| 225 |
+
Step 12260/50462 | Loss: 2.6675 | LR: 4.37e-04 | 8383.5ms/step | 7.2 samples/s
|
| 226 |
+
Step 12270/50462 | Loss: 2.7184 | LR: 4.37e-04 | 5629.0ms/step | 10.7 samples/s
|
| 227 |
+
Step 12280/50462 | Loss: 2.6815 | LR: 4.37e-04 | 5635.1ms/step | 10.6 samples/s
|
| 228 |
+
Step 12290/50462 | Loss: 2.7288 | LR: 4.36e-04 | 5638.3ms/step | 10.6 samples/s
|
| 229 |
+
Step 12300/50462 | Loss: 2.7355 | LR: 4.36e-04 | 5637.1ms/step | 10.6 samples/s
|
| 230 |
+
Step 12310/50462 | Loss: 2.7291 | LR: 4.36e-04 | 5638.1ms/step | 10.6 samples/s
|
| 231 |
+
Step 12320/50462 | Loss: 2.6975 | LR: 4.36e-04 | 5637.8ms/step | 10.6 samples/s
|
| 232 |
+
Step 12330/50462 | Loss: 2.7134 | LR: 4.36e-04 | 5637.4ms/step | 10.6 samples/s
|
| 233 |
+
Step 12340/50462 | Loss: 2.6923 | LR: 4.36e-04 | 5637.9ms/step | 10.6 samples/s
|
| 234 |
+
Step 12350/50462 | Loss: 2.7155 | LR: 4.36e-04 | 5636.6ms/step | 10.6 samples/s
|
| 235 |
+
Step 12360/50462 | Loss: 2.7111 | LR: 4.36e-04 | 5639.0ms/step | 10.6 samples/s
|
| 236 |
+
Step 12370/50462 | Loss: 2.7175 | LR: 4.36e-04 | 5636.5ms/step | 10.6 samples/s
|
| 237 |
+
Step 12380/50462 | Loss: 2.7681 | LR: 4.35e-04 | 5639.3ms/step | 10.6 samples/s
|
| 238 |
+
Step 12390/50462 | Loss: 2.7098 | LR: 4.35e-04 | 5638.5ms/step | 10.6 samples/s
|
| 239 |
+
Step 12400/50462 | Loss: 2.6669 | LR: 4.35e-04 | 5639.2ms/step | 10.6 samples/s
|
| 240 |
+
Step 12410/50462 | Loss: 2.7098 | LR: 4.35e-04 | 5638.9ms/step | 10.6 samples/s
|
| 241 |
+
Step 12420/50462 | Loss: 2.7075 | LR: 4.35e-04 | 5639.7ms/step | 10.6 samples/s
|
| 242 |
+
Step 12430/50462 | Loss: 2.6992 | LR: 4.35e-04 | 5639.2ms/step | 10.6 samples/s
|
| 243 |
+
Step 12440/50462 | Loss: 2.7315 | LR: 4.35e-04 | 5638.8ms/step | 10.6 samples/s
|
| 244 |
+
Step 12450/50462 | Loss: 2.6956 | LR: 4.35e-04 | 5639.5ms/step | 10.6 samples/s
|
| 245 |
+
Step 12460/50462 | Loss: 2.7096 | LR: 4.35e-04 | 5640.2ms/step | 10.6 samples/s
|
| 246 |
+
Step 12470/50462 | Loss: 2.7177 | LR: 4.35e-04 | 5639.1ms/step | 10.6 samples/s
|
| 247 |
+
Step 12480/50462 | Loss: 2.7254 | LR: 4.34e-04 | 5639.4ms/step | 10.6 samples/s
|
| 248 |
+
Step 12490/50462 | Loss: 2.7152 | LR: 4.34e-04 | 5638.3ms/step | 10.6 samples/s
|
| 249 |
+
Step 12500/50462 | Loss: 2.7136 | LR: 4.34e-04 | 5637.0ms/step | 10.6 samples/s
|
| 250 |
+
Validation at step 12500: Loss = 2.7509, Perplexity = 15.66
|
| 251 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-12500
|
| 252 |
+
Step 12510/50462 | Loss: 2.6697 | LR: 4.34e-04 | 8368.0ms/step | 7.2 samples/s
|
| 253 |
+
Step 12520/50462 | Loss: 2.7041 | LR: 4.34e-04 | 5630.3ms/step | 10.7 samples/s
|
| 254 |
+
Step 12530/50462 | Loss: 2.7109 | LR: 4.34e-04 | 5636.5ms/step | 10.6 samples/s
|
| 255 |
+
Step 12540/50462 | Loss: 2.6784 | LR: 4.34e-04 | 5637.5ms/step | 10.6 samples/s
|
| 256 |
+
Step 12550/50462 | Loss: 2.7024 | LR: 4.34e-04 | 5638.8ms/step | 10.6 samples/s
|
| 257 |
+
Step 12560/50462 | Loss: 2.6931 | LR: 4.34e-04 | 5638.8ms/step | 10.6 samples/s
|
| 258 |
+
Step 12570/50462 | Loss: 2.6884 | LR: 4.33e-04 | 5637.4ms/step | 10.6 samples/s
|
| 259 |
+
Step 12580/50462 | Loss: 2.6848 | LR: 4.33e-04 | 5640.3ms/step | 10.6 samples/s
|
| 260 |
+
Step 12590/50462 | Loss: 2.7059 | LR: 4.33e-04 | 5639.3ms/step | 10.6 samples/s
|
| 261 |
+
Step 12600/50462 | Loss: 2.7072 | LR: 4.33e-04 | 5638.7ms/step | 10.6 samples/s
|
| 262 |
+
Step 12610/50462 | Loss: 2.7055 | LR: 4.33e-04 | 5640.0ms/step | 10.6 samples/s
|
| 263 |
+
Step 12620/50462 | Loss: 2.6985 | LR: 4.33e-04 | 5639.7ms/step | 10.6 samples/s
|
| 264 |
+
Step 12630/50462 | Loss: 2.7075 | LR: 4.33e-04 | 5637.0ms/step | 10.6 samples/s
|
| 265 |
+
Step 12640/50462 | Loss: 2.7099 | LR: 4.33e-04 | 5637.9ms/step | 10.6 samples/s
|
| 266 |
+
Step 12650/50462 | Loss: 2.7097 | LR: 4.33e-04 | 5638.6ms/step | 10.6 samples/s
|
| 267 |
+
Step 12660/50462 | Loss: 2.6667 | LR: 4.32e-04 | 5638.6ms/step | 10.6 samples/s
|
| 268 |
+
Step 12670/50462 | Loss: 2.7050 | LR: 4.32e-04 | 5639.0ms/step | 10.6 samples/s
|
| 269 |
+
Step 12680/50462 | Loss: 2.6851 | LR: 4.32e-04 | 5639.9ms/step | 10.6 samples/s
|
| 270 |
+
Step 12690/50462 | Loss: 2.6891 | LR: 4.32e-04 | 5640.2ms/step | 10.6 samples/s
|
| 271 |
+
Step 12700/50462 | Loss: 2.6817 | LR: 4.32e-04 | 5639.8ms/step | 10.6 samples/s
|
| 272 |
+
Step 12710/50462 | Loss: 2.6736 | LR: 4.32e-04 | 5641.0ms/step | 10.6 samples/s
|
| 273 |
+
Step 12720/50462 | Loss: 2.6906 | LR: 4.32e-04 | 5639.2ms/step | 10.6 samples/s
|
| 274 |
+
Step 12730/50462 | Loss: 2.7160 | LR: 4.32e-04 | 5640.1ms/step | 10.6 samples/s
|
| 275 |
+
Step 12740/50462 | Loss: 2.6925 | LR: 4.32e-04 | 5639.7ms/step | 10.6 samples/s
|
| 276 |
+
Step 12750/50462 | Loss: 2.6902 | LR: 4.31e-04 | 5640.5ms/step | 10.6 samples/s
|
| 277 |
+
Validation at step 12750: Loss = 2.7480, Perplexity = 15.61
|
| 278 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-12750
|
| 279 |
+
Step 12760/50462 | Loss: 2.7178 | LR: 4.31e-04 | 8366.6ms/step | 7.2 samples/s
|
| 280 |
+
Step 12770/50462 | Loss: 2.7105 | LR: 4.31e-04 | 5626.2ms/step | 10.7 samples/s
|
| 281 |
+
Step 12780/50462 | Loss: 2.6892 | LR: 4.31e-04 | 5634.1ms/step | 10.6 samples/s
|
| 282 |
+
Step 12790/50462 | Loss: 2.6927 | LR: 4.31e-04 | 5649.6ms/step | 10.6 samples/s
|
| 283 |
+
Step 12800/50462 | Loss: 2.7041 | LR: 4.31e-04 | 5647.6ms/step | 10.6 samples/s
|
| 284 |
+
Step 12810/50462 | Loss: 2.7128 | LR: 4.31e-04 | 5646.8ms/step | 10.6 samples/s
|
| 285 |
+
Step 12820/50462 | Loss: 2.7024 | LR: 4.31e-04 | 5647.2ms/step | 10.6 samples/s
|
| 286 |
+
Step 12830/50462 | Loss: 2.6945 | LR: 4.31e-04 | 5649.1ms/step | 10.6 samples/s
|
| 287 |
+
Step 12840/50462 | Loss: 2.6898 | LR: 4.31e-04 | 5646.9ms/step | 10.6 samples/s
|
| 288 |
+
Step 12850/50462 | Loss: 2.7150 | LR: 4.30e-04 | 5648.3ms/step | 10.6 samples/s
|
| 289 |
+
Step 12860/50462 | Loss: 2.7605 | LR: 4.30e-04 | 5644.8ms/step | 10.6 samples/s
|
| 290 |
+
Step 12870/50462 | Loss: 2.6795 | LR: 4.30e-04 | 5645.9ms/step | 10.6 samples/s
|
| 291 |
+
Step 12880/50462 | Loss: 2.7200 | LR: 4.30e-04 | 5647.0ms/step | 10.6 samples/s
|
| 292 |
+
Step 12890/50462 | Loss: 2.7309 | LR: 4.30e-04 | 5635.5ms/step | 10.6 samples/s
|
| 293 |
+
Step 12900/50462 | Loss: 2.6920 | LR: 4.30e-04 | 5635.8ms/step | 10.6 samples/s
|
| 294 |
+
Step 12910/50462 | Loss: 2.6885 | LR: 4.30e-04 | 5642.0ms/step | 10.6 samples/s
|
| 295 |
+
Step 12920/50462 | Loss: 2.7058 | LR: 4.30e-04 | 5645.4ms/step | 10.6 samples/s
|
| 296 |
+
Step 12930/50462 | Loss: 2.7011 | LR: 4.30e-04 | 5646.1ms/step | 10.6 samples/s
|
| 297 |
+
Step 12940/50462 | Loss: 2.7012 | LR: 4.29e-04 | 5646.6ms/step | 10.6 samples/s
|
| 298 |
+
Step 12950/50462 | Loss: 2.6862 | LR: 4.29e-04 | 5641.7ms/step | 10.6 samples/s
|
| 299 |
+
Step 12960/50462 | Loss: 2.6712 | LR: 4.29e-04 | 5640.8ms/step | 10.6 samples/s
|
| 300 |
+
Step 12970/50462 | Loss: 2.6992 | LR: 4.29e-04 | 5644.4ms/step | 10.6 samples/s
|
| 301 |
+
Step 12980/50462 | Loss: 2.6826 | LR: 4.29e-04 | 5646.4ms/step | 10.6 samples/s
|
| 302 |
+
Step 12990/50462 | Loss: 2.6624 | LR: 4.29e-04 | 5643.9ms/step | 10.6 samples/s
|
| 303 |
+
Step 13000/50462 | Loss: 2.7072 | LR: 4.29e-04 | 5644.5ms/step | 10.6 samples/s
|
| 304 |
+
Validation at step 13000: Loss = 2.7452, Perplexity = 15.57
|
| 305 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-13000
|
| 306 |
+
Step 13010/50462 | Loss: 2.7087 | LR: 4.29e-04 | 8430.9ms/step | 7.1 samples/s
|
| 307 |
+
Step 13020/50462 | Loss: 2.7035 | LR: 4.29e-04 | 5636.0ms/step | 10.6 samples/s
|
| 308 |
+
Step 13030/50462 | Loss: 2.6612 | LR: 4.28e-04 | 5637.9ms/step | 10.6 samples/s
|
| 309 |
+
Step 13040/50462 | Loss: 2.6868 | LR: 4.28e-04 | 5637.7ms/step | 10.6 samples/s
|
| 310 |
+
Step 13050/50462 | Loss: 2.7020 | LR: 4.28e-04 | 5639.0ms/step | 10.6 samples/s
|
| 311 |
+
Step 13060/50462 | Loss: 2.6835 | LR: 4.28e-04 | 5640.6ms/step | 10.6 samples/s
|
| 312 |
+
Step 13070/50462 | Loss: 2.6975 | LR: 4.28e-04 | 5640.5ms/step | 10.6 samples/s
|
| 313 |
+
Step 13080/50462 | Loss: 2.6663 | LR: 4.28e-04 | 5638.7ms/step | 10.6 samples/s
|
| 314 |
+
Step 13090/50462 | Loss: 2.6980 | LR: 4.28e-04 | 5638.9ms/step | 10.6 samples/s
|
| 315 |
+
Step 13100/50462 | Loss: 2.7095 | LR: 4.28e-04 | 5640.0ms/step | 10.6 samples/s
|
| 316 |
+
Step 13110/50462 | Loss: 2.6990 | LR: 4.28e-04 | 5640.7ms/step | 10.6 samples/s
|
| 317 |
+
Step 13120/50462 | Loss: 2.6762 | LR: 4.27e-04 | 5638.4ms/step | 10.6 samples/s
|
| 318 |
+
Step 13130/50462 | Loss: 2.6855 | LR: 4.27e-04 | 5637.8ms/step | 10.6 samples/s
|
| 319 |
+
Step 13140/50462 | Loss: 2.6784 | LR: 4.27e-04 | 5637.3ms/step | 10.6 samples/s
|
| 320 |
+
Step 13150/50462 | Loss: 2.6935 | LR: 4.27e-04 | 5639.2ms/step | 10.6 samples/s
|
| 321 |
+
Step 13160/50462 | Loss: 2.7033 | LR: 4.27e-04 | 5637.5ms/step | 10.6 samples/s
|
| 322 |
+
Step 13170/50462 | Loss: 2.6324 | LR: 4.27e-04 | 5639.1ms/step | 10.6 samples/s
|
| 323 |
+
Step 13180/50462 | Loss: 2.7152 | LR: 4.27e-04 | 5640.2ms/step | 10.6 samples/s
|
| 324 |
+
Step 13190/50462 | Loss: 2.6310 | LR: 4.27e-04 | 5639.8ms/step | 10.6 samples/s
|
| 325 |
+
Step 13200/50462 | Loss: 2.7208 | LR: 4.27e-04 | 5641.2ms/step | 10.6 samples/s
|
| 326 |
+
Step 13210/50462 | Loss: 2.6577 | LR: 4.26e-04 | 5642.0ms/step | 10.6 samples/s
|
| 327 |
+
Step 13220/50462 | Loss: 2.7141 | LR: 4.26e-04 | 5641.1ms/step | 10.6 samples/s
|
| 328 |
+
Step 13230/50462 | Loss: 2.6278 | LR: 4.26e-04 | 5639.8ms/step | 10.6 samples/s
|
| 329 |
+
Step 13240/50462 | Loss: 2.7089 | LR: 4.26e-04 | 5638.2ms/step | 10.6 samples/s
|
| 330 |
+
Step 13250/50462 | Loss: 2.6795 | LR: 4.26e-04 | 5634.8ms/step | 10.6 samples/s
|
| 331 |
+
Validation at step 13250: Loss = 2.7395, Perplexity = 15.48
|
| 332 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-13250
|
| 333 |
+
Step 13260/50462 | Loss: 2.6344 | LR: 4.26e-04 | 8390.3ms/step | 7.2 samples/s
|
| 334 |
+
Step 13270/50462 | Loss: 2.6942 | LR: 4.26e-04 | 5635.6ms/step | 10.6 samples/s
|
| 335 |
+
Step 13280/50462 | Loss: 2.6995 | LR: 4.26e-04 | 5636.4ms/step | 10.6 samples/s
|
| 336 |
+
Step 13290/50462 | Loss: 2.6941 | LR: 4.26e-04 | 5639.4ms/step | 10.6 samples/s
|
| 337 |
+
Step 13300/50462 | Loss: 2.6853 | LR: 4.25e-04 | 5637.6ms/step | 10.6 samples/s
|
| 338 |
+
Step 13310/50462 | Loss: 2.6773 | LR: 4.25e-04 | 5637.8ms/step | 10.6 samples/s
|
| 339 |
+
Step 13320/50462 | Loss: 2.6918 | LR: 4.25e-04 | 5638.3ms/step | 10.6 samples/s
|
| 340 |
+
Step 13330/50462 | Loss: 2.7092 | LR: 4.25e-04 | 5638.8ms/step | 10.6 samples/s
|
| 341 |
+
Step 13340/50462 | Loss: 2.6727 | LR: 4.25e-04 | 5638.8ms/step | 10.6 samples/s
|
| 342 |
+
Step 13350/50462 | Loss: 2.6845 | LR: 4.25e-04 | 5637.5ms/step | 10.6 samples/s
|
| 343 |
+
Step 13360/50462 | Loss: 2.6827 | LR: 4.25e-04 | 5637.6ms/step | 10.6 samples/s
|
| 344 |
+
Step 13370/50462 | Loss: 2.7254 | LR: 4.25e-04 | 5638.6ms/step | 10.6 samples/s
|
| 345 |
+
Step 13380/50462 | Loss: 2.6825 | LR: 4.25e-04 | 5636.3ms/step | 10.6 samples/s
|
| 346 |
+
Step 13390/50462 | Loss: 2.6745 | LR: 4.24e-04 | 5637.4ms/step | 10.6 samples/s
|
| 347 |
+
Step 13400/50462 | Loss: 2.7167 | LR: 4.24e-04 | 5633.9ms/step | 10.6 samples/s
|
| 348 |
+
Step 13410/50462 | Loss: 2.6443 | LR: 4.24e-04 | 5637.3ms/step | 10.6 samples/s
|
| 349 |
+
Step 13420/50462 | Loss: 2.6787 | LR: 4.24e-04 | 5639.8ms/step | 10.6 samples/s
|
| 350 |
+
Step 13430/50462 | Loss: 2.7108 | LR: 4.24e-04 | 5635.9ms/step | 10.6 samples/s
|
| 351 |
+
Step 13440/50462 | Loss: 2.6802 | LR: 4.24e-04 | 5641.9ms/step | 10.6 samples/s
|
| 352 |
+
Step 13450/50462 | Loss: 2.7312 | LR: 4.24e-04 | 5638.6ms/step | 10.6 samples/s
|
| 353 |
+
Step 13460/50462 | Loss: 2.6788 | LR: 4.24e-04 | 5636.7ms/step | 10.6 samples/s
|
| 354 |
+
Step 13470/50462 | Loss: 2.6803 | LR: 4.23e-04 | 5639.1ms/step | 10.6 samples/s
|
| 355 |
+
Step 13480/50462 | Loss: 2.6908 | LR: 4.23e-04 | 5639.0ms/step | 10.6 samples/s
|
| 356 |
+
Step 13490/50462 | Loss: 2.6883 | LR: 4.23e-04 | 5639.8ms/step | 10.6 samples/s
|
| 357 |
+
Step 13500/50462 | Loss: 2.7029 | LR: 4.23e-04 | 5638.1ms/step | 10.6 samples/s
|
| 358 |
+
Validation at step 13500: Loss = 2.7361, Perplexity = 15.43
|
| 359 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-13500
|
| 360 |
+
Step 13510/50462 | Loss: 2.6593 | LR: 4.23e-04 | 8385.9ms/step | 7.2 samples/s
|
| 361 |
+
Step 13520/50462 | Loss: 2.7098 | LR: 4.23e-04 | 5628.9ms/step | 10.7 samples/s
|
| 362 |
+
Step 13530/50462 | Loss: 2.6718 | LR: 4.23e-04 | 5634.3ms/step | 10.6 samples/s
|
| 363 |
+
Step 13540/50462 | Loss: 2.6783 | LR: 4.23e-04 | 5638.9ms/step | 10.6 samples/s
|
| 364 |
+
Step 13550/50462 | Loss: 2.6606 | LR: 4.23e-04 | 5644.4ms/step | 10.6 samples/s
|
| 365 |
+
Step 13560/50462 | Loss: 2.6996 | LR: 4.22e-04 | 5644.4ms/step | 10.6 samples/s
|
| 366 |
+
Step 13570/50462 | Loss: 2.6735 | LR: 4.22e-04 | 5645.1ms/step | 10.6 samples/s
|
| 367 |
+
Step 13580/50462 | Loss: 2.6864 | LR: 4.22e-04 | 5645.7ms/step | 10.6 samples/s
|
epfl_final/SparseAtt/wandb/run-20260824_034449-rgmkfpnl/files/requirements.txt
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
| 142 |
+
entmax==1.3
|
epfl_final/SparseAtt/wandb/run-20260824_034449-rgmkfpnl/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-08-24T03:44:49.182308Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"configs/p2_1p2b_dense_seed70_5gpu.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/TokenPrune.git",
|
| 14 |
+
"commit": "5637886f0cc760009989b14e68c6ae06d4827418"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "a100-train-5-gpu-0-5",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python",
|
| 20 |
+
"cpu_count": 64,
|
| 21 |
+
"cpu_count_logical": 128,
|
| 22 |
+
"gpu": "NVIDIA A100-SXM4-80GB",
|
| 23 |
+
"gpu_count": 5,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "7679362727936",
|
| 27 |
+
"used": "4180563054592"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1081884958720"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 36 |
+
"memoryTotal": "85899345920",
|
| 37 |
+
"cudaCores": 6912,
|
| 38 |
+
"architecture": "Ampere",
|
| 39 |
+
"uuid": "GPU-b8ef56c4-1127-15f2-894f-367e4d65de9a"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 43 |
+
"memoryTotal": "85899345920",
|
| 44 |
+
"cudaCores": 6912,
|
| 45 |
+
"architecture": "Ampere",
|
| 46 |
+
"uuid": "GPU-09f84fa1-0922-384a-dd6d-cf437a21a2da"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 50 |
+
"memoryTotal": "85899345920",
|
| 51 |
+
"cudaCores": 6912,
|
| 52 |
+
"architecture": "Ampere",
|
| 53 |
+
"uuid": "GPU-c0cfe7a4-315b-3a86-0b75-ed455b493ee6"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 57 |
+
"memoryTotal": "85899345920",
|
| 58 |
+
"cudaCores": 6912,
|
| 59 |
+
"architecture": "Ampere",
|
| 60 |
+
"uuid": "GPU-e915521e-9f1b-cf27-6665-31ccb43a583a"
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
"name": "NVIDIA A100-SXM4-80GB",
|
| 64 |
+
"memoryTotal": "85899345920",
|
| 65 |
+
"cudaCores": 6912,
|
| 66 |
+
"architecture": "Ampere",
|
| 67 |
+
"uuid": "GPU-628e4b10-6e84-e74c-352c-9b3202dde384"
|
| 68 |
+
}
|
| 69 |
+
],
|
| 70 |
+
"cudaVersion": "13.0",
|
| 71 |
+
"writerId": "rwp633doqp8vjm43yf6x1kfupwhg64sv"
|
| 72 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260824_034449-rgmkfpnl/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260824_034449-rgmkfpnl/logs/debug.log
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-08-24 03:44:49,225 INFO MainThread:235 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-08-24 03:44:49,225 INFO MainThread:235 [wandb_setup.py:_flush():81] Configure stats pid to 235
|
| 3 |
+
2026-08-24 03:44:49,225 INFO MainThread:235 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-08-24 03:44:49,225 INFO MainThread:235 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260824_034449-rgmkfpnl/logs/debug.log
|
| 5 |
+
2026-08-24 03:44:49,225 INFO MainThread:235 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260824_034449-rgmkfpnl/logs/debug-internal.log
|
| 6 |
+
2026-08-24 03:44:49,225 INFO MainThread:235 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-08-24 03:44:49,225 INFO MainThread:235 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-10250', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 12, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 70, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b dense 8k seed70 5gpu resume', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'dense', 'seed70'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-08-24 03:44:49,225 INFO MainThread:235 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-08-24 03:44:50,606 INFO MainThread:235 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-08-24 03:44:51,143 INFO MainThread:235 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-08-24 03:44:51,146 INFO MainThread:235 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-08-24 03:44:51,221 INFO MainThread:235 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-08-24 03:44:51,791 INFO MainThread:235 [wandb_init.py:init():1051] run resumed
|
| 15 |
+
2026-08-24 03:44:51,792 INFO MainThread:235 [wandb_init.py:init():1056] starting run threads in backend
|
| 16 |
+
2026-08-24 03:44:52,186 INFO MainThread:235 [wandb_run.py:_console_start():2554] atexit reg
|
| 17 |
+
2026-08-24 03:44:52,186 INFO MainThread:235 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 18 |
+
2026-08-24 03:44:52,186 INFO MainThread:235 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 19 |
+
2026-08-24 03:44:52,186 INFO MainThread:235 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 20 |
+
2026-08-24 03:44:52,200 INFO MainThread:235 [wandb_init.py:init():1094] run started, returning control to user process
|
epfl_final/SparseAtt/wandb/run-20260824_034449-rgmkfpnl/run-rgmkfpnl.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:326c950bebdade42ed15a1c3d699d768205d1ed427a927c34f516d98c52f9fbe
|
| 3 |
+
size 3801088
|
epfl_final/SparseAtt/wandb/run-20260824_034710-i8ewyy95/files/output.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260824_034710-i8ewyy95/files/requirements.txt
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
| 142 |
+
entmax==1.3
|
epfl_final/SparseAtt/wandb/run-20260824_034710-i8ewyy95/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-08-24T03:47:10.691081Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"configs/p2_1p2b_mixed_seed70_6gpu.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/TokenPrune.git",
|
| 14 |
+
"commit": "5637886f0cc760009989b14e68c6ae06d4827418"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "h100-train-6-gpu-0-33",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 23 |
+
"gpu_count": 6,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "9374931091456",
|
| 27 |
+
"used": "5339950206976"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1623005626368"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-6fb12770-6b51-9595-91c9-0d20801e3896"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 43 |
+
"memoryTotal": "85520809984",
|
| 44 |
+
"cudaCores": 16896,
|
| 45 |
+
"architecture": "Hopper",
|
| 46 |
+
"uuid": "GPU-3ea6c840-350f-c68c-7562-6e0affa4e153"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 50 |
+
"memoryTotal": "85520809984",
|
| 51 |
+
"cudaCores": 16896,
|
| 52 |
+
"architecture": "Hopper",
|
| 53 |
+
"uuid": "GPU-5cb9f8b7-4a13-4f85-d73e-6bcbebebefb8"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 57 |
+
"memoryTotal": "85520809984",
|
| 58 |
+
"cudaCores": 16896,
|
| 59 |
+
"architecture": "Hopper",
|
| 60 |
+
"uuid": "GPU-32209e2e-7887-e730-7611-251622a8e78f"
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 64 |
+
"memoryTotal": "85520809984",
|
| 65 |
+
"cudaCores": 16896,
|
| 66 |
+
"architecture": "Hopper",
|
| 67 |
+
"uuid": "GPU-3512802a-d501-0a3c-9812-4ab9deb3a3a9"
|
| 68 |
+
},
|
| 69 |
+
{
|
| 70 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 71 |
+
"memoryTotal": "85520809984",
|
| 72 |
+
"cudaCores": 16896,
|
| 73 |
+
"architecture": "Hopper",
|
| 74 |
+
"uuid": "GPU-24436def-beee-c9f1-bd05-c97df556de08"
|
| 75 |
+
}
|
| 76 |
+
],
|
| 77 |
+
"cudaVersion": "13.0",
|
| 78 |
+
"writerId": "doavzx0bpjkeo998wvxe0q2op1yhgtss"
|
| 79 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260824_034710-i8ewyy95/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260824_034710-i8ewyy95/logs/debug.log
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-08-24 03:47:10,720 INFO MainThread:211 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-08-24 03:47:10,721 INFO MainThread:211 [wandb_setup.py:_flush():81] Configure stats pid to 211
|
| 3 |
+
2026-08-24 03:47:10,721 INFO MainThread:211 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-08-24 03:47:10,721 INFO MainThread:211 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260824_034710-i8ewyy95/logs/debug.log
|
| 5 |
+
2026-08-24 03:47:10,721 INFO MainThread:211 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260824_034710-i8ewyy95/logs/debug-internal.log
|
| 6 |
+
2026-08-24 03:47:10,721 INFO MainThread:211 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-08-24 03:47:10,721 INFO MainThread:211 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-24000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 70, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b mixed 8k seed70 6gpu resume', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'mixed', 'seed70'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed', 'sparse_mask_seed': 70, 'sparse_mixed_nrb_set': [63, 47, 31, 21, 11], 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-08-24 03:47:10,721 INFO MainThread:211 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-08-24 03:47:11,529 INFO MainThread:211 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-08-24 03:47:11,944 INFO MainThread:211 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-08-24 03:47:11,946 INFO MainThread:211 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-08-24 03:47:11,969 INFO MainThread:211 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-08-24 03:47:12,360 INFO MainThread:211 [wandb_init.py:init():1051] run resumed
|
| 15 |
+
2026-08-24 03:47:12,361 INFO MainThread:211 [wandb_init.py:init():1056] starting run threads in backend
|
| 16 |
+
2026-08-24 03:47:12,578 INFO MainThread:211 [wandb_run.py:_console_start():2554] atexit reg
|
| 17 |
+
2026-08-24 03:47:12,578 INFO MainThread:211 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 18 |
+
2026-08-24 03:47:12,578 INFO MainThread:211 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 19 |
+
2026-08-24 03:47:12,578 INFO MainThread:211 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 20 |
+
2026-08-24 03:47:12,585 INFO MainThread:211 [wandb_init.py:init():1094] run started, returning control to user process
|
epfl_final/SparseAtt/wandb/run-20260824_034710-i8ewyy95/run-i8ewyy95.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6e6de7f86ec1d2901af91867d5a82de58b7880c8f6bc2cac11849758635126db
|
| 3 |
+
size 18415616
|
epfl_final/SparseAtt/wandb/run-20260824_132232-rgmkfpnl/files/output.log
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Starting training...
|
| 2 |
+
Validation at step 13500: Loss = 2.6953, Perplexity = 14.81
|
| 3 |
+
wandb: WARNING Tried to log to step 13500 that is less than the current step 13571. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 4 |
+
Switching attention mode for step 13501: dense (dense step interval)
|
| 5 |
+
Step 13510/50462 | Loss: 2.4722 | LR: 4.23e-04 | 54319.4ms/step | 1.1 samples/s
|
| 6 |
+
wandb: WARNING Tried to log to step 13510 that is less than the current step 13571. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 7 |
+
Step 13520/50462 | Loss: 2.6271 | LR: 4.23e-04 | 5264.0ms/step | 11.4 samples/s
|
| 8 |
+
wandb: WARNING Tried to log to step 13520 that is less than the current step 13571. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 9 |
+
Step 13530/50462 | Loss: 2.6168 | LR: 4.23e-04 | 5274.7ms/step | 11.4 samples/s
|
| 10 |
+
wandb: WARNING Tried to log to step 13530 that is less than the current step 13571. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 11 |
+
Step 13540/50462 | Loss: 2.6109 | LR: 4.23e-04 | 5270.0ms/step | 11.4 samples/s
|
| 12 |
+
wandb: WARNING Tried to log to step 13540 that is less than the current step 13571. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 13 |
+
Step 13550/50462 | Loss: 2.6349 | LR: 4.23e-04 | 5272.2ms/step | 11.4 samples/s
|
| 14 |
+
wandb: WARNING Tried to log to step 13550 that is less than the current step 13571. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 15 |
+
Step 13560/50462 | Loss: 2.6069 | LR: 4.22e-04 | 5280.6ms/step | 11.4 samples/s
|
| 16 |
+
wandb: WARNING Tried to log to step 13560 that is less than the current step 13571. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 17 |
+
Step 13570/50462 | Loss: 2.5938 | LR: 4.22e-04 | 5275.3ms/step | 11.4 samples/s
|
| 18 |
+
wandb: WARNING Tried to log to step 13570 that is less than the current step 13571. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 19 |
+
Step 13580/50462 | Loss: 2.6238 | LR: 4.22e-04 | 5278.3ms/step | 11.4 samples/s
|
| 20 |
+
Step 13590/50462 | Loss: 2.6244 | LR: 4.22e-04 | 5276.8ms/step | 11.4 samples/s
|
| 21 |
+
Step 13600/50462 | Loss: 2.6255 | LR: 4.22e-04 | 5281.6ms/step | 11.4 samples/s
|
| 22 |
+
Step 13610/50462 | Loss: 2.6189 | LR: 4.22e-04 | 5277.3ms/step | 11.4 samples/s
|
| 23 |
+
Step 13620/50462 | Loss: 2.6123 | LR: 4.22e-04 | 5267.3ms/step | 11.4 samples/s
|
| 24 |
+
Step 13630/50462 | Loss: 2.5956 | LR: 4.22e-04 | 5280.2ms/step | 11.4 samples/s
|
| 25 |
+
Step 13640/50462 | Loss: 2.6044 | LR: 4.22e-04 | 5284.6ms/step | 11.4 samples/s
|
| 26 |
+
Step 13650/50462 | Loss: 2.5911 | LR: 4.21e-04 | 5277.2ms/step | 11.4 samples/s
|
| 27 |
+
Step 13660/50462 | Loss: 2.5980 | LR: 4.21e-04 | 5282.2ms/step | 11.4 samples/s
|
| 28 |
+
Step 13670/50462 | Loss: 2.6001 | LR: 4.21e-04 | 5274.3ms/step | 11.4 samples/s
|
| 29 |
+
Step 13680/50462 | Loss: 2.5996 | LR: 4.21e-04 | 5276.5ms/step | 11.4 samples/s
|
| 30 |
+
Step 13690/50462 | Loss: 2.5965 | LR: 4.21e-04 | 5270.3ms/step | 11.4 samples/s
|
| 31 |
+
Step 13700/50462 | Loss: 2.6122 | LR: 4.21e-04 | 5272.4ms/step | 11.4 samples/s
|
| 32 |
+
Step 13710/50462 | Loss: 2.5976 | LR: 4.21e-04 | 5281.6ms/step | 11.4 samples/s
|
| 33 |
+
Step 13720/50462 | Loss: 2.5974 | LR: 4.21e-04 | 5274.9ms/step | 11.4 samples/s
|
| 34 |
+
Step 13730/50462 | Loss: 2.6090 | LR: 4.21e-04 | 5269.0ms/step | 11.4 samples/s
|
| 35 |
+
Step 13740/50462 | Loss: 2.6212 | LR: 4.20e-04 | 5271.2ms/step | 11.4 samples/s
|
| 36 |
+
Step 13750/50462 | Loss: 2.6233 | LR: 4.20e-04 | 5279.3ms/step | 11.4 samples/s
|
| 37 |
+
Validation at step 13750: Loss = 2.6969, Perplexity = 14.83
|
| 38 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-13750
|
| 39 |
+
Step 13760/50462 | Loss: 2.6068 | LR: 4.20e-04 | 10192.1ms/step | 5.9 samples/s
|
| 40 |
+
Step 13770/50462 | Loss: 2.6057 | LR: 4.20e-04 | 5285.9ms/step | 11.4 samples/s
|
| 41 |
+
Step 13780/50462 | Loss: 2.6062 | LR: 4.20e-04 | 5290.7ms/step | 11.3 samples/s
|
| 42 |
+
Step 13790/50462 | Loss: 2.6155 | LR: 4.20e-04 | 5297.9ms/step | 11.3 samples/s
|
| 43 |
+
Step 13800/50462 | Loss: 2.5954 | LR: 4.20e-04 | 5295.6ms/step | 11.3 samples/s
|
| 44 |
+
Step 13810/50462 | Loss: 2.5958 | LR: 4.20e-04 | 5299.0ms/step | 11.3 samples/s
|
| 45 |
+
Step 13820/50462 | Loss: 2.6024 | LR: 4.19e-04 | 5297.2ms/step | 11.3 samples/s
|
| 46 |
+
Step 13830/50462 | Loss: 2.5973 | LR: 4.19e-04 | 5304.4ms/step | 11.3 samples/s
|
| 47 |
+
Step 13840/50462 | Loss: 2.6033 | LR: 4.19e-04 | 5310.4ms/step | 11.3 samples/s
|
| 48 |
+
Step 13850/50462 | Loss: 2.5919 | LR: 4.19e-04 | 5290.7ms/step | 11.3 samples/s
|
| 49 |
+
Step 13860/50462 | Loss: 2.5948 | LR: 4.19e-04 | 5297.0ms/step | 11.3 samples/s
|
| 50 |
+
Step 13870/50462 | Loss: 2.6241 | LR: 4.19e-04 | 5296.4ms/step | 11.3 samples/s
|
| 51 |
+
Step 13880/50462 | Loss: 2.6233 | LR: 4.19e-04 | 5300.5ms/step | 11.3 samples/s
|
| 52 |
+
Step 13890/50462 | Loss: 2.6220 | LR: 4.19e-04 | 5300.9ms/step | 11.3 samples/s
|
| 53 |
+
Step 13900/50462 | Loss: 2.6202 | LR: 4.19e-04 | 5300.6ms/step | 11.3 samples/s
|
| 54 |
+
Step 13910/50462 | Loss: 2.6004 | LR: 4.18e-04 | 5298.3ms/step | 11.3 samples/s
|
| 55 |
+
Step 13920/50462 | Loss: 2.6166 | LR: 4.18e-04 | 5294.9ms/step | 11.3 samples/s
|
| 56 |
+
Step 13930/50462 | Loss: 2.5898 | LR: 4.18e-04 | 5292.7ms/step | 11.3 samples/s
|
| 57 |
+
Step 13940/50462 | Loss: 2.6027 | LR: 4.18e-04 | 5284.0ms/step | 11.4 samples/s
|
| 58 |
+
Step 13950/50462 | Loss: 2.5709 | LR: 4.18e-04 | 5293.2ms/step | 11.3 samples/s
|
| 59 |
+
Step 13960/50462 | Loss: 2.5982 | LR: 4.18e-04 | 5287.4ms/step | 11.3 samples/s
|
| 60 |
+
Step 13970/50462 | Loss: 2.6064 | LR: 4.18e-04 | 5290.6ms/step | 11.3 samples/s
|
| 61 |
+
Step 13980/50462 | Loss: 2.5810 | LR: 4.18e-04 | 5298.9ms/step | 11.3 samples/s
|
| 62 |
+
Step 13990/50462 | Loss: 2.6062 | LR: 4.17e-04 | 5306.8ms/step | 11.3 samples/s
|
| 63 |
+
Step 14000/50462 | Loss: 2.5894 | LR: 4.17e-04 | 5306.1ms/step | 11.3 samples/s
|
| 64 |
+
Validation at step 14000: Loss = 2.6971, Perplexity = 14.84
|
| 65 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-14000
|
| 66 |
+
Step 14010/50462 | Loss: 2.6314 | LR: 4.17e-04 | 10219.2ms/step | 5.9 samples/s
|
| 67 |
+
Step 14020/50462 | Loss: 2.5881 | LR: 4.17e-04 | 5289.3ms/step | 11.3 samples/s
|
| 68 |
+
Step 14030/50462 | Loss: 2.6019 | LR: 4.17e-04 | 5296.4ms/step | 11.3 samples/s
|
| 69 |
+
Step 14040/50462 | Loss: 2.6026 | LR: 4.17e-04 | 5298.8ms/step | 11.3 samples/s
|
| 70 |
+
Step 14050/50462 | Loss: 2.6193 | LR: 4.17e-04 | 5294.1ms/step | 11.3 samples/s
|
| 71 |
+
Step 14060/50462 | Loss: 2.6171 | LR: 4.17e-04 | 5295.9ms/step | 11.3 samples/s
|
| 72 |
+
Step 14070/50462 | Loss: 2.5912 | LR: 4.17e-04 | 5299.9ms/step | 11.3 samples/s
|
| 73 |
+
Step 14080/50462 | Loss: 2.5981 | LR: 4.16e-04 | 5279.6ms/step | 11.4 samples/s
|
| 74 |
+
Step 14090/50462 | Loss: 2.6275 | LR: 4.16e-04 | 5280.1ms/step | 11.4 samples/s
|
| 75 |
+
Step 14100/50462 | Loss: 2.5862 | LR: 4.16e-04 | 5296.0ms/step | 11.3 samples/s
|
| 76 |
+
Step 14110/50462 | Loss: 2.5917 | LR: 4.16e-04 | 5289.9ms/step | 11.3 samples/s
|
| 77 |
+
Step 14120/50462 | Loss: 2.6106 | LR: 4.16e-04 | 5295.7ms/step | 11.3 samples/s
|
| 78 |
+
Step 14130/50462 | Loss: 2.5950 | LR: 4.16e-04 | 5289.9ms/step | 11.3 samples/s
|
| 79 |
+
Step 14140/50462 | Loss: 2.6024 | LR: 4.16e-04 | 5296.1ms/step | 11.3 samples/s
|
| 80 |
+
Step 14150/50462 | Loss: 2.5973 | LR: 4.16e-04 | 5294.2ms/step | 11.3 samples/s
|
| 81 |
+
Step 14160/50462 | Loss: 2.6174 | LR: 4.15e-04 | 5290.3ms/step | 11.3 samples/s
|
| 82 |
+
Step 14170/50462 | Loss: 2.6277 | LR: 4.15e-04 | 5292.4ms/step | 11.3 samples/s
|
| 83 |
+
Step 14180/50462 | Loss: 2.6080 | LR: 4.15e-04 | 5290.4ms/step | 11.3 samples/s
|
| 84 |
+
Step 14190/50462 | Loss: 2.6026 | LR: 4.15e-04 | 5301.3ms/step | 11.3 samples/s
|
| 85 |
+
Step 14200/50462 | Loss: 2.6050 | LR: 4.15e-04 | 5276.9ms/step | 11.4 samples/s
|
| 86 |
+
Step 14210/50462 | Loss: 2.5928 | LR: 4.15e-04 | 5287.1ms/step | 11.3 samples/s
|
| 87 |
+
Step 14220/50462 | Loss: 2.6176 | LR: 4.15e-04 | 5295.8ms/step | 11.3 samples/s
|
| 88 |
+
Step 14230/50462 | Loss: 2.6042 | LR: 4.15e-04 | 5285.7ms/step | 11.4 samples/s
|
| 89 |
+
Step 14240/50462 | Loss: 2.6180 | LR: 4.15e-04 | 5277.8ms/step | 11.4 samples/s
|
| 90 |
+
Step 14250/50462 | Loss: 2.5955 | LR: 4.14e-04 | 5283.2ms/step | 11.4 samples/s
|
| 91 |
+
Validation at step 14250: Loss = 2.6976, Perplexity = 14.84
|
| 92 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-14250
|
| 93 |
+
Step 14260/50462 | Loss: 2.5887 | LR: 4.14e-04 | 10155.1ms/step | 5.9 samples/s
|
| 94 |
+
Step 14270/50462 | Loss: 2.5967 | LR: 4.14e-04 | 5273.3ms/step | 11.4 samples/s
|
| 95 |
+
Step 14280/50462 | Loss: 2.5996 | LR: 4.14e-04 | 5284.9ms/step | 11.4 samples/s
|
| 96 |
+
Step 14290/50462 | Loss: 2.5783 | LR: 4.14e-04 | 5289.6ms/step | 11.3 samples/s
|
| 97 |
+
Step 14300/50462 | Loss: 2.6125 | LR: 4.14e-04 | 5293.4ms/step | 11.3 samples/s
|
| 98 |
+
Step 14310/50462 | Loss: 2.5902 | LR: 4.14e-04 | 5296.2ms/step | 11.3 samples/s
|
| 99 |
+
Step 14320/50462 | Loss: 2.5892 | LR: 4.14e-04 | 5292.2ms/step | 11.3 samples/s
|
| 100 |
+
Step 14330/50462 | Loss: 2.6244 | LR: 4.13e-04 | 5297.1ms/step | 11.3 samples/s
|
| 101 |
+
Step 14340/50462 | Loss: 2.5663 | LR: 4.13e-04 | 5288.7ms/step | 11.3 samples/s
|
| 102 |
+
Step 14350/50462 | Loss: 2.5948 | LR: 4.13e-04 | 5293.3ms/step | 11.3 samples/s
|
| 103 |
+
Step 14360/50462 | Loss: 2.6006 | LR: 4.13e-04 | 5291.7ms/step | 11.3 samples/s
|
| 104 |
+
Step 14370/50462 | Loss: 2.5762 | LR: 4.13e-04 | 5293.3ms/step | 11.3 samples/s
|
| 105 |
+
Step 14380/50462 | Loss: 2.5736 | LR: 4.13e-04 | 5296.3ms/step | 11.3 samples/s
|
| 106 |
+
Step 14390/50462 | Loss: 2.5527 | LR: 4.13e-04 | 5280.8ms/step | 11.4 samples/s
|
| 107 |
+
Step 14400/50462 | Loss: 2.6021 | LR: 4.13e-04 | 5291.8ms/step | 11.3 samples/s
|
| 108 |
+
Step 14410/50462 | Loss: 2.6074 | LR: 4.12e-04 | 5299.8ms/step | 11.3 samples/s
|
| 109 |
+
Step 14420/50462 | Loss: 2.5956 | LR: 4.12e-04 | 5295.0ms/step | 11.3 samples/s
|
| 110 |
+
Step 14430/50462 | Loss: 2.5700 | LR: 4.12e-04 | 5295.4ms/step | 11.3 samples/s
|
| 111 |
+
Step 14440/50462 | Loss: 2.5924 | LR: 4.12e-04 | 5294.8ms/step | 11.3 samples/s
|
epfl_final/SparseAtt/wandb/run-20260824_132232-rgmkfpnl/files/requirements.txt
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
| 142 |
+
entmax==1.3
|
epfl_final/SparseAtt/wandb/run-20260824_132232-rgmkfpnl/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-08-24T13:22:32.395457Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"configs/p2_1p2b_dense_seed70_2gpu.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/TokenPrune.git",
|
| 14 |
+
"commit": "5637886f0cc760009989b14e68c6ae06d4827418"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "h100-workspace-2-gpu-0-10",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 23 |
+
"gpu_count": 2,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "9374931091456",
|
| 27 |
+
"used": "6343249899520"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1623005732864"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-51f13867-a72c-7b65-ff60-661787f0c18a"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 43 |
+
"memoryTotal": "85520809984",
|
| 44 |
+
"cudaCores": 16896,
|
| 45 |
+
"architecture": "Hopper",
|
| 46 |
+
"uuid": "GPU-bab1e315-c967-43cf-bdfb-cac1a8dfdfac"
|
| 47 |
+
}
|
| 48 |
+
],
|
| 49 |
+
"cudaVersion": "13.0",
|
| 50 |
+
"writerId": "id2qbdffzvi2lv1j596shx4dhbm3sd2i"
|
| 51 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260824_132232-rgmkfpnl/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260824_132232-rgmkfpnl/logs/debug.log
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-08-24 13:22:32,434 INFO MainThread:229 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-08-24 13:22:32,434 INFO MainThread:229 [wandb_setup.py:_flush():81] Configure stats pid to 229
|
| 3 |
+
2026-08-24 13:22:32,435 INFO MainThread:229 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-08-24 13:22:32,435 INFO MainThread:229 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260824_132232-rgmkfpnl/logs/debug.log
|
| 5 |
+
2026-08-24 13:22:32,435 INFO MainThread:229 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260824_132232-rgmkfpnl/logs/debug-internal.log
|
| 6 |
+
2026-08-24 13:22:32,435 INFO MainThread:229 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-08-24 13:22:32,435 INFO MainThread:229 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-13500', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 15, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 70, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b dense 8k seed70 2gpu resume', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'dense', 'seed70'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-08-24 13:22:32,435 INFO MainThread:229 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-08-24 13:22:33,314 INFO MainThread:229 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-08-24 13:22:33,845 INFO MainThread:229 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-08-24 13:22:33,848 INFO MainThread:229 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-08-24 13:22:33,878 INFO MainThread:229 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-08-24 13:22:34,330 INFO MainThread:229 [wandb_init.py:init():1051] run resumed
|
| 15 |
+
2026-08-24 13:22:34,333 INFO MainThread:229 [wandb_init.py:init():1056] starting run threads in backend
|
| 16 |
+
2026-08-24 13:22:34,709 INFO MainThread:229 [wandb_run.py:_console_start():2554] atexit reg
|
| 17 |
+
2026-08-24 13:22:34,709 INFO MainThread:229 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 18 |
+
2026-08-24 13:22:34,709 INFO MainThread:229 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 19 |
+
2026-08-24 13:22:34,709 INFO MainThread:229 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 20 |
+
2026-08-24 13:22:34,717 INFO MainThread:229 [wandb_init.py:init():1094] run started, returning control to user process
|
epfl_final/SparseAtt/wandb/run-20260824_132232-rgmkfpnl/run-rgmkfpnl.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3aeb172670ffd5dfe0a3794c3e11c6476db51eebbb297890c37d152fee14a059
|
| 3 |
+
size 1146880
|
epfl_final/SparseAtt/wandb/run-20260824_181605-i8ewyy95/files/config.yaml
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.26.1
|
| 4 |
+
e:
|
| 5 |
+
yzsnfmnyz8zlr1rq2pyszw88pg06lgmv:
|
| 6 |
+
args:
|
| 7 |
+
- --config
|
| 8 |
+
- configs/p2_1p2b_mixed_seed70_6gpu.json
|
| 9 |
+
codePath: train.py
|
| 10 |
+
codePathLocal: train.py
|
| 11 |
+
cpu_count: 96
|
| 12 |
+
cpu_count_logical: 192
|
| 13 |
+
cudaVersion: "13.0"
|
| 14 |
+
disk:
|
| 15 |
+
/:
|
| 16 |
+
total: "9374931091456"
|
| 17 |
+
used: "4851041062912"
|
| 18 |
+
email: ruben.navasardyan@mail.utoronto.ca
|
| 19 |
+
executable: /mloscratch/homes/navasard/conda/bin/python
|
| 20 |
+
git:
|
| 21 |
+
commit: 5637886f0cc760009989b14e68c6ae06d4827418
|
| 22 |
+
remote: https://github.com/rubentium/TokenPrune.git
|
| 23 |
+
gpu: NVIDIA H100 80GB HBM3
|
| 24 |
+
gpu_count: 6
|
| 25 |
+
gpu_nvidia:
|
| 26 |
+
- architecture: Hopper
|
| 27 |
+
cudaCores: 16896
|
| 28 |
+
memoryTotal: "85520809984"
|
| 29 |
+
name: NVIDIA H100 80GB HBM3
|
| 30 |
+
uuid: GPU-7004b604-21a2-890e-0312-3eee1eb93c67
|
| 31 |
+
- architecture: Hopper
|
| 32 |
+
cudaCores: 16896
|
| 33 |
+
memoryTotal: "85520809984"
|
| 34 |
+
name: NVIDIA H100 80GB HBM3
|
| 35 |
+
uuid: GPU-3f5bace4-0066-71eb-863d-73a3a6d79e3b
|
| 36 |
+
- architecture: Hopper
|
| 37 |
+
cudaCores: 16896
|
| 38 |
+
memoryTotal: "85520809984"
|
| 39 |
+
name: NVIDIA H100 80GB HBM3
|
| 40 |
+
uuid: GPU-1aef7113-cfe5-3f49-53bb-a84f3c5abfa0
|
| 41 |
+
- architecture: Hopper
|
| 42 |
+
cudaCores: 16896
|
| 43 |
+
memoryTotal: "85520809984"
|
| 44 |
+
name: NVIDIA H100 80GB HBM3
|
| 45 |
+
uuid: GPU-5e17ddff-2e95-8c68-b074-b0189417e834
|
| 46 |
+
- architecture: Hopper
|
| 47 |
+
cudaCores: 16896
|
| 48 |
+
memoryTotal: "85520809984"
|
| 49 |
+
name: NVIDIA H100 80GB HBM3
|
| 50 |
+
uuid: GPU-61e64705-1798-f2b5-568f-877549238703
|
| 51 |
+
- architecture: Hopper
|
| 52 |
+
cudaCores: 16896
|
| 53 |
+
memoryTotal: "85520809984"
|
| 54 |
+
name: NVIDIA H100 80GB HBM3
|
| 55 |
+
uuid: GPU-9cd8a6e5-05e9-cfde-0a28-d8058797b3bf
|
| 56 |
+
host: h100-train-6-gpu-0-34
|
| 57 |
+
memory:
|
| 58 |
+
total: "1623005732864"
|
| 59 |
+
os: Linux-6.8.0-124-generic-x86_64-with-glibc2.35
|
| 60 |
+
program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
|
| 61 |
+
python: CPython 3.13.13
|
| 62 |
+
root: /mloscratch/homes/navasard/sparse/SparseAtt
|
| 63 |
+
startedAt: "2026-08-24T18:16:05.392668Z"
|
| 64 |
+
writerId: yzsnfmnyz8zlr1rq2pyszw88pg06lgmv
|
| 65 |
+
m: []
|
| 66 |
+
python_version: 3.13.13
|
| 67 |
+
t:
|
| 68 |
+
"1":
|
| 69 |
+
- 1
|
| 70 |
+
- 5
|
| 71 |
+
- 11
|
| 72 |
+
- 49
|
| 73 |
+
- 51
|
| 74 |
+
- 53
|
| 75 |
+
- 71
|
| 76 |
+
"2":
|
| 77 |
+
- 1
|
| 78 |
+
- 5
|
| 79 |
+
- 11
|
| 80 |
+
- 49
|
| 81 |
+
- 51
|
| 82 |
+
- 53
|
| 83 |
+
- 71
|
| 84 |
+
"3":
|
| 85 |
+
- 2
|
| 86 |
+
- 5
|
| 87 |
+
- 13
|
| 88 |
+
- 14
|
| 89 |
+
- 15
|
| 90 |
+
- 16
|
| 91 |
+
- 61
|
| 92 |
+
- 62
|
| 93 |
+
"4": 3.13.13
|
| 94 |
+
"5": 0.26.1
|
| 95 |
+
"6": 5.7.0
|
| 96 |
+
"12": 0.26.1
|
| 97 |
+
"13": linux-x86_64
|
| 98 |
+
attention_dropout:
|
| 99 |
+
value: null
|
| 100 |
+
attention_type:
|
| 101 |
+
value: sparse_stochastic
|
| 102 |
+
checkpoint_activations:
|
| 103 |
+
value: true
|
| 104 |
+
dense_mode:
|
| 105 |
+
value: false
|
| 106 |
+
dense_step_intervals:
|
| 107 |
+
value: null
|
| 108 |
+
eval_batches:
|
| 109 |
+
value: 50
|
| 110 |
+
eval_steps:
|
| 111 |
+
value: 250
|
| 112 |
+
force_new_run:
|
| 113 |
+
value: false
|
| 114 |
+
gradient_accumulation_steps:
|
| 115 |
+
value: 10
|
| 116 |
+
head_dim:
|
| 117 |
+
value: null
|
| 118 |
+
hidden_size:
|
| 119 |
+
value: 1536
|
| 120 |
+
init_from_scratch:
|
| 121 |
+
value: true
|
| 122 |
+
intermediate_size:
|
| 123 |
+
value: 6144
|
| 124 |
+
learning_rate:
|
| 125 |
+
value: 0.0005
|
| 126 |
+
logging_steps:
|
| 127 |
+
value: 10
|
| 128 |
+
lr_scheduler:
|
| 129 |
+
value: cosine
|
| 130 |
+
max_grad_norm:
|
| 131 |
+
value: 1
|
| 132 |
+
max_seq_length:
|
| 133 |
+
value: 8192
|
| 134 |
+
max_steps:
|
| 135 |
+
value: 50462
|
| 136 |
+
mixed_precision:
|
| 137 |
+
value: bf16
|
| 138 |
+
model_dtype:
|
| 139 |
+
value: bfloat16
|
| 140 |
+
model_path:
|
| 141 |
+
value: /mloscratch/homes/navasard/qwen3_06b_base
|
| 142 |
+
num_attention_heads:
|
| 143 |
+
value: 12
|
| 144 |
+
num_epochs:
|
| 145 |
+
value: 1
|
| 146 |
+
num_hidden_layers:
|
| 147 |
+
value: null
|
| 148 |
+
num_key_value_heads:
|
| 149 |
+
value: 4
|
| 150 |
+
num_workers:
|
| 151 |
+
value: 8
|
| 152 |
+
output_dir:
|
| 153 |
+
value: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107
|
| 154 |
+
per_device_batch_size:
|
| 155 |
+
value: 1
|
| 156 |
+
resume_from_checkpoint:
|
| 157 |
+
value: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-38000
|
| 158 |
+
rope_train_position_scale_set:
|
| 159 |
+
value: null
|
| 160 |
+
save_steps:
|
| 161 |
+
value: 250
|
| 162 |
+
seed:
|
| 163 |
+
value: 70
|
| 164 |
+
sparse_block_size:
|
| 165 |
+
value: 128
|
| 166 |
+
sparse_layer_indices:
|
| 167 |
+
value: null
|
| 168 |
+
sparse_mask_cache_dir:
|
| 169 |
+
value: /mloscratch/homes/navasard/sparse/mask_cache_8k_mixed
|
| 170 |
+
sparse_mask_pool_size:
|
| 171 |
+
value: 10000
|
| 172 |
+
sparse_mask_seed:
|
| 173 |
+
value: 70
|
| 174 |
+
sparse_mixed_nrb_set:
|
| 175 |
+
value:
|
| 176 |
+
- 63
|
| 177 |
+
- 47
|
| 178 |
+
- 31
|
| 179 |
+
- 21
|
| 180 |
+
- 11
|
| 181 |
+
sparse_num_random_blocks:
|
| 182 |
+
value: 35
|
| 183 |
+
sparse_val_dense_compare:
|
| 184 |
+
value: false
|
| 185 |
+
strategy:
|
| 186 |
+
value: ddp
|
| 187 |
+
train_path:
|
| 188 |
+
value: /mloscratch/homes/navasard/data/fineweb_large/tokenized
|
| 189 |
+
val_path:
|
| 190 |
+
value: null
|
| 191 |
+
verbose:
|
| 192 |
+
value: true
|
| 193 |
+
wandb_enabled:
|
| 194 |
+
value: true
|
| 195 |
+
wandb_name:
|
| 196 |
+
value: 1.2b mixed 8k seed70 6gpu resume
|
| 197 |
+
wandb_project:
|
| 198 |
+
value: stochastic_sparsity
|
| 199 |
+
wandb_tags:
|
| 200 |
+
value:
|
| 201 |
+
- qwen3
|
| 202 |
+
- 1.2b
|
| 203 |
+
- 8k
|
| 204 |
+
- p2
|
| 205 |
+
- mixed
|
| 206 |
+
- seed70
|
| 207 |
+
warmup_steps:
|
| 208 |
+
value: 750
|
| 209 |
+
weight_decay:
|
| 210 |
+
value: 0.01
|
epfl_final/SparseAtt/wandb/run-20260824_181605-i8ewyy95/files/output.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260824_181605-i8ewyy95/files/requirements.txt
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
| 142 |
+
entmax==1.3
|
epfl_final/SparseAtt/wandb/run-20260824_181605-i8ewyy95/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-08-24T18:16:05.392668Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"configs/p2_1p2b_mixed_seed70_6gpu.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/TokenPrune.git",
|
| 14 |
+
"commit": "5637886f0cc760009989b14e68c6ae06d4827418"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "h100-train-6-gpu-0-34",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 23 |
+
"gpu_count": 6,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "9374931091456",
|
| 27 |
+
"used": "4851041062912"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1623005732864"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-7004b604-21a2-890e-0312-3eee1eb93c67"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 43 |
+
"memoryTotal": "85520809984",
|
| 44 |
+
"cudaCores": 16896,
|
| 45 |
+
"architecture": "Hopper",
|
| 46 |
+
"uuid": "GPU-3f5bace4-0066-71eb-863d-73a3a6d79e3b"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 50 |
+
"memoryTotal": "85520809984",
|
| 51 |
+
"cudaCores": 16896,
|
| 52 |
+
"architecture": "Hopper",
|
| 53 |
+
"uuid": "GPU-1aef7113-cfe5-3f49-53bb-a84f3c5abfa0"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 57 |
+
"memoryTotal": "85520809984",
|
| 58 |
+
"cudaCores": 16896,
|
| 59 |
+
"architecture": "Hopper",
|
| 60 |
+
"uuid": "GPU-5e17ddff-2e95-8c68-b074-b0189417e834"
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 64 |
+
"memoryTotal": "85520809984",
|
| 65 |
+
"cudaCores": 16896,
|
| 66 |
+
"architecture": "Hopper",
|
| 67 |
+
"uuid": "GPU-61e64705-1798-f2b5-568f-877549238703"
|
| 68 |
+
},
|
| 69 |
+
{
|
| 70 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 71 |
+
"memoryTotal": "85520809984",
|
| 72 |
+
"cudaCores": 16896,
|
| 73 |
+
"architecture": "Hopper",
|
| 74 |
+
"uuid": "GPU-9cd8a6e5-05e9-cfde-0a28-d8058797b3bf"
|
| 75 |
+
}
|
| 76 |
+
],
|
| 77 |
+
"cudaVersion": "13.0",
|
| 78 |
+
"writerId": "yzsnfmnyz8zlr1rq2pyszw88pg06lgmv"
|
| 79 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260824_181605-i8ewyy95/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"speed/samples_per_sec":21.985407094904552,"train/epoch":0.9702350283381554,"eval/uses_dense_attention":0,"train/learning_rate":1.1253848138212951e-06,"eval/uses_sparse_attention":1,"eval/loss":2.5281137384119488,"_wandb":{"runtime":156363},"train/loss":2.5299893152713775,"speed/ms_per_step":2729.08296585083,"eval/dense_interval_start":-1,"speed/steps_per_sec":0.36642345158174255,"_step":48960,"train/global_step":48960,"_timestamp":1.7876265843495035e+09,"_runtime":156363.039635395,"eval/dense_interval_end":-1,"_wandb.runtime":25091,"eval/dense_interval_active":0,"eval/perplexity":12.529849258791515}
|
epfl_final/SparseAtt/wandb/run-20260824_181605-i8ewyy95/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260824_181605-i8ewyy95/logs/debug.log
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-08-24 18:16:05,445 INFO MainThread:213 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-08-24 18:16:05,445 INFO MainThread:213 [wandb_setup.py:_flush():81] Configure stats pid to 213
|
| 3 |
+
2026-08-24 18:16:05,445 INFO MainThread:213 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-08-24 18:16:05,446 INFO MainThread:213 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260824_181605-i8ewyy95/logs/debug.log
|
| 5 |
+
2026-08-24 18:16:05,446 INFO MainThread:213 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260824_181605-i8ewyy95/logs/debug-internal.log
|
| 6 |
+
2026-08-24 18:16:05,446 INFO MainThread:213 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-08-24 18:16:05,446 INFO MainThread:213 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-38000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 70, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b mixed 8k seed70 6gpu resume', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'mixed', 'seed70'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed', 'sparse_mask_seed': 70, 'sparse_mixed_nrb_set': [63, 47, 31, 21, 11], 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-08-24 18:16:05,446 INFO MainThread:213 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-08-24 18:16:06,235 INFO MainThread:213 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-08-24 18:16:06,700 INFO MainThread:213 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-08-24 18:16:06,703 INFO MainThread:213 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-08-24 18:16:06,765 INFO MainThread:213 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-08-24 18:16:07,697 INFO MainThread:213 [wandb_init.py:init():1051] run resumed
|
| 15 |
+
2026-08-24 18:16:07,700 INFO MainThread:213 [wandb_init.py:init():1056] starting run threads in backend
|
| 16 |
+
2026-08-24 18:16:07,977 INFO MainThread:213 [wandb_run.py:_console_start():2554] atexit reg
|
| 17 |
+
2026-08-24 18:16:07,977 INFO MainThread:213 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 18 |
+
2026-08-24 18:16:07,977 INFO MainThread:213 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 19 |
+
2026-08-24 18:16:07,977 INFO MainThread:213 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 20 |
+
2026-08-24 18:16:07,991 INFO MainThread:213 [wandb_init.py:init():1094] run started, returning control to user process
|
| 21 |
+
2026-08-25 02:56:41,739 INFO MainThread:213 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/i8ewyy95
|
| 22 |
+
2026-08-25 02:56:41,739 INFO MainThread:213 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
|
| 23 |
+
2026-08-25 02:56:41,740 INFO MainThread:213 [wandb_run.py:_restore():2502] restore
|
| 24 |
+
2026-08-25 02:56:41,740 INFO MainThread:213 [wandb_run.py:_restore():2508] restore done
|
| 25 |
+
2026-08-25 02:56:43,202 INFO MainThread:213 [wandb_run.py:_footer_sync_info():3889] logging synced files
|
epfl_final/SparseAtt/wandb/run-20260824_181605-i8ewyy95/run-i8ewyy95.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:eac9650514d1410247d303fe4c0737559a7b6e6f0152ec85ba86d064ea8a465a
|
| 3 |
+
size 14297474
|
epfl_final/SparseAtt/wandb/run-20260824_181749-rgmkfpnl/files/output.log
ADDED
|
@@ -0,0 +1,708 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Starting training...
|
| 2 |
+
Validation at step 13500: Loss = 2.6953, Perplexity = 14.81
|
| 3 |
+
wandb: WARNING Tried to log to step 13500 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 4 |
+
Switching attention mode for step 13501: dense (dense step interval)
|
| 5 |
+
Step 13510/50462 | Loss: 2.4722 | LR: 4.23e-04 | 42665.0ms/step | 1.4 samples/s
|
| 6 |
+
wandb: WARNING Tried to log to step 13510 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 7 |
+
Step 13520/50462 | Loss: 2.6271 | LR: 4.23e-04 | 5239.1ms/step | 11.5 samples/s
|
| 8 |
+
wandb: WARNING Tried to log to step 13520 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 9 |
+
Step 13530/50462 | Loss: 2.6167 | LR: 4.23e-04 | 5244.6ms/step | 11.4 samples/s
|
| 10 |
+
wandb: WARNING Tried to log to step 13530 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 11 |
+
Step 13540/50462 | Loss: 2.6109 | LR: 4.23e-04 | 5249.7ms/step | 11.4 samples/s
|
| 12 |
+
wandb: WARNING Tried to log to step 13540 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 13 |
+
Step 13550/50462 | Loss: 2.6348 | LR: 4.23e-04 | 5254.4ms/step | 11.4 samples/s
|
| 14 |
+
wandb: WARNING Tried to log to step 13550 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 15 |
+
Step 13560/50462 | Loss: 2.6069 | LR: 4.22e-04 | 5257.1ms/step | 11.4 samples/s
|
| 16 |
+
wandb: WARNING Tried to log to step 13560 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 17 |
+
Step 13570/50462 | Loss: 2.5938 | LR: 4.22e-04 | 5253.4ms/step | 11.4 samples/s
|
| 18 |
+
wandb: WARNING Tried to log to step 13570 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 19 |
+
Step 13580/50462 | Loss: 2.6239 | LR: 4.22e-04 | 5255.6ms/step | 11.4 samples/s
|
| 20 |
+
wandb: WARNING Tried to log to step 13580 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 21 |
+
Step 13590/50462 | Loss: 2.6244 | LR: 4.22e-04 | 5252.6ms/step | 11.4 samples/s
|
| 22 |
+
wandb: WARNING Tried to log to step 13590 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 23 |
+
Step 13600/50462 | Loss: 2.6255 | LR: 4.22e-04 | 5253.2ms/step | 11.4 samples/s
|
| 24 |
+
wandb: WARNING Tried to log to step 13600 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 25 |
+
Step 13610/50462 | Loss: 2.6189 | LR: 4.22e-04 | 5252.0ms/step | 11.4 samples/s
|
| 26 |
+
wandb: WARNING Tried to log to step 13610 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 27 |
+
Step 13620/50462 | Loss: 2.6123 | LR: 4.22e-04 | 5254.3ms/step | 11.4 samples/s
|
| 28 |
+
wandb: WARNING Tried to log to step 13620 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 29 |
+
Step 13630/50462 | Loss: 2.5956 | LR: 4.22e-04 | 5253.1ms/step | 11.4 samples/s
|
| 30 |
+
wandb: WARNING Tried to log to step 13630 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 31 |
+
Step 13640/50462 | Loss: 2.6044 | LR: 4.22e-04 | 5251.2ms/step | 11.4 samples/s
|
| 32 |
+
wandb: WARNING Tried to log to step 13640 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 33 |
+
Step 13650/50462 | Loss: 2.5911 | LR: 4.21e-04 | 5244.5ms/step | 11.4 samples/s
|
| 34 |
+
wandb: WARNING Tried to log to step 13650 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 35 |
+
Step 13660/50462 | Loss: 2.5980 | LR: 4.21e-04 | 5253.5ms/step | 11.4 samples/s
|
| 36 |
+
wandb: WARNING Tried to log to step 13660 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 37 |
+
Step 13670/50462 | Loss: 2.6001 | LR: 4.21e-04 | 5254.5ms/step | 11.4 samples/s
|
| 38 |
+
wandb: WARNING Tried to log to step 13670 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 39 |
+
Step 13680/50462 | Loss: 2.5996 | LR: 4.21e-04 | 5255.0ms/step | 11.4 samples/s
|
| 40 |
+
wandb: WARNING Tried to log to step 13680 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 41 |
+
Step 13690/50462 | Loss: 2.5965 | LR: 4.21e-04 | 5256.4ms/step | 11.4 samples/s
|
| 42 |
+
wandb: WARNING Tried to log to step 13690 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 43 |
+
Step 13700/50462 | Loss: 2.6121 | LR: 4.21e-04 | 5259.7ms/step | 11.4 samples/s
|
| 44 |
+
wandb: WARNING Tried to log to step 13700 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 45 |
+
Step 13710/50462 | Loss: 2.5976 | LR: 4.21e-04 | 5248.0ms/step | 11.4 samples/s
|
| 46 |
+
wandb: WARNING Tried to log to step 13710 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 47 |
+
Step 13720/50462 | Loss: 2.5975 | LR: 4.21e-04 | 5255.3ms/step | 11.4 samples/s
|
| 48 |
+
wandb: WARNING Tried to log to step 13720 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 49 |
+
Step 13730/50462 | Loss: 2.6090 | LR: 4.21e-04 | 5254.4ms/step | 11.4 samples/s
|
| 50 |
+
wandb: WARNING Tried to log to step 13730 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 51 |
+
Step 13740/50462 | Loss: 2.6212 | LR: 4.20e-04 | 5247.3ms/step | 11.4 samples/s
|
| 52 |
+
wandb: WARNING Tried to log to step 13740 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 53 |
+
Step 13750/50462 | Loss: 2.6234 | LR: 4.20e-04 | 5247.5ms/step | 11.4 samples/s
|
| 54 |
+
wandb: WARNING Tried to log to step 13750 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 55 |
+
Validation at step 13750: Loss = 2.6969, Perplexity = 14.83
|
| 56 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-13750
|
| 57 |
+
Step 13760/50462 | Loss: 2.6068 | LR: 4.20e-04 | 10144.4ms/step | 5.9 samples/s
|
| 58 |
+
wandb: WARNING Tried to log to step 13760 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 59 |
+
Step 13770/50462 | Loss: 2.6058 | LR: 4.20e-04 | 5251.4ms/step | 11.4 samples/s
|
| 60 |
+
wandb: WARNING Tried to log to step 13770 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 61 |
+
Step 13780/50462 | Loss: 2.6062 | LR: 4.20e-04 | 5255.5ms/step | 11.4 samples/s
|
| 62 |
+
wandb: WARNING Tried to log to step 13780 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 63 |
+
Step 13790/50462 | Loss: 2.6155 | LR: 4.20e-04 | 5252.7ms/step | 11.4 samples/s
|
| 64 |
+
wandb: WARNING Tried to log to step 13790 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 65 |
+
Step 13800/50462 | Loss: 2.5953 | LR: 4.20e-04 | 5256.6ms/step | 11.4 samples/s
|
| 66 |
+
wandb: WARNING Tried to log to step 13800 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 67 |
+
Step 13810/50462 | Loss: 2.5958 | LR: 4.20e-04 | 5255.2ms/step | 11.4 samples/s
|
| 68 |
+
wandb: WARNING Tried to log to step 13810 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 69 |
+
Step 13820/50462 | Loss: 2.6024 | LR: 4.19e-04 | 5259.2ms/step | 11.4 samples/s
|
| 70 |
+
wandb: WARNING Tried to log to step 13820 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 71 |
+
Step 13830/50462 | Loss: 2.5974 | LR: 4.19e-04 | 5254.6ms/step | 11.4 samples/s
|
| 72 |
+
wandb: WARNING Tried to log to step 13830 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 73 |
+
Step 13840/50462 | Loss: 2.6033 | LR: 4.19e-04 | 5248.3ms/step | 11.4 samples/s
|
| 74 |
+
wandb: WARNING Tried to log to step 13840 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 75 |
+
Step 13850/50462 | Loss: 2.5919 | LR: 4.19e-04 | 5252.1ms/step | 11.4 samples/s
|
| 76 |
+
wandb: WARNING Tried to log to step 13850 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 77 |
+
Step 13860/50462 | Loss: 2.5948 | LR: 4.19e-04 | 5257.5ms/step | 11.4 samples/s
|
| 78 |
+
wandb: WARNING Tried to log to step 13860 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 79 |
+
Step 13870/50462 | Loss: 2.6241 | LR: 4.19e-04 | 5257.4ms/step | 11.4 samples/s
|
| 80 |
+
wandb: WARNING Tried to log to step 13870 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 81 |
+
Step 13880/50462 | Loss: 2.6233 | LR: 4.19e-04 | 5259.3ms/step | 11.4 samples/s
|
| 82 |
+
wandb: WARNING Tried to log to step 13880 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 83 |
+
Step 13890/50462 | Loss: 2.6220 | LR: 4.19e-04 | 5259.8ms/step | 11.4 samples/s
|
| 84 |
+
wandb: WARNING Tried to log to step 13890 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 85 |
+
Step 13900/50462 | Loss: 2.6202 | LR: 4.19e-04 | 5260.9ms/step | 11.4 samples/s
|
| 86 |
+
wandb: WARNING Tried to log to step 13900 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 87 |
+
Step 13910/50462 | Loss: 2.6004 | LR: 4.18e-04 | 5258.9ms/step | 11.4 samples/s
|
| 88 |
+
wandb: WARNING Tried to log to step 13910 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 89 |
+
Step 13920/50462 | Loss: 2.6166 | LR: 4.18e-04 | 5265.4ms/step | 11.4 samples/s
|
| 90 |
+
wandb: WARNING Tried to log to step 13920 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 91 |
+
Step 13930/50462 | Loss: 2.5898 | LR: 4.18e-04 | 5260.2ms/step | 11.4 samples/s
|
| 92 |
+
wandb: WARNING Tried to log to step 13930 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 93 |
+
Step 13940/50462 | Loss: 2.6027 | LR: 4.18e-04 | 5260.6ms/step | 11.4 samples/s
|
| 94 |
+
wandb: WARNING Tried to log to step 13940 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 95 |
+
Step 13950/50462 | Loss: 2.5709 | LR: 4.18e-04 | 5252.1ms/step | 11.4 samples/s
|
| 96 |
+
wandb: WARNING Tried to log to step 13950 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 97 |
+
Step 13960/50462 | Loss: 2.5982 | LR: 4.18e-04 | 5255.4ms/step | 11.4 samples/s
|
| 98 |
+
wandb: WARNING Tried to log to step 13960 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 99 |
+
Step 13970/50462 | Loss: 2.6064 | LR: 4.18e-04 | 5258.2ms/step | 11.4 samples/s
|
| 100 |
+
wandb: WARNING Tried to log to step 13970 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 101 |
+
Step 13980/50462 | Loss: 2.5810 | LR: 4.18e-04 | 5258.7ms/step | 11.4 samples/s
|
| 102 |
+
wandb: WARNING Tried to log to step 13980 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 103 |
+
Step 13990/50462 | Loss: 2.6062 | LR: 4.17e-04 | 5259.1ms/step | 11.4 samples/s
|
| 104 |
+
wandb: WARNING Tried to log to step 13990 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 105 |
+
Step 14000/50462 | Loss: 2.5894 | LR: 4.17e-04 | 5258.5ms/step | 11.4 samples/s
|
| 106 |
+
wandb: WARNING Tried to log to step 14000 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 107 |
+
Validation at step 14000: Loss = 2.6970, Perplexity = 14.84
|
| 108 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-14000
|
| 109 |
+
Step 14010/50462 | Loss: 2.6314 | LR: 4.17e-04 | 10243.6ms/step | 5.9 samples/s
|
| 110 |
+
wandb: WARNING Tried to log to step 14010 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 111 |
+
Step 14020/50462 | Loss: 2.5881 | LR: 4.17e-04 | 5260.5ms/step | 11.4 samples/s
|
| 112 |
+
wandb: WARNING Tried to log to step 14020 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 113 |
+
Step 14030/50462 | Loss: 2.6019 | LR: 4.17e-04 | 5260.0ms/step | 11.4 samples/s
|
| 114 |
+
wandb: WARNING Tried to log to step 14030 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 115 |
+
Step 14040/50462 | Loss: 2.6027 | LR: 4.17e-04 | 5267.5ms/step | 11.4 samples/s
|
| 116 |
+
wandb: WARNING Tried to log to step 14040 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 117 |
+
Step 14050/50462 | Loss: 2.6193 | LR: 4.17e-04 | 5256.1ms/step | 11.4 samples/s
|
| 118 |
+
wandb: WARNING Tried to log to step 14050 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 119 |
+
Step 14060/50462 | Loss: 2.6171 | LR: 4.17e-04 | 5260.8ms/step | 11.4 samples/s
|
| 120 |
+
wandb: WARNING Tried to log to step 14060 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 121 |
+
Step 14070/50462 | Loss: 2.5913 | LR: 4.17e-04 | 5263.1ms/step | 11.4 samples/s
|
| 122 |
+
wandb: WARNING Tried to log to step 14070 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 123 |
+
Step 14080/50462 | Loss: 2.5981 | LR: 4.16e-04 | 5260.8ms/step | 11.4 samples/s
|
| 124 |
+
wandb: WARNING Tried to log to step 14080 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 125 |
+
Step 14090/50462 | Loss: 2.6275 | LR: 4.16e-04 | 5256.8ms/step | 11.4 samples/s
|
| 126 |
+
wandb: WARNING Tried to log to step 14090 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 127 |
+
Step 14100/50462 | Loss: 2.5862 | LR: 4.16e-04 | 5256.7ms/step | 11.4 samples/s
|
| 128 |
+
wandb: WARNING Tried to log to step 14100 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 129 |
+
Step 14110/50462 | Loss: 2.5918 | LR: 4.16e-04 | 5260.0ms/step | 11.4 samples/s
|
| 130 |
+
wandb: WARNING Tried to log to step 14110 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 131 |
+
Step 14120/50462 | Loss: 2.6106 | LR: 4.16e-04 | 5258.7ms/step | 11.4 samples/s
|
| 132 |
+
wandb: WARNING Tried to log to step 14120 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 133 |
+
Step 14130/50462 | Loss: 2.5951 | LR: 4.16e-04 | 5256.7ms/step | 11.4 samples/s
|
| 134 |
+
wandb: WARNING Tried to log to step 14130 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 135 |
+
Step 14140/50462 | Loss: 2.6023 | LR: 4.16e-04 | 5255.5ms/step | 11.4 samples/s
|
| 136 |
+
wandb: WARNING Tried to log to step 14140 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 137 |
+
Step 14150/50462 | Loss: 2.5973 | LR: 4.16e-04 | 5260.8ms/step | 11.4 samples/s
|
| 138 |
+
wandb: WARNING Tried to log to step 14150 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 139 |
+
Step 14160/50462 | Loss: 2.6174 | LR: 4.15e-04 | 5264.1ms/step | 11.4 samples/s
|
| 140 |
+
wandb: WARNING Tried to log to step 14160 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 141 |
+
Step 14170/50462 | Loss: 2.6277 | LR: 4.15e-04 | 5254.9ms/step | 11.4 samples/s
|
| 142 |
+
wandb: WARNING Tried to log to step 14170 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 143 |
+
Step 14180/50462 | Loss: 2.6080 | LR: 4.15e-04 | 5256.5ms/step | 11.4 samples/s
|
| 144 |
+
wandb: WARNING Tried to log to step 14180 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 145 |
+
Step 14190/50462 | Loss: 2.6026 | LR: 4.15e-04 | 5260.0ms/step | 11.4 samples/s
|
| 146 |
+
wandb: WARNING Tried to log to step 14190 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 147 |
+
Step 14200/50462 | Loss: 2.6050 | LR: 4.15e-04 | 5258.2ms/step | 11.4 samples/s
|
| 148 |
+
wandb: WARNING Tried to log to step 14200 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 149 |
+
Step 14210/50462 | Loss: 2.5927 | LR: 4.15e-04 | 5255.1ms/step | 11.4 samples/s
|
| 150 |
+
wandb: WARNING Tried to log to step 14210 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 151 |
+
Step 14220/50462 | Loss: 2.6176 | LR: 4.15e-04 | 5258.9ms/step | 11.4 samples/s
|
| 152 |
+
wandb: WARNING Tried to log to step 14220 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 153 |
+
Step 14230/50462 | Loss: 2.6042 | LR: 4.15e-04 | 5254.0ms/step | 11.4 samples/s
|
| 154 |
+
wandb: WARNING Tried to log to step 14230 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 155 |
+
Step 14240/50462 | Loss: 2.6180 | LR: 4.15e-04 | 5260.6ms/step | 11.4 samples/s
|
| 156 |
+
wandb: WARNING Tried to log to step 14240 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 157 |
+
Step 14250/50462 | Loss: 2.5955 | LR: 4.14e-04 | 5260.3ms/step | 11.4 samples/s
|
| 158 |
+
wandb: WARNING Tried to log to step 14250 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 159 |
+
Validation at step 14250: Loss = 2.6976, Perplexity = 14.84
|
| 160 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-14250
|
| 161 |
+
Step 14260/50462 | Loss: 2.5888 | LR: 4.14e-04 | 10241.3ms/step | 5.9 samples/s
|
| 162 |
+
wandb: WARNING Tried to log to step 14260 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 163 |
+
Step 14270/50462 | Loss: 2.5967 | LR: 4.14e-04 | 5254.9ms/step | 11.4 samples/s
|
| 164 |
+
wandb: WARNING Tried to log to step 14270 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 165 |
+
Step 14280/50462 | Loss: 2.5995 | LR: 4.14e-04 | 5258.1ms/step | 11.4 samples/s
|
| 166 |
+
wandb: WARNING Tried to log to step 14280 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 167 |
+
Step 14290/50462 | Loss: 2.5783 | LR: 4.14e-04 | 5260.4ms/step | 11.4 samples/s
|
| 168 |
+
wandb: WARNING Tried to log to step 14290 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 169 |
+
Step 14300/50462 | Loss: 2.6126 | LR: 4.14e-04 | 5259.8ms/step | 11.4 samples/s
|
| 170 |
+
wandb: WARNING Tried to log to step 14300 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 171 |
+
Step 14310/50462 | Loss: 2.5903 | LR: 4.14e-04 | 5261.9ms/step | 11.4 samples/s
|
| 172 |
+
wandb: WARNING Tried to log to step 14310 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 173 |
+
Step 14320/50462 | Loss: 2.5893 | LR: 4.14e-04 | 5261.1ms/step | 11.4 samples/s
|
| 174 |
+
wandb: WARNING Tried to log to step 14320 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 175 |
+
Step 14330/50462 | Loss: 2.6244 | LR: 4.13e-04 | 5259.1ms/step | 11.4 samples/s
|
| 176 |
+
wandb: WARNING Tried to log to step 14330 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 177 |
+
Step 14340/50462 | Loss: 2.5662 | LR: 4.13e-04 | 5263.1ms/step | 11.4 samples/s
|
| 178 |
+
wandb: WARNING Tried to log to step 14340 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 179 |
+
Step 14350/50462 | Loss: 2.5948 | LR: 4.13e-04 | 5259.3ms/step | 11.4 samples/s
|
| 180 |
+
wandb: WARNING Tried to log to step 14350 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 181 |
+
Step 14360/50462 | Loss: 2.6006 | LR: 4.13e-04 | 5260.1ms/step | 11.4 samples/s
|
| 182 |
+
wandb: WARNING Tried to log to step 14360 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 183 |
+
Step 14370/50462 | Loss: 2.5762 | LR: 4.13e-04 | 5263.3ms/step | 11.4 samples/s
|
| 184 |
+
wandb: WARNING Tried to log to step 14370 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 185 |
+
Step 14380/50462 | Loss: 2.5736 | LR: 4.13e-04 | 5264.1ms/step | 11.4 samples/s
|
| 186 |
+
wandb: WARNING Tried to log to step 14380 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 187 |
+
Step 14390/50462 | Loss: 2.5528 | LR: 4.13e-04 | 5263.3ms/step | 11.4 samples/s
|
| 188 |
+
wandb: WARNING Tried to log to step 14390 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 189 |
+
Step 14400/50462 | Loss: 2.6021 | LR: 4.13e-04 | 5263.9ms/step | 11.4 samples/s
|
| 190 |
+
wandb: WARNING Tried to log to step 14400 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 191 |
+
Step 14410/50462 | Loss: 2.6074 | LR: 4.12e-04 | 5258.8ms/step | 11.4 samples/s
|
| 192 |
+
wandb: WARNING Tried to log to step 14410 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 193 |
+
Step 14420/50462 | Loss: 2.5956 | LR: 4.12e-04 | 5263.4ms/step | 11.4 samples/s
|
| 194 |
+
wandb: WARNING Tried to log to step 14420 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 195 |
+
Step 14430/50462 | Loss: 2.5700 | LR: 4.12e-04 | 5259.9ms/step | 11.4 samples/s
|
| 196 |
+
wandb: WARNING Tried to log to step 14430 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 197 |
+
Step 14440/50462 | Loss: 2.5924 | LR: 4.12e-04 | 5257.0ms/step | 11.4 samples/s
|
| 198 |
+
Step 14450/50462 | Loss: 2.5934 | LR: 4.12e-04 | 5255.8ms/step | 11.4 samples/s
|
| 199 |
+
Step 14460/50462 | Loss: 2.5592 | LR: 4.12e-04 | 5262.2ms/step | 11.4 samples/s
|
| 200 |
+
Step 14470/50462 | Loss: 2.6006 | LR: 4.12e-04 | 5259.4ms/step | 11.4 samples/s
|
| 201 |
+
Step 14480/50462 | Loss: 2.5971 | LR: 4.12e-04 | 5261.0ms/step | 11.4 samples/s
|
| 202 |
+
Step 14490/50462 | Loss: 2.6076 | LR: 4.12e-04 | 5259.1ms/step | 11.4 samples/s
|
| 203 |
+
Step 14500/50462 | Loss: 2.6152 | LR: 4.11e-04 | 5256.3ms/step | 11.4 samples/s
|
| 204 |
+
Validation at step 14500: Loss = 2.6974, Perplexity = 14.84
|
| 205 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-14500
|
| 206 |
+
Step 14510/50462 | Loss: 2.5892 | LR: 4.11e-04 | 10193.0ms/step | 5.9 samples/s
|
| 207 |
+
Step 14520/50462 | Loss: 2.6036 | LR: 4.11e-04 | 5257.9ms/step | 11.4 samples/s
|
| 208 |
+
Step 14530/50462 | Loss: 2.6103 | LR: 4.11e-04 | 5258.8ms/step | 11.4 samples/s
|
| 209 |
+
Step 14540/50462 | Loss: 2.6010 | LR: 4.11e-04 | 5258.9ms/step | 11.4 samples/s
|
| 210 |
+
Step 14550/50462 | Loss: 2.5856 | LR: 4.11e-04 | 5259.0ms/step | 11.4 samples/s
|
| 211 |
+
Step 14560/50462 | Loss: 2.5904 | LR: 4.11e-04 | 5259.4ms/step | 11.4 samples/s
|
| 212 |
+
Step 14570/50462 | Loss: 2.6005 | LR: 4.11e-04 | 5254.1ms/step | 11.4 samples/s
|
| 213 |
+
Step 14580/50462 | Loss: 2.5816 | LR: 4.10e-04 | 5254.4ms/step | 11.4 samples/s
|
| 214 |
+
Step 14590/50462 | Loss: 2.5995 | LR: 4.10e-04 | 5255.8ms/step | 11.4 samples/s
|
| 215 |
+
Step 14600/50462 | Loss: 2.6015 | LR: 4.10e-04 | 5257.9ms/step | 11.4 samples/s
|
| 216 |
+
Step 14610/50462 | Loss: 2.5945 | LR: 4.10e-04 | 5255.4ms/step | 11.4 samples/s
|
| 217 |
+
Step 14620/50462 | Loss: 2.5857 | LR: 4.10e-04 | 5254.0ms/step | 11.4 samples/s
|
| 218 |
+
Step 14630/50462 | Loss: 2.5577 | LR: 4.10e-04 | 5259.2ms/step | 11.4 samples/s
|
| 219 |
+
Step 14640/50462 | Loss: 2.5768 | LR: 4.10e-04 | 5250.5ms/step | 11.4 samples/s
|
| 220 |
+
Step 14650/50462 | Loss: 2.5587 | LR: 4.10e-04 | 5254.8ms/step | 11.4 samples/s
|
| 221 |
+
Step 14660/50462 | Loss: 2.5813 | LR: 4.09e-04 | 5252.0ms/step | 11.4 samples/s
|
| 222 |
+
Step 14670/50462 | Loss: 2.6041 | LR: 4.09e-04 | 5258.1ms/step | 11.4 samples/s
|
| 223 |
+
Step 14680/50462 | Loss: 2.6009 | LR: 4.09e-04 | 5255.6ms/step | 11.4 samples/s
|
| 224 |
+
Step 14690/50462 | Loss: 2.5820 | LR: 4.09e-04 | 5255.0ms/step | 11.4 samples/s
|
| 225 |
+
Step 14700/50462 | Loss: 2.5994 | LR: 4.09e-04 | 5254.7ms/step | 11.4 samples/s
|
| 226 |
+
Step 14710/50462 | Loss: 2.5968 | LR: 4.09e-04 | 5249.7ms/step | 11.4 samples/s
|
| 227 |
+
Step 14720/50462 | Loss: 2.6091 | LR: 4.09e-04 | 5251.2ms/step | 11.4 samples/s
|
| 228 |
+
Step 14730/50462 | Loss: 2.5932 | LR: 4.09e-04 | 5238.0ms/step | 11.5 samples/s
|
| 229 |
+
Step 14740/50462 | Loss: 2.5991 | LR: 4.08e-04 | 5231.4ms/step | 11.5 samples/s
|
| 230 |
+
Step 14750/50462 | Loss: 2.5984 | LR: 4.08e-04 | 5236.3ms/step | 11.5 samples/s
|
| 231 |
+
Validation at step 14750: Loss = 2.6969, Perplexity = 14.83
|
| 232 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-14750
|
| 233 |
+
Step 14760/50462 | Loss: 2.5961 | LR: 4.08e-04 | 10177.7ms/step | 5.9 samples/s
|
| 234 |
+
Step 14770/50462 | Loss: 2.6188 | LR: 4.08e-04 | 5254.2ms/step | 11.4 samples/s
|
| 235 |
+
Step 14780/50462 | Loss: 2.6042 | LR: 4.08e-04 | 5251.1ms/step | 11.4 samples/s
|
| 236 |
+
Step 14790/50462 | Loss: 2.5840 | LR: 4.08e-04 | 5258.2ms/step | 11.4 samples/s
|
| 237 |
+
Step 14800/50462 | Loss: 2.5821 | LR: 4.08e-04 | 5258.7ms/step | 11.4 samples/s
|
| 238 |
+
Step 14810/50462 | Loss: 2.6035 | LR: 4.08e-04 | 5253.7ms/step | 11.4 samples/s
|
| 239 |
+
Step 14820/50462 | Loss: 2.5770 | LR: 4.08e-04 | 5262.4ms/step | 11.4 samples/s
|
| 240 |
+
Step 14830/50462 | Loss: 2.5833 | LR: 4.07e-04 | 5259.5ms/step | 11.4 samples/s
|
| 241 |
+
Step 14840/50462 | Loss: 2.6310 | LR: 4.07e-04 | 5259.3ms/step | 11.4 samples/s
|
| 242 |
+
Step 14850/50462 | Loss: 2.6318 | LR: 4.07e-04 | 5260.0ms/step | 11.4 samples/s
|
| 243 |
+
Step 14860/50462 | Loss: 2.6036 | LR: 4.07e-04 | 5258.6ms/step | 11.4 samples/s
|
| 244 |
+
Step 14870/50462 | Loss: 2.6005 | LR: 4.07e-04 | 5261.1ms/step | 11.4 samples/s
|
| 245 |
+
Step 14880/50462 | Loss: 2.5679 | LR: 4.07e-04 | 5259.7ms/step | 11.4 samples/s
|
| 246 |
+
Step 14890/50462 | Loss: 2.5928 | LR: 4.07e-04 | 5258.9ms/step | 11.4 samples/s
|
| 247 |
+
Step 14900/50462 | Loss: 2.6090 | LR: 4.07e-04 | 5256.3ms/step | 11.4 samples/s
|
| 248 |
+
Step 14910/50462 | Loss: 2.5979 | LR: 4.06e-04 | 5254.5ms/step | 11.4 samples/s
|
| 249 |
+
Step 14920/50462 | Loss: 2.5907 | LR: 4.06e-04 | 5252.1ms/step | 11.4 samples/s
|
| 250 |
+
Step 14930/50462 | Loss: 2.5941 | LR: 4.06e-04 | 5257.0ms/step | 11.4 samples/s
|
| 251 |
+
Step 14940/50462 | Loss: 2.5892 | LR: 4.06e-04 | 5255.4ms/step | 11.4 samples/s
|
| 252 |
+
Step 14950/50462 | Loss: 2.5902 | LR: 4.06e-04 | 5252.3ms/step | 11.4 samples/s
|
| 253 |
+
Step 14960/50462 | Loss: 2.6008 | LR: 4.06e-04 | 5255.7ms/step | 11.4 samples/s
|
| 254 |
+
Step 14970/50462 | Loss: 2.5571 | LR: 4.06e-04 | 5253.0ms/step | 11.4 samples/s
|
| 255 |
+
Step 14980/50462 | Loss: 2.5836 | LR: 4.06e-04 | 5252.5ms/step | 11.4 samples/s
|
| 256 |
+
Step 14990/50462 | Loss: 2.5911 | LR: 4.05e-04 | 5254.0ms/step | 11.4 samples/s
|
| 257 |
+
Step 15000/50462 | Loss: 2.5963 | LR: 4.05e-04 | 5254.1ms/step | 11.4 samples/s
|
| 258 |
+
Validation at step 15000: Loss = 2.6955, Perplexity = 14.81
|
| 259 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-15000
|
| 260 |
+
Step 15010/50462 | Loss: 2.5961 | LR: 4.05e-04 | 10196.7ms/step | 5.9 samples/s
|
| 261 |
+
Step 15020/50462 | Loss: 2.5905 | LR: 4.05e-04 | 5249.4ms/step | 11.4 samples/s
|
| 262 |
+
Step 15030/50462 | Loss: 2.5922 | LR: 4.05e-04 | 5256.5ms/step | 11.4 samples/s
|
| 263 |
+
Step 15040/50462 | Loss: 2.5921 | LR: 4.05e-04 | 5252.1ms/step | 11.4 samples/s
|
| 264 |
+
Step 15050/50462 | Loss: 2.6171 | LR: 4.05e-04 | 5251.7ms/step | 11.4 samples/s
|
| 265 |
+
Step 15060/50462 | Loss: 2.5734 | LR: 4.05e-04 | 5255.5ms/step | 11.4 samples/s
|
| 266 |
+
Step 15070/50462 | Loss: 2.5866 | LR: 4.04e-04 | 5254.1ms/step | 11.4 samples/s
|
| 267 |
+
Step 15080/50462 | Loss: 2.6219 | LR: 4.04e-04 | 5259.6ms/step | 11.4 samples/s
|
| 268 |
+
Step 15090/50462 | Loss: 2.5759 | LR: 4.04e-04 | 5252.8ms/step | 11.4 samples/s
|
| 269 |
+
Step 15100/50462 | Loss: 2.6238 | LR: 4.04e-04 | 5253.0ms/step | 11.4 samples/s
|
| 270 |
+
Step 15110/50462 | Loss: 2.5872 | LR: 4.04e-04 | 5249.0ms/step | 11.4 samples/s
|
| 271 |
+
Step 15120/50462 | Loss: 2.6010 | LR: 4.04e-04 | 5250.9ms/step | 11.4 samples/s
|
| 272 |
+
Step 15130/50462 | Loss: 2.5726 | LR: 4.04e-04 | 5252.6ms/step | 11.4 samples/s
|
| 273 |
+
Step 15140/50462 | Loss: 2.6049 | LR: 4.04e-04 | 5256.4ms/step | 11.4 samples/s
|
| 274 |
+
Step 15150/50462 | Loss: 2.5988 | LR: 4.03e-04 | 5256.9ms/step | 11.4 samples/s
|
| 275 |
+
Step 15160/50462 | Loss: 2.5711 | LR: 4.03e-04 | 5249.4ms/step | 11.4 samples/s
|
| 276 |
+
Step 15170/50462 | Loss: 2.6170 | LR: 4.03e-04 | 5251.6ms/step | 11.4 samples/s
|
| 277 |
+
Step 15180/50462 | Loss: 2.6008 | LR: 4.03e-04 | 5253.6ms/step | 11.4 samples/s
|
| 278 |
+
Step 15190/50462 | Loss: 2.5808 | LR: 4.03e-04 | 5249.2ms/step | 11.4 samples/s
|
| 279 |
+
Step 15200/50462 | Loss: 2.5941 | LR: 4.03e-04 | 5256.5ms/step | 11.4 samples/s
|
| 280 |
+
Step 15210/50462 | Loss: 2.6047 | LR: 4.03e-04 | 5251.9ms/step | 11.4 samples/s
|
| 281 |
+
Step 15220/50462 | Loss: 2.5929 | LR: 4.03e-04 | 5249.6ms/step | 11.4 samples/s
|
| 282 |
+
Step 15230/50462 | Loss: 2.5720 | LR: 4.02e-04 | 5250.9ms/step | 11.4 samples/s
|
| 283 |
+
Step 15240/50462 | Loss: 2.6031 | LR: 4.02e-04 | 5253.3ms/step | 11.4 samples/s
|
| 284 |
+
Step 15250/50462 | Loss: 2.5804 | LR: 4.02e-04 | 5256.4ms/step | 11.4 samples/s
|
| 285 |
+
Validation at step 15250: Loss = 2.6934, Perplexity = 14.78
|
| 286 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-15250
|
| 287 |
+
Step 15260/50462 | Loss: 2.6015 | LR: 4.02e-04 | 10149.0ms/step | 5.9 samples/s
|
| 288 |
+
Step 15270/50462 | Loss: 2.6012 | LR: 4.02e-04 | 5251.1ms/step | 11.4 samples/s
|
| 289 |
+
Step 15280/50462 | Loss: 2.5760 | LR: 4.02e-04 | 5255.9ms/step | 11.4 samples/s
|
| 290 |
+
Step 15290/50462 | Loss: 2.5770 | LR: 4.02e-04 | 5255.8ms/step | 11.4 samples/s
|
| 291 |
+
Step 15300/50462 | Loss: 2.5675 | LR: 4.02e-04 | 5257.3ms/step | 11.4 samples/s
|
| 292 |
+
Step 15310/50462 | Loss: 2.6076 | LR: 4.01e-04 | 5254.8ms/step | 11.4 samples/s
|
| 293 |
+
Step 15320/50462 | Loss: 2.5929 | LR: 4.01e-04 | 5255.1ms/step | 11.4 samples/s
|
| 294 |
+
Step 15330/50462 | Loss: 2.5974 | LR: 4.01e-04 | 5254.5ms/step | 11.4 samples/s
|
| 295 |
+
Step 15340/50462 | Loss: 2.5763 | LR: 4.01e-04 | 5260.8ms/step | 11.4 samples/s
|
| 296 |
+
Step 15350/50462 | Loss: 2.5890 | LR: 4.01e-04 | 5262.7ms/step | 11.4 samples/s
|
| 297 |
+
Step 15360/50462 | Loss: 2.5774 | LR: 4.01e-04 | 5263.4ms/step | 11.4 samples/s
|
| 298 |
+
Step 15370/50462 | Loss: 2.6042 | LR: 4.01e-04 | 5256.6ms/step | 11.4 samples/s
|
| 299 |
+
Step 15380/50462 | Loss: 2.5976 | LR: 4.01e-04 | 5251.8ms/step | 11.4 samples/s
|
| 300 |
+
Step 15390/50462 | Loss: 2.5890 | LR: 4.00e-04 | 5250.7ms/step | 11.4 samples/s
|
| 301 |
+
Step 15400/50462 | Loss: 2.6028 | LR: 4.00e-04 | 5251.2ms/step | 11.4 samples/s
|
| 302 |
+
Step 15410/50462 | Loss: 2.5913 | LR: 4.00e-04 | 5250.8ms/step | 11.4 samples/s
|
| 303 |
+
Step 15420/50462 | Loss: 2.5939 | LR: 4.00e-04 | 5253.8ms/step | 11.4 samples/s
|
| 304 |
+
Step 15430/50462 | Loss: 2.5854 | LR: 4.00e-04 | 5251.9ms/step | 11.4 samples/s
|
| 305 |
+
Step 15440/50462 | Loss: 2.5980 | LR: 4.00e-04 | 5254.1ms/step | 11.4 samples/s
|
| 306 |
+
Step 15450/50462 | Loss: 2.5673 | LR: 4.00e-04 | 5254.7ms/step | 11.4 samples/s
|
| 307 |
+
Step 15460/50462 | Loss: 2.5870 | LR: 4.00e-04 | 5255.9ms/step | 11.4 samples/s
|
| 308 |
+
Step 15470/50462 | Loss: 2.5876 | LR: 3.99e-04 | 5256.6ms/step | 11.4 samples/s
|
| 309 |
+
Step 15480/50462 | Loss: 2.6031 | LR: 3.99e-04 | 5254.3ms/step | 11.4 samples/s
|
| 310 |
+
Step 15490/50462 | Loss: 2.5735 | LR: 3.99e-04 | 5252.9ms/step | 11.4 samples/s
|
| 311 |
+
Step 15500/50462 | Loss: 2.5920 | LR: 3.99e-04 | 5251.3ms/step | 11.4 samples/s
|
| 312 |
+
Validation at step 15500: Loss = 2.6935, Perplexity = 14.78
|
| 313 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-15500
|
| 314 |
+
Step 15510/50462 | Loss: 2.5917 | LR: 3.99e-04 | 10104.6ms/step | 5.9 samples/s
|
| 315 |
+
Step 15520/50462 | Loss: 2.5826 | LR: 3.99e-04 | 5249.9ms/step | 11.4 samples/s
|
| 316 |
+
Step 15530/50462 | Loss: 2.6022 | LR: 3.99e-04 | 5254.4ms/step | 11.4 samples/s
|
| 317 |
+
Step 15540/50462 | Loss: 2.5862 | LR: 3.99e-04 | 5254.4ms/step | 11.4 samples/s
|
| 318 |
+
Step 15550/50462 | Loss: 2.5800 | LR: 3.98e-04 | 5256.5ms/step | 11.4 samples/s
|
| 319 |
+
Step 15560/50462 | Loss: 2.5930 | LR: 3.98e-04 | 5258.5ms/step | 11.4 samples/s
|
| 320 |
+
Step 15570/50462 | Loss: 2.6109 | LR: 3.98e-04 | 5262.4ms/step | 11.4 samples/s
|
| 321 |
+
Step 15580/50462 | Loss: 2.5938 | LR: 3.98e-04 | 5258.5ms/step | 11.4 samples/s
|
| 322 |
+
Step 15590/50462 | Loss: 2.5845 | LR: 3.98e-04 | 5258.1ms/step | 11.4 samples/s
|
| 323 |
+
Step 15600/50462 | Loss: 2.5621 | LR: 3.98e-04 | 5260.0ms/step | 11.4 samples/s
|
| 324 |
+
Step 15610/50462 | Loss: 2.5745 | LR: 3.98e-04 | 5254.4ms/step | 11.4 samples/s
|
| 325 |
+
Step 15620/50462 | Loss: 2.5914 | LR: 3.98e-04 | 5256.8ms/step | 11.4 samples/s
|
| 326 |
+
Step 15630/50462 | Loss: 2.6048 | LR: 3.97e-04 | 5254.9ms/step | 11.4 samples/s
|
| 327 |
+
Step 15640/50462 | Loss: 2.5648 | LR: 3.97e-04 | 5260.7ms/step | 11.4 samples/s
|
| 328 |
+
Step 15650/50462 | Loss: 2.5650 | LR: 3.97e-04 | 5258.1ms/step | 11.4 samples/s
|
| 329 |
+
Step 15660/50462 | Loss: 2.5839 | LR: 3.97e-04 | 5258.1ms/step | 11.4 samples/s
|
| 330 |
+
Step 15670/50462 | Loss: 2.6103 | LR: 3.97e-04 | 5254.7ms/step | 11.4 samples/s
|
| 331 |
+
Step 15680/50462 | Loss: 2.5890 | LR: 3.97e-04 | 5253.8ms/step | 11.4 samples/s
|
| 332 |
+
Step 15690/50462 | Loss: 2.6038 | LR: 3.97e-04 | 5248.8ms/step | 11.4 samples/s
|
| 333 |
+
Step 15700/50462 | Loss: 2.5974 | LR: 3.96e-04 | 5251.9ms/step | 11.4 samples/s
|
| 334 |
+
Step 15710/50462 | Loss: 2.5917 | LR: 3.96e-04 | 5254.6ms/step | 11.4 samples/s
|
| 335 |
+
Step 15720/50462 | Loss: 2.6000 | LR: 3.96e-04 | 5260.3ms/step | 11.4 samples/s
|
| 336 |
+
Step 15730/50462 | Loss: 2.5923 | LR: 3.96e-04 | 5257.7ms/step | 11.4 samples/s
|
| 337 |
+
Step 15740/50462 | Loss: 2.5743 | LR: 3.96e-04 | 5260.8ms/step | 11.4 samples/s
|
| 338 |
+
Step 15750/50462 | Loss: 2.6047 | LR: 3.96e-04 | 5261.8ms/step | 11.4 samples/s
|
| 339 |
+
Validation at step 15750: Loss = 2.6929, Perplexity = 14.78
|
| 340 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-15750
|
| 341 |
+
Step 15760/50462 | Loss: 2.5691 | LR: 3.96e-04 | 10106.5ms/step | 5.9 samples/s
|
| 342 |
+
Step 15770/50462 | Loss: 2.5990 | LR: 3.96e-04 | 5237.2ms/step | 11.5 samples/s
|
| 343 |
+
Step 15780/50462 | Loss: 2.5820 | LR: 3.95e-04 | 5238.2ms/step | 11.5 samples/s
|
| 344 |
+
Step 15790/50462 | Loss: 2.6089 | LR: 3.95e-04 | 5238.5ms/step | 11.5 samples/s
|
| 345 |
+
Step 15800/50462 | Loss: 2.6076 | LR: 3.95e-04 | 5233.4ms/step | 11.5 samples/s
|
| 346 |
+
Step 15810/50462 | Loss: 2.5743 | LR: 3.95e-04 | 5231.5ms/step | 11.5 samples/s
|
| 347 |
+
Step 15820/50462 | Loss: 2.5922 | LR: 3.95e-04 | 5232.6ms/step | 11.5 samples/s
|
| 348 |
+
Step 15830/50462 | Loss: 2.5755 | LR: 3.95e-04 | 5235.4ms/step | 11.5 samples/s
|
| 349 |
+
Step 15840/50462 | Loss: 2.5793 | LR: 3.95e-04 | 5234.9ms/step | 11.5 samples/s
|
| 350 |
+
Step 15850/50462 | Loss: 2.5882 | LR: 3.95e-04 | 5234.8ms/step | 11.5 samples/s
|
| 351 |
+
Step 15860/50462 | Loss: 2.5818 | LR: 3.94e-04 | 5238.2ms/step | 11.5 samples/s
|
| 352 |
+
Step 15870/50462 | Loss: 2.5719 | LR: 3.94e-04 | 5238.2ms/step | 11.5 samples/s
|
| 353 |
+
Step 15880/50462 | Loss: 2.5906 | LR: 3.94e-04 | 5241.1ms/step | 11.4 samples/s
|
| 354 |
+
Step 15890/50462 | Loss: 2.5650 | LR: 3.94e-04 | 5242.1ms/step | 11.4 samples/s
|
| 355 |
+
Step 15900/50462 | Loss: 2.5709 | LR: 3.94e-04 | 5242.5ms/step | 11.4 samples/s
|
| 356 |
+
Step 15910/50462 | Loss: 2.5951 | LR: 3.94e-04 | 5237.8ms/step | 11.5 samples/s
|
| 357 |
+
Step 15920/50462 | Loss: 2.5806 | LR: 3.94e-04 | 5235.6ms/step | 11.5 samples/s
|
| 358 |
+
Step 15930/50462 | Loss: 2.5731 | LR: 3.94e-04 | 5232.9ms/step | 11.5 samples/s
|
| 359 |
+
Step 15940/50462 | Loss: 2.5848 | LR: 3.93e-04 | 5239.7ms/step | 11.5 samples/s
|
| 360 |
+
Step 15950/50462 | Loss: 2.6084 | LR: 3.93e-04 | 5235.3ms/step | 11.5 samples/s
|
| 361 |
+
Step 15960/50462 | Loss: 2.5868 | LR: 3.93e-04 | 5235.1ms/step | 11.5 samples/s
|
| 362 |
+
Step 15970/50462 | Loss: 2.5861 | LR: 3.93e-04 | 5229.6ms/step | 11.5 samples/s
|
| 363 |
+
Step 15980/50462 | Loss: 2.5803 | LR: 3.93e-04 | 5242.2ms/step | 11.4 samples/s
|
| 364 |
+
Step 15990/50462 | Loss: 2.6005 | LR: 3.93e-04 | 5231.1ms/step | 11.5 samples/s
|
| 365 |
+
Step 16000/50462 | Loss: 2.5985 | LR: 3.93e-04 | 5230.0ms/step | 11.5 samples/s
|
| 366 |
+
Validation at step 16000: Loss = 2.6900, Perplexity = 14.73
|
| 367 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-16000
|
| 368 |
+
Step 16010/50462 | Loss: 2.5732 | LR: 3.92e-04 | 10152.8ms/step | 5.9 samples/s
|
| 369 |
+
Step 16020/50462 | Loss: 2.5728 | LR: 3.92e-04 | 5257.9ms/step | 11.4 samples/s
|
| 370 |
+
Step 16030/50462 | Loss: 2.6043 | LR: 3.92e-04 | 5256.7ms/step | 11.4 samples/s
|
| 371 |
+
Step 16040/50462 | Loss: 2.5809 | LR: 3.92e-04 | 5244.4ms/step | 11.4 samples/s
|
| 372 |
+
Step 16050/50462 | Loss: 2.5976 | LR: 3.92e-04 | 5238.3ms/step | 11.5 samples/s
|
| 373 |
+
Step 16060/50462 | Loss: 2.5926 | LR: 3.92e-04 | 5242.6ms/step | 11.4 samples/s
|
| 374 |
+
Step 16070/50462 | Loss: 2.6094 | LR: 3.92e-04 | 5240.2ms/step | 11.4 samples/s
|
| 375 |
+
Step 16080/50462 | Loss: 2.5933 | LR: 3.92e-04 | 5238.8ms/step | 11.5 samples/s
|
| 376 |
+
Step 16090/50462 | Loss: 2.5859 | LR: 3.91e-04 | 5245.6ms/step | 11.4 samples/s
|
| 377 |
+
Step 16100/50462 | Loss: 2.5837 | LR: 3.91e-04 | 5238.2ms/step | 11.5 samples/s
|
| 378 |
+
Step 16110/50462 | Loss: 2.5723 | LR: 3.91e-04 | 5235.5ms/step | 11.5 samples/s
|
| 379 |
+
Step 16120/50462 | Loss: 2.5728 | LR: 3.91e-04 | 5228.8ms/step | 11.5 samples/s
|
| 380 |
+
Step 16130/50462 | Loss: 2.5771 | LR: 3.91e-04 | 5226.0ms/step | 11.5 samples/s
|
| 381 |
+
Step 16140/50462 | Loss: 2.5951 | LR: 3.91e-04 | 5232.3ms/step | 11.5 samples/s
|
| 382 |
+
Step 16150/50462 | Loss: 2.5761 | LR: 3.91e-04 | 5231.6ms/step | 11.5 samples/s
|
| 383 |
+
Step 16160/50462 | Loss: 2.5733 | LR: 3.91e-04 | 5238.3ms/step | 11.5 samples/s
|
| 384 |
+
Step 16170/50462 | Loss: 2.5755 | LR: 3.90e-04 | 5230.3ms/step | 11.5 samples/s
|
| 385 |
+
Step 16180/50462 | Loss: 2.5875 | LR: 3.90e-04 | 5245.1ms/step | 11.4 samples/s
|
| 386 |
+
Step 16190/50462 | Loss: 2.5954 | LR: 3.90e-04 | 5230.6ms/step | 11.5 samples/s
|
| 387 |
+
Step 16200/50462 | Loss: 2.5650 | LR: 3.90e-04 | 5229.7ms/step | 11.5 samples/s
|
| 388 |
+
Step 16210/50462 | Loss: 2.5966 | LR: 3.90e-04 | 5225.9ms/step | 11.5 samples/s
|
| 389 |
+
Step 16220/50462 | Loss: 2.5780 | LR: 3.90e-04 | 5227.6ms/step | 11.5 samples/s
|
| 390 |
+
Step 16230/50462 | Loss: 2.5901 | LR: 3.90e-04 | 5229.8ms/step | 11.5 samples/s
|
| 391 |
+
Step 16240/50462 | Loss: 2.5868 | LR: 3.89e-04 | 5230.4ms/step | 11.5 samples/s
|
| 392 |
+
Step 16250/50462 | Loss: 2.5653 | LR: 3.89e-04 | 5229.8ms/step | 11.5 samples/s
|
| 393 |
+
Validation at step 16250: Loss = 2.6891, Perplexity = 14.72
|
| 394 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-16250
|
| 395 |
+
Step 16260/50462 | Loss: 2.5994 | LR: 3.89e-04 | 10099.7ms/step | 5.9 samples/s
|
| 396 |
+
Step 16270/50462 | Loss: 2.5961 | LR: 3.89e-04 | 5231.1ms/step | 11.5 samples/s
|
| 397 |
+
Step 16280/50462 | Loss: 2.5651 | LR: 3.89e-04 | 5234.7ms/step | 11.5 samples/s
|
| 398 |
+
Step 16290/50462 | Loss: 2.5972 | LR: 3.89e-04 | 5243.3ms/step | 11.4 samples/s
|
| 399 |
+
Step 16300/50462 | Loss: 2.6193 | LR: 3.89e-04 | 5233.5ms/step | 11.5 samples/s
|
| 400 |
+
Step 16310/50462 | Loss: 2.6008 | LR: 3.89e-04 | 5237.5ms/step | 11.5 samples/s
|
| 401 |
+
Step 16320/50462 | Loss: 2.5744 | LR: 3.88e-04 | 5229.5ms/step | 11.5 samples/s
|
| 402 |
+
Step 16330/50462 | Loss: 2.6035 | LR: 3.88e-04 | 5239.6ms/step | 11.5 samples/s
|
| 403 |
+
Step 16340/50462 | Loss: 2.6097 | LR: 3.88e-04 | 5231.0ms/step | 11.5 samples/s
|
| 404 |
+
Step 16350/50462 | Loss: 2.5874 | LR: 3.88e-04 | 5229.2ms/step | 11.5 samples/s
|
| 405 |
+
Step 16360/50462 | Loss: 2.5836 | LR: 3.88e-04 | 5225.2ms/step | 11.5 samples/s
|
| 406 |
+
Step 16370/50462 | Loss: 2.6065 | LR: 3.88e-04 | 5226.6ms/step | 11.5 samples/s
|
| 407 |
+
Step 16380/50462 | Loss: 2.5605 | LR: 3.88e-04 | 5235.1ms/step | 11.5 samples/s
|
| 408 |
+
Step 16390/50462 | Loss: 2.5982 | LR: 3.88e-04 | 5224.3ms/step | 11.5 samples/s
|
| 409 |
+
Step 16400/50462 | Loss: 2.5626 | LR: 3.87e-04 | 5230.9ms/step | 11.5 samples/s
|
| 410 |
+
Step 16410/50462 | Loss: 2.5759 | LR: 3.87e-04 | 5231.7ms/step | 11.5 samples/s
|
| 411 |
+
Step 16420/50462 | Loss: 2.5934 | LR: 3.87e-04 | 5235.5ms/step | 11.5 samples/s
|
| 412 |
+
Step 16430/50462 | Loss: 2.5680 | LR: 3.87e-04 | 5232.4ms/step | 11.5 samples/s
|
| 413 |
+
Step 16440/50462 | Loss: 2.5954 | LR: 3.87e-04 | 5227.1ms/step | 11.5 samples/s
|
| 414 |
+
Step 16450/50462 | Loss: 2.6783 | LR: 3.87e-04 | 5226.5ms/step | 11.5 samples/s
|
| 415 |
+
Step 16460/50462 | Loss: 2.6648 | LR: 3.87e-04 | 5228.0ms/step | 11.5 samples/s
|
| 416 |
+
Step 16470/50462 | Loss: 2.6929 | LR: 3.86e-04 | 5228.7ms/step | 11.5 samples/s
|
| 417 |
+
Step 16480/50462 | Loss: 2.6569 | LR: 3.86e-04 | 5235.1ms/step | 11.5 samples/s
|
| 418 |
+
Step 16490/50462 | Loss: 2.6710 | LR: 3.86e-04 | 5233.3ms/step | 11.5 samples/s
|
| 419 |
+
Step 16500/50462 | Loss: 2.6969 | LR: 3.86e-04 | 5233.0ms/step | 11.5 samples/s
|
| 420 |
+
Validation at step 16500: Loss = 2.6783, Perplexity = 14.56
|
| 421 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-16500
|
| 422 |
+
Step 16510/50462 | Loss: 2.6662 | LR: 3.86e-04 | 10141.6ms/step | 5.9 samples/s
|
| 423 |
+
Step 16520/50462 | Loss: 2.6532 | LR: 3.86e-04 | 5232.3ms/step | 11.5 samples/s
|
| 424 |
+
Step 16530/50462 | Loss: 2.6691 | LR: 3.86e-04 | 5230.9ms/step | 11.5 samples/s
|
| 425 |
+
Step 16540/50462 | Loss: 2.6758 | LR: 3.86e-04 | 5234.3ms/step | 11.5 samples/s
|
| 426 |
+
Step 16550/50462 | Loss: 2.6622 | LR: 3.85e-04 | 5228.7ms/step | 11.5 samples/s
|
| 427 |
+
Step 16560/50462 | Loss: 2.6443 | LR: 3.85e-04 | 5228.2ms/step | 11.5 samples/s
|
| 428 |
+
Step 16570/50462 | Loss: 2.6606 | LR: 3.85e-04 | 5228.2ms/step | 11.5 samples/s
|
| 429 |
+
Step 16580/50462 | Loss: 2.6622 | LR: 3.85e-04 | 5229.1ms/step | 11.5 samples/s
|
| 430 |
+
Step 16590/50462 | Loss: 2.6694 | LR: 3.85e-04 | 5229.4ms/step | 11.5 samples/s
|
| 431 |
+
Step 16600/50462 | Loss: 2.6484 | LR: 3.85e-04 | 5220.8ms/step | 11.5 samples/s
|
| 432 |
+
Step 16610/50462 | Loss: 2.6436 | LR: 3.85e-04 | 5234.1ms/step | 11.5 samples/s
|
| 433 |
+
Step 16620/50462 | Loss: 2.6728 | LR: 3.84e-04 | 5228.4ms/step | 11.5 samples/s
|
| 434 |
+
Step 16630/50462 | Loss: 2.6473 | LR: 3.84e-04 | 5219.3ms/step | 11.5 samples/s
|
| 435 |
+
Step 16640/50462 | Loss: 2.6741 | LR: 3.84e-04 | 5228.0ms/step | 11.5 samples/s
|
| 436 |
+
Step 16650/50462 | Loss: 2.6729 | LR: 3.84e-04 | 5230.1ms/step | 11.5 samples/s
|
| 437 |
+
Step 16660/50462 | Loss: 2.6525 | LR: 3.84e-04 | 5229.4ms/step | 11.5 samples/s
|
| 438 |
+
Step 16670/50462 | Loss: 2.6642 | LR: 3.84e-04 | 5232.4ms/step | 11.5 samples/s
|
| 439 |
+
Step 16680/50462 | Loss: 2.6518 | LR: 3.84e-04 | 5233.6ms/step | 11.5 samples/s
|
| 440 |
+
Step 16690/50462 | Loss: 2.6672 | LR: 3.84e-04 | 5233.7ms/step | 11.5 samples/s
|
| 441 |
+
Step 16700/50462 | Loss: 2.6535 | LR: 3.83e-04 | 5230.1ms/step | 11.5 samples/s
|
| 442 |
+
Step 16710/50462 | Loss: 2.6896 | LR: 3.83e-04 | 5232.8ms/step | 11.5 samples/s
|
| 443 |
+
Step 16720/50462 | Loss: 2.6610 | LR: 3.83e-04 | 5232.1ms/step | 11.5 samples/s
|
| 444 |
+
Step 16730/50462 | Loss: 2.6708 | LR: 3.83e-04 | 5233.8ms/step | 11.5 samples/s
|
| 445 |
+
Step 16740/50462 | Loss: 2.6713 | LR: 3.83e-04 | 5233.3ms/step | 11.5 samples/s
|
| 446 |
+
Step 16750/50462 | Loss: 2.6428 | LR: 3.83e-04 | 5230.7ms/step | 11.5 samples/s
|
| 447 |
+
Validation at step 16750: Loss = 2.6730, Perplexity = 14.48
|
| 448 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-16750
|
| 449 |
+
Step 16760/50462 | Loss: 2.6969 | LR: 3.83e-04 | 10165.9ms/step | 5.9 samples/s
|
| 450 |
+
Step 16770/50462 | Loss: 2.6650 | LR: 3.82e-04 | 5256.1ms/step | 11.4 samples/s
|
| 451 |
+
Step 16780/50462 | Loss: 2.6560 | LR: 3.82e-04 | 5257.2ms/step | 11.4 samples/s
|
| 452 |
+
Step 16790/50462 | Loss: 2.6474 | LR: 3.82e-04 | 5257.1ms/step | 11.4 samples/s
|
| 453 |
+
Step 16800/50462 | Loss: 2.6592 | LR: 3.82e-04 | 5255.6ms/step | 11.4 samples/s
|
| 454 |
+
Step 16810/50462 | Loss: 2.6480 | LR: 3.82e-04 | 5256.7ms/step | 11.4 samples/s
|
| 455 |
+
Step 16820/50462 | Loss: 2.6679 | LR: 3.82e-04 | 5257.8ms/step | 11.4 samples/s
|
| 456 |
+
Step 16830/50462 | Loss: 2.6607 | LR: 3.82e-04 | 5256.9ms/step | 11.4 samples/s
|
| 457 |
+
Step 16840/50462 | Loss: 2.6475 | LR: 3.82e-04 | 5259.6ms/step | 11.4 samples/s
|
| 458 |
+
Step 16850/50462 | Loss: 2.6324 | LR: 3.81e-04 | 5258.2ms/step | 11.4 samples/s
|
| 459 |
+
Step 16860/50462 | Loss: 2.6650 | LR: 3.81e-04 | 5255.6ms/step | 11.4 samples/s
|
| 460 |
+
Step 16870/50462 | Loss: 2.6795 | LR: 3.81e-04 | 5251.4ms/step | 11.4 samples/s
|
| 461 |
+
Step 16880/50462 | Loss: 2.6527 | LR: 3.81e-04 | 5248.5ms/step | 11.4 samples/s
|
| 462 |
+
Step 16890/50462 | Loss: 2.6748 | LR: 3.81e-04 | 5254.9ms/step | 11.4 samples/s
|
| 463 |
+
Step 16900/50462 | Loss: 2.6418 | LR: 3.81e-04 | 5254.9ms/step | 11.4 samples/s
|
| 464 |
+
Step 16910/50462 | Loss: 2.6807 | LR: 3.81e-04 | 5254.5ms/step | 11.4 samples/s
|
| 465 |
+
Step 16920/50462 | Loss: 2.6627 | LR: 3.80e-04 | 5245.8ms/step | 11.4 samples/s
|
| 466 |
+
Step 16930/50462 | Loss: 2.6625 | LR: 3.80e-04 | 5251.4ms/step | 11.4 samples/s
|
| 467 |
+
Step 16940/50462 | Loss: 2.6637 | LR: 3.80e-04 | 5251.6ms/step | 11.4 samples/s
|
| 468 |
+
Step 16950/50462 | Loss: 2.6541 | LR: 3.80e-04 | 5251.6ms/step | 11.4 samples/s
|
| 469 |
+
Step 16960/50462 | Loss: 2.6514 | LR: 3.80e-04 | 5250.5ms/step | 11.4 samples/s
|
| 470 |
+
Step 16970/50462 | Loss: 2.6738 | LR: 3.80e-04 | 5250.0ms/step | 11.4 samples/s
|
| 471 |
+
Step 16980/50462 | Loss: 2.6814 | LR: 3.80e-04 | 5250.4ms/step | 11.4 samples/s
|
| 472 |
+
Step 16990/50462 | Loss: 2.6625 | LR: 3.79e-04 | 5251.5ms/step | 11.4 samples/s
|
| 473 |
+
Step 17000/50462 | Loss: 2.6530 | LR: 3.79e-04 | 5250.1ms/step | 11.4 samples/s
|
| 474 |
+
Validation at step 17000: Loss = 2.6670, Perplexity = 14.40
|
| 475 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-17000
|
| 476 |
+
Step 17010/50462 | Loss: 2.6488 | LR: 3.79e-04 | 10172.4ms/step | 5.9 samples/s
|
| 477 |
+
Step 17020/50462 | Loss: 2.6560 | LR: 3.79e-04 | 5252.0ms/step | 11.4 samples/s
|
| 478 |
+
Step 17030/50462 | Loss: 2.6591 | LR: 3.79e-04 | 5253.5ms/step | 11.4 samples/s
|
| 479 |
+
Step 17040/50462 | Loss: 2.6780 | LR: 3.79e-04 | 5250.9ms/step | 11.4 samples/s
|
| 480 |
+
Step 17050/50462 | Loss: 2.6338 | LR: 3.79e-04 | 5252.9ms/step | 11.4 samples/s
|
| 481 |
+
Step 17060/50462 | Loss: 2.6589 | LR: 3.79e-04 | 5249.8ms/step | 11.4 samples/s
|
| 482 |
+
Step 17070/50462 | Loss: 2.6402 | LR: 3.78e-04 | 5255.6ms/step | 11.4 samples/s
|
| 483 |
+
Step 17080/50462 | Loss: 2.6374 | LR: 3.78e-04 | 5253.7ms/step | 11.4 samples/s
|
| 484 |
+
Step 17090/50462 | Loss: 2.6449 | LR: 3.78e-04 | 5255.4ms/step | 11.4 samples/s
|
| 485 |
+
Step 17100/50462 | Loss: 2.6580 | LR: 3.78e-04 | 5250.7ms/step | 11.4 samples/s
|
| 486 |
+
Step 17110/50462 | Loss: 2.6451 | LR: 3.78e-04 | 5251.8ms/step | 11.4 samples/s
|
| 487 |
+
Step 17120/50462 | Loss: 2.6504 | LR: 3.78e-04 | 5246.6ms/step | 11.4 samples/s
|
| 488 |
+
Step 17130/50462 | Loss: 2.6430 | LR: 3.78e-04 | 5253.6ms/step | 11.4 samples/s
|
| 489 |
+
Step 17140/50462 | Loss: 2.6290 | LR: 3.77e-04 | 5247.7ms/step | 11.4 samples/s
|
| 490 |
+
Step 17150/50462 | Loss: 2.6552 | LR: 3.77e-04 | 5248.3ms/step | 11.4 samples/s
|
| 491 |
+
Step 17160/50462 | Loss: 2.6506 | LR: 3.77e-04 | 5249.6ms/step | 11.4 samples/s
|
| 492 |
+
Step 17170/50462 | Loss: 2.6390 | LR: 3.77e-04 | 5253.0ms/step | 11.4 samples/s
|
| 493 |
+
Step 17180/50462 | Loss: 2.6875 | LR: 3.77e-04 | 5250.3ms/step | 11.4 samples/s
|
| 494 |
+
Step 17190/50462 | Loss: 2.6423 | LR: 3.77e-04 | 5249.1ms/step | 11.4 samples/s
|
| 495 |
+
Step 17200/50462 | Loss: 2.6407 | LR: 3.77e-04 | 5250.5ms/step | 11.4 samples/s
|
| 496 |
+
Step 17210/50462 | Loss: 2.6438 | LR: 3.77e-04 | 5248.3ms/step | 11.4 samples/s
|
| 497 |
+
Step 17220/50462 | Loss: 2.6455 | LR: 3.76e-04 | 5248.4ms/step | 11.4 samples/s
|
| 498 |
+
Step 17230/50462 | Loss: 2.6607 | LR: 3.76e-04 | 5253.3ms/step | 11.4 samples/s
|
| 499 |
+
Step 17240/50462 | Loss: 2.6410 | LR: 3.76e-04 | 5248.1ms/step | 11.4 samples/s
|
| 500 |
+
Step 17250/50462 | Loss: 2.6437 | LR: 3.76e-04 | 5252.3ms/step | 11.4 samples/s
|
| 501 |
+
Validation at step 17250: Loss = 2.6628, Perplexity = 14.34
|
| 502 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-17250
|
| 503 |
+
Step 17260/50462 | Loss: 2.6405 | LR: 3.76e-04 | 10173.7ms/step | 5.9 samples/s
|
| 504 |
+
Step 17270/50462 | Loss: 2.6494 | LR: 3.76e-04 | 5250.0ms/step | 11.4 samples/s
|
| 505 |
+
Step 17280/50462 | Loss: 2.6384 | LR: 3.76e-04 | 5253.0ms/step | 11.4 samples/s
|
| 506 |
+
Step 17290/50462 | Loss: 2.6252 | LR: 3.75e-04 | 5252.2ms/step | 11.4 samples/s
|
| 507 |
+
Step 17300/50462 | Loss: 2.6299 | LR: 3.75e-04 | 5253.6ms/step | 11.4 samples/s
|
| 508 |
+
Step 17310/50462 | Loss: 2.6536 | LR: 3.75e-04 | 5253.3ms/step | 11.4 samples/s
|
| 509 |
+
Step 17320/50462 | Loss: 2.6200 | LR: 3.75e-04 | 5256.8ms/step | 11.4 samples/s
|
| 510 |
+
Step 17330/50462 | Loss: 2.6409 | LR: 3.75e-04 | 5254.5ms/step | 11.4 samples/s
|
| 511 |
+
Step 17340/50462 | Loss: 2.6635 | LR: 3.75e-04 | 5259.4ms/step | 11.4 samples/s
|
| 512 |
+
Step 17350/50462 | Loss: 2.6307 | LR: 3.75e-04 | 5253.3ms/step | 11.4 samples/s
|
| 513 |
+
Step 17360/50462 | Loss: 2.6770 | LR: 3.74e-04 | 5251.1ms/step | 11.4 samples/s
|
| 514 |
+
Step 17370/50462 | Loss: 2.6522 | LR: 3.74e-04 | 5245.6ms/step | 11.4 samples/s
|
| 515 |
+
Step 17380/50462 | Loss: 2.6659 | LR: 3.74e-04 | 5249.0ms/step | 11.4 samples/s
|
| 516 |
+
Step 17390/50462 | Loss: 2.6695 | LR: 3.74e-04 | 5250.2ms/step | 11.4 samples/s
|
| 517 |
+
Step 17400/50462 | Loss: 2.6566 | LR: 3.74e-04 | 5255.3ms/step | 11.4 samples/s
|
| 518 |
+
Step 17410/50462 | Loss: 2.6340 | LR: 3.74e-04 | 5252.1ms/step | 11.4 samples/s
|
| 519 |
+
Step 17420/50462 | Loss: 2.6584 | LR: 3.74e-04 | 5252.8ms/step | 11.4 samples/s
|
| 520 |
+
Step 17430/50462 | Loss: 2.6131 | LR: 3.74e-04 | 5255.4ms/step | 11.4 samples/s
|
| 521 |
+
Step 17440/50462 | Loss: 2.6714 | LR: 3.73e-04 | 5254.2ms/step | 11.4 samples/s
|
| 522 |
+
Step 17450/50462 | Loss: 2.6423 | LR: 3.73e-04 | 5252.5ms/step | 11.4 samples/s
|
| 523 |
+
Step 17460/50462 | Loss: 2.6522 | LR: 3.73e-04 | 5248.4ms/step | 11.4 samples/s
|
| 524 |
+
Step 17470/50462 | Loss: 2.6580 | LR: 3.73e-04 | 5252.1ms/step | 11.4 samples/s
|
| 525 |
+
Step 17480/50462 | Loss: 2.6617 | LR: 3.73e-04 | 5252.4ms/step | 11.4 samples/s
|
| 526 |
+
Step 17490/50462 | Loss: 2.6718 | LR: 3.73e-04 | 5253.1ms/step | 11.4 samples/s
|
| 527 |
+
Step 17500/50462 | Loss: 2.6603 | LR: 3.73e-04 | 5258.6ms/step | 11.4 samples/s
|
| 528 |
+
Validation at step 17500: Loss = 2.6592, Perplexity = 14.28
|
| 529 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-17500
|
| 530 |
+
Step 17510/50462 | Loss: 2.6720 | LR: 3.72e-04 | 10142.9ms/step | 5.9 samples/s
|
| 531 |
+
Step 17520/50462 | Loss: 2.6499 | LR: 3.72e-04 | 5249.2ms/step | 11.4 samples/s
|
| 532 |
+
Step 17530/50462 | Loss: 2.6210 | LR: 3.72e-04 | 5251.1ms/step | 11.4 samples/s
|
| 533 |
+
Step 17540/50462 | Loss: 2.6730 | LR: 3.72e-04 | 5250.4ms/step | 11.4 samples/s
|
| 534 |
+
Step 17550/50462 | Loss: 2.6690 | LR: 3.72e-04 | 5249.9ms/step | 11.4 samples/s
|
| 535 |
+
Step 17560/50462 | Loss: 2.6138 | LR: 3.72e-04 | 5252.5ms/step | 11.4 samples/s
|
| 536 |
+
Step 17570/50462 | Loss: 2.6407 | LR: 3.72e-04 | 5250.2ms/step | 11.4 samples/s
|
| 537 |
+
Step 17580/50462 | Loss: 2.6359 | LR: 3.71e-04 | 5252.8ms/step | 11.4 samples/s
|
| 538 |
+
Step 17590/50462 | Loss: 2.6282 | LR: 3.71e-04 | 5248.8ms/step | 11.4 samples/s
|
| 539 |
+
Step 17600/50462 | Loss: 2.6619 | LR: 3.71e-04 | 5250.8ms/step | 11.4 samples/s
|
| 540 |
+
Step 17610/50462 | Loss: 2.6151 | LR: 3.71e-04 | 5257.4ms/step | 11.4 samples/s
|
| 541 |
+
Step 17620/50462 | Loss: 2.6598 | LR: 3.71e-04 | 5250.4ms/step | 11.4 samples/s
|
| 542 |
+
Step 17630/50462 | Loss: 2.6456 | LR: 3.71e-04 | 5250.6ms/step | 11.4 samples/s
|
| 543 |
+
Step 17640/50462 | Loss: 2.6312 | LR: 3.71e-04 | 5249.1ms/step | 11.4 samples/s
|
| 544 |
+
Step 17650/50462 | Loss: 2.6589 | LR: 3.70e-04 | 5253.0ms/step | 11.4 samples/s
|
| 545 |
+
Step 17660/50462 | Loss: 2.6411 | LR: 3.70e-04 | 5250.6ms/step | 11.4 samples/s
|
| 546 |
+
Step 17670/50462 | Loss: 2.6482 | LR: 3.70e-04 | 5256.3ms/step | 11.4 samples/s
|
| 547 |
+
Step 17680/50462 | Loss: 2.6723 | LR: 3.70e-04 | 5250.4ms/step | 11.4 samples/s
|
| 548 |
+
Step 17690/50462 | Loss: 2.6560 | LR: 3.70e-04 | 5251.1ms/step | 11.4 samples/s
|
| 549 |
+
Step 17700/50462 | Loss: 2.6382 | LR: 3.70e-04 | 5248.1ms/step | 11.4 samples/s
|
| 550 |
+
Step 17710/50462 | Loss: 2.6354 | LR: 3.70e-04 | 5254.4ms/step | 11.4 samples/s
|
| 551 |
+
Step 17720/50462 | Loss: 2.6418 | LR: 3.69e-04 | 5253.3ms/step | 11.4 samples/s
|
| 552 |
+
Step 17730/50462 | Loss: 2.6437 | LR: 3.69e-04 | 5254.1ms/step | 11.4 samples/s
|
| 553 |
+
Step 17740/50462 | Loss: 2.6566 | LR: 3.69e-04 | 5250.2ms/step | 11.4 samples/s
|
| 554 |
+
Step 17750/50462 | Loss: 2.6687 | LR: 3.69e-04 | 5252.4ms/step | 11.4 samples/s
|
| 555 |
+
Validation at step 17750: Loss = 2.6547, Perplexity = 14.22
|
| 556 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-17750
|
| 557 |
+
Step 17760/50462 | Loss: 2.6500 | LR: 3.69e-04 | 10154.7ms/step | 5.9 samples/s
|
| 558 |
+
Step 17770/50462 | Loss: 2.6273 | LR: 3.69e-04 | 5228.6ms/step | 11.5 samples/s
|
| 559 |
+
Step 17780/50462 | Loss: 2.6318 | LR: 3.69e-04 | 5230.5ms/step | 11.5 samples/s
|
| 560 |
+
Step 17790/50462 | Loss: 2.6299 | LR: 3.69e-04 | 5232.2ms/step | 11.5 samples/s
|
| 561 |
+
Step 17800/50462 | Loss: 2.6275 | LR: 3.68e-04 | 5230.8ms/step | 11.5 samples/s
|
| 562 |
+
Step 17810/50462 | Loss: 2.6502 | LR: 3.68e-04 | 5229.2ms/step | 11.5 samples/s
|
| 563 |
+
Step 17820/50462 | Loss: 2.6246 | LR: 3.68e-04 | 5230.9ms/step | 11.5 samples/s
|
| 564 |
+
Step 17830/50462 | Loss: 2.6453 | LR: 3.68e-04 | 5234.3ms/step | 11.5 samples/s
|
| 565 |
+
Step 17840/50462 | Loss: 2.6452 | LR: 3.68e-04 | 5222.7ms/step | 11.5 samples/s
|
| 566 |
+
Step 17850/50462 | Loss: 2.6334 | LR: 3.68e-04 | 5230.4ms/step | 11.5 samples/s
|
| 567 |
+
Step 17860/50462 | Loss: 2.6423 | LR: 3.68e-04 | 5223.4ms/step | 11.5 samples/s
|
| 568 |
+
Step 17870/50462 | Loss: 2.6414 | LR: 3.67e-04 | 5228.9ms/step | 11.5 samples/s
|
| 569 |
+
Step 17880/50462 | Loss: 2.6315 | LR: 3.67e-04 | 5226.8ms/step | 11.5 samples/s
|
| 570 |
+
Step 17890/50462 | Loss: 2.6652 | LR: 3.67e-04 | 5224.9ms/step | 11.5 samples/s
|
| 571 |
+
Step 17900/50462 | Loss: 2.6311 | LR: 3.67e-04 | 5221.1ms/step | 11.5 samples/s
|
| 572 |
+
Step 17910/50462 | Loss: 2.6430 | LR: 3.67e-04 | 5223.9ms/step | 11.5 samples/s
|
| 573 |
+
Step 17920/50462 | Loss: 2.6547 | LR: 3.67e-04 | 5231.8ms/step | 11.5 samples/s
|
| 574 |
+
Step 17930/50462 | Loss: 2.6255 | LR: 3.67e-04 | 5225.5ms/step | 11.5 samples/s
|
| 575 |
+
Step 17940/50462 | Loss: 2.6618 | LR: 3.66e-04 | 5226.0ms/step | 11.5 samples/s
|
| 576 |
+
Step 17950/50462 | Loss: 2.6619 | LR: 3.66e-04 | 5228.3ms/step | 11.5 samples/s
|
| 577 |
+
Step 17960/50462 | Loss: 2.6457 | LR: 3.66e-04 | 5237.6ms/step | 11.5 samples/s
|
| 578 |
+
Step 17970/50462 | Loss: 2.5900 | LR: 3.66e-04 | 5225.7ms/step | 11.5 samples/s
|
| 579 |
+
Step 17980/50462 | Loss: 2.6449 | LR: 3.66e-04 | 5227.1ms/step | 11.5 samples/s
|
| 580 |
+
Step 17990/50462 | Loss: 2.6251 | LR: 3.66e-04 | 5230.2ms/step | 11.5 samples/s
|
| 581 |
+
Step 18000/50462 | Loss: 2.6323 | LR: 3.66e-04 | 5229.0ms/step | 11.5 samples/s
|
| 582 |
+
Validation at step 18000: Loss = 2.6514, Perplexity = 14.17
|
| 583 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-18000
|
| 584 |
+
Step 18010/50462 | Loss: 2.6503 | LR: 3.65e-04 | 10111.0ms/step | 5.9 samples/s
|
| 585 |
+
Step 18020/50462 | Loss: 2.6422 | LR: 3.65e-04 | 5251.2ms/step | 11.4 samples/s
|
| 586 |
+
Step 18030/50462 | Loss: 2.6458 | LR: 3.65e-04 | 5255.8ms/step | 11.4 samples/s
|
| 587 |
+
Step 18040/50462 | Loss: 2.6288 | LR: 3.65e-04 | 5246.3ms/step | 11.4 samples/s
|
| 588 |
+
Step 18050/50462 | Loss: 2.6522 | LR: 3.65e-04 | 5250.1ms/step | 11.4 samples/s
|
| 589 |
+
Step 18060/50462 | Loss: 2.6368 | LR: 3.65e-04 | 5253.2ms/step | 11.4 samples/s
|
| 590 |
+
Step 18070/50462 | Loss: 2.6324 | LR: 3.65e-04 | 5254.7ms/step | 11.4 samples/s
|
| 591 |
+
Step 18080/50462 | Loss: 2.6278 | LR: 3.64e-04 | 5248.6ms/step | 11.4 samples/s
|
| 592 |
+
Step 18090/50462 | Loss: 2.6333 | LR: 3.64e-04 | 5252.0ms/step | 11.4 samples/s
|
| 593 |
+
Step 18100/50462 | Loss: 2.6510 | LR: 3.64e-04 | 5252.4ms/step | 11.4 samples/s
|
| 594 |
+
Step 18110/50462 | Loss: 2.6447 | LR: 3.64e-04 | 5254.2ms/step | 11.4 samples/s
|
| 595 |
+
Step 18120/50462 | Loss: 2.6260 | LR: 3.64e-04 | 5255.3ms/step | 11.4 samples/s
|
| 596 |
+
Step 18130/50462 | Loss: 2.6412 | LR: 3.64e-04 | 5250.4ms/step | 11.4 samples/s
|
| 597 |
+
Step 18140/50462 | Loss: 2.6437 | LR: 3.64e-04 | 5252.8ms/step | 11.4 samples/s
|
| 598 |
+
Step 18150/50462 | Loss: 2.6419 | LR: 3.63e-04 | 5255.1ms/step | 11.4 samples/s
|
| 599 |
+
Step 18160/50462 | Loss: 2.6495 | LR: 3.63e-04 | 5247.5ms/step | 11.4 samples/s
|
| 600 |
+
Step 18170/50462 | Loss: 2.6320 | LR: 3.63e-04 | 5249.9ms/step | 11.4 samples/s
|
| 601 |
+
Step 18180/50462 | Loss: 2.6394 | LR: 3.63e-04 | 5254.1ms/step | 11.4 samples/s
|
| 602 |
+
Step 18190/50462 | Loss: 2.6502 | LR: 3.63e-04 | 5251.5ms/step | 11.4 samples/s
|
| 603 |
+
Step 18200/50462 | Loss: 2.6213 | LR: 3.63e-04 | 5250.4ms/step | 11.4 samples/s
|
| 604 |
+
Step 18210/50462 | Loss: 2.6139 | LR: 3.63e-04 | 5257.4ms/step | 11.4 samples/s
|
| 605 |
+
Step 18220/50462 | Loss: 2.6543 | LR: 3.62e-04 | 5242.3ms/step | 11.4 samples/s
|
| 606 |
+
Step 18230/50462 | Loss: 2.6328 | LR: 3.62e-04 | 5251.2ms/step | 11.4 samples/s
|
| 607 |
+
Step 18240/50462 | Loss: 2.6410 | LR: 3.62e-04 | 5252.4ms/step | 11.4 samples/s
|
| 608 |
+
Step 18250/50462 | Loss: 2.6347 | LR: 3.62e-04 | 5256.6ms/step | 11.4 samples/s
|
| 609 |
+
Validation at step 18250: Loss = 2.6481, Perplexity = 14.13
|
| 610 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-18250
|
| 611 |
+
Step 18260/50462 | Loss: 2.6413 | LR: 3.62e-04 | 10092.8ms/step | 5.9 samples/s
|
| 612 |
+
Step 18270/50462 | Loss: 2.6195 | LR: 3.62e-04 | 5257.1ms/step | 11.4 samples/s
|
| 613 |
+
Step 18280/50462 | Loss: 2.6301 | LR: 3.62e-04 | 5260.2ms/step | 11.4 samples/s
|
| 614 |
+
Step 18290/50462 | Loss: 2.6350 | LR: 3.62e-04 | 5254.7ms/step | 11.4 samples/s
|
| 615 |
+
Step 18300/50462 | Loss: 2.6181 | LR: 3.61e-04 | 5247.3ms/step | 11.4 samples/s
|
| 616 |
+
Step 18310/50462 | Loss: 2.6135 | LR: 3.61e-04 | 5253.4ms/step | 11.4 samples/s
|
| 617 |
+
Step 18320/50462 | Loss: 2.6189 | LR: 3.61e-04 | 5251.8ms/step | 11.4 samples/s
|
| 618 |
+
Step 18330/50462 | Loss: 2.6410 | LR: 3.61e-04 | 5250.5ms/step | 11.4 samples/s
|
| 619 |
+
Step 18340/50462 | Loss: 2.6603 | LR: 3.61e-04 | 5250.3ms/step | 11.4 samples/s
|
| 620 |
+
Step 18350/50462 | Loss: 2.6313 | LR: 3.61e-04 | 5246.0ms/step | 11.4 samples/s
|
| 621 |
+
Step 18360/50462 | Loss: 2.6398 | LR: 3.61e-04 | 5249.1ms/step | 11.4 samples/s
|
| 622 |
+
Step 18370/50462 | Loss: 2.6141 | LR: 3.60e-04 | 5251.8ms/step | 11.4 samples/s
|
| 623 |
+
Step 18380/50462 | Loss: 2.6303 | LR: 3.60e-04 | 5249.6ms/step | 11.4 samples/s
|
| 624 |
+
Step 18390/50462 | Loss: 2.6404 | LR: 3.60e-04 | 5252.8ms/step | 11.4 samples/s
|
| 625 |
+
Step 18400/50462 | Loss: 2.6474 | LR: 3.60e-04 | 5251.3ms/step | 11.4 samples/s
|
| 626 |
+
Step 18410/50462 | Loss: 2.6348 | LR: 3.60e-04 | 5255.1ms/step | 11.4 samples/s
|
| 627 |
+
Step 18420/50462 | Loss: 2.6236 | LR: 3.60e-04 | 5249.8ms/step | 11.4 samples/s
|
| 628 |
+
Step 18430/50462 | Loss: 2.6730 | LR: 3.60e-04 | 5253.9ms/step | 11.4 samples/s
|
| 629 |
+
Step 18440/50462 | Loss: 2.6279 | LR: 3.59e-04 | 5248.2ms/step | 11.4 samples/s
|
| 630 |
+
Step 18450/50462 | Loss: 2.6321 | LR: 3.59e-04 | 5252.9ms/step | 11.4 samples/s
|
| 631 |
+
Step 18460/50462 | Loss: 2.6222 | LR: 3.59e-04 | 5250.5ms/step | 11.4 samples/s
|
| 632 |
+
Step 18470/50462 | Loss: 2.6298 | LR: 3.59e-04 | 5254.6ms/step | 11.4 samples/s
|
| 633 |
+
Step 18480/50462 | Loss: 2.6228 | LR: 3.59e-04 | 5250.4ms/step | 11.4 samples/s
|
| 634 |
+
Step 18490/50462 | Loss: 2.6252 | LR: 3.59e-04 | 5245.3ms/step | 11.4 samples/s
|
| 635 |
+
Step 18500/50462 | Loss: 2.6440 | LR: 3.59e-04 | 5249.4ms/step | 11.4 samples/s
|
| 636 |
+
Validation at step 18500: Loss = 2.6449, Perplexity = 14.08
|
| 637 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-18500
|
| 638 |
+
Step 18510/50462 | Loss: 2.6599 | LR: 3.58e-04 | 10124.3ms/step | 5.9 samples/s
|
| 639 |
+
Step 18520/50462 | Loss: 2.6227 | LR: 3.58e-04 | 5230.6ms/step | 11.5 samples/s
|
| 640 |
+
Step 18530/50462 | Loss: 2.6243 | LR: 3.58e-04 | 5227.3ms/step | 11.5 samples/s
|
| 641 |
+
Step 18540/50462 | Loss: 2.6135 | LR: 3.58e-04 | 5228.1ms/step | 11.5 samples/s
|
| 642 |
+
Step 18550/50462 | Loss: 2.6253 | LR: 3.58e-04 | 5229.5ms/step | 11.5 samples/s
|
| 643 |
+
Step 18560/50462 | Loss: 2.6301 | LR: 3.58e-04 | 5232.2ms/step | 11.5 samples/s
|
| 644 |
+
Step 18570/50462 | Loss: 2.6390 | LR: 3.58e-04 | 5231.8ms/step | 11.5 samples/s
|
| 645 |
+
Step 18580/50462 | Loss: 2.6244 | LR: 3.57e-04 | 5223.8ms/step | 11.5 samples/s
|
| 646 |
+
Step 18590/50462 | Loss: 2.6267 | LR: 3.57e-04 | 5219.4ms/step | 11.5 samples/s
|
| 647 |
+
Step 18600/50462 | Loss: 2.6421 | LR: 3.57e-04 | 5238.2ms/step | 11.5 samples/s
|
| 648 |
+
Step 18610/50462 | Loss: 2.6310 | LR: 3.57e-04 | 5253.0ms/step | 11.4 samples/s
|
| 649 |
+
Step 18620/50462 | Loss: 2.6210 | LR: 3.57e-04 | 5247.2ms/step | 11.4 samples/s
|
| 650 |
+
Step 18630/50462 | Loss: 2.6248 | LR: 3.57e-04 | 5254.0ms/step | 11.4 samples/s
|
| 651 |
+
Step 18640/50462 | Loss: 2.6173 | LR: 3.57e-04 | 5249.4ms/step | 11.4 samples/s
|
| 652 |
+
Step 18650/50462 | Loss: 2.6290 | LR: 3.56e-04 | 5253.7ms/step | 11.4 samples/s
|
| 653 |
+
Step 18660/50462 | Loss: 2.6287 | LR: 3.56e-04 | 5246.8ms/step | 11.4 samples/s
|
| 654 |
+
Step 18670/50462 | Loss: 2.6159 | LR: 3.56e-04 | 5252.0ms/step | 11.4 samples/s
|
| 655 |
+
Step 18680/50462 | Loss: 2.6228 | LR: 3.56e-04 | 5257.6ms/step | 11.4 samples/s
|
| 656 |
+
Step 18690/50462 | Loss: 2.6265 | LR: 3.56e-04 | 5247.9ms/step | 11.4 samples/s
|
| 657 |
+
Step 18700/50462 | Loss: 2.6232 | LR: 3.56e-04 | 5254.7ms/step | 11.4 samples/s
|
| 658 |
+
Step 18710/50462 | Loss: 2.6366 | LR: 3.56e-04 | 5244.9ms/step | 11.4 samples/s
|
| 659 |
+
Step 18720/50462 | Loss: 2.6347 | LR: 3.55e-04 | 5248.5ms/step | 11.4 samples/s
|
| 660 |
+
Step 18730/50462 | Loss: 2.6132 | LR: 3.55e-04 | 5254.3ms/step | 11.4 samples/s
|
| 661 |
+
Step 18740/50462 | Loss: 2.6292 | LR: 3.55e-04 | 5257.1ms/step | 11.4 samples/s
|
| 662 |
+
Step 18750/50462 | Loss: 2.6268 | LR: 3.55e-04 | 5254.7ms/step | 11.4 samples/s
|
| 663 |
+
Validation at step 18750: Loss = 2.6417, Perplexity = 14.04
|
| 664 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-18750
|
| 665 |
+
Step 18760/50462 | Loss: 2.6438 | LR: 3.55e-04 | 10176.4ms/step | 5.9 samples/s
|
| 666 |
+
Step 18770/50462 | Loss: 2.6242 | LR: 3.55e-04 | 5252.1ms/step | 11.4 samples/s
|
| 667 |
+
Step 18780/50462 | Loss: 2.6241 | LR: 3.55e-04 | 5250.9ms/step | 11.4 samples/s
|
| 668 |
+
Step 18790/50462 | Loss: 2.6496 | LR: 3.54e-04 | 5248.4ms/step | 11.4 samples/s
|
| 669 |
+
Step 18800/50462 | Loss: 2.6185 | LR: 3.54e-04 | 5245.6ms/step | 11.4 samples/s
|
| 670 |
+
Step 18810/50462 | Loss: 2.6404 | LR: 3.54e-04 | 5252.5ms/step | 11.4 samples/s
|
| 671 |
+
Step 18820/50462 | Loss: 2.6365 | LR: 3.54e-04 | 5247.5ms/step | 11.4 samples/s
|
| 672 |
+
Step 18830/50462 | Loss: 2.6339 | LR: 3.54e-04 | 5249.8ms/step | 11.4 samples/s
|
| 673 |
+
Step 18840/50462 | Loss: 2.6336 | LR: 3.54e-04 | 5247.8ms/step | 11.4 samples/s
|
| 674 |
+
Step 18850/50462 | Loss: 2.6368 | LR: 3.54e-04 | 5250.2ms/step | 11.4 samples/s
|
| 675 |
+
Step 18860/50462 | Loss: 2.6482 | LR: 3.53e-04 | 5251.5ms/step | 11.4 samples/s
|
| 676 |
+
Step 18870/50462 | Loss: 2.5955 | LR: 3.53e-04 | 5250.2ms/step | 11.4 samples/s
|
| 677 |
+
Step 18880/50462 | Loss: 2.6281 | LR: 3.53e-04 | 5250.6ms/step | 11.4 samples/s
|
| 678 |
+
Step 18890/50462 | Loss: 2.6123 | LR: 3.53e-04 | 5245.5ms/step | 11.4 samples/s
|
| 679 |
+
Step 18900/50462 | Loss: 2.6470 | LR: 3.53e-04 | 5255.0ms/step | 11.4 samples/s
|
| 680 |
+
Step 18910/50462 | Loss: 2.6380 | LR: 3.53e-04 | 5254.9ms/step | 11.4 samples/s
|
| 681 |
+
Step 18920/50462 | Loss: 2.6131 | LR: 3.53e-04 | 5251.3ms/step | 11.4 samples/s
|
| 682 |
+
Step 18930/50462 | Loss: 2.6396 | LR: 3.52e-04 | 5244.7ms/step | 11.4 samples/s
|
| 683 |
+
Step 18940/50462 | Loss: 2.6569 | LR: 3.52e-04 | 5247.4ms/step | 11.4 samples/s
|
| 684 |
+
Step 18950/50462 | Loss: 2.6368 | LR: 3.52e-04 | 5246.1ms/step | 11.4 samples/s
|
| 685 |
+
Step 18960/50462 | Loss: 2.6420 | LR: 3.52e-04 | 5253.2ms/step | 11.4 samples/s
|
| 686 |
+
Step 18970/50462 | Loss: 2.6348 | LR: 3.52e-04 | 5251.9ms/step | 11.4 samples/s
|
| 687 |
+
Step 18980/50462 | Loss: 2.6316 | LR: 3.52e-04 | 5248.7ms/step | 11.4 samples/s
|
| 688 |
+
Step 18990/50462 | Loss: 2.6176 | LR: 3.52e-04 | 5255.9ms/step | 11.4 samples/s
|
| 689 |
+
Step 19000/50462 | Loss: 2.6013 | LR: 3.51e-04 | 5249.3ms/step | 11.4 samples/s
|
| 690 |
+
Validation at step 19000: Loss = 2.6393, Perplexity = 14.00
|
| 691 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-19000
|
| 692 |
+
Step 19010/50462 | Loss: 2.6277 | LR: 3.51e-04 | 10113.9ms/step | 5.9 samples/s
|
| 693 |
+
Step 19020/50462 | Loss: 2.6279 | LR: 3.51e-04 | 5242.6ms/step | 11.4 samples/s
|
| 694 |
+
Step 19030/50462 | Loss: 2.6434 | LR: 3.51e-04 | 5249.5ms/step | 11.4 samples/s
|
| 695 |
+
Step 19040/50462 | Loss: 2.6177 | LR: 3.51e-04 | 5240.9ms/step | 11.4 samples/s
|
| 696 |
+
Step 19050/50462 | Loss: 2.6221 | LR: 3.51e-04 | 5229.6ms/step | 11.5 samples/s
|
| 697 |
+
Step 19060/50462 | Loss: 2.6198 | LR: 3.50e-04 | 5228.0ms/step | 11.5 samples/s
|
| 698 |
+
Step 19070/50462 | Loss: 2.6231 | LR: 3.50e-04 | 5244.7ms/step | 11.4 samples/s
|
| 699 |
+
Step 19080/50462 | Loss: 2.5970 | LR: 3.50e-04 | 5225.9ms/step | 11.5 samples/s
|
| 700 |
+
Step 19090/50462 | Loss: 2.6259 | LR: 3.50e-04 | 5228.2ms/step | 11.5 samples/s
|
| 701 |
+
Step 19100/50462 | Loss: 2.6249 | LR: 3.50e-04 | 5229.4ms/step | 11.5 samples/s
|
| 702 |
+
Step 19110/50462 | Loss: 2.6293 | LR: 3.50e-04 | 5238.8ms/step | 11.5 samples/s
|
| 703 |
+
Step 19120/50462 | Loss: 2.6366 | LR: 3.50e-04 | 5235.9ms/step | 11.5 samples/s
|
| 704 |
+
Step 19130/50462 | Loss: 2.6336 | LR: 3.49e-04 | 5228.9ms/step | 11.5 samples/s
|
| 705 |
+
Step 19140/50462 | Loss: 2.6194 | LR: 3.49e-04 | 5230.3ms/step | 11.5 samples/s
|
| 706 |
+
Step 19150/50462 | Loss: 2.6282 | LR: 3.49e-04 | 5228.5ms/step | 11.5 samples/s
|
| 707 |
+
Step 19160/50462 | Loss: 2.6331 | LR: 3.49e-04 | 5228.4ms/step | 11.5 samples/s
|
| 708 |
+
Step 19170/50462 | Loss: 2.6337 | LR: 3.49e-04 | 5235.1ms/step | 11.5 samples/s
|
epfl_final/SparseAtt/wandb/run-20260824_181749-rgmkfpnl/files/requirements.txt
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
| 142 |
+
entmax==1.3
|
epfl_final/SparseAtt/wandb/run-20260824_181749-rgmkfpnl/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-08-24T18:17:49.946622Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"configs/p2_1p2b_dense_seed70_2gpu.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/TokenPrune.git",
|
| 14 |
+
"commit": "5637886f0cc760009989b14e68c6ae06d4827418"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "h100-workspace-2-gpu-0-11",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 23 |
+
"gpu_count": 2,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "9374931091456",
|
| 27 |
+
"used": "5347748433920"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1623005626368"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-3ea6c840-350f-c68c-7562-6e0affa4e153"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 43 |
+
"memoryTotal": "85520809984",
|
| 44 |
+
"cudaCores": 16896,
|
| 45 |
+
"architecture": "Hopper",
|
| 46 |
+
"uuid": "GPU-5cb9f8b7-4a13-4f85-d73e-6bcbebebefb8"
|
| 47 |
+
}
|
| 48 |
+
],
|
| 49 |
+
"cudaVersion": "13.0",
|
| 50 |
+
"writerId": "bd4vrfznh1qxakmu3p2w0ytkgev282d5"
|
| 51 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260824_181749-rgmkfpnl/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260824_181749-rgmkfpnl/logs/debug.log
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-08-24 18:17:49,963 INFO MainThread:217 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-08-24 18:17:49,963 INFO MainThread:217 [wandb_setup.py:_flush():81] Configure stats pid to 217
|
| 3 |
+
2026-08-24 18:17:49,963 INFO MainThread:217 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-08-24 18:17:49,963 INFO MainThread:217 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260824_181749-rgmkfpnl/logs/debug.log
|
| 5 |
+
2026-08-24 18:17:49,963 INFO MainThread:217 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260824_181749-rgmkfpnl/logs/debug-internal.log
|
| 6 |
+
2026-08-24 18:17:49,963 INFO MainThread:217 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-08-24 18:17:49,963 INFO MainThread:217 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-13500', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 15, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 70, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b dense 8k seed70 2gpu resume', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'dense', 'seed70'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-08-24 18:17:49,963 INFO MainThread:217 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-08-24 18:17:50,492 INFO MainThread:217 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-08-24 18:17:50,908 INFO MainThread:217 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-08-24 18:17:50,911 INFO MainThread:217 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-08-24 18:17:50,933 INFO MainThread:217 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-08-24 18:17:51,444 INFO MainThread:217 [wandb_init.py:init():1051] run resumed
|
| 15 |
+
2026-08-24 18:17:51,445 INFO MainThread:217 [wandb_init.py:init():1056] starting run threads in backend
|
| 16 |
+
2026-08-24 18:17:51,676 INFO MainThread:217 [wandb_run.py:_console_start():2554] atexit reg
|
| 17 |
+
2026-08-24 18:17:51,676 INFO MainThread:217 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 18 |
+
2026-08-24 18:17:51,676 INFO MainThread:217 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 19 |
+
2026-08-24 18:17:51,676 INFO MainThread:217 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 20 |
+
2026-08-24 18:17:51,686 INFO MainThread:217 [wandb_init.py:init():1094] run started, returning control to user process
|
epfl_final/SparseAtt/wandb/run-20260824_181749-rgmkfpnl/run-rgmkfpnl.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7b7a618989ec80ddc2c217ed32040bf91590deb402ed4164e5ce3574a39084a9
|
| 3 |
+
size 6488064
|
epfl_final/SparseAtt/wandb/run-20260825_031743-qctui00g/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-08-25T03:17:43.801752Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"configs/p2_1p2b_dense_seed71_6gpu.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/TokenPrune.git",
|
| 14 |
+
"commit": "5637886f0cc760009989b14e68c6ae06d4827418"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "h100-train-6-gpu-0-35",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 23 |
+
"gpu_count": 6,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "9374931091456",
|
| 27 |
+
"used": "5347751227392"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1623005626368"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-572b71c8-9d82-4b1c-e4f6-ed5616fd870a"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 43 |
+
"memoryTotal": "85520809984",
|
| 44 |
+
"cudaCores": 16896,
|
| 45 |
+
"architecture": "Hopper",
|
| 46 |
+
"uuid": "GPU-3ea6c840-350f-c68c-7562-6e0affa4e153"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 50 |
+
"memoryTotal": "85520809984",
|
| 51 |
+
"cudaCores": 16896,
|
| 52 |
+
"architecture": "Hopper",
|
| 53 |
+
"uuid": "GPU-32209e2e-7887-e730-7611-251622a8e78f"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 57 |
+
"memoryTotal": "85520809984",
|
| 58 |
+
"cudaCores": 16896,
|
| 59 |
+
"architecture": "Hopper",
|
| 60 |
+
"uuid": "GPU-3512802a-d501-0a3c-9812-4ab9deb3a3a9"
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 64 |
+
"memoryTotal": "85520809984",
|
| 65 |
+
"cudaCores": 16896,
|
| 66 |
+
"architecture": "Hopper",
|
| 67 |
+
"uuid": "GPU-24436def-beee-c9f1-bd05-c97df556de08"
|
| 68 |
+
},
|
| 69 |
+
{
|
| 70 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 71 |
+
"memoryTotal": "85520809984",
|
| 72 |
+
"cudaCores": 16896,
|
| 73 |
+
"architecture": "Hopper",
|
| 74 |
+
"uuid": "GPU-7b827c86-854a-af51-b5d3-6aaa43307d4a"
|
| 75 |
+
}
|
| 76 |
+
],
|
| 77 |
+
"cudaVersion": "13.0",
|
| 78 |
+
"writerId": "o0h4kd2w2pyo3izb1phpr1huw44s4rk3"
|
| 79 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260825_031743-qctui00g/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|