EPFL final checkpoints and W&B history 1051-1100 of 1463
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +7 -0
- epfl_final/SparseAtt/wandb/run-20260825_031743-qctui00g/files/output.log +1188 -0
- epfl_final/SparseAtt/wandb/run-20260825_031743-qctui00g/files/requirements.txt +142 -0
- epfl_final/SparseAtt/wandb/run-20260825_031743-qctui00g/run-qctui00g.wandb +3 -0
- epfl_final/SparseAtt/wandb/run-20260825_031836-rgmkfpnl/files/output.log +198 -0
- epfl_final/SparseAtt/wandb/run-20260825_031836-rgmkfpnl/files/requirements.txt +142 -0
- epfl_final/SparseAtt/wandb/run-20260825_031836-rgmkfpnl/files/wandb-metadata.json +51 -0
- epfl_final/SparseAtt/wandb/run-20260825_031836-rgmkfpnl/logs/debug-internal.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260825_031836-rgmkfpnl/logs/debug.log +20 -0
- epfl_final/SparseAtt/wandb/run-20260825_031836-rgmkfpnl/run-rgmkfpnl.wandb +3 -0
- epfl_final/SparseAtt/wandb/run-20260825_134623-qctui00g/files/output.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260825_134623-qctui00g/files/requirements.txt +142 -0
- epfl_final/SparseAtt/wandb/run-20260825_134623-qctui00g/files/wandb-metadata.json +79 -0
- epfl_final/SparseAtt/wandb/run-20260825_134623-qctui00g/logs/debug-internal.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260825_134623-qctui00g/logs/debug.log +20 -0
- epfl_final/SparseAtt/wandb/run-20260825_134623-qctui00g/run-qctui00g.wandb +3 -0
- epfl_final/SparseAtt/wandb/run-20260825_135931-rgmkfpnl/files/output.log +501 -0
- epfl_final/SparseAtt/wandb/run-20260825_135931-rgmkfpnl/files/requirements.txt +142 -0
- epfl_final/SparseAtt/wandb/run-20260825_135931-rgmkfpnl/files/wandb-metadata.json +51 -0
- epfl_final/SparseAtt/wandb/run-20260825_135931-rgmkfpnl/logs/debug-internal.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260825_135931-rgmkfpnl/logs/debug.log +20 -0
- epfl_final/SparseAtt/wandb/run-20260825_135931-rgmkfpnl/run-rgmkfpnl.wandb +3 -0
- epfl_final/SparseAtt/wandb/run-20260825_192830-6sx9f1kf/files/output.log +309 -0
- epfl_final/SparseAtt/wandb/run-20260825_192830-6sx9f1kf/files/requirements.txt +142 -0
- epfl_final/SparseAtt/wandb/run-20260825_192830-6sx9f1kf/files/wandb-metadata.json +44 -0
- epfl_final/SparseAtt/wandb/run-20260825_192830-6sx9f1kf/logs/debug-internal.log +573 -0
- epfl_final/SparseAtt/wandb/run-20260825_192830-6sx9f1kf/logs/debug.log +19 -0
- epfl_final/SparseAtt/wandb/run-20260825_192830-6sx9f1kf/run-6sx9f1kf.wandb +3 -0
- epfl_final/SparseAtt/wandb/run-20260825_204951-qctui00g/files/output.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260825_204951-qctui00g/files/requirements.txt +142 -0
- epfl_final/SparseAtt/wandb/run-20260825_204951-qctui00g/files/wandb-metadata.json +79 -0
- epfl_final/SparseAtt/wandb/run-20260825_204951-qctui00g/logs/debug-internal.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260825_204951-qctui00g/logs/debug.log +20 -0
- epfl_final/SparseAtt/wandb/run-20260825_204951-qctui00g/run-qctui00g.wandb +3 -0
- epfl_final/SparseAtt/wandb/run-20260825_205215-yl9g6w7y/files/output.log +7 -0
- epfl_final/SparseAtt/wandb/run-20260825_205215-yl9g6w7y/files/requirements.txt +142 -0
- epfl_final/SparseAtt/wandb/run-20260825_205215-yl9g6w7y/files/wandb-metadata.json +44 -0
- epfl_final/SparseAtt/wandb/run-20260825_205215-yl9g6w7y/logs/debug-internal.log +41 -0
- epfl_final/SparseAtt/wandb/run-20260825_205215-yl9g6w7y/logs/debug.log +19 -0
- epfl_final/SparseAtt/wandb/run-20260825_205215-yl9g6w7y/run-yl9g6w7y.wandb +0 -0
- epfl_final/SparseAtt/wandb/run-20260825_211848-rgmkfpnl/files/output.log +411 -0
- epfl_final/SparseAtt/wandb/run-20260825_211848-rgmkfpnl/files/requirements.txt +142 -0
- epfl_final/SparseAtt/wandb/run-20260825_211848-rgmkfpnl/files/wandb-metadata.json +51 -0
- epfl_final/SparseAtt/wandb/run-20260825_211848-rgmkfpnl/logs/debug-internal.log +0 -0
- epfl_final/SparseAtt/wandb/run-20260825_211848-rgmkfpnl/logs/debug.log +20 -0
- epfl_final/SparseAtt/wandb/run-20260825_211848-rgmkfpnl/run-rgmkfpnl.wandb +3 -0
- epfl_final/SparseAtt/wandb/run-20260826_034845-rgmkfpnl/files/output.log +429 -0
- epfl_final/SparseAtt/wandb/run-20260826_034845-rgmkfpnl/files/requirements.txt +142 -0
- epfl_final/SparseAtt/wandb/run-20260826_034845-rgmkfpnl/files/wandb-metadata.json +51 -0
- epfl_final/SparseAtt/wandb/run-20260826_034845-rgmkfpnl/logs/debug-internal.log +0 -0
.gitattributes
CHANGED
|
@@ -139,3 +139,10 @@ epfl_final/SparseAtt/wandb/run-20260824_034710-i8ewyy95/run-i8ewyy95.wandb filte
|
|
| 139 |
epfl_final/SparseAtt/wandb/run-20260824_132232-rgmkfpnl/run-rgmkfpnl.wandb filter=lfs diff=lfs merge=lfs -text
|
| 140 |
epfl_final/SparseAtt/wandb/run-20260824_181605-i8ewyy95/run-i8ewyy95.wandb filter=lfs diff=lfs merge=lfs -text
|
| 141 |
epfl_final/SparseAtt/wandb/run-20260824_181749-rgmkfpnl/run-rgmkfpnl.wandb filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 139 |
epfl_final/SparseAtt/wandb/run-20260824_132232-rgmkfpnl/run-rgmkfpnl.wandb filter=lfs diff=lfs merge=lfs -text
|
| 140 |
epfl_final/SparseAtt/wandb/run-20260824_181605-i8ewyy95/run-i8ewyy95.wandb filter=lfs diff=lfs merge=lfs -text
|
| 141 |
epfl_final/SparseAtt/wandb/run-20260824_181749-rgmkfpnl/run-rgmkfpnl.wandb filter=lfs diff=lfs merge=lfs -text
|
| 142 |
+
epfl_final/SparseAtt/wandb/run-20260825_031743-qctui00g/run-qctui00g.wandb filter=lfs diff=lfs merge=lfs -text
|
| 143 |
+
epfl_final/SparseAtt/wandb/run-20260825_031836-rgmkfpnl/run-rgmkfpnl.wandb filter=lfs diff=lfs merge=lfs -text
|
| 144 |
+
epfl_final/SparseAtt/wandb/run-20260825_134623-qctui00g/run-qctui00g.wandb filter=lfs diff=lfs merge=lfs -text
|
| 145 |
+
epfl_final/SparseAtt/wandb/run-20260825_135931-rgmkfpnl/run-rgmkfpnl.wandb filter=lfs diff=lfs merge=lfs -text
|
| 146 |
+
epfl_final/SparseAtt/wandb/run-20260825_192830-6sx9f1kf/run-6sx9f1kf.wandb filter=lfs diff=lfs merge=lfs -text
|
| 147 |
+
epfl_final/SparseAtt/wandb/run-20260825_204951-qctui00g/run-qctui00g.wandb filter=lfs diff=lfs merge=lfs -text
|
| 148 |
+
epfl_final/SparseAtt/wandb/run-20260825_211848-rgmkfpnl/run-rgmkfpnl.wandb filter=lfs diff=lfs merge=lfs -text
|
epfl_final/SparseAtt/wandb/run-20260825_031743-qctui00g/files/output.log
ADDED
|
@@ -0,0 +1,1188 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Starting training...
|
| 2 |
+
Validation at step 2500: Loss = 3.1156, Perplexity = 22.55
|
| 3 |
+
wandb: WARNING Tried to log to step 2500 that is less than the current step 2611. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 4 |
+
Switching attention mode for step 2501: dense (dense step interval)
|
| 5 |
+
Step 2510/50462 | Loss: 3.0854 | LR: 4.98e-04 | 13125.4ms/step | 4.6 samples/s
|
| 6 |
+
wandb: WARNING Tried to log to step 2510 that is less than the current step 2611. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 7 |
+
Step 2520/50462 | Loss: 3.1177 | LR: 4.98e-04 | 1891.4ms/step | 31.7 samples/s
|
| 8 |
+
wandb: WARNING Tried to log to step 2520 that is less than the current step 2611. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 9 |
+
Step 2530/50462 | Loss: 3.1660 | LR: 4.98e-04 | 1896.0ms/step | 31.6 samples/s
|
| 10 |
+
wandb: WARNING Tried to log to step 2530 that is less than the current step 2611. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 11 |
+
Step 2540/50462 | Loss: 3.1500 | LR: 4.98e-04 | 1900.0ms/step | 31.6 samples/s
|
| 12 |
+
wandb: WARNING Tried to log to step 2540 that is less than the current step 2611. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 13 |
+
Step 2550/50462 | Loss: 3.1158 | LR: 4.98e-04 | 1900.5ms/step | 31.6 samples/s
|
| 14 |
+
wandb: WARNING Tried to log to step 2550 that is less than the current step 2611. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 15 |
+
Step 2560/50462 | Loss: 3.0982 | LR: 4.98e-04 | 1903.8ms/step | 31.5 samples/s
|
| 16 |
+
wandb: WARNING Tried to log to step 2560 that is less than the current step 2611. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 17 |
+
Step 2570/50462 | Loss: 3.1327 | LR: 4.98e-04 | 1905.9ms/step | 31.5 samples/s
|
| 18 |
+
wandb: WARNING Tried to log to step 2570 that is less than the current step 2611. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 19 |
+
Step 2580/50462 | Loss: 3.0830 | LR: 4.98e-04 | 1904.9ms/step | 31.5 samples/s
|
| 20 |
+
wandb: WARNING Tried to log to step 2580 that is less than the current step 2611. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 21 |
+
Step 2590/50462 | Loss: 3.1023 | LR: 4.98e-04 | 1907.9ms/step | 31.4 samples/s
|
| 22 |
+
wandb: WARNING Tried to log to step 2590 that is less than the current step 2611. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 23 |
+
Step 2600/50462 | Loss: 3.1342 | LR: 4.98e-04 | 1905.4ms/step | 31.5 samples/s
|
| 24 |
+
wandb: WARNING Tried to log to step 2600 that is less than the current step 2611. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 25 |
+
Step 2610/50462 | Loss: 3.0733 | LR: 4.98e-04 | 1905.7ms/step | 31.5 samples/s
|
| 26 |
+
wandb: WARNING Tried to log to step 2610 that is less than the current step 2611. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 27 |
+
Step 2620/50462 | Loss: 3.1236 | LR: 4.98e-04 | 1904.3ms/step | 31.5 samples/s
|
| 28 |
+
Step 2630/50462 | Loss: 3.1401 | LR: 4.98e-04 | 1906.8ms/step | 31.5 samples/s
|
| 29 |
+
Step 2640/50462 | Loss: 3.0964 | LR: 4.98e-04 | 1909.4ms/step | 31.4 samples/s
|
| 30 |
+
Step 2650/50462 | Loss: 3.0968 | LR: 4.98e-04 | 1905.7ms/step | 31.5 samples/s
|
| 31 |
+
Step 2660/50462 | Loss: 3.0760 | LR: 4.98e-04 | 1908.6ms/step | 31.4 samples/s
|
| 32 |
+
Step 2670/50462 | Loss: 3.1006 | LR: 4.98e-04 | 1908.2ms/step | 31.4 samples/s
|
| 33 |
+
Step 2680/50462 | Loss: 3.1140 | LR: 4.98e-04 | 1907.4ms/step | 31.5 samples/s
|
| 34 |
+
Step 2690/50462 | Loss: 3.0751 | LR: 4.98e-04 | 1904.9ms/step | 31.5 samples/s
|
| 35 |
+
Step 2700/50462 | Loss: 3.1206 | LR: 4.98e-04 | 1906.1ms/step | 31.5 samples/s
|
| 36 |
+
Step 2710/50462 | Loss: 3.1392 | LR: 4.98e-04 | 1908.8ms/step | 31.4 samples/s
|
| 37 |
+
Step 2720/50462 | Loss: 3.0849 | LR: 4.98e-04 | 1909.1ms/step | 31.4 samples/s
|
| 38 |
+
Step 2730/50462 | Loss: 3.0740 | LR: 4.98e-04 | 1907.2ms/step | 31.5 samples/s
|
| 39 |
+
Step 2740/50462 | Loss: 3.1060 | LR: 4.98e-04 | 1907.6ms/step | 31.5 samples/s
|
| 40 |
+
Step 2750/50462 | Loss: 3.0526 | LR: 4.98e-04 | 1903.5ms/step | 31.5 samples/s
|
| 41 |
+
Validation at step 2750: Loss = 3.0782, Perplexity = 21.72
|
| 42 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-2750
|
| 43 |
+
Step 2760/50462 | Loss: 3.0767 | LR: 4.98e-04 | 3440.7ms/step | 17.4 samples/s
|
| 44 |
+
Step 2770/50462 | Loss: 3.1286 | LR: 4.98e-04 | 1899.0ms/step | 31.6 samples/s
|
| 45 |
+
Step 2780/50462 | Loss: 3.0682 | LR: 4.98e-04 | 1903.9ms/step | 31.5 samples/s
|
| 46 |
+
Step 2790/50462 | Loss: 3.1283 | LR: 4.98e-04 | 1904.8ms/step | 31.5 samples/s
|
| 47 |
+
Step 2800/50462 | Loss: 3.0816 | LR: 4.98e-04 | 1909.5ms/step | 31.4 samples/s
|
| 48 |
+
Step 2810/50462 | Loss: 3.1023 | LR: 4.98e-04 | 1909.4ms/step | 31.4 samples/s
|
| 49 |
+
Step 2820/50462 | Loss: 3.0978 | LR: 4.98e-04 | 1910.5ms/step | 31.4 samples/s
|
| 50 |
+
Step 2830/50462 | Loss: 3.0747 | LR: 4.98e-04 | 1910.8ms/step | 31.4 samples/s
|
| 51 |
+
Step 2840/50462 | Loss: 3.0866 | LR: 4.98e-04 | 1908.5ms/step | 31.4 samples/s
|
| 52 |
+
Step 2850/50462 | Loss: 3.1138 | LR: 4.98e-04 | 1911.0ms/step | 31.4 samples/s
|
| 53 |
+
Step 2860/50462 | Loss: 3.1015 | LR: 4.98e-04 | 1909.6ms/step | 31.4 samples/s
|
| 54 |
+
Step 2870/50462 | Loss: 3.1098 | LR: 4.98e-04 | 1911.0ms/step | 31.4 samples/s
|
| 55 |
+
Step 2880/50462 | Loss: 3.1027 | LR: 4.98e-04 | 1908.9ms/step | 31.4 samples/s
|
| 56 |
+
Step 2890/50462 | Loss: 3.0486 | LR: 4.98e-04 | 1911.5ms/step | 31.4 samples/s
|
| 57 |
+
Step 2900/50462 | Loss: 3.0688 | LR: 4.98e-04 | 1910.9ms/step | 31.4 samples/s
|
| 58 |
+
Step 2910/50462 | Loss: 3.0731 | LR: 4.98e-04 | 1908.6ms/step | 31.4 samples/s
|
| 59 |
+
Step 2920/50462 | Loss: 3.0471 | LR: 4.98e-04 | 1908.9ms/step | 31.4 samples/s
|
| 60 |
+
Step 2930/50462 | Loss: 3.0629 | LR: 4.98e-04 | 1913.7ms/step | 31.4 samples/s
|
| 61 |
+
Step 2940/50462 | Loss: 3.0769 | LR: 4.98e-04 | 1913.8ms/step | 31.4 samples/s
|
| 62 |
+
Step 2950/50462 | Loss: 3.0599 | LR: 4.98e-04 | 1911.4ms/step | 31.4 samples/s
|
| 63 |
+
Step 2960/50462 | Loss: 3.0353 | LR: 4.98e-04 | 1912.6ms/step | 31.4 samples/s
|
| 64 |
+
Step 2970/50462 | Loss: 3.0840 | LR: 4.98e-04 | 1909.6ms/step | 31.4 samples/s
|
| 65 |
+
Step 2980/50462 | Loss: 3.0567 | LR: 4.98e-04 | 1908.9ms/step | 31.4 samples/s
|
| 66 |
+
Step 2990/50462 | Loss: 3.1263 | LR: 4.97e-04 | 1907.9ms/step | 31.4 samples/s
|
| 67 |
+
Step 3000/50462 | Loss: 3.0461 | LR: 4.97e-04 | 1910.3ms/step | 31.4 samples/s
|
| 68 |
+
Validation at step 3000: Loss = 3.0502, Perplexity = 21.12
|
| 69 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-3000
|
| 70 |
+
Step 3010/50462 | Loss: 3.0719 | LR: 4.97e-04 | 3298.7ms/step | 18.2 samples/s
|
| 71 |
+
Step 3020/50462 | Loss: 3.0375 | LR: 4.97e-04 | 1901.6ms/step | 31.6 samples/s
|
| 72 |
+
Step 3030/50462 | Loss: 3.0948 | LR: 4.97e-04 | 1904.3ms/step | 31.5 samples/s
|
| 73 |
+
Step 3040/50462 | Loss: 3.0497 | LR: 4.97e-04 | 1908.1ms/step | 31.4 samples/s
|
| 74 |
+
Step 3050/50462 | Loss: 3.0545 | LR: 4.97e-04 | 1908.9ms/step | 31.4 samples/s
|
| 75 |
+
Step 3060/50462 | Loss: 3.0760 | LR: 4.97e-04 | 1908.0ms/step | 31.4 samples/s
|
| 76 |
+
Step 3070/50462 | Loss: 3.0567 | LR: 4.97e-04 | 1910.1ms/step | 31.4 samples/s
|
| 77 |
+
Step 3080/50462 | Loss: 3.0653 | LR: 4.97e-04 | 1910.7ms/step | 31.4 samples/s
|
| 78 |
+
Step 3090/50462 | Loss: 3.0543 | LR: 4.97e-04 | 1912.2ms/step | 31.4 samples/s
|
| 79 |
+
Step 3100/50462 | Loss: 3.0325 | LR: 4.97e-04 | 1909.6ms/step | 31.4 samples/s
|
| 80 |
+
Step 3110/50462 | Loss: 3.0323 | LR: 4.97e-04 | 1908.5ms/step | 31.4 samples/s
|
| 81 |
+
Step 3120/50462 | Loss: 3.0297 | LR: 4.97e-04 | 1908.0ms/step | 31.4 samples/s
|
| 82 |
+
Step 3130/50462 | Loss: 3.0222 | LR: 4.97e-04 | 1906.8ms/step | 31.5 samples/s
|
| 83 |
+
Step 3140/50462 | Loss: 3.0669 | LR: 4.97e-04 | 1908.4ms/step | 31.4 samples/s
|
| 84 |
+
Step 3150/50462 | Loss: 3.0647 | LR: 4.97e-04 | 1911.8ms/step | 31.4 samples/s
|
| 85 |
+
Step 3160/50462 | Loss: 3.0484 | LR: 4.97e-04 | 1914.2ms/step | 31.3 samples/s
|
| 86 |
+
Step 3170/50462 | Loss: 3.0301 | LR: 4.97e-04 | 1914.9ms/step | 31.3 samples/s
|
| 87 |
+
Step 3180/50462 | Loss: 3.0207 | LR: 4.97e-04 | 1915.6ms/step | 31.3 samples/s
|
| 88 |
+
Step 3190/50462 | Loss: 3.0142 | LR: 4.97e-04 | 1915.5ms/step | 31.3 samples/s
|
| 89 |
+
Step 3200/50462 | Loss: 3.0447 | LR: 4.97e-04 | 1912.1ms/step | 31.4 samples/s
|
| 90 |
+
Step 3210/50462 | Loss: 3.0084 | LR: 4.97e-04 | 1914.0ms/step | 31.3 samples/s
|
| 91 |
+
Step 3220/50462 | Loss: 3.0159 | LR: 4.97e-04 | 1913.6ms/step | 31.4 samples/s
|
| 92 |
+
Step 3230/50462 | Loss: 3.0350 | LR: 4.97e-04 | 1914.5ms/step | 31.3 samples/s
|
| 93 |
+
Step 3240/50462 | Loss: 2.9973 | LR: 4.97e-04 | 1912.4ms/step | 31.4 samples/s
|
| 94 |
+
Step 3250/50462 | Loss: 3.0368 | LR: 4.97e-04 | 1913.2ms/step | 31.4 samples/s
|
| 95 |
+
Validation at step 3250: Loss = 3.0207, Perplexity = 20.51
|
| 96 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-3250
|
| 97 |
+
Step 3260/50462 | Loss: 3.0022 | LR: 4.97e-04 | 3298.6ms/step | 18.2 samples/s
|
| 98 |
+
Step 3270/50462 | Loss: 3.0150 | LR: 4.97e-04 | 1899.9ms/step | 31.6 samples/s
|
| 99 |
+
Step 3280/50462 | Loss: 3.0681 | LR: 4.97e-04 | 1907.2ms/step | 31.5 samples/s
|
| 100 |
+
Step 3290/50462 | Loss: 3.0322 | LR: 4.97e-04 | 1906.9ms/step | 31.5 samples/s
|
| 101 |
+
Step 3300/50462 | Loss: 3.0271 | LR: 4.97e-04 | 1908.0ms/step | 31.4 samples/s
|
| 102 |
+
Step 3310/50462 | Loss: 3.0427 | LR: 4.97e-04 | 1908.2ms/step | 31.4 samples/s
|
| 103 |
+
Step 3320/50462 | Loss: 2.9960 | LR: 4.97e-04 | 1905.3ms/step | 31.5 samples/s
|
| 104 |
+
Step 3330/50462 | Loss: 2.9897 | LR: 4.97e-04 | 1906.0ms/step | 31.5 samples/s
|
| 105 |
+
Step 3340/50462 | Loss: 3.0382 | LR: 4.97e-04 | 1912.5ms/step | 31.4 samples/s
|
| 106 |
+
Step 3350/50462 | Loss: 3.0206 | LR: 4.97e-04 | 1909.9ms/step | 31.4 samples/s
|
| 107 |
+
Step 3360/50462 | Loss: 3.0777 | LR: 4.97e-04 | 1908.9ms/step | 31.4 samples/s
|
| 108 |
+
Step 3370/50462 | Loss: 3.0276 | LR: 4.97e-04 | 1909.4ms/step | 31.4 samples/s
|
| 109 |
+
Step 3380/50462 | Loss: 3.0753 | LR: 4.97e-04 | 1907.5ms/step | 31.5 samples/s
|
| 110 |
+
Step 3390/50462 | Loss: 3.0186 | LR: 4.97e-04 | 1909.4ms/step | 31.4 samples/s
|
| 111 |
+
Step 3400/50462 | Loss: 2.9841 | LR: 4.97e-04 | 1908.7ms/step | 31.4 samples/s
|
| 112 |
+
Step 3410/50462 | Loss: 3.0091 | LR: 4.96e-04 | 1910.3ms/step | 31.4 samples/s
|
| 113 |
+
Step 3420/50462 | Loss: 3.0469 | LR: 4.96e-04 | 1911.2ms/step | 31.4 samples/s
|
| 114 |
+
Step 3430/50462 | Loss: 2.9463 | LR: 4.96e-04 | 1909.9ms/step | 31.4 samples/s
|
| 115 |
+
Step 3440/50462 | Loss: 3.0141 | LR: 4.96e-04 | 1909.9ms/step | 31.4 samples/s
|
| 116 |
+
Step 3450/50462 | Loss: 2.9965 | LR: 4.96e-04 | 1913.2ms/step | 31.4 samples/s
|
| 117 |
+
Step 3460/50462 | Loss: 2.9916 | LR: 4.96e-04 | 1905.8ms/step | 31.5 samples/s
|
| 118 |
+
Step 3470/50462 | Loss: 3.0137 | LR: 4.96e-04 | 1906.0ms/step | 31.5 samples/s
|
| 119 |
+
Step 3480/50462 | Loss: 3.0528 | LR: 4.96e-04 | 1909.5ms/step | 31.4 samples/s
|
| 120 |
+
Step 3490/50462 | Loss: 2.9815 | LR: 4.96e-04 | 1908.8ms/step | 31.4 samples/s
|
| 121 |
+
Step 3500/50462 | Loss: 3.0195 | LR: 4.96e-04 | 1909.5ms/step | 31.4 samples/s
|
| 122 |
+
Validation at step 3500: Loss = 2.9997, Perplexity = 20.08
|
| 123 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-3500
|
| 124 |
+
Step 3510/50462 | Loss: 3.0214 | LR: 4.96e-04 | 3254.6ms/step | 18.4 samples/s
|
| 125 |
+
Step 3520/50462 | Loss: 3.0286 | LR: 4.96e-04 | 1902.7ms/step | 31.5 samples/s
|
| 126 |
+
Step 3530/50462 | Loss: 3.0108 | LR: 4.96e-04 | 1906.1ms/step | 31.5 samples/s
|
| 127 |
+
Step 3540/50462 | Loss: 3.0082 | LR: 4.96e-04 | 1910.5ms/step | 31.4 samples/s
|
| 128 |
+
Step 3550/50462 | Loss: 3.0235 | LR: 4.96e-04 | 1908.5ms/step | 31.4 samples/s
|
| 129 |
+
Step 3560/50462 | Loss: 3.0217 | LR: 4.96e-04 | 1909.5ms/step | 31.4 samples/s
|
| 130 |
+
Step 3570/50462 | Loss: 3.0493 | LR: 4.96e-04 | 1911.7ms/step | 31.4 samples/s
|
| 131 |
+
Step 3580/50462 | Loss: 3.0204 | LR: 4.96e-04 | 1912.7ms/step | 31.4 samples/s
|
| 132 |
+
Step 3590/50462 | Loss: 2.9945 | LR: 4.96e-04 | 1912.0ms/step | 31.4 samples/s
|
| 133 |
+
Step 3600/50462 | Loss: 2.9825 | LR: 4.96e-04 | 1912.5ms/step | 31.4 samples/s
|
| 134 |
+
Step 3610/50462 | Loss: 2.9897 | LR: 4.96e-04 | 1911.1ms/step | 31.4 samples/s
|
| 135 |
+
Step 3620/50462 | Loss: 3.0097 | LR: 4.96e-04 | 1912.1ms/step | 31.4 samples/s
|
| 136 |
+
Step 3630/50462 | Loss: 2.9718 | LR: 4.96e-04 | 1911.5ms/step | 31.4 samples/s
|
| 137 |
+
Step 3640/50462 | Loss: 2.9449 | LR: 4.96e-04 | 1914.0ms/step | 31.3 samples/s
|
| 138 |
+
Step 3650/50462 | Loss: 3.0318 | LR: 4.96e-04 | 1913.0ms/step | 31.4 samples/s
|
| 139 |
+
Step 3660/50462 | Loss: 2.9952 | LR: 4.96e-04 | 1914.5ms/step | 31.3 samples/s
|
| 140 |
+
Step 3670/50462 | Loss: 3.0259 | LR: 4.96e-04 | 1911.8ms/step | 31.4 samples/s
|
| 141 |
+
Step 3680/50462 | Loss: 3.0018 | LR: 4.96e-04 | 1913.5ms/step | 31.4 samples/s
|
| 142 |
+
Step 3690/50462 | Loss: 2.9555 | LR: 4.96e-04 | 1915.4ms/step | 31.3 samples/s
|
| 143 |
+
Step 3700/50462 | Loss: 2.9638 | LR: 4.96e-04 | 1915.7ms/step | 31.3 samples/s
|
| 144 |
+
Step 3710/50462 | Loss: 2.9746 | LR: 4.96e-04 | 1911.3ms/step | 31.4 samples/s
|
| 145 |
+
Step 3720/50462 | Loss: 2.9771 | LR: 4.96e-04 | 1912.3ms/step | 31.4 samples/s
|
| 146 |
+
Step 3730/50462 | Loss: 3.0034 | LR: 4.96e-04 | 1909.8ms/step | 31.4 samples/s
|
| 147 |
+
Step 3740/50462 | Loss: 3.0235 | LR: 4.96e-04 | 1913.2ms/step | 31.4 samples/s
|
| 148 |
+
Step 3750/50462 | Loss: 2.9867 | LR: 4.96e-04 | 1911.9ms/step | 31.4 samples/s
|
| 149 |
+
Validation at step 3750: Loss = 2.9766, Perplexity = 19.62
|
| 150 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-3750
|
| 151 |
+
Step 3760/50462 | Loss: 2.9753 | LR: 4.95e-04 | 3275.8ms/step | 18.3 samples/s
|
| 152 |
+
Step 3770/50462 | Loss: 2.9529 | LR: 4.95e-04 | 1901.5ms/step | 31.6 samples/s
|
| 153 |
+
Step 3780/50462 | Loss: 2.9433 | LR: 4.95e-04 | 1906.5ms/step | 31.5 samples/s
|
| 154 |
+
Step 3790/50462 | Loss: 2.9548 | LR: 4.95e-04 | 1910.3ms/step | 31.4 samples/s
|
| 155 |
+
Step 3800/50462 | Loss: 2.9861 | LR: 4.95e-04 | 1912.2ms/step | 31.4 samples/s
|
| 156 |
+
Step 3810/50462 | Loss: 2.9591 | LR: 4.95e-04 | 1909.6ms/step | 31.4 samples/s
|
| 157 |
+
Step 3820/50462 | Loss: 2.9513 | LR: 4.95e-04 | 1902.6ms/step | 31.5 samples/s
|
| 158 |
+
Step 3830/50462 | Loss: 2.9790 | LR: 4.95e-04 | 1906.8ms/step | 31.5 samples/s
|
| 159 |
+
Step 3840/50462 | Loss: 3.0273 | LR: 4.95e-04 | 1910.1ms/step | 31.4 samples/s
|
| 160 |
+
Step 3850/50462 | Loss: 3.0103 | LR: 4.95e-04 | 1910.2ms/step | 31.4 samples/s
|
| 161 |
+
Step 3860/50462 | Loss: 2.9923 | LR: 4.95e-04 | 1909.4ms/step | 31.4 samples/s
|
| 162 |
+
Step 3870/50462 | Loss: 2.9827 | LR: 4.95e-04 | 1912.7ms/step | 31.4 samples/s
|
| 163 |
+
Step 3880/50462 | Loss: 2.9852 | LR: 4.95e-04 | 1911.9ms/step | 31.4 samples/s
|
| 164 |
+
Step 3890/50462 | Loss: 2.9473 | LR: 4.95e-04 | 1912.4ms/step | 31.4 samples/s
|
| 165 |
+
Step 3900/50462 | Loss: 2.9623 | LR: 4.95e-04 | 1913.1ms/step | 31.4 samples/s
|
| 166 |
+
Step 3910/50462 | Loss: 3.0118 | LR: 4.95e-04 | 1912.6ms/step | 31.4 samples/s
|
| 167 |
+
Step 3920/50462 | Loss: 2.9947 | LR: 4.95e-04 | 1911.9ms/step | 31.4 samples/s
|
| 168 |
+
Step 3930/50462 | Loss: 2.9783 | LR: 4.95e-04 | 1913.3ms/step | 31.4 samples/s
|
| 169 |
+
Step 3940/50462 | Loss: 2.9784 | LR: 4.95e-04 | 1913.5ms/step | 31.4 samples/s
|
| 170 |
+
Step 3950/50462 | Loss: 2.9475 | LR: 4.95e-04 | 1913.2ms/step | 31.4 samples/s
|
| 171 |
+
Step 3960/50462 | Loss: 2.9594 | LR: 4.95e-04 | 1913.0ms/step | 31.4 samples/s
|
| 172 |
+
Step 3970/50462 | Loss: 2.9685 | LR: 4.95e-04 | 1912.9ms/step | 31.4 samples/s
|
| 173 |
+
Step 3980/50462 | Loss: 2.9664 | LR: 4.95e-04 | 1911.3ms/step | 31.4 samples/s
|
| 174 |
+
Step 3990/50462 | Loss: 2.9888 | LR: 4.95e-04 | 1913.2ms/step | 31.4 samples/s
|
| 175 |
+
Step 4000/50462 | Loss: 2.9662 | LR: 4.95e-04 | 1928.9ms/step | 31.1 samples/s
|
| 176 |
+
Validation at step 4000: Loss = 2.9579, Perplexity = 19.26
|
| 177 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-4000
|
| 178 |
+
Step 4010/50462 | Loss: 2.9768 | LR: 4.95e-04 | 3341.0ms/step | 18.0 samples/s
|
| 179 |
+
Step 4020/50462 | Loss: 2.9856 | LR: 4.95e-04 | 1899.9ms/step | 31.6 samples/s
|
| 180 |
+
Step 4030/50462 | Loss: 3.0025 | LR: 4.95e-04 | 1903.4ms/step | 31.5 samples/s
|
| 181 |
+
Step 4040/50462 | Loss: 2.9680 | LR: 4.95e-04 | 1910.4ms/step | 31.4 samples/s
|
| 182 |
+
Step 4050/50462 | Loss: 2.9945 | LR: 4.95e-04 | 1908.3ms/step | 31.4 samples/s
|
| 183 |
+
Step 4060/50462 | Loss: 2.9508 | LR: 4.95e-04 | 1913.2ms/step | 31.4 samples/s
|
| 184 |
+
Step 4070/50462 | Loss: 2.9837 | LR: 4.95e-04 | 1911.7ms/step | 31.4 samples/s
|
| 185 |
+
Step 4080/50462 | Loss: 2.9413 | LR: 4.94e-04 | 1912.8ms/step | 31.4 samples/s
|
| 186 |
+
Step 4090/50462 | Loss: 2.9119 | LR: 4.94e-04 | 1909.9ms/step | 31.4 samples/s
|
| 187 |
+
Step 4100/50462 | Loss: 3.0032 | LR: 4.94e-04 | 1908.2ms/step | 31.4 samples/s
|
| 188 |
+
Step 4110/50462 | Loss: 2.9365 | LR: 4.94e-04 | 1909.0ms/step | 31.4 samples/s
|
| 189 |
+
Step 4120/50462 | Loss: 2.9925 | LR: 4.94e-04 | 1909.3ms/step | 31.4 samples/s
|
| 190 |
+
Step 4130/50462 | Loss: 2.9468 | LR: 4.94e-04 | 1910.0ms/step | 31.4 samples/s
|
| 191 |
+
Step 4140/50462 | Loss: 2.9404 | LR: 4.94e-04 | 1911.0ms/step | 31.4 samples/s
|
| 192 |
+
Step 4150/50462 | Loss: 2.9303 | LR: 4.94e-04 | 1911.5ms/step | 31.4 samples/s
|
| 193 |
+
Step 4160/50462 | Loss: 2.9605 | LR: 4.94e-04 | 1912.7ms/step | 31.4 samples/s
|
| 194 |
+
Step 4170/50462 | Loss: 2.9402 | LR: 4.94e-04 | 1912.8ms/step | 31.4 samples/s
|
| 195 |
+
Step 4180/50462 | Loss: 2.9434 | LR: 4.94e-04 | 1913.5ms/step | 31.4 samples/s
|
| 196 |
+
Step 4190/50462 | Loss: 2.9102 | LR: 4.94e-04 | 1911.5ms/step | 31.4 samples/s
|
| 197 |
+
Step 4200/50462 | Loss: 2.9414 | LR: 4.94e-04 | 1912.1ms/step | 31.4 samples/s
|
| 198 |
+
Step 4210/50462 | Loss: 2.9585 | LR: 4.94e-04 | 1912.0ms/step | 31.4 samples/s
|
| 199 |
+
Step 4220/50462 | Loss: 2.9214 | LR: 4.94e-04 | 1910.8ms/step | 31.4 samples/s
|
| 200 |
+
Step 4230/50462 | Loss: 2.9410 | LR: 4.94e-04 | 1912.9ms/step | 31.4 samples/s
|
| 201 |
+
Step 4240/50462 | Loss: 2.9154 | LR: 4.94e-04 | 1912.2ms/step | 31.4 samples/s
|
| 202 |
+
Step 4250/50462 | Loss: 2.9842 | LR: 4.94e-04 | 1907.7ms/step | 31.5 samples/s
|
| 203 |
+
Validation at step 4250: Loss = 2.9398, Perplexity = 18.91
|
| 204 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-4250
|
| 205 |
+
Step 4260/50462 | Loss: 2.9586 | LR: 4.94e-04 | 3270.9ms/step | 18.3 samples/s
|
| 206 |
+
Step 4270/50462 | Loss: 2.9843 | LR: 4.94e-04 | 1904.6ms/step | 31.5 samples/s
|
| 207 |
+
Step 4280/50462 | Loss: 3.0208 | LR: 4.94e-04 | 1908.7ms/step | 31.4 samples/s
|
| 208 |
+
Step 4290/50462 | Loss: 2.9869 | LR: 4.94e-04 | 1910.2ms/step | 31.4 samples/s
|
| 209 |
+
Step 4300/50462 | Loss: 2.9139 | LR: 4.94e-04 | 1909.4ms/step | 31.4 samples/s
|
| 210 |
+
Step 4310/50462 | Loss: 2.9807 | LR: 4.94e-04 | 1911.0ms/step | 31.4 samples/s
|
| 211 |
+
Step 4320/50462 | Loss: 2.9735 | LR: 4.94e-04 | 1911.5ms/step | 31.4 samples/s
|
| 212 |
+
Step 4330/50462 | Loss: 2.9135 | LR: 4.94e-04 | 1911.4ms/step | 31.4 samples/s
|
| 213 |
+
Step 4340/50462 | Loss: 2.9616 | LR: 4.94e-04 | 1913.6ms/step | 31.4 samples/s
|
| 214 |
+
Step 4350/50462 | Loss: 2.9579 | LR: 4.94e-04 | 1911.6ms/step | 31.4 samples/s
|
| 215 |
+
Step 4360/50462 | Loss: 2.9508 | LR: 4.94e-04 | 1907.5ms/step | 31.5 samples/s
|
| 216 |
+
Step 4370/50462 | Loss: 2.9531 | LR: 4.93e-04 | 1908.1ms/step | 31.4 samples/s
|
| 217 |
+
Step 4380/50462 | Loss: 2.9606 | LR: 4.93e-04 | 1906.3ms/step | 31.5 samples/s
|
| 218 |
+
Step 4390/50462 | Loss: 2.9917 | LR: 4.93e-04 | 1908.3ms/step | 31.4 samples/s
|
| 219 |
+
Step 4400/50462 | Loss: 2.9509 | LR: 4.93e-04 | 1909.2ms/step | 31.4 samples/s
|
| 220 |
+
Step 4410/50462 | Loss: 2.9611 | LR: 4.93e-04 | 1905.5ms/step | 31.5 samples/s
|
| 221 |
+
Step 4420/50462 | Loss: 2.9348 | LR: 4.93e-04 | 1906.7ms/step | 31.5 samples/s
|
| 222 |
+
Step 4430/50462 | Loss: 2.9028 | LR: 4.93e-04 | 1905.7ms/step | 31.5 samples/s
|
| 223 |
+
Step 4440/50462 | Loss: 2.9230 | LR: 4.93e-04 | 1904.7ms/step | 31.5 samples/s
|
| 224 |
+
Step 4450/50462 | Loss: 2.9192 | LR: 4.93e-04 | 1907.5ms/step | 31.5 samples/s
|
| 225 |
+
Step 4460/50462 | Loss: 2.9549 | LR: 4.93e-04 | 1908.5ms/step | 31.4 samples/s
|
| 226 |
+
Step 4470/50462 | Loss: 2.9919 | LR: 4.93e-04 | 1907.8ms/step | 31.5 samples/s
|
| 227 |
+
Step 4480/50462 | Loss: 2.9310 | LR: 4.93e-04 | 1904.2ms/step | 31.5 samples/s
|
| 228 |
+
Step 4490/50462 | Loss: 2.9158 | LR: 4.93e-04 | 1906.8ms/step | 31.5 samples/s
|
| 229 |
+
Step 4500/50462 | Loss: 2.9560 | LR: 4.93e-04 | 1904.4ms/step | 31.5 samples/s
|
| 230 |
+
Validation at step 4500: Loss = 2.9253, Perplexity = 18.64
|
| 231 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-4500
|
| 232 |
+
Step 4510/50462 | Loss: 2.9614 | LR: 4.93e-04 | 3235.1ms/step | 18.5 samples/s
|
| 233 |
+
Step 4520/50462 | Loss: 2.9044 | LR: 4.93e-04 | 1898.2ms/step | 31.6 samples/s
|
| 234 |
+
Step 4530/50462 | Loss: 2.9534 | LR: 4.93e-04 | 1905.9ms/step | 31.5 samples/s
|
| 235 |
+
Step 4540/50462 | Loss: 2.8904 | LR: 4.93e-04 | 1903.3ms/step | 31.5 samples/s
|
| 236 |
+
Step 4550/50462 | Loss: 2.9252 | LR: 4.93e-04 | 1904.1ms/step | 31.5 samples/s
|
| 237 |
+
Step 4560/50462 | Loss: 2.8996 | LR: 4.93e-04 | 1907.1ms/step | 31.5 samples/s
|
| 238 |
+
Step 4570/50462 | Loss: 2.9315 | LR: 4.93e-04 | 1904.5ms/step | 31.5 samples/s
|
| 239 |
+
Step 4580/50462 | Loss: 2.9316 | LR: 4.93e-04 | 1903.1ms/step | 31.5 samples/s
|
| 240 |
+
Step 4590/50462 | Loss: 2.9244 | LR: 4.93e-04 | 1905.6ms/step | 31.5 samples/s
|
| 241 |
+
Step 4600/50462 | Loss: 2.9241 | LR: 4.93e-04 | 1904.2ms/step | 31.5 samples/s
|
| 242 |
+
Step 4610/50462 | Loss: 2.8848 | LR: 4.93e-04 | 1901.7ms/step | 31.6 samples/s
|
| 243 |
+
Step 4620/50462 | Loss: 2.9280 | LR: 4.93e-04 | 1904.1ms/step | 31.5 samples/s
|
| 244 |
+
Step 4630/50462 | Loss: 2.8894 | LR: 4.93e-04 | 1902.3ms/step | 31.5 samples/s
|
| 245 |
+
Step 4640/50462 | Loss: 2.9400 | LR: 4.92e-04 | 1904.3ms/step | 31.5 samples/s
|
| 246 |
+
Step 4650/50462 | Loss: 2.9528 | LR: 4.92e-04 | 1906.8ms/step | 31.5 samples/s
|
| 247 |
+
Step 4660/50462 | Loss: 2.9020 | LR: 4.92e-04 | 1906.7ms/step | 31.5 samples/s
|
| 248 |
+
Step 4670/50462 | Loss: 2.9320 | LR: 4.92e-04 | 1908.3ms/step | 31.4 samples/s
|
| 249 |
+
Step 4680/50462 | Loss: 2.9560 | LR: 4.92e-04 | 1907.4ms/step | 31.5 samples/s
|
| 250 |
+
Step 4690/50462 | Loss: 2.9201 | LR: 4.92e-04 | 1908.4ms/step | 31.4 samples/s
|
| 251 |
+
Step 4700/50462 | Loss: 2.9300 | LR: 4.92e-04 | 1908.8ms/step | 31.4 samples/s
|
| 252 |
+
Step 4710/50462 | Loss: 2.9102 | LR: 4.92e-04 | 1910.1ms/step | 31.4 samples/s
|
| 253 |
+
Step 4720/50462 | Loss: 2.9033 | LR: 4.92e-04 | 1910.4ms/step | 31.4 samples/s
|
| 254 |
+
Step 4730/50462 | Loss: 2.9384 | LR: 4.92e-04 | 1909.9ms/step | 31.4 samples/s
|
| 255 |
+
Step 4740/50462 | Loss: 2.9790 | LR: 4.92e-04 | 1909.2ms/step | 31.4 samples/s
|
| 256 |
+
Step 4750/50462 | Loss: 2.9566 | LR: 4.92e-04 | 1912.8ms/step | 31.4 samples/s
|
| 257 |
+
Validation at step 4750: Loss = 2.9131, Perplexity = 18.41
|
| 258 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-4750
|
| 259 |
+
Step 4760/50462 | Loss: 2.9196 | LR: 4.92e-04 | 3271.2ms/step | 18.3 samples/s
|
| 260 |
+
Step 4770/50462 | Loss: 2.9065 | LR: 4.92e-04 | 1899.3ms/step | 31.6 samples/s
|
| 261 |
+
Step 4780/50462 | Loss: 2.8979 | LR: 4.92e-04 | 1902.5ms/step | 31.5 samples/s
|
| 262 |
+
Step 4790/50462 | Loss: 2.9276 | LR: 4.92e-04 | 1908.6ms/step | 31.4 samples/s
|
| 263 |
+
Step 4800/50462 | Loss: 2.9130 | LR: 4.92e-04 | 1906.6ms/step | 31.5 samples/s
|
| 264 |
+
Step 4810/50462 | Loss: 2.9546 | LR: 4.92e-04 | 1902.8ms/step | 31.5 samples/s
|
| 265 |
+
Step 4820/50462 | Loss: 2.9556 | LR: 4.92e-04 | 1903.9ms/step | 31.5 samples/s
|
| 266 |
+
Step 4830/50462 | Loss: 2.9193 | LR: 4.92e-04 | 1904.7ms/step | 31.5 samples/s
|
| 267 |
+
Step 4840/50462 | Loss: 2.9282 | LR: 4.92e-04 | 1906.9ms/step | 31.5 samples/s
|
| 268 |
+
Step 4850/50462 | Loss: 2.9236 | LR: 4.92e-04 | 1908.5ms/step | 31.4 samples/s
|
| 269 |
+
Step 4860/50462 | Loss: 2.8812 | LR: 4.92e-04 | 1905.7ms/step | 31.5 samples/s
|
| 270 |
+
Step 4870/50462 | Loss: 2.8688 | LR: 4.92e-04 | 1907.6ms/step | 31.5 samples/s
|
| 271 |
+
Step 4880/50462 | Loss: 2.8962 | LR: 4.92e-04 | 1909.0ms/step | 31.4 samples/s
|
| 272 |
+
Step 4890/50462 | Loss: 2.9300 | LR: 4.91e-04 | 1908.0ms/step | 31.4 samples/s
|
| 273 |
+
Step 4900/50462 | Loss: 2.9287 | LR: 4.91e-04 | 1911.4ms/step | 31.4 samples/s
|
| 274 |
+
Step 4910/50462 | Loss: 2.9489 | LR: 4.91e-04 | 1910.5ms/step | 31.4 samples/s
|
| 275 |
+
Step 4920/50462 | Loss: 2.8821 | LR: 4.91e-04 | 1906.8ms/step | 31.5 samples/s
|
| 276 |
+
Step 4930/50462 | Loss: 2.8659 | LR: 4.91e-04 | 1905.4ms/step | 31.5 samples/s
|
| 277 |
+
Step 4940/50462 | Loss: 2.9004 | LR: 4.91e-04 | 1908.4ms/step | 31.4 samples/s
|
| 278 |
+
Step 4950/50462 | Loss: 2.8864 | LR: 4.91e-04 | 1904.3ms/step | 31.5 samples/s
|
| 279 |
+
Step 4960/50462 | Loss: 2.9025 | LR: 4.91e-04 | 1907.1ms/step | 31.5 samples/s
|
| 280 |
+
Step 4970/50462 | Loss: 2.9113 | LR: 4.91e-04 | 1905.7ms/step | 31.5 samples/s
|
| 281 |
+
Step 4980/50462 | Loss: 2.8673 | LR: 4.91e-04 | 1908.2ms/step | 31.4 samples/s
|
| 282 |
+
Step 4990/50462 | Loss: 2.8910 | LR: 4.91e-04 | 1911.5ms/step | 31.4 samples/s
|
| 283 |
+
Step 5000/50462 | Loss: 2.9331 | LR: 4.91e-04 | 1903.0ms/step | 31.5 samples/s
|
| 284 |
+
Validation at step 5000: Loss = 2.8970, Perplexity = 18.12
|
| 285 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-5000
|
| 286 |
+
Step 5010/50462 | Loss: 2.9010 | LR: 4.91e-04 | 3267.4ms/step | 18.4 samples/s
|
| 287 |
+
Step 5020/50462 | Loss: 2.9161 | LR: 4.91e-04 | 1902.5ms/step | 31.5 samples/s
|
| 288 |
+
Step 5030/50462 | Loss: 2.8765 | LR: 4.91e-04 | 1906.0ms/step | 31.5 samples/s
|
| 289 |
+
Step 5040/50462 | Loss: 2.9250 | LR: 4.91e-04 | 1903.8ms/step | 31.5 samples/s
|
| 290 |
+
Step 5050/50462 | Loss: 2.8811 | LR: 4.91e-04 | 1901.5ms/step | 31.6 samples/s
|
| 291 |
+
Step 5060/50462 | Loss: 2.9517 | LR: 4.91e-04 | 1909.0ms/step | 31.4 samples/s
|
| 292 |
+
Step 5070/50462 | Loss: 2.9142 | LR: 4.91e-04 | 1906.5ms/step | 31.5 samples/s
|
| 293 |
+
Step 5080/50462 | Loss: 2.8689 | LR: 4.91e-04 | 1907.8ms/step | 31.5 samples/s
|
| 294 |
+
Step 5090/50462 | Loss: 2.8741 | LR: 4.91e-04 | 1912.5ms/step | 31.4 samples/s
|
| 295 |
+
Step 5100/50462 | Loss: 2.9199 | LR: 4.91e-04 | 1911.0ms/step | 31.4 samples/s
|
| 296 |
+
Step 5110/50462 | Loss: 2.9283 | LR: 4.91e-04 | 1909.7ms/step | 31.4 samples/s
|
| 297 |
+
Step 5120/50462 | Loss: 2.9286 | LR: 4.91e-04 | 1911.7ms/step | 31.4 samples/s
|
| 298 |
+
Step 5130/50462 | Loss: 2.9014 | LR: 4.90e-04 | 1908.8ms/step | 31.4 samples/s
|
| 299 |
+
Step 5140/50462 | Loss: 2.9162 | LR: 4.90e-04 | 1910.9ms/step | 31.4 samples/s
|
| 300 |
+
Step 5150/50462 | Loss: 2.8705 | LR: 4.90e-04 | 1909.9ms/step | 31.4 samples/s
|
| 301 |
+
Step 5160/50462 | Loss: 2.9088 | LR: 4.90e-04 | 1910.5ms/step | 31.4 samples/s
|
| 302 |
+
Step 5170/50462 | Loss: 2.8631 | LR: 4.90e-04 | 1911.6ms/step | 31.4 samples/s
|
| 303 |
+
Step 5180/50462 | Loss: 2.8751 | LR: 4.90e-04 | 1914.2ms/step | 31.3 samples/s
|
| 304 |
+
Step 5190/50462 | Loss: 2.9103 | LR: 4.90e-04 | 1913.6ms/step | 31.4 samples/s
|
| 305 |
+
Step 5200/50462 | Loss: 2.9401 | LR: 4.90e-04 | 1915.5ms/step | 31.3 samples/s
|
| 306 |
+
Step 5210/50462 | Loss: 2.8899 | LR: 4.90e-04 | 1908.8ms/step | 31.4 samples/s
|
| 307 |
+
Step 5220/50462 | Loss: 2.8783 | LR: 4.90e-04 | 1911.9ms/step | 31.4 samples/s
|
| 308 |
+
Step 5230/50462 | Loss: 2.9317 | LR: 4.90e-04 | 1912.3ms/step | 31.4 samples/s
|
| 309 |
+
Step 5240/50462 | Loss: 2.8692 | LR: 4.90e-04 | 1908.7ms/step | 31.4 samples/s
|
| 310 |
+
Step 5250/50462 | Loss: 2.8519 | LR: 4.90e-04 | 1910.9ms/step | 31.4 samples/s
|
| 311 |
+
Validation at step 5250: Loss = 2.8830, Perplexity = 17.87
|
| 312 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-5250
|
| 313 |
+
Step 5260/50462 | Loss: 2.9073 | LR: 4.90e-04 | 3356.4ms/step | 17.9 samples/s
|
| 314 |
+
Step 5270/50462 | Loss: 2.8713 | LR: 4.90e-04 | 1903.6ms/step | 31.5 samples/s
|
| 315 |
+
Step 5280/50462 | Loss: 2.8778 | LR: 4.90e-04 | 1904.5ms/step | 31.5 samples/s
|
| 316 |
+
Step 5290/50462 | Loss: 2.8994 | LR: 4.90e-04 | 1905.1ms/step | 31.5 samples/s
|
| 317 |
+
Step 5300/50462 | Loss: 2.8453 | LR: 4.90e-04 | 1903.2ms/step | 31.5 samples/s
|
| 318 |
+
Step 5310/50462 | Loss: 2.8873 | LR: 4.90e-04 | 1903.4ms/step | 31.5 samples/s
|
| 319 |
+
Step 5320/50462 | Loss: 2.8643 | LR: 4.90e-04 | 1904.0ms/step | 31.5 samples/s
|
| 320 |
+
Step 5330/50462 | Loss: 2.9292 | LR: 4.90e-04 | 1908.7ms/step | 31.4 samples/s
|
| 321 |
+
Step 5340/50462 | Loss: 2.8901 | LR: 4.90e-04 | 1908.7ms/step | 31.4 samples/s
|
| 322 |
+
Step 5350/50462 | Loss: 2.9035 | LR: 4.90e-04 | 1914.8ms/step | 31.3 samples/s
|
| 323 |
+
Step 5360/50462 | Loss: 2.8727 | LR: 4.89e-04 | 1911.7ms/step | 31.4 samples/s
|
| 324 |
+
Step 5370/50462 | Loss: 2.8817 | LR: 4.89e-04 | 1911.1ms/step | 31.4 samples/s
|
| 325 |
+
Step 5380/50462 | Loss: 2.8822 | LR: 4.89e-04 | 1910.4ms/step | 31.4 samples/s
|
| 326 |
+
Step 5390/50462 | Loss: 2.8988 | LR: 4.89e-04 | 1908.0ms/step | 31.4 samples/s
|
| 327 |
+
Step 5400/50462 | Loss: 2.8896 | LR: 4.89e-04 | 1907.7ms/step | 31.5 samples/s
|
| 328 |
+
Step 5410/50462 | Loss: 2.9301 | LR: 4.89e-04 | 1908.7ms/step | 31.4 samples/s
|
| 329 |
+
Step 5420/50462 | Loss: 2.8864 | LR: 4.89e-04 | 1906.5ms/step | 31.5 samples/s
|
| 330 |
+
Step 5430/50462 | Loss: 2.9120 | LR: 4.89e-04 | 1910.6ms/step | 31.4 samples/s
|
| 331 |
+
Step 5440/50462 | Loss: 2.9044 | LR: 4.89e-04 | 1903.6ms/step | 31.5 samples/s
|
| 332 |
+
Step 5450/50462 | Loss: 2.8894 | LR: 4.89e-04 | 1905.2ms/step | 31.5 samples/s
|
| 333 |
+
Step 5460/50462 | Loss: 2.8657 | LR: 4.89e-04 | 1905.0ms/step | 31.5 samples/s
|
| 334 |
+
Step 5470/50462 | Loss: 2.8855 | LR: 4.89e-04 | 1906.8ms/step | 31.5 samples/s
|
| 335 |
+
Step 5480/50462 | Loss: 2.8533 | LR: 4.89e-04 | 1904.5ms/step | 31.5 samples/s
|
| 336 |
+
Step 5490/50462 | Loss: 2.9164 | LR: 4.89e-04 | 1904.2ms/step | 31.5 samples/s
|
| 337 |
+
Step 5500/50462 | Loss: 2.9329 | LR: 4.89e-04 | 1905.2ms/step | 31.5 samples/s
|
| 338 |
+
Validation at step 5500: Loss = 2.8726, Perplexity = 17.68
|
| 339 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-5500
|
| 340 |
+
Step 5510/50462 | Loss: 2.9126 | LR: 4.89e-04 | 3341.8ms/step | 18.0 samples/s
|
| 341 |
+
Step 5520/50462 | Loss: 2.9081 | LR: 4.89e-04 | 1900.4ms/step | 31.6 samples/s
|
| 342 |
+
Step 5530/50462 | Loss: 2.9110 | LR: 4.89e-04 | 1905.1ms/step | 31.5 samples/s
|
| 343 |
+
Step 5540/50462 | Loss: 2.8469 | LR: 4.89e-04 | 1905.3ms/step | 31.5 samples/s
|
| 344 |
+
Step 5550/50462 | Loss: 2.8808 | LR: 4.89e-04 | 1906.8ms/step | 31.5 samples/s
|
| 345 |
+
Step 5560/50462 | Loss: 2.8446 | LR: 4.89e-04 | 1909.5ms/step | 31.4 samples/s
|
| 346 |
+
Step 5570/50462 | Loss: 2.8776 | LR: 4.88e-04 | 1911.4ms/step | 31.4 samples/s
|
| 347 |
+
Step 5580/50462 | Loss: 2.8873 | LR: 4.88e-04 | 1915.6ms/step | 31.3 samples/s
|
| 348 |
+
Step 5590/50462 | Loss: 2.8521 | LR: 4.88e-04 | 1910.7ms/step | 31.4 samples/s
|
| 349 |
+
Step 5600/50462 | Loss: 2.9043 | LR: 4.88e-04 | 1911.7ms/step | 31.4 samples/s
|
| 350 |
+
Step 5610/50462 | Loss: 2.8708 | LR: 4.88e-04 | 1911.6ms/step | 31.4 samples/s
|
| 351 |
+
Step 5620/50462 | Loss: 2.8910 | LR: 4.88e-04 | 1905.1ms/step | 31.5 samples/s
|
| 352 |
+
Step 5630/50462 | Loss: 2.8991 | LR: 4.88e-04 | 1907.0ms/step | 31.5 samples/s
|
| 353 |
+
Step 5640/50462 | Loss: 2.8497 | LR: 4.88e-04 | 1905.7ms/step | 31.5 samples/s
|
| 354 |
+
Step 5650/50462 | Loss: 2.8644 | LR: 4.88e-04 | 1901.6ms/step | 31.6 samples/s
|
| 355 |
+
Step 5660/50462 | Loss: 2.9214 | LR: 4.88e-04 | 1905.7ms/step | 31.5 samples/s
|
| 356 |
+
Step 5670/50462 | Loss: 2.8923 | LR: 4.88e-04 | 1904.0ms/step | 31.5 samples/s
|
| 357 |
+
Step 5680/50462 | Loss: 2.8829 | LR: 4.88e-04 | 1905.9ms/step | 31.5 samples/s
|
| 358 |
+
Step 5690/50462 | Loss: 2.9228 | LR: 4.88e-04 | 1906.9ms/step | 31.5 samples/s
|
| 359 |
+
Step 5700/50462 | Loss: 2.8309 | LR: 4.88e-04 | 1909.2ms/step | 31.4 samples/s
|
| 360 |
+
Step 5710/50462 | Loss: 2.8787 | LR: 4.88e-04 | 1910.3ms/step | 31.4 samples/s
|
| 361 |
+
Step 5720/50462 | Loss: 2.8839 | LR: 4.88e-04 | 1908.2ms/step | 31.4 samples/s
|
| 362 |
+
Step 5730/50462 | Loss: 2.8529 | LR: 4.88e-04 | 1910.0ms/step | 31.4 samples/s
|
| 363 |
+
Step 5740/50462 | Loss: 2.8451 | LR: 4.88e-04 | 1910.5ms/step | 31.4 samples/s
|
| 364 |
+
Step 5750/50462 | Loss: 2.8658 | LR: 4.88e-04 | 1907.4ms/step | 31.5 samples/s
|
| 365 |
+
Validation at step 5750: Loss = 2.8616, Perplexity = 17.49
|
| 366 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-5750
|
| 367 |
+
Step 5760/50462 | Loss: 2.9061 | LR: 4.88e-04 | 3276.1ms/step | 18.3 samples/s
|
| 368 |
+
Step 5770/50462 | Loss: 2.8922 | LR: 4.88e-04 | 1900.5ms/step | 31.6 samples/s
|
| 369 |
+
Step 5780/50462 | Loss: 2.8283 | LR: 4.87e-04 | 1902.4ms/step | 31.5 samples/s
|
| 370 |
+
Step 5790/50462 | Loss: 2.8608 | LR: 4.87e-04 | 1907.0ms/step | 31.5 samples/s
|
| 371 |
+
Step 5800/50462 | Loss: 2.9390 | LR: 4.87e-04 | 1905.8ms/step | 31.5 samples/s
|
| 372 |
+
Step 5810/50462 | Loss: 2.8393 | LR: 4.87e-04 | 1901.1ms/step | 31.6 samples/s
|
| 373 |
+
Step 5820/50462 | Loss: 2.8539 | LR: 4.87e-04 | 1902.6ms/step | 31.5 samples/s
|
| 374 |
+
Step 5830/50462 | Loss: 2.8797 | LR: 4.87e-04 | 1907.6ms/step | 31.5 samples/s
|
| 375 |
+
Step 5840/50462 | Loss: 2.8588 | LR: 4.87e-04 | 1909.1ms/step | 31.4 samples/s
|
| 376 |
+
Step 5850/50462 | Loss: 2.8711 | LR: 4.87e-04 | 1909.4ms/step | 31.4 samples/s
|
| 377 |
+
Step 5860/50462 | Loss: 2.8979 | LR: 4.87e-04 | 1911.9ms/step | 31.4 samples/s
|
| 378 |
+
Step 5870/50462 | Loss: 2.8966 | LR: 4.87e-04 | 1912.9ms/step | 31.4 samples/s
|
| 379 |
+
Step 5880/50462 | Loss: 2.8616 | LR: 4.87e-04 | 1909.3ms/step | 31.4 samples/s
|
| 380 |
+
Step 5890/50462 | Loss: 2.8905 | LR: 4.87e-04 | 1910.0ms/step | 31.4 samples/s
|
| 381 |
+
Step 5900/50462 | Loss: 2.8632 | LR: 4.87e-04 | 1904.7ms/step | 31.5 samples/s
|
| 382 |
+
Step 5910/50462 | Loss: 2.8724 | LR: 4.87e-04 | 1903.2ms/step | 31.5 samples/s
|
| 383 |
+
Step 5920/50462 | Loss: 2.8758 | LR: 4.87e-04 | 1902.7ms/step | 31.5 samples/s
|
| 384 |
+
Step 5930/50462 | Loss: 2.8550 | LR: 4.87e-04 | 1904.9ms/step | 31.5 samples/s
|
| 385 |
+
Step 5940/50462 | Loss: 2.8726 | LR: 4.87e-04 | 1906.8ms/step | 31.5 samples/s
|
| 386 |
+
Step 5950/50462 | Loss: 2.8490 | LR: 4.87e-04 | 1901.9ms/step | 31.5 samples/s
|
| 387 |
+
Step 5960/50462 | Loss: 2.8510 | LR: 4.87e-04 | 1906.3ms/step | 31.5 samples/s
|
| 388 |
+
Step 5970/50462 | Loss: 2.9040 | LR: 4.87e-04 | 1907.2ms/step | 31.5 samples/s
|
| 389 |
+
Step 5980/50462 | Loss: 2.8304 | LR: 4.86e-04 | 1907.9ms/step | 31.4 samples/s
|
| 390 |
+
Step 5990/50462 | Loss: 2.8380 | LR: 4.86e-04 | 1911.0ms/step | 31.4 samples/s
|
| 391 |
+
Step 6000/50462 | Loss: 2.8613 | LR: 4.86e-04 | 1911.2ms/step | 31.4 samples/s
|
| 392 |
+
Validation at step 6000: Loss = 2.8499, Perplexity = 17.29
|
| 393 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-6000
|
| 394 |
+
Step 6010/50462 | Loss: 2.8945 | LR: 4.86e-04 | 3252.9ms/step | 18.4 samples/s
|
| 395 |
+
Step 6020/50462 | Loss: 2.8740 | LR: 4.86e-04 | 1901.5ms/step | 31.6 samples/s
|
| 396 |
+
Step 6030/50462 | Loss: 2.8862 | LR: 4.86e-04 | 1898.8ms/step | 31.6 samples/s
|
| 397 |
+
Step 6040/50462 | Loss: 2.8376 | LR: 4.86e-04 | 1907.8ms/step | 31.5 samples/s
|
| 398 |
+
Step 6050/50462 | Loss: 2.8654 | LR: 4.86e-04 | 1901.2ms/step | 31.6 samples/s
|
| 399 |
+
Step 6060/50462 | Loss: 2.8284 | LR: 4.86e-04 | 1905.1ms/step | 31.5 samples/s
|
| 400 |
+
Step 6070/50462 | Loss: 2.8190 | LR: 4.86e-04 | 1905.1ms/step | 31.5 samples/s
|
| 401 |
+
Step 6080/50462 | Loss: 2.8347 | LR: 4.86e-04 | 1905.4ms/step | 31.5 samples/s
|
| 402 |
+
Step 6090/50462 | Loss: 2.9083 | LR: 4.86e-04 | 1905.8ms/step | 31.5 samples/s
|
| 403 |
+
Step 6100/50462 | Loss: 2.8295 | LR: 4.86e-04 | 1904.1ms/step | 31.5 samples/s
|
| 404 |
+
Step 6110/50462 | Loss: 2.8720 | LR: 4.86e-04 | 1910.0ms/step | 31.4 samples/s
|
| 405 |
+
Step 6120/50462 | Loss: 2.8291 | LR: 4.86e-04 | 1907.9ms/step | 31.4 samples/s
|
| 406 |
+
Step 6130/50462 | Loss: 2.8743 | LR: 4.86e-04 | 1906.7ms/step | 31.5 samples/s
|
| 407 |
+
Step 6140/50462 | Loss: 2.8616 | LR: 4.86e-04 | 1909.6ms/step | 31.4 samples/s
|
| 408 |
+
Step 6150/50462 | Loss: 2.8826 | LR: 4.86e-04 | 1900.8ms/step | 31.6 samples/s
|
| 409 |
+
Step 6160/50462 | Loss: 2.8287 | LR: 4.86e-04 | 1906.3ms/step | 31.5 samples/s
|
| 410 |
+
Step 6170/50462 | Loss: 2.8475 | LR: 4.85e-04 | 1906.8ms/step | 31.5 samples/s
|
| 411 |
+
Step 6180/50462 | Loss: 2.8726 | LR: 4.85e-04 | 1909.6ms/step | 31.4 samples/s
|
| 412 |
+
Step 6190/50462 | Loss: 2.8733 | LR: 4.85e-04 | 1914.1ms/step | 31.3 samples/s
|
| 413 |
+
Step 6200/50462 | Loss: 2.8899 | LR: 4.85e-04 | 1913.0ms/step | 31.4 samples/s
|
| 414 |
+
Step 6210/50462 | Loss: 2.8516 | LR: 4.85e-04 | 1911.2ms/step | 31.4 samples/s
|
| 415 |
+
Step 6220/50462 | Loss: 2.8719 | LR: 4.85e-04 | 1910.6ms/step | 31.4 samples/s
|
| 416 |
+
Step 6230/50462 | Loss: 2.8264 | LR: 4.85e-04 | 1914.2ms/step | 31.3 samples/s
|
| 417 |
+
Step 6240/50462 | Loss: 2.8763 | LR: 4.85e-04 | 1916.0ms/step | 31.3 samples/s
|
| 418 |
+
Step 6250/50462 | Loss: 2.8129 | LR: 4.85e-04 | 1912.6ms/step | 31.4 samples/s
|
| 419 |
+
Validation at step 6250: Loss = 2.8391, Perplexity = 17.10
|
| 420 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-6250
|
| 421 |
+
Step 6260/50462 | Loss: 2.8384 | LR: 4.85e-04 | 3259.9ms/step | 18.4 samples/s
|
| 422 |
+
Step 6270/50462 | Loss: 2.8714 | LR: 4.85e-04 | 1898.7ms/step | 31.6 samples/s
|
| 423 |
+
Step 6280/50462 | Loss: 2.8504 | LR: 4.85e-04 | 1904.6ms/step | 31.5 samples/s
|
| 424 |
+
Step 6290/50462 | Loss: 2.8868 | LR: 4.85e-04 | 1906.5ms/step | 31.5 samples/s
|
| 425 |
+
Step 6300/50462 | Loss: 2.8623 | LR: 4.85e-04 | 1905.7ms/step | 31.5 samples/s
|
| 426 |
+
Step 6310/50462 | Loss: 2.8417 | LR: 4.85e-04 | 1903.2ms/step | 31.5 samples/s
|
| 427 |
+
Step 6320/50462 | Loss: 2.8038 | LR: 4.85e-04 | 1900.7ms/step | 31.6 samples/s
|
| 428 |
+
Step 6330/50462 | Loss: 2.8440 | LR: 4.85e-04 | 1904.8ms/step | 31.5 samples/s
|
| 429 |
+
Step 6340/50462 | Loss: 2.8311 | LR: 4.85e-04 | 1908.6ms/step | 31.4 samples/s
|
| 430 |
+
Step 6350/50462 | Loss: 2.8459 | LR: 4.85e-04 | 1906.0ms/step | 31.5 samples/s
|
| 431 |
+
Step 6360/50462 | Loss: 2.8448 | LR: 4.84e-04 | 1909.7ms/step | 31.4 samples/s
|
| 432 |
+
Step 6370/50462 | Loss: 2.8272 | LR: 4.84e-04 | 1907.4ms/step | 31.5 samples/s
|
| 433 |
+
Step 6380/50462 | Loss: 2.8177 | LR: 4.84e-04 | 1911.5ms/step | 31.4 samples/s
|
| 434 |
+
Step 6390/50462 | Loss: 2.8185 | LR: 4.84e-04 | 1911.0ms/step | 31.4 samples/s
|
| 435 |
+
Step 6400/50462 | Loss: 2.8507 | LR: 4.84e-04 | 1907.0ms/step | 31.5 samples/s
|
| 436 |
+
Step 6410/50462 | Loss: 2.8460 | LR: 4.84e-04 | 1906.1ms/step | 31.5 samples/s
|
| 437 |
+
Step 6420/50462 | Loss: 2.8787 | LR: 4.84e-04 | 1910.4ms/step | 31.4 samples/s
|
| 438 |
+
Step 6430/50462 | Loss: 2.8529 | LR: 4.84e-04 | 1904.9ms/step | 31.5 samples/s
|
| 439 |
+
Step 6440/50462 | Loss: 2.8686 | LR: 4.84e-04 | 1903.6ms/step | 31.5 samples/s
|
| 440 |
+
Step 6450/50462 | Loss: 2.8407 | LR: 4.84e-04 | 1904.7ms/step | 31.5 samples/s
|
| 441 |
+
Step 6460/50462 | Loss: 2.8338 | LR: 4.84e-04 | 1908.1ms/step | 31.4 samples/s
|
| 442 |
+
Step 6470/50462 | Loss: 2.8730 | LR: 4.84e-04 | 1907.7ms/step | 31.5 samples/s
|
| 443 |
+
Step 6480/50462 | Loss: 2.8583 | LR: 4.84e-04 | 1909.1ms/step | 31.4 samples/s
|
| 444 |
+
Step 6490/50462 | Loss: 2.8352 | LR: 4.84e-04 | 1910.0ms/step | 31.4 samples/s
|
| 445 |
+
Step 6500/50462 | Loss: 2.8616 | LR: 4.84e-04 | 1909.8ms/step | 31.4 samples/s
|
| 446 |
+
Validation at step 6500: Loss = 2.8310, Perplexity = 16.96
|
| 447 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-6500
|
| 448 |
+
Step 6510/50462 | Loss: 2.8527 | LR: 4.84e-04 | 3225.7ms/step | 18.6 samples/s
|
| 449 |
+
Step 6520/50462 | Loss: 2.8864 | LR: 4.84e-04 | 1899.8ms/step | 31.6 samples/s
|
| 450 |
+
Step 6530/50462 | Loss: 2.8098 | LR: 4.84e-04 | 1904.2ms/step | 31.5 samples/s
|
| 451 |
+
Step 6540/50462 | Loss: 2.8360 | LR: 4.83e-04 | 1903.4ms/step | 31.5 samples/s
|
| 452 |
+
Step 6550/50462 | Loss: 2.8720 | LR: 4.83e-04 | 1902.6ms/step | 31.5 samples/s
|
| 453 |
+
Step 6560/50462 | Loss: 2.8186 | LR: 4.83e-04 | 1907.9ms/step | 31.4 samples/s
|
| 454 |
+
Step 6570/50462 | Loss: 2.8518 | LR: 4.83e-04 | 1905.5ms/step | 31.5 samples/s
|
| 455 |
+
Step 6580/50462 | Loss: 2.8295 | LR: 4.83e-04 | 1907.6ms/step | 31.5 samples/s
|
| 456 |
+
Step 6590/50462 | Loss: 2.8413 | LR: 4.83e-04 | 1906.7ms/step | 31.5 samples/s
|
| 457 |
+
Step 6600/50462 | Loss: 2.8173 | LR: 4.83e-04 | 1906.9ms/step | 31.5 samples/s
|
| 458 |
+
Step 6610/50462 | Loss: 2.8142 | LR: 4.83e-04 | 1909.8ms/step | 31.4 samples/s
|
| 459 |
+
Step 6620/50462 | Loss: 2.8261 | LR: 4.83e-04 | 1909.0ms/step | 31.4 samples/s
|
| 460 |
+
Step 6630/50462 | Loss: 2.8357 | LR: 4.83e-04 | 1910.7ms/step | 31.4 samples/s
|
| 461 |
+
Step 6640/50462 | Loss: 2.8785 | LR: 4.83e-04 | 1911.4ms/step | 31.4 samples/s
|
| 462 |
+
Step 6650/50462 | Loss: 2.8286 | LR: 4.83e-04 | 1910.6ms/step | 31.4 samples/s
|
| 463 |
+
Step 6660/50462 | Loss: 2.8723 | LR: 4.83e-04 | 1910.1ms/step | 31.4 samples/s
|
| 464 |
+
Step 6670/50462 | Loss: 2.8477 | LR: 4.83e-04 | 1910.9ms/step | 31.4 samples/s
|
| 465 |
+
Step 6680/50462 | Loss: 2.8176 | LR: 4.83e-04 | 1911.4ms/step | 31.4 samples/s
|
| 466 |
+
Step 6690/50462 | Loss: 2.8454 | LR: 4.83e-04 | 1909.0ms/step | 31.4 samples/s
|
| 467 |
+
Step 6700/50462 | Loss: 2.7940 | LR: 4.83e-04 | 1905.3ms/step | 31.5 samples/s
|
| 468 |
+
Step 6710/50462 | Loss: 2.8033 | LR: 4.82e-04 | 1906.6ms/step | 31.5 samples/s
|
| 469 |
+
Step 6720/50462 | Loss: 2.8165 | LR: 4.82e-04 | 1906.8ms/step | 31.5 samples/s
|
| 470 |
+
Step 6730/50462 | Loss: 2.8232 | LR: 4.82e-04 | 1905.2ms/step | 31.5 samples/s
|
| 471 |
+
Step 6740/50462 | Loss: 2.8289 | LR: 4.82e-04 | 1905.4ms/step | 31.5 samples/s
|
| 472 |
+
Step 6750/50462 | Loss: 2.7961 | LR: 4.82e-04 | 1900.9ms/step | 31.6 samples/s
|
| 473 |
+
Validation at step 6750: Loss = 2.8233, Perplexity = 16.83
|
| 474 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-6750
|
| 475 |
+
Step 6760/50462 | Loss: 2.8261 | LR: 4.82e-04 | 3254.7ms/step | 18.4 samples/s
|
| 476 |
+
Step 6770/50462 | Loss: 2.8570 | LR: 4.82e-04 | 1900.5ms/step | 31.6 samples/s
|
| 477 |
+
Step 6780/50462 | Loss: 2.8591 | LR: 4.82e-04 | 1906.0ms/step | 31.5 samples/s
|
| 478 |
+
Step 6790/50462 | Loss: 2.8426 | LR: 4.82e-04 | 1906.5ms/step | 31.5 samples/s
|
| 479 |
+
Step 6800/50462 | Loss: 2.8364 | LR: 4.82e-04 | 1906.4ms/step | 31.5 samples/s
|
| 480 |
+
Step 6810/50462 | Loss: 2.8256 | LR: 4.82e-04 | 1903.6ms/step | 31.5 samples/s
|
| 481 |
+
Step 6820/50462 | Loss: 2.8438 | LR: 4.82e-04 | 1908.0ms/step | 31.4 samples/s
|
| 482 |
+
Step 6830/50462 | Loss: 2.7954 | LR: 4.82e-04 | 1903.8ms/step | 31.5 samples/s
|
| 483 |
+
Step 6840/50462 | Loss: 2.8461 | LR: 4.82e-04 | 1905.4ms/step | 31.5 samples/s
|
| 484 |
+
Step 6850/50462 | Loss: 2.8408 | LR: 4.82e-04 | 1909.6ms/step | 31.4 samples/s
|
| 485 |
+
Step 6860/50462 | Loss: 2.7894 | LR: 4.82e-04 | 1903.3ms/step | 31.5 samples/s
|
| 486 |
+
Step 6870/50462 | Loss: 2.8244 | LR: 4.82e-04 | 1907.6ms/step | 31.5 samples/s
|
| 487 |
+
Step 6880/50462 | Loss: 2.8271 | LR: 4.81e-04 | 1909.0ms/step | 31.4 samples/s
|
| 488 |
+
Step 6890/50462 | Loss: 2.8389 | LR: 4.81e-04 | 1909.9ms/step | 31.4 samples/s
|
| 489 |
+
Step 6900/50462 | Loss: 2.7838 | LR: 4.81e-04 | 1907.0ms/step | 31.5 samples/s
|
| 490 |
+
Step 6910/50462 | Loss: 2.8171 | LR: 4.81e-04 | 1906.0ms/step | 31.5 samples/s
|
| 491 |
+
Step 6920/50462 | Loss: 2.7901 | LR: 4.81e-04 | 1904.2ms/step | 31.5 samples/s
|
| 492 |
+
Step 6930/50462 | Loss: 2.8239 | LR: 4.81e-04 | 1908.2ms/step | 31.4 samples/s
|
| 493 |
+
Step 6940/50462 | Loss: 2.8258 | LR: 4.81e-04 | 1906.9ms/step | 31.5 samples/s
|
| 494 |
+
Step 6950/50462 | Loss: 2.8364 | LR: 4.81e-04 | 1908.5ms/step | 31.4 samples/s
|
| 495 |
+
Step 6960/50462 | Loss: 2.8124 | LR: 4.81e-04 | 1909.3ms/step | 31.4 samples/s
|
| 496 |
+
Step 6970/50462 | Loss: 2.8410 | LR: 4.81e-04 | 1905.9ms/step | 31.5 samples/s
|
| 497 |
+
Step 6980/50462 | Loss: 2.8550 | LR: 4.81e-04 | 1910.2ms/step | 31.4 samples/s
|
| 498 |
+
Step 6990/50462 | Loss: 2.8084 | LR: 4.81e-04 | 1905.5ms/step | 31.5 samples/s
|
| 499 |
+
Step 7000/50462 | Loss: 2.8774 | LR: 4.81e-04 | 1909.5ms/step | 31.4 samples/s
|
| 500 |
+
Validation at step 7000: Loss = 2.8141, Perplexity = 16.68
|
| 501 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-7000
|
| 502 |
+
Step 7010/50462 | Loss: 2.8189 | LR: 4.81e-04 | 3219.9ms/step | 18.6 samples/s
|
| 503 |
+
Step 7020/50462 | Loss: 2.8102 | LR: 4.81e-04 | 1902.1ms/step | 31.5 samples/s
|
| 504 |
+
Step 7030/50462 | Loss: 2.8494 | LR: 4.81e-04 | 1900.9ms/step | 31.6 samples/s
|
| 505 |
+
Step 7040/50462 | Loss: 2.8045 | LR: 4.81e-04 | 1903.8ms/step | 31.5 samples/s
|
| 506 |
+
Step 7050/50462 | Loss: 2.8540 | LR: 4.80e-04 | 1906.0ms/step | 31.5 samples/s
|
| 507 |
+
Step 7060/50462 | Loss: 2.7788 | LR: 4.80e-04 | 1902.8ms/step | 31.5 samples/s
|
| 508 |
+
Step 7070/50462 | Loss: 2.8222 | LR: 4.80e-04 | 1905.4ms/step | 31.5 samples/s
|
| 509 |
+
Step 7080/50462 | Loss: 2.7850 | LR: 4.80e-04 | 1907.0ms/step | 31.5 samples/s
|
| 510 |
+
Step 7090/50462 | Loss: 2.8190 | LR: 4.80e-04 | 1905.4ms/step | 31.5 samples/s
|
| 511 |
+
Step 7100/50462 | Loss: 2.7950 | LR: 4.80e-04 | 1904.8ms/step | 31.5 samples/s
|
| 512 |
+
Step 7110/50462 | Loss: 2.8443 | LR: 4.80e-04 | 1907.8ms/step | 31.5 samples/s
|
| 513 |
+
Step 7120/50462 | Loss: 2.8313 | LR: 4.80e-04 | 1911.1ms/step | 31.4 samples/s
|
| 514 |
+
Step 7130/50462 | Loss: 2.7977 | LR: 4.80e-04 | 1910.7ms/step | 31.4 samples/s
|
| 515 |
+
Step 7140/50462 | Loss: 2.7924 | LR: 4.80e-04 | 1911.0ms/step | 31.4 samples/s
|
| 516 |
+
Step 7150/50462 | Loss: 2.8165 | LR: 4.80e-04 | 1911.8ms/step | 31.4 samples/s
|
| 517 |
+
Step 7160/50462 | Loss: 2.8622 | LR: 4.80e-04 | 1910.9ms/step | 31.4 samples/s
|
| 518 |
+
Step 7170/50462 | Loss: 2.8303 | LR: 4.80e-04 | 1912.7ms/step | 31.4 samples/s
|
| 519 |
+
Step 7180/50462 | Loss: 2.7851 | LR: 4.80e-04 | 1910.2ms/step | 31.4 samples/s
|
| 520 |
+
Step 7190/50462 | Loss: 2.8107 | LR: 4.80e-04 | 1911.7ms/step | 31.4 samples/s
|
| 521 |
+
Step 7200/50462 | Loss: 2.8132 | LR: 4.80e-04 | 1908.7ms/step | 31.4 samples/s
|
| 522 |
+
Step 7210/50462 | Loss: 2.8482 | LR: 4.79e-04 | 1909.2ms/step | 31.4 samples/s
|
| 523 |
+
Step 7220/50462 | Loss: 2.8516 | LR: 4.79e-04 | 1913.3ms/step | 31.4 samples/s
|
| 524 |
+
Step 7230/50462 | Loss: 2.8126 | LR: 4.79e-04 | 1912.6ms/step | 31.4 samples/s
|
| 525 |
+
Step 7240/50462 | Loss: 2.7784 | LR: 4.79e-04 | 1915.3ms/step | 31.3 samples/s
|
| 526 |
+
Step 7250/50462 | Loss: 2.7772 | LR: 4.79e-04 | 1914.4ms/step | 31.3 samples/s
|
| 527 |
+
Validation at step 7250: Loss = 2.8065, Perplexity = 16.55
|
| 528 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-7250
|
| 529 |
+
Step 7260/50462 | Loss: 2.8282 | LR: 4.79e-04 | 3278.3ms/step | 18.3 samples/s
|
| 530 |
+
Step 7270/50462 | Loss: 2.8307 | LR: 4.79e-04 | 1900.5ms/step | 31.6 samples/s
|
| 531 |
+
Step 7280/50462 | Loss: 2.8252 | LR: 4.79e-04 | 1906.4ms/step | 31.5 samples/s
|
| 532 |
+
Step 7290/50462 | Loss: 2.8545 | LR: 4.79e-04 | 1906.9ms/step | 31.5 samples/s
|
| 533 |
+
Step 7300/50462 | Loss: 2.8470 | LR: 4.79e-04 | 1904.1ms/step | 31.5 samples/s
|
| 534 |
+
Step 7310/50462 | Loss: 2.8172 | LR: 4.79e-04 | 1907.3ms/step | 31.5 samples/s
|
| 535 |
+
Step 7320/50462 | Loss: 2.8118 | LR: 4.79e-04 | 1907.6ms/step | 31.5 samples/s
|
| 536 |
+
Step 7330/50462 | Loss: 2.7904 | LR: 4.79e-04 | 1909.1ms/step | 31.4 samples/s
|
| 537 |
+
Step 7340/50462 | Loss: 2.7968 | LR: 4.79e-04 | 1909.3ms/step | 31.4 samples/s
|
| 538 |
+
Step 7350/50462 | Loss: 2.7958 | LR: 4.79e-04 | 1912.4ms/step | 31.4 samples/s
|
| 539 |
+
Step 7360/50462 | Loss: 2.7975 | LR: 4.79e-04 | 1906.2ms/step | 31.5 samples/s
|
| 540 |
+
Step 7370/50462 | Loss: 2.8257 | LR: 4.78e-04 | 1903.6ms/step | 31.5 samples/s
|
| 541 |
+
Step 7380/50462 | Loss: 2.7949 | LR: 4.78e-04 | 1908.5ms/step | 31.4 samples/s
|
| 542 |
+
Step 7390/50462 | Loss: 2.7450 | LR: 4.78e-04 | 1908.2ms/step | 31.4 samples/s
|
| 543 |
+
Step 7400/50462 | Loss: 2.7792 | LR: 4.78e-04 | 1906.6ms/step | 31.5 samples/s
|
| 544 |
+
Step 7410/50462 | Loss: 2.8203 | LR: 4.78e-04 | 1906.2ms/step | 31.5 samples/s
|
| 545 |
+
Step 7420/50462 | Loss: 2.8639 | LR: 4.78e-04 | 1905.0ms/step | 31.5 samples/s
|
| 546 |
+
Step 7430/50462 | Loss: 2.7685 | LR: 4.78e-04 | 1901.8ms/step | 31.5 samples/s
|
| 547 |
+
Step 7440/50462 | Loss: 2.8128 | LR: 4.78e-04 | 1907.5ms/step | 31.5 samples/s
|
| 548 |
+
Step 7450/50462 | Loss: 2.8177 | LR: 4.78e-04 | 1909.6ms/step | 31.4 samples/s
|
| 549 |
+
Step 7460/50462 | Loss: 2.7821 | LR: 4.78e-04 | 1912.1ms/step | 31.4 samples/s
|
| 550 |
+
Step 7470/50462 | Loss: 2.8533 | LR: 4.78e-04 | 1906.2ms/step | 31.5 samples/s
|
| 551 |
+
Step 7480/50462 | Loss: 2.8392 | LR: 4.78e-04 | 1909.4ms/step | 31.4 samples/s
|
| 552 |
+
Step 7490/50462 | Loss: 2.8163 | LR: 4.78e-04 | 1908.2ms/step | 31.4 samples/s
|
| 553 |
+
Step 7500/50462 | Loss: 2.8200 | LR: 4.78e-04 | 1910.2ms/step | 31.4 samples/s
|
| 554 |
+
Validation at step 7500: Loss = 2.7980, Perplexity = 16.41
|
| 555 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-7500
|
| 556 |
+
Step 7510/50462 | Loss: 2.8070 | LR: 4.78e-04 | 3256.8ms/step | 18.4 samples/s
|
| 557 |
+
Step 7520/50462 | Loss: 2.7929 | LR: 4.77e-04 | 1902.7ms/step | 31.5 samples/s
|
| 558 |
+
Step 7530/50462 | Loss: 2.8381 | LR: 4.77e-04 | 1901.7ms/step | 31.6 samples/s
|
| 559 |
+
Step 7540/50462 | Loss: 2.7603 | LR: 4.77e-04 | 1903.7ms/step | 31.5 samples/s
|
| 560 |
+
Step 7550/50462 | Loss: 2.7977 | LR: 4.77e-04 | 1904.2ms/step | 31.5 samples/s
|
| 561 |
+
Step 7560/50462 | Loss: 2.7890 | LR: 4.77e-04 | 1900.6ms/step | 31.6 samples/s
|
| 562 |
+
Step 7570/50462 | Loss: 2.8085 | LR: 4.77e-04 | 1904.6ms/step | 31.5 samples/s
|
| 563 |
+
Step 7580/50462 | Loss: 2.8050 | LR: 4.77e-04 | 1904.3ms/step | 31.5 samples/s
|
| 564 |
+
Step 7590/50462 | Loss: 2.7736 | LR: 4.77e-04 | 1906.3ms/step | 31.5 samples/s
|
| 565 |
+
Step 7600/50462 | Loss: 2.7648 | LR: 4.77e-04 | 1903.0ms/step | 31.5 samples/s
|
| 566 |
+
Step 7610/50462 | Loss: 2.8151 | LR: 4.77e-04 | 1905.6ms/step | 31.5 samples/s
|
| 567 |
+
Step 7620/50462 | Loss: 2.7439 | LR: 4.77e-04 | 1907.2ms/step | 31.5 samples/s
|
| 568 |
+
Step 7630/50462 | Loss: 2.8161 | LR: 4.77e-04 | 1908.2ms/step | 31.4 samples/s
|
| 569 |
+
Step 7640/50462 | Loss: 2.7606 | LR: 4.77e-04 | 1909.7ms/step | 31.4 samples/s
|
| 570 |
+
Step 7650/50462 | Loss: 2.8095 | LR: 4.77e-04 | 1907.7ms/step | 31.5 samples/s
|
| 571 |
+
Step 7660/50462 | Loss: 2.8400 | LR: 4.77e-04 | 1904.9ms/step | 31.5 samples/s
|
| 572 |
+
Step 7670/50462 | Loss: 2.7708 | LR: 4.76e-04 | 1901.8ms/step | 31.5 samples/s
|
| 573 |
+
Step 7680/50462 | Loss: 2.8090 | LR: 4.76e-04 | 1906.6ms/step | 31.5 samples/s
|
| 574 |
+
Step 7690/50462 | Loss: 2.7726 | LR: 4.76e-04 | 1907.1ms/step | 31.5 samples/s
|
| 575 |
+
Step 7700/50462 | Loss: 2.7793 | LR: 4.76e-04 | 1910.8ms/step | 31.4 samples/s
|
| 576 |
+
Step 7710/50462 | Loss: 2.7923 | LR: 4.76e-04 | 1912.4ms/step | 31.4 samples/s
|
| 577 |
+
Step 7720/50462 | Loss: 2.8010 | LR: 4.76e-04 | 1910.4ms/step | 31.4 samples/s
|
| 578 |
+
Step 7730/50462 | Loss: 2.8040 | LR: 4.76e-04 | 1912.8ms/step | 31.4 samples/s
|
| 579 |
+
Step 7740/50462 | Loss: 2.8156 | LR: 4.76e-04 | 1911.1ms/step | 31.4 samples/s
|
| 580 |
+
Step 7750/50462 | Loss: 2.7804 | LR: 4.76e-04 | 1907.0ms/step | 31.5 samples/s
|
| 581 |
+
Validation at step 7750: Loss = 2.7913, Perplexity = 16.30
|
| 582 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-7750
|
| 583 |
+
Step 7760/50462 | Loss: 2.8072 | LR: 4.76e-04 | 3265.1ms/step | 18.4 samples/s
|
| 584 |
+
Step 7770/50462 | Loss: 2.7764 | LR: 4.76e-04 | 1900.2ms/step | 31.6 samples/s
|
| 585 |
+
Step 7780/50462 | Loss: 2.8189 | LR: 4.76e-04 | 1903.2ms/step | 31.5 samples/s
|
| 586 |
+
Step 7790/50462 | Loss: 2.7683 | LR: 4.76e-04 | 1908.7ms/step | 31.4 samples/s
|
| 587 |
+
Step 7800/50462 | Loss: 2.8349 | LR: 4.76e-04 | 1907.7ms/step | 31.5 samples/s
|
| 588 |
+
Step 7810/50462 | Loss: 2.8447 | LR: 4.76e-04 | 1909.4ms/step | 31.4 samples/s
|
| 589 |
+
Step 7820/50462 | Loss: 2.7702 | LR: 4.75e-04 | 1909.0ms/step | 31.4 samples/s
|
| 590 |
+
Step 7830/50462 | Loss: 2.7745 | LR: 4.75e-04 | 1906.3ms/step | 31.5 samples/s
|
| 591 |
+
Step 7840/50462 | Loss: 2.8135 | LR: 4.75e-04 | 1912.6ms/step | 31.4 samples/s
|
| 592 |
+
Step 7850/50462 | Loss: 2.7991 | LR: 4.75e-04 | 1910.0ms/step | 31.4 samples/s
|
| 593 |
+
Step 7860/50462 | Loss: 2.7878 | LR: 4.75e-04 | 1909.5ms/step | 31.4 samples/s
|
| 594 |
+
Step 7870/50462 | Loss: 2.7853 | LR: 4.75e-04 | 1908.5ms/step | 31.4 samples/s
|
| 595 |
+
Step 7880/50462 | Loss: 2.8056 | LR: 4.75e-04 | 1911.2ms/step | 31.4 samples/s
|
| 596 |
+
Step 7890/50462 | Loss: 2.8050 | LR: 4.75e-04 | 1911.8ms/step | 31.4 samples/s
|
| 597 |
+
Step 7900/50462 | Loss: 2.7589 | LR: 4.75e-04 | 1911.5ms/step | 31.4 samples/s
|
| 598 |
+
Step 7910/50462 | Loss: 2.8438 | LR: 4.75e-04 | 1908.3ms/step | 31.4 samples/s
|
| 599 |
+
Step 7920/50462 | Loss: 2.7026 | LR: 4.75e-04 | 1912.2ms/step | 31.4 samples/s
|
| 600 |
+
Step 7930/50462 | Loss: 2.7848 | LR: 4.75e-04 | 1911.0ms/step | 31.4 samples/s
|
| 601 |
+
Step 7940/50462 | Loss: 2.8149 | LR: 4.75e-04 | 1907.1ms/step | 31.5 samples/s
|
| 602 |
+
Step 7950/50462 | Loss: 2.8003 | LR: 4.75e-04 | 1907.7ms/step | 31.5 samples/s
|
| 603 |
+
Step 7960/50462 | Loss: 2.8151 | LR: 4.74e-04 | 1908.3ms/step | 31.4 samples/s
|
| 604 |
+
Step 7970/50462 | Loss: 2.7871 | LR: 4.74e-04 | 1904.3ms/step | 31.5 samples/s
|
| 605 |
+
Step 7980/50462 | Loss: 2.7570 | LR: 4.74e-04 | 1908.8ms/step | 31.4 samples/s
|
| 606 |
+
Step 7990/50462 | Loss: 2.7815 | LR: 4.74e-04 | 1910.7ms/step | 31.4 samples/s
|
| 607 |
+
Step 8000/50462 | Loss: 2.7966 | LR: 4.74e-04 | 1908.1ms/step | 31.4 samples/s
|
| 608 |
+
Validation at step 8000: Loss = 2.7848, Perplexity = 16.20
|
| 609 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-8000
|
| 610 |
+
Step 8010/50462 | Loss: 2.8172 | LR: 4.74e-04 | 3267.2ms/step | 18.4 samples/s
|
| 611 |
+
Step 8020/50462 | Loss: 2.7287 | LR: 4.74e-04 | 1898.5ms/step | 31.6 samples/s
|
| 612 |
+
Step 8030/50462 | Loss: 2.8033 | LR: 4.74e-04 | 1900.5ms/step | 31.6 samples/s
|
| 613 |
+
Step 8040/50462 | Loss: 2.7560 | LR: 4.74e-04 | 1905.9ms/step | 31.5 samples/s
|
| 614 |
+
Step 8050/50462 | Loss: 2.8055 | LR: 4.74e-04 | 1905.6ms/step | 31.5 samples/s
|
| 615 |
+
Step 8060/50462 | Loss: 2.8483 | LR: 4.74e-04 | 1906.2ms/step | 31.5 samples/s
|
| 616 |
+
Step 8070/50462 | Loss: 2.8000 | LR: 4.74e-04 | 1908.8ms/step | 31.4 samples/s
|
| 617 |
+
Step 8080/50462 | Loss: 2.8090 | LR: 4.74e-04 | 1906.4ms/step | 31.5 samples/s
|
| 618 |
+
Step 8090/50462 | Loss: 2.8273 | LR: 4.74e-04 | 1905.4ms/step | 31.5 samples/s
|
| 619 |
+
Step 8100/50462 | Loss: 2.7630 | LR: 4.74e-04 | 1908.8ms/step | 31.4 samples/s
|
| 620 |
+
Step 8110/50462 | Loss: 2.8195 | LR: 4.73e-04 | 1906.0ms/step | 31.5 samples/s
|
| 621 |
+
Step 8120/50462 | Loss: 2.8095 | LR: 4.73e-04 | 1903.8ms/step | 31.5 samples/s
|
| 622 |
+
Step 8130/50462 | Loss: 2.7818 | LR: 4.73e-04 | 1907.0ms/step | 31.5 samples/s
|
| 623 |
+
Step 8140/50462 | Loss: 2.7952 | LR: 4.73e-04 | 1911.1ms/step | 31.4 samples/s
|
| 624 |
+
Step 8150/50462 | Loss: 2.8425 | LR: 4.73e-04 | 1904.3ms/step | 31.5 samples/s
|
| 625 |
+
Step 8160/50462 | Loss: 2.7912 | LR: 4.73e-04 | 1907.5ms/step | 31.5 samples/s
|
| 626 |
+
Step 8170/50462 | Loss: 2.7879 | LR: 4.73e-04 | 1908.9ms/step | 31.4 samples/s
|
| 627 |
+
Step 8180/50462 | Loss: 2.7429 | LR: 4.73e-04 | 1908.9ms/step | 31.4 samples/s
|
| 628 |
+
Step 8190/50462 | Loss: 2.7735 | LR: 4.73e-04 | 1906.0ms/step | 31.5 samples/s
|
| 629 |
+
Step 8200/50462 | Loss: 2.7673 | LR: 4.73e-04 | 1907.7ms/step | 31.5 samples/s
|
| 630 |
+
Step 8210/50462 | Loss: 2.7643 | LR: 4.73e-04 | 1909.3ms/step | 31.4 samples/s
|
| 631 |
+
Step 8220/50462 | Loss: 2.7362 | LR: 4.73e-04 | 1908.1ms/step | 31.4 samples/s
|
| 632 |
+
Step 8230/50462 | Loss: 2.8094 | LR: 4.73e-04 | 1905.2ms/step | 31.5 samples/s
|
| 633 |
+
Step 8240/50462 | Loss: 2.7879 | LR: 4.73e-04 | 1906.5ms/step | 31.5 samples/s
|
| 634 |
+
Step 8250/50462 | Loss: 2.8130 | LR: 4.72e-04 | 1905.0ms/step | 31.5 samples/s
|
| 635 |
+
Validation at step 8250: Loss = 2.7772, Perplexity = 16.07
|
| 636 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-8250
|
| 637 |
+
Step 8260/50462 | Loss: 2.7920 | LR: 4.72e-04 | 3211.9ms/step | 18.7 samples/s
|
| 638 |
+
Step 8270/50462 | Loss: 2.7824 | LR: 4.72e-04 | 1896.9ms/step | 31.6 samples/s
|
| 639 |
+
Step 8280/50462 | Loss: 2.7716 | LR: 4.72e-04 | 1901.7ms/step | 31.6 samples/s
|
| 640 |
+
Step 8290/50462 | Loss: 2.7952 | LR: 4.72e-04 | 1902.1ms/step | 31.5 samples/s
|
| 641 |
+
Step 8300/50462 | Loss: 2.8156 | LR: 4.72e-04 | 1906.2ms/step | 31.5 samples/s
|
| 642 |
+
Step 8310/50462 | Loss: 2.7694 | LR: 4.72e-04 | 1907.1ms/step | 31.5 samples/s
|
| 643 |
+
Step 8320/50462 | Loss: 2.7838 | LR: 4.72e-04 | 1906.9ms/step | 31.5 samples/s
|
| 644 |
+
Step 8330/50462 | Loss: 2.7440 | LR: 4.72e-04 | 1903.4ms/step | 31.5 samples/s
|
| 645 |
+
Step 8340/50462 | Loss: 2.7836 | LR: 4.72e-04 | 1908.0ms/step | 31.4 samples/s
|
| 646 |
+
Step 8350/50462 | Loss: 2.7781 | LR: 4.72e-04 | 1906.8ms/step | 31.5 samples/s
|
| 647 |
+
Step 8360/50462 | Loss: 2.7844 | LR: 4.72e-04 | 1908.3ms/step | 31.4 samples/s
|
| 648 |
+
Step 8370/50462 | Loss: 2.7601 | LR: 4.72e-04 | 1908.8ms/step | 31.4 samples/s
|
| 649 |
+
Step 8380/50462 | Loss: 2.7807 | LR: 4.71e-04 | 1909.8ms/step | 31.4 samples/s
|
| 650 |
+
Step 8390/50462 | Loss: 2.7687 | LR: 4.71e-04 | 1905.2ms/step | 31.5 samples/s
|
| 651 |
+
Step 8400/50462 | Loss: 2.7875 | LR: 4.71e-04 | 1903.0ms/step | 31.5 samples/s
|
| 652 |
+
Step 8410/50462 | Loss: 2.7714 | LR: 4.71e-04 | 1906.8ms/step | 31.5 samples/s
|
| 653 |
+
Step 8420/50462 | Loss: 2.7874 | LR: 4.71e-04 | 1909.1ms/step | 31.4 samples/s
|
| 654 |
+
Step 8430/50462 | Loss: 2.7697 | LR: 4.71e-04 | 1904.5ms/step | 31.5 samples/s
|
| 655 |
+
Step 8440/50462 | Loss: 2.7614 | LR: 4.71e-04 | 1906.0ms/step | 31.5 samples/s
|
| 656 |
+
Step 8450/50462 | Loss: 2.7776 | LR: 4.71e-04 | 1908.4ms/step | 31.4 samples/s
|
| 657 |
+
Step 8460/50462 | Loss: 2.7881 | LR: 4.71e-04 | 1907.1ms/step | 31.5 samples/s
|
| 658 |
+
Step 8470/50462 | Loss: 2.7652 | LR: 4.71e-04 | 1904.7ms/step | 31.5 samples/s
|
| 659 |
+
Step 8480/50462 | Loss: 2.8250 | LR: 4.71e-04 | 1906.9ms/step | 31.5 samples/s
|
| 660 |
+
Step 8490/50462 | Loss: 2.7584 | LR: 4.71e-04 | 1909.3ms/step | 31.4 samples/s
|
| 661 |
+
Step 8500/50462 | Loss: 2.7796 | LR: 4.71e-04 | 1908.7ms/step | 31.4 samples/s
|
| 662 |
+
Validation at step 8500: Loss = 2.7709, Perplexity = 15.97
|
| 663 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-8500
|
| 664 |
+
Step 8510/50462 | Loss: 2.7559 | LR: 4.71e-04 | 3247.3ms/step | 18.5 samples/s
|
| 665 |
+
Step 8520/50462 | Loss: 2.7829 | LR: 4.70e-04 | 1897.5ms/step | 31.6 samples/s
|
| 666 |
+
Step 8530/50462 | Loss: 2.7819 | LR: 4.70e-04 | 1904.8ms/step | 31.5 samples/s
|
| 667 |
+
Step 8540/50462 | Loss: 2.7773 | LR: 4.70e-04 | 1905.9ms/step | 31.5 samples/s
|
| 668 |
+
Step 8550/50462 | Loss: 2.7987 | LR: 4.70e-04 | 1908.1ms/step | 31.4 samples/s
|
| 669 |
+
Step 8560/50462 | Loss: 2.7371 | LR: 4.70e-04 | 1907.9ms/step | 31.4 samples/s
|
| 670 |
+
Step 8570/50462 | Loss: 2.7930 | LR: 4.70e-04 | 1908.8ms/step | 31.4 samples/s
|
| 671 |
+
Step 8580/50462 | Loss: 2.8013 | LR: 4.70e-04 | 1909.2ms/step | 31.4 samples/s
|
| 672 |
+
Step 8590/50462 | Loss: 2.7697 | LR: 4.70e-04 | 1911.6ms/step | 31.4 samples/s
|
| 673 |
+
Step 8600/50462 | Loss: 2.7281 | LR: 4.70e-04 | 1911.1ms/step | 31.4 samples/s
|
| 674 |
+
Step 8610/50462 | Loss: 2.7604 | LR: 4.70e-04 | 1912.1ms/step | 31.4 samples/s
|
| 675 |
+
Step 8620/50462 | Loss: 2.7681 | LR: 4.70e-04 | 1911.0ms/step | 31.4 samples/s
|
| 676 |
+
Step 8630/50462 | Loss: 2.7910 | LR: 4.70e-04 | 1909.8ms/step | 31.4 samples/s
|
| 677 |
+
Step 8640/50462 | Loss: 2.7787 | LR: 4.70e-04 | 1910.0ms/step | 31.4 samples/s
|
| 678 |
+
Step 8650/50462 | Loss: 2.7774 | LR: 4.69e-04 | 1912.0ms/step | 31.4 samples/s
|
| 679 |
+
Step 8660/50462 | Loss: 2.7582 | LR: 4.69e-04 | 1909.7ms/step | 31.4 samples/s
|
| 680 |
+
Step 8670/50462 | Loss: 2.7834 | LR: 4.69e-04 | 1907.9ms/step | 31.4 samples/s
|
| 681 |
+
Step 8680/50462 | Loss: 2.7608 | LR: 4.69e-04 | 1910.2ms/step | 31.4 samples/s
|
| 682 |
+
Step 8690/50462 | Loss: 2.7856 | LR: 4.69e-04 | 1908.4ms/step | 31.4 samples/s
|
| 683 |
+
Step 8700/50462 | Loss: 2.7630 | LR: 4.69e-04 | 1912.0ms/step | 31.4 samples/s
|
| 684 |
+
Step 8710/50462 | Loss: 2.7843 | LR: 4.69e-04 | 1909.8ms/step | 31.4 samples/s
|
| 685 |
+
Step 8720/50462 | Loss: 2.7488 | LR: 4.69e-04 | 1910.7ms/step | 31.4 samples/s
|
| 686 |
+
Step 8730/50462 | Loss: 2.8155 | LR: 4.69e-04 | 1910.2ms/step | 31.4 samples/s
|
| 687 |
+
Step 8740/50462 | Loss: 2.7242 | LR: 4.69e-04 | 1907.7ms/step | 31.5 samples/s
|
| 688 |
+
Step 8750/50462 | Loss: 2.7342 | LR: 4.69e-04 | 1906.2ms/step | 31.5 samples/s
|
| 689 |
+
Validation at step 8750: Loss = 2.7651, Perplexity = 15.88
|
| 690 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-8750
|
| 691 |
+
Step 8760/50462 | Loss: 2.7935 | LR: 4.69e-04 | 3312.4ms/step | 18.1 samples/s
|
| 692 |
+
Step 8770/50462 | Loss: 2.7650 | LR: 4.69e-04 | 1897.3ms/step | 31.6 samples/s
|
| 693 |
+
Step 8780/50462 | Loss: 2.7697 | LR: 4.68e-04 | 1903.5ms/step | 31.5 samples/s
|
| 694 |
+
Step 8790/50462 | Loss: 2.7561 | LR: 4.68e-04 | 1904.7ms/step | 31.5 samples/s
|
| 695 |
+
Step 8800/50462 | Loss: 2.7681 | LR: 4.68e-04 | 1905.0ms/step | 31.5 samples/s
|
| 696 |
+
Step 8810/50462 | Loss: 2.8030 | LR: 4.68e-04 | 1909.0ms/step | 31.4 samples/s
|
| 697 |
+
Step 8820/50462 | Loss: 2.7834 | LR: 4.68e-04 | 1904.3ms/step | 31.5 samples/s
|
| 698 |
+
Step 8830/50462 | Loss: 2.7452 | LR: 4.68e-04 | 1908.4ms/step | 31.4 samples/s
|
| 699 |
+
Step 8840/50462 | Loss: 2.7517 | LR: 4.68e-04 | 1908.1ms/step | 31.4 samples/s
|
| 700 |
+
Step 8850/50462 | Loss: 2.7856 | LR: 4.68e-04 | 1908.2ms/step | 31.4 samples/s
|
| 701 |
+
Step 8860/50462 | Loss: 2.7755 | LR: 4.68e-04 | 1909.0ms/step | 31.4 samples/s
|
| 702 |
+
Step 8870/50462 | Loss: 2.7911 | LR: 4.68e-04 | 1908.7ms/step | 31.4 samples/s
|
| 703 |
+
Step 8880/50462 | Loss: 2.7225 | LR: 4.68e-04 | 1909.5ms/step | 31.4 samples/s
|
| 704 |
+
Step 8890/50462 | Loss: 2.7243 | LR: 4.68e-04 | 1905.5ms/step | 31.5 samples/s
|
| 705 |
+
Step 8900/50462 | Loss: 2.8048 | LR: 4.68e-04 | 1904.8ms/step | 31.5 samples/s
|
| 706 |
+
Step 8910/50462 | Loss: 2.7581 | LR: 4.67e-04 | 1909.3ms/step | 31.4 samples/s
|
| 707 |
+
Step 8920/50462 | Loss: 2.7356 | LR: 4.67e-04 | 1908.1ms/step | 31.4 samples/s
|
| 708 |
+
Step 8930/50462 | Loss: 2.8099 | LR: 4.67e-04 | 1908.5ms/step | 31.4 samples/s
|
| 709 |
+
Step 8940/50462 | Loss: 2.8052 | LR: 4.67e-04 | 1911.2ms/step | 31.4 samples/s
|
| 710 |
+
Step 8950/50462 | Loss: 2.8184 | LR: 4.67e-04 | 1907.1ms/step | 31.5 samples/s
|
| 711 |
+
Step 8960/50462 | Loss: 2.7392 | LR: 4.67e-04 | 1903.6ms/step | 31.5 samples/s
|
| 712 |
+
Step 8970/50462 | Loss: 2.7744 | LR: 4.67e-04 | 1908.2ms/step | 31.4 samples/s
|
| 713 |
+
Step 8980/50462 | Loss: 2.7717 | LR: 4.67e-04 | 1908.0ms/step | 31.4 samples/s
|
| 714 |
+
Step 8990/50462 | Loss: 2.7695 | LR: 4.67e-04 | 1906.0ms/step | 31.5 samples/s
|
| 715 |
+
Step 9000/50462 | Loss: 2.7498 | LR: 4.67e-04 | 1897.7ms/step | 31.6 samples/s
|
| 716 |
+
Validation at step 9000: Loss = 2.7594, Perplexity = 15.79
|
| 717 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-9000
|
| 718 |
+
Step 9010/50462 | Loss: 2.7941 | LR: 4.67e-04 | 3237.6ms/step | 18.5 samples/s
|
| 719 |
+
Step 9020/50462 | Loss: 2.7457 | LR: 4.67e-04 | 1896.4ms/step | 31.6 samples/s
|
| 720 |
+
Step 9030/50462 | Loss: 2.7852 | LR: 4.67e-04 | 1899.7ms/step | 31.6 samples/s
|
| 721 |
+
Step 9040/50462 | Loss: 2.8126 | LR: 4.66e-04 | 1905.5ms/step | 31.5 samples/s
|
| 722 |
+
Step 9050/50462 | Loss: 2.7836 | LR: 4.66e-04 | 1908.4ms/step | 31.4 samples/s
|
| 723 |
+
Step 9060/50462 | Loss: 2.7653 | LR: 4.66e-04 | 1904.2ms/step | 31.5 samples/s
|
| 724 |
+
Step 9070/50462 | Loss: 2.7706 | LR: 4.66e-04 | 1907.3ms/step | 31.5 samples/s
|
| 725 |
+
Step 9080/50462 | Loss: 2.7038 | LR: 4.66e-04 | 1907.7ms/step | 31.5 samples/s
|
| 726 |
+
Step 9090/50462 | Loss: 2.7580 | LR: 4.66e-04 | 1911.7ms/step | 31.4 samples/s
|
| 727 |
+
Step 9100/50462 | Loss: 2.8208 | LR: 4.66e-04 | 1905.4ms/step | 31.5 samples/s
|
| 728 |
+
Step 9110/50462 | Loss: 2.7382 | LR: 4.66e-04 | 1907.7ms/step | 31.5 samples/s
|
| 729 |
+
Step 9120/50462 | Loss: 2.7561 | LR: 4.66e-04 | 1908.3ms/step | 31.4 samples/s
|
| 730 |
+
Step 9130/50462 | Loss: 2.7467 | LR: 4.66e-04 | 1906.2ms/step | 31.5 samples/s
|
| 731 |
+
Step 9140/50462 | Loss: 2.7317 | LR: 4.66e-04 | 1904.2ms/step | 31.5 samples/s
|
| 732 |
+
Step 9150/50462 | Loss: 2.7442 | LR: 4.66e-04 | 1908.6ms/step | 31.4 samples/s
|
| 733 |
+
Step 9160/50462 | Loss: 2.7676 | LR: 4.66e-04 | 1908.5ms/step | 31.4 samples/s
|
| 734 |
+
Step 9170/50462 | Loss: 2.7584 | LR: 4.65e-04 | 1907.6ms/step | 31.5 samples/s
|
| 735 |
+
Step 9180/50462 | Loss: 2.7204 | LR: 4.65e-04 | 1909.1ms/step | 31.4 samples/s
|
| 736 |
+
Step 9190/50462 | Loss: 2.7923 | LR: 4.65e-04 | 1909.0ms/step | 31.4 samples/s
|
| 737 |
+
Step 9200/50462 | Loss: 2.7260 | LR: 4.65e-04 | 1898.1ms/step | 31.6 samples/s
|
| 738 |
+
Step 9210/50462 | Loss: 2.7434 | LR: 4.65e-04 | 1899.9ms/step | 31.6 samples/s
|
| 739 |
+
Step 9220/50462 | Loss: 2.7510 | LR: 4.65e-04 | 1903.8ms/step | 31.5 samples/s
|
| 740 |
+
Step 9230/50462 | Loss: 2.7771 | LR: 4.65e-04 | 1903.7ms/step | 31.5 samples/s
|
| 741 |
+
Step 9240/50462 | Loss: 2.7887 | LR: 4.65e-04 | 1906.4ms/step | 31.5 samples/s
|
| 742 |
+
Step 9250/50462 | Loss: 2.7800 | LR: 4.65e-04 | 1905.7ms/step | 31.5 samples/s
|
| 743 |
+
Validation at step 9250: Loss = 2.7523, Perplexity = 15.68
|
| 744 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-9250
|
| 745 |
+
Step 9260/50462 | Loss: 2.7292 | LR: 4.65e-04 | 3254.5ms/step | 18.4 samples/s
|
| 746 |
+
Step 9270/50462 | Loss: 2.7619 | LR: 4.65e-04 | 1903.0ms/step | 31.5 samples/s
|
| 747 |
+
Step 9280/50462 | Loss: 2.7798 | LR: 4.65e-04 | 1907.1ms/step | 31.5 samples/s
|
| 748 |
+
Step 9290/50462 | Loss: 2.7142 | LR: 4.64e-04 | 1907.7ms/step | 31.5 samples/s
|
| 749 |
+
Step 9300/50462 | Loss: 2.7110 | LR: 4.64e-04 | 1908.3ms/step | 31.4 samples/s
|
| 750 |
+
Step 9310/50462 | Loss: 2.7181 | LR: 4.64e-04 | 1906.0ms/step | 31.5 samples/s
|
| 751 |
+
Step 9320/50462 | Loss: 2.7427 | LR: 4.64e-04 | 1906.7ms/step | 31.5 samples/s
|
| 752 |
+
Step 9330/50462 | Loss: 2.7284 | LR: 4.64e-04 | 1908.7ms/step | 31.4 samples/s
|
| 753 |
+
Step 9340/50462 | Loss: 2.7578 | LR: 4.64e-04 | 1910.6ms/step | 31.4 samples/s
|
| 754 |
+
Step 9350/50462 | Loss: 2.7552 | LR: 4.64e-04 | 1899.7ms/step | 31.6 samples/s
|
| 755 |
+
Step 9360/50462 | Loss: 2.7633 | LR: 4.64e-04 | 1902.9ms/step | 31.5 samples/s
|
| 756 |
+
Step 9370/50462 | Loss: 2.7613 | LR: 4.64e-04 | 1907.7ms/step | 31.5 samples/s
|
| 757 |
+
Step 9380/50462 | Loss: 2.7546 | LR: 4.64e-04 | 1904.9ms/step | 31.5 samples/s
|
| 758 |
+
Step 9390/50462 | Loss: 2.7513 | LR: 4.64e-04 | 1905.1ms/step | 31.5 samples/s
|
| 759 |
+
Step 9400/50462 | Loss: 2.7956 | LR: 4.64e-04 | 1905.4ms/step | 31.5 samples/s
|
| 760 |
+
Step 9410/50462 | Loss: 2.7203 | LR: 4.63e-04 | 1905.6ms/step | 31.5 samples/s
|
| 761 |
+
Step 9420/50462 | Loss: 2.7115 | LR: 4.63e-04 | 1906.3ms/step | 31.5 samples/s
|
| 762 |
+
Step 9430/50462 | Loss: 2.8152 | LR: 4.63e-04 | 1906.7ms/step | 31.5 samples/s
|
| 763 |
+
Step 9440/50462 | Loss: 2.7316 | LR: 4.63e-04 | 1904.1ms/step | 31.5 samples/s
|
| 764 |
+
Step 9450/50462 | Loss: 2.7429 | LR: 4.63e-04 | 1905.3ms/step | 31.5 samples/s
|
| 765 |
+
Step 9460/50462 | Loss: 2.7250 | LR: 4.63e-04 | 1905.5ms/step | 31.5 samples/s
|
| 766 |
+
Step 9470/50462 | Loss: 2.7973 | LR: 4.63e-04 | 1906.4ms/step | 31.5 samples/s
|
| 767 |
+
Step 9480/50462 | Loss: 2.7494 | LR: 4.63e-04 | 1906.0ms/step | 31.5 samples/s
|
| 768 |
+
Step 9490/50462 | Loss: 2.7590 | LR: 4.63e-04 | 1904.6ms/step | 31.5 samples/s
|
| 769 |
+
Step 9500/50462 | Loss: 2.7360 | LR: 4.63e-04 | 1907.0ms/step | 31.5 samples/s
|
| 770 |
+
Validation at step 9500: Loss = 2.7482, Perplexity = 15.61
|
| 771 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-9500
|
| 772 |
+
Step 9510/50462 | Loss: 2.7350 | LR: 4.63e-04 | 3264.1ms/step | 18.4 samples/s
|
| 773 |
+
Step 9520/50462 | Loss: 2.8165 | LR: 4.63e-04 | 1900.9ms/step | 31.6 samples/s
|
| 774 |
+
Step 9530/50462 | Loss: 2.7482 | LR: 4.62e-04 | 1904.4ms/step | 31.5 samples/s
|
| 775 |
+
Step 9540/50462 | Loss: 2.7788 | LR: 4.62e-04 | 1906.1ms/step | 31.5 samples/s
|
| 776 |
+
Step 9550/50462 | Loss: 2.7755 | LR: 4.62e-04 | 1907.0ms/step | 31.5 samples/s
|
| 777 |
+
Step 9560/50462 | Loss: 2.7346 | LR: 4.62e-04 | 1906.0ms/step | 31.5 samples/s
|
| 778 |
+
Step 9570/50462 | Loss: 2.7558 | LR: 4.62e-04 | 1906.1ms/step | 31.5 samples/s
|
| 779 |
+
Step 9580/50462 | Loss: 2.7353 | LR: 4.62e-04 | 1906.4ms/step | 31.5 samples/s
|
| 780 |
+
Step 9590/50462 | Loss: 2.7506 | LR: 4.62e-04 | 1905.7ms/step | 31.5 samples/s
|
| 781 |
+
Step 9600/50462 | Loss: 2.7669 | LR: 4.62e-04 | 1906.6ms/step | 31.5 samples/s
|
| 782 |
+
Step 9610/50462 | Loss: 2.7469 | LR: 4.62e-04 | 1905.2ms/step | 31.5 samples/s
|
| 783 |
+
Step 9620/50462 | Loss: 2.7973 | LR: 4.62e-04 | 1905.9ms/step | 31.5 samples/s
|
| 784 |
+
Step 9630/50462 | Loss: 2.7646 | LR: 4.62e-04 | 1906.8ms/step | 31.5 samples/s
|
| 785 |
+
Step 9640/50462 | Loss: 2.7764 | LR: 4.62e-04 | 1906.5ms/step | 31.5 samples/s
|
| 786 |
+
Step 9650/50462 | Loss: 2.7172 | LR: 4.61e-04 | 1906.5ms/step | 31.5 samples/s
|
| 787 |
+
Step 9660/50462 | Loss: 2.7661 | LR: 4.61e-04 | 1909.2ms/step | 31.4 samples/s
|
| 788 |
+
Step 9670/50462 | Loss: 2.7551 | LR: 4.61e-04 | 1907.1ms/step | 31.5 samples/s
|
| 789 |
+
Step 9680/50462 | Loss: 2.7623 | LR: 4.61e-04 | 1908.5ms/step | 31.4 samples/s
|
| 790 |
+
Step 9690/50462 | Loss: 2.7207 | LR: 4.61e-04 | 1907.9ms/step | 31.4 samples/s
|
| 791 |
+
Step 9700/50462 | Loss: 2.7638 | LR: 4.61e-04 | 1908.9ms/step | 31.4 samples/s
|
| 792 |
+
Step 9710/50462 | Loss: 2.7696 | LR: 4.61e-04 | 1907.2ms/step | 31.5 samples/s
|
| 793 |
+
Step 9720/50462 | Loss: 2.7589 | LR: 4.61e-04 | 1907.6ms/step | 31.5 samples/s
|
| 794 |
+
Step 9730/50462 | Loss: 2.7538 | LR: 4.61e-04 | 1908.9ms/step | 31.4 samples/s
|
| 795 |
+
Step 9740/50462 | Loss: 2.7486 | LR: 4.61e-04 | 1909.6ms/step | 31.4 samples/s
|
| 796 |
+
Step 9750/50462 | Loss: 2.7164 | LR: 4.61e-04 | 1907.8ms/step | 31.4 samples/s
|
| 797 |
+
Validation at step 9750: Loss = 2.7434, Perplexity = 15.54
|
| 798 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-9750
|
| 799 |
+
Step 9760/50462 | Loss: 2.7845 | LR: 4.61e-04 | 3286.6ms/step | 18.3 samples/s
|
| 800 |
+
Step 9770/50462 | Loss: 2.7670 | LR: 4.60e-04 | 1900.9ms/step | 31.6 samples/s
|
| 801 |
+
Step 9780/50462 | Loss: 2.8044 | LR: 4.60e-04 | 1905.5ms/step | 31.5 samples/s
|
| 802 |
+
Step 9790/50462 | Loss: 2.7648 | LR: 4.60e-04 | 1905.1ms/step | 31.5 samples/s
|
| 803 |
+
Step 9800/50462 | Loss: 2.6993 | LR: 4.60e-04 | 1903.4ms/step | 31.5 samples/s
|
| 804 |
+
Step 9810/50462 | Loss: 2.7577 | LR: 4.60e-04 | 1903.8ms/step | 31.5 samples/s
|
| 805 |
+
Step 9820/50462 | Loss: 2.7468 | LR: 4.60e-04 | 1900.0ms/step | 31.6 samples/s
|
| 806 |
+
Step 9830/50462 | Loss: 2.7526 | LR: 4.60e-04 | 1903.3ms/step | 31.5 samples/s
|
| 807 |
+
Step 9840/50462 | Loss: 2.7705 | LR: 4.60e-04 | 1905.3ms/step | 31.5 samples/s
|
| 808 |
+
Step 9850/50462 | Loss: 2.7589 | LR: 4.60e-04 | 1901.9ms/step | 31.5 samples/s
|
| 809 |
+
Step 9860/50462 | Loss: 2.7294 | LR: 4.60e-04 | 1904.0ms/step | 31.5 samples/s
|
| 810 |
+
Step 9870/50462 | Loss: 2.7909 | LR: 4.60e-04 | 1906.2ms/step | 31.5 samples/s
|
| 811 |
+
Step 9880/50462 | Loss: 2.7479 | LR: 4.60e-04 | 1903.2ms/step | 31.5 samples/s
|
| 812 |
+
Step 9890/50462 | Loss: 2.7262 | LR: 4.59e-04 | 1904.0ms/step | 31.5 samples/s
|
| 813 |
+
Step 9900/50462 | Loss: 2.7513 | LR: 4.59e-04 | 1904.6ms/step | 31.5 samples/s
|
| 814 |
+
Step 9910/50462 | Loss: 2.7440 | LR: 4.59e-04 | 1902.4ms/step | 31.5 samples/s
|
| 815 |
+
Step 9920/50462 | Loss: 2.7493 | LR: 4.59e-04 | 1902.5ms/step | 31.5 samples/s
|
| 816 |
+
Step 9930/50462 | Loss: 2.7480 | LR: 4.59e-04 | 1905.8ms/step | 31.5 samples/s
|
| 817 |
+
Step 9940/50462 | Loss: 2.7672 | LR: 4.59e-04 | 1905.7ms/step | 31.5 samples/s
|
| 818 |
+
Step 9950/50462 | Loss: 2.7411 | LR: 4.59e-04 | 1902.5ms/step | 31.5 samples/s
|
| 819 |
+
Step 9960/50462 | Loss: 2.7275 | LR: 4.59e-04 | 1904.3ms/step | 31.5 samples/s
|
| 820 |
+
Step 9970/50462 | Loss: 2.7658 | LR: 4.59e-04 | 1906.7ms/step | 31.5 samples/s
|
| 821 |
+
Step 9980/50462 | Loss: 2.7555 | LR: 4.59e-04 | 1902.8ms/step | 31.5 samples/s
|
| 822 |
+
Step 9990/50462 | Loss: 2.7070 | LR: 4.59e-04 | 1902.8ms/step | 31.5 samples/s
|
| 823 |
+
Step 10000/50462 | Loss: 2.7342 | LR: 4.58e-04 | 1904.5ms/step | 31.5 samples/s
|
| 824 |
+
Validation at step 10000: Loss = 2.7366, Perplexity = 15.43
|
| 825 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-10000
|
| 826 |
+
Step 10010/50462 | Loss: 2.7676 | LR: 4.58e-04 | 3241.4ms/step | 18.5 samples/s
|
| 827 |
+
Step 10020/50462 | Loss: 2.7480 | LR: 4.58e-04 | 1897.3ms/step | 31.6 samples/s
|
| 828 |
+
Step 10030/50462 | Loss: 2.7321 | LR: 4.58e-04 | 1901.0ms/step | 31.6 samples/s
|
| 829 |
+
Step 10040/50462 | Loss: 2.7642 | LR: 4.58e-04 | 1903.2ms/step | 31.5 samples/s
|
| 830 |
+
Step 10050/50462 | Loss: 2.7040 | LR: 4.58e-04 | 1903.8ms/step | 31.5 samples/s
|
| 831 |
+
Step 10060/50462 | Loss: 2.7254 | LR: 4.58e-04 | 1903.6ms/step | 31.5 samples/s
|
| 832 |
+
Step 10070/50462 | Loss: 2.7766 | LR: 4.58e-04 | 1902.7ms/step | 31.5 samples/s
|
| 833 |
+
Step 10080/50462 | Loss: 2.7352 | LR: 4.58e-04 | 1904.8ms/step | 31.5 samples/s
|
| 834 |
+
Step 10090/50462 | Loss: 2.7672 | LR: 4.58e-04 | 1906.1ms/step | 31.5 samples/s
|
| 835 |
+
Step 10100/50462 | Loss: 2.7301 | LR: 4.58e-04 | 1907.2ms/step | 31.5 samples/s
|
| 836 |
+
Step 10110/50462 | Loss: 2.7437 | LR: 4.58e-04 | 1908.3ms/step | 31.4 samples/s
|
| 837 |
+
Step 10120/50462 | Loss: 2.7487 | LR: 4.57e-04 | 1907.7ms/step | 31.5 samples/s
|
| 838 |
+
Step 10130/50462 | Loss: 2.7847 | LR: 4.57e-04 | 1905.7ms/step | 31.5 samples/s
|
| 839 |
+
Step 10140/50462 | Loss: 2.7277 | LR: 4.57e-04 | 1902.9ms/step | 31.5 samples/s
|
| 840 |
+
Step 10150/50462 | Loss: 2.7305 | LR: 4.57e-04 | 1904.2ms/step | 31.5 samples/s
|
| 841 |
+
Step 10160/50462 | Loss: 2.7588 | LR: 4.57e-04 | 1907.0ms/step | 31.5 samples/s
|
| 842 |
+
Step 10170/50462 | Loss: 2.7003 | LR: 4.57e-04 | 1909.6ms/step | 31.4 samples/s
|
| 843 |
+
Step 10180/50462 | Loss: 2.7429 | LR: 4.57e-04 | 1907.2ms/step | 31.5 samples/s
|
| 844 |
+
Step 10190/50462 | Loss: 2.7340 | LR: 4.57e-04 | 1906.4ms/step | 31.5 samples/s
|
| 845 |
+
Step 10200/50462 | Loss: 2.6784 | LR: 4.57e-04 | 1903.8ms/step | 31.5 samples/s
|
| 846 |
+
Step 10210/50462 | Loss: 2.7421 | LR: 4.57e-04 | 1908.1ms/step | 31.4 samples/s
|
| 847 |
+
Step 10220/50462 | Loss: 2.7544 | LR: 4.57e-04 | 1906.9ms/step | 31.5 samples/s
|
| 848 |
+
Step 10230/50462 | Loss: 2.7533 | LR: 4.56e-04 | 1908.1ms/step | 31.4 samples/s
|
| 849 |
+
Step 10240/50462 | Loss: 2.7729 | LR: 4.56e-04 | 1906.1ms/step | 31.5 samples/s
|
| 850 |
+
Step 10250/50462 | Loss: 2.7165 | LR: 4.56e-04 | 1906.8ms/step | 31.5 samples/s
|
| 851 |
+
Validation at step 10250: Loss = 2.7322, Perplexity = 15.37
|
| 852 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-10250
|
| 853 |
+
Step 10260/50462 | Loss: 2.7253 | LR: 4.56e-04 | 3252.0ms/step | 18.5 samples/s
|
| 854 |
+
Step 10270/50462 | Loss: 2.7371 | LR: 4.56e-04 | 1898.4ms/step | 31.6 samples/s
|
| 855 |
+
Step 10280/50462 | Loss: 2.7416 | LR: 4.56e-04 | 1902.9ms/step | 31.5 samples/s
|
| 856 |
+
Step 10290/50462 | Loss: 2.7437 | LR: 4.56e-04 | 1906.2ms/step | 31.5 samples/s
|
| 857 |
+
Step 10300/50462 | Loss: 2.7378 | LR: 4.56e-04 | 1906.8ms/step | 31.5 samples/s
|
| 858 |
+
Step 10310/50462 | Loss: 2.7304 | LR: 4.56e-04 | 1904.5ms/step | 31.5 samples/s
|
| 859 |
+
Step 10320/50462 | Loss: 2.6956 | LR: 4.56e-04 | 1905.0ms/step | 31.5 samples/s
|
| 860 |
+
Step 10330/50462 | Loss: 2.7192 | LR: 4.56e-04 | 1907.4ms/step | 31.5 samples/s
|
| 861 |
+
Step 10340/50462 | Loss: 2.7515 | LR: 4.55e-04 | 1904.0ms/step | 31.5 samples/s
|
| 862 |
+
Step 10350/50462 | Loss: 2.7115 | LR: 4.55e-04 | 1901.6ms/step | 31.6 samples/s
|
| 863 |
+
Step 10360/50462 | Loss: 2.7115 | LR: 4.55e-04 | 1905.3ms/step | 31.5 samples/s
|
| 864 |
+
Step 10370/50462 | Loss: 2.7353 | LR: 4.55e-04 | 1906.0ms/step | 31.5 samples/s
|
| 865 |
+
Step 10380/50462 | Loss: 2.7483 | LR: 4.55e-04 | 1903.6ms/step | 31.5 samples/s
|
| 866 |
+
Step 10390/50462 | Loss: 2.7346 | LR: 4.55e-04 | 1907.7ms/step | 31.5 samples/s
|
| 867 |
+
Step 10400/50462 | Loss: 2.7305 | LR: 4.55e-04 | 1907.5ms/step | 31.5 samples/s
|
| 868 |
+
Step 10410/50462 | Loss: 2.7406 | LR: 4.55e-04 | 1905.2ms/step | 31.5 samples/s
|
| 869 |
+
Step 10420/50462 | Loss: 2.7512 | LR: 4.55e-04 | 1902.6ms/step | 31.5 samples/s
|
| 870 |
+
Step 10430/50462 | Loss: 2.7403 | LR: 4.55e-04 | 1904.6ms/step | 31.5 samples/s
|
| 871 |
+
Step 10440/50462 | Loss: 2.7040 | LR: 4.55e-04 | 1907.1ms/step | 31.5 samples/s
|
| 872 |
+
Step 10450/50462 | Loss: 2.7770 | LR: 4.54e-04 | 1906.4ms/step | 31.5 samples/s
|
| 873 |
+
Step 10460/50462 | Loss: 2.7405 | LR: 4.54e-04 | 1908.8ms/step | 31.4 samples/s
|
| 874 |
+
Step 10470/50462 | Loss: 2.7166 | LR: 4.54e-04 | 1904.8ms/step | 31.5 samples/s
|
| 875 |
+
Step 10480/50462 | Loss: 2.7546 | LR: 4.54e-04 | 1908.4ms/step | 31.4 samples/s
|
| 876 |
+
Step 10490/50462 | Loss: 2.7498 | LR: 4.54e-04 | 1907.5ms/step | 31.5 samples/s
|
| 877 |
+
Step 10500/50462 | Loss: 2.7171 | LR: 4.54e-04 | 1906.1ms/step | 31.5 samples/s
|
| 878 |
+
Validation at step 10500: Loss = 2.7252, Perplexity = 15.26
|
| 879 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-10500
|
| 880 |
+
Step 10510/50462 | Loss: 2.7982 | LR: 4.54e-04 | 3240.8ms/step | 18.5 samples/s
|
| 881 |
+
Step 10520/50462 | Loss: 2.7495 | LR: 4.54e-04 | 1897.0ms/step | 31.6 samples/s
|
| 882 |
+
Step 10530/50462 | Loss: 2.7109 | LR: 4.54e-04 | 1897.7ms/step | 31.6 samples/s
|
| 883 |
+
Step 10540/50462 | Loss: 2.7104 | LR: 4.54e-04 | 1901.1ms/step | 31.6 samples/s
|
| 884 |
+
Step 10550/50462 | Loss: 2.7405 | LR: 4.54e-04 | 1903.5ms/step | 31.5 samples/s
|
| 885 |
+
Step 10560/50462 | Loss: 2.7277 | LR: 4.53e-04 | 1904.3ms/step | 31.5 samples/s
|
| 886 |
+
Step 10570/50462 | Loss: 2.7395 | LR: 4.53e-04 | 1900.5ms/step | 31.6 samples/s
|
| 887 |
+
Step 10580/50462 | Loss: 2.7417 | LR: 4.53e-04 | 1903.6ms/step | 31.5 samples/s
|
| 888 |
+
Step 10590/50462 | Loss: 2.7496 | LR: 4.53e-04 | 1906.9ms/step | 31.5 samples/s
|
| 889 |
+
Step 10600/50462 | Loss: 2.7664 | LR: 4.53e-04 | 1902.9ms/step | 31.5 samples/s
|
| 890 |
+
Step 10610/50462 | Loss: 2.6942 | LR: 4.53e-04 | 1905.5ms/step | 31.5 samples/s
|
| 891 |
+
Step 10620/50462 | Loss: 2.7274 | LR: 4.53e-04 | 1904.9ms/step | 31.5 samples/s
|
| 892 |
+
Step 10630/50462 | Loss: 2.7232 | LR: 4.53e-04 | 1904.8ms/step | 31.5 samples/s
|
| 893 |
+
Step 10640/50462 | Loss: 2.7427 | LR: 4.53e-04 | 1907.4ms/step | 31.5 samples/s
|
| 894 |
+
Step 10650/50462 | Loss: 2.7150 | LR: 4.53e-04 | 1906.8ms/step | 31.5 samples/s
|
| 895 |
+
Step 10660/50462 | Loss: 2.7154 | LR: 4.53e-04 | 1906.9ms/step | 31.5 samples/s
|
| 896 |
+
Step 10670/50462 | Loss: 2.7761 | LR: 4.52e-04 | 1906.4ms/step | 31.5 samples/s
|
| 897 |
+
Step 10680/50462 | Loss: 2.7439 | LR: 4.52e-04 | 1907.2ms/step | 31.5 samples/s
|
| 898 |
+
Step 10690/50462 | Loss: 2.7421 | LR: 4.52e-04 | 1906.5ms/step | 31.5 samples/s
|
| 899 |
+
Step 10700/50462 | Loss: 2.7434 | LR: 4.52e-04 | 1907.3ms/step | 31.5 samples/s
|
| 900 |
+
Step 10710/50462 | Loss: 2.7423 | LR: 4.52e-04 | 1910.1ms/step | 31.4 samples/s
|
| 901 |
+
Step 10720/50462 | Loss: 2.7285 | LR: 4.52e-04 | 1907.3ms/step | 31.5 samples/s
|
| 902 |
+
Step 10730/50462 | Loss: 2.7732 | LR: 4.52e-04 | 1906.9ms/step | 31.5 samples/s
|
| 903 |
+
Step 10740/50462 | Loss: 2.7554 | LR: 4.52e-04 | 1906.8ms/step | 31.5 samples/s
|
| 904 |
+
Step 10750/50462 | Loss: 2.7219 | LR: 4.52e-04 | 1910.9ms/step | 31.4 samples/s
|
| 905 |
+
Validation at step 10750: Loss = 2.7204, Perplexity = 15.19
|
| 906 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-10750
|
| 907 |
+
Step 10760/50462 | Loss: 2.7203 | LR: 4.52e-04 | 3256.1ms/step | 18.4 samples/s
|
| 908 |
+
Step 10770/50462 | Loss: 2.7182 | LR: 4.52e-04 | 1902.4ms/step | 31.5 samples/s
|
| 909 |
+
Step 10780/50462 | Loss: 2.7431 | LR: 4.51e-04 | 1903.4ms/step | 31.5 samples/s
|
| 910 |
+
Step 10790/50462 | Loss: 2.6923 | LR: 4.51e-04 | 1906.3ms/step | 31.5 samples/s
|
| 911 |
+
Step 10800/50462 | Loss: 2.7244 | LR: 4.51e-04 | 1906.0ms/step | 31.5 samples/s
|
| 912 |
+
Step 10810/50462 | Loss: 2.7663 | LR: 4.51e-04 | 1905.9ms/step | 31.5 samples/s
|
| 913 |
+
Step 10820/50462 | Loss: 2.7117 | LR: 4.51e-04 | 1906.1ms/step | 31.5 samples/s
|
| 914 |
+
Step 10830/50462 | Loss: 2.7397 | LR: 4.51e-04 | 1907.7ms/step | 31.5 samples/s
|
| 915 |
+
Step 10840/50462 | Loss: 2.7615 | LR: 4.51e-04 | 1906.1ms/step | 31.5 samples/s
|
| 916 |
+
Step 10850/50462 | Loss: 2.6945 | LR: 4.51e-04 | 1909.7ms/step | 31.4 samples/s
|
| 917 |
+
Step 10860/50462 | Loss: 2.7049 | LR: 4.51e-04 | 1907.5ms/step | 31.5 samples/s
|
| 918 |
+
Step 10870/50462 | Loss: 2.7419 | LR: 4.51e-04 | 1904.9ms/step | 31.5 samples/s
|
| 919 |
+
Step 10880/50462 | Loss: 2.7269 | LR: 4.50e-04 | 1908.0ms/step | 31.4 samples/s
|
| 920 |
+
Step 10890/50462 | Loss: 2.6858 | LR: 4.50e-04 | 1907.5ms/step | 31.5 samples/s
|
| 921 |
+
Step 10900/50462 | Loss: 2.6987 | LR: 4.50e-04 | 1908.2ms/step | 31.4 samples/s
|
| 922 |
+
Step 10910/50462 | Loss: 2.7110 | LR: 4.50e-04 | 1908.5ms/step | 31.4 samples/s
|
| 923 |
+
Step 10920/50462 | Loss: 2.7468 | LR: 4.50e-04 | 1908.3ms/step | 31.4 samples/s
|
| 924 |
+
Step 10930/50462 | Loss: 2.6844 | LR: 4.50e-04 | 1906.9ms/step | 31.5 samples/s
|
| 925 |
+
Step 10940/50462 | Loss: 2.7307 | LR: 4.50e-04 | 1907.4ms/step | 31.5 samples/s
|
| 926 |
+
Step 10950/50462 | Loss: 2.7091 | LR: 4.50e-04 | 1905.7ms/step | 31.5 samples/s
|
| 927 |
+
Step 10960/50462 | Loss: 2.7241 | LR: 4.50e-04 | 1906.0ms/step | 31.5 samples/s
|
| 928 |
+
Step 10970/50462 | Loss: 2.7098 | LR: 4.50e-04 | 1909.6ms/step | 31.4 samples/s
|
| 929 |
+
Step 10980/50462 | Loss: 2.7138 | LR: 4.50e-04 | 1906.7ms/step | 31.5 samples/s
|
| 930 |
+
Step 10990/50462 | Loss: 2.6696 | LR: 4.49e-04 | 1910.9ms/step | 31.4 samples/s
|
| 931 |
+
Step 11000/50462 | Loss: 2.7315 | LR: 4.49e-04 | 1908.7ms/step | 31.4 samples/s
|
| 932 |
+
Validation at step 11000: Loss = 2.7175, Perplexity = 15.14
|
| 933 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-11000
|
| 934 |
+
Step 11010/50462 | Loss: 2.7114 | LR: 4.49e-04 | 3249.2ms/step | 18.5 samples/s
|
| 935 |
+
Step 11020/50462 | Loss: 2.7443 | LR: 4.49e-04 | 1900.3ms/step | 31.6 samples/s
|
| 936 |
+
Step 11030/50462 | Loss: 2.6878 | LR: 4.49e-04 | 1904.6ms/step | 31.5 samples/s
|
| 937 |
+
Step 11040/50462 | Loss: 2.7319 | LR: 4.49e-04 | 1907.8ms/step | 31.4 samples/s
|
| 938 |
+
Step 11050/50462 | Loss: 2.7222 | LR: 4.49e-04 | 1905.4ms/step | 31.5 samples/s
|
| 939 |
+
Step 11060/50462 | Loss: 2.7124 | LR: 4.49e-04 | 1906.5ms/step | 31.5 samples/s
|
| 940 |
+
Step 11070/50462 | Loss: 2.7242 | LR: 4.49e-04 | 1905.6ms/step | 31.5 samples/s
|
| 941 |
+
Step 11080/50462 | Loss: 2.7200 | LR: 4.49e-04 | 1906.5ms/step | 31.5 samples/s
|
| 942 |
+
Step 11090/50462 | Loss: 2.7610 | LR: 4.48e-04 | 1907.7ms/step | 31.5 samples/s
|
| 943 |
+
Step 11100/50462 | Loss: 2.7066 | LR: 4.48e-04 | 1907.1ms/step | 31.5 samples/s
|
| 944 |
+
Step 11110/50462 | Loss: 2.7334 | LR: 4.48e-04 | 1905.6ms/step | 31.5 samples/s
|
| 945 |
+
Step 11120/50462 | Loss: 2.7171 | LR: 4.48e-04 | 1907.2ms/step | 31.5 samples/s
|
| 946 |
+
Step 11130/50462 | Loss: 2.7053 | LR: 4.48e-04 | 1909.1ms/step | 31.4 samples/s
|
| 947 |
+
Step 11140/50462 | Loss: 2.7152 | LR: 4.48e-04 | 1909.3ms/step | 31.4 samples/s
|
| 948 |
+
Step 11150/50462 | Loss: 2.7333 | LR: 4.48e-04 | 1905.5ms/step | 31.5 samples/s
|
| 949 |
+
Step 11160/50462 | Loss: 2.7156 | LR: 4.48e-04 | 1908.4ms/step | 31.4 samples/s
|
| 950 |
+
Step 11170/50462 | Loss: 2.6912 | LR: 4.48e-04 | 1907.3ms/step | 31.5 samples/s
|
| 951 |
+
Step 11180/50462 | Loss: 2.7471 | LR: 4.48e-04 | 1907.1ms/step | 31.5 samples/s
|
| 952 |
+
Step 11190/50462 | Loss: 2.7233 | LR: 4.48e-04 | 1907.6ms/step | 31.5 samples/s
|
| 953 |
+
Step 11200/50462 | Loss: 2.7452 | LR: 4.47e-04 | 1908.8ms/step | 31.4 samples/s
|
| 954 |
+
Step 11210/50462 | Loss: 2.7654 | LR: 4.47e-04 | 1908.6ms/step | 31.4 samples/s
|
| 955 |
+
Step 11220/50462 | Loss: 2.7053 | LR: 4.47e-04 | 1909.3ms/step | 31.4 samples/s
|
| 956 |
+
Step 11230/50462 | Loss: 2.7073 | LR: 4.47e-04 | 1907.4ms/step | 31.5 samples/s
|
| 957 |
+
Step 11240/50462 | Loss: 2.6920 | LR: 4.47e-04 | 1907.1ms/step | 31.5 samples/s
|
| 958 |
+
Step 11250/50462 | Loss: 2.7401 | LR: 4.47e-04 | 1907.8ms/step | 31.4 samples/s
|
| 959 |
+
Validation at step 11250: Loss = 2.7114, Perplexity = 15.05
|
| 960 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-11250
|
| 961 |
+
Step 11260/50462 | Loss: 2.7073 | LR: 4.47e-04 | 3270.4ms/step | 18.3 samples/s
|
| 962 |
+
Step 11270/50462 | Loss: 2.7455 | LR: 4.47e-04 | 1897.5ms/step | 31.6 samples/s
|
| 963 |
+
Step 11280/50462 | Loss: 2.7084 | LR: 4.47e-04 | 1897.8ms/step | 31.6 samples/s
|
| 964 |
+
Step 11290/50462 | Loss: 2.7001 | LR: 4.47e-04 | 1899.0ms/step | 31.6 samples/s
|
| 965 |
+
Step 11300/50462 | Loss: 2.7208 | LR: 4.46e-04 | 1905.4ms/step | 31.5 samples/s
|
| 966 |
+
Step 11310/50462 | Loss: 2.7138 | LR: 4.46e-04 | 1904.6ms/step | 31.5 samples/s
|
| 967 |
+
Step 11320/50462 | Loss: 2.7185 | LR: 4.46e-04 | 1904.0ms/step | 31.5 samples/s
|
| 968 |
+
Step 11330/50462 | Loss: 2.6712 | LR: 4.46e-04 | 1907.0ms/step | 31.5 samples/s
|
| 969 |
+
Step 11340/50462 | Loss: 2.7619 | LR: 4.46e-04 | 1902.0ms/step | 31.5 samples/s
|
| 970 |
+
Step 11350/50462 | Loss: 2.7384 | LR: 4.46e-04 | 1903.4ms/step | 31.5 samples/s
|
| 971 |
+
Step 11360/50462 | Loss: 2.7255 | LR: 4.46e-04 | 1904.7ms/step | 31.5 samples/s
|
| 972 |
+
Step 11370/50462 | Loss: 2.7442 | LR: 4.46e-04 | 1907.2ms/step | 31.5 samples/s
|
| 973 |
+
Step 11380/50462 | Loss: 2.7487 | LR: 4.46e-04 | 1907.3ms/step | 31.5 samples/s
|
| 974 |
+
Step 11390/50462 | Loss: 2.7313 | LR: 4.46e-04 | 1906.6ms/step | 31.5 samples/s
|
| 975 |
+
Step 11400/50462 | Loss: 2.7398 | LR: 4.45e-04 | 1901.7ms/step | 31.6 samples/s
|
| 976 |
+
Step 11410/50462 | Loss: 2.7207 | LR: 4.45e-04 | 1905.6ms/step | 31.5 samples/s
|
| 977 |
+
Step 11420/50462 | Loss: 2.7296 | LR: 4.45e-04 | 1903.8ms/step | 31.5 samples/s
|
| 978 |
+
Step 11430/50462 | Loss: 2.7005 | LR: 4.45e-04 | 1905.8ms/step | 31.5 samples/s
|
| 979 |
+
Step 11440/50462 | Loss: 2.7290 | LR: 4.45e-04 | 1906.7ms/step | 31.5 samples/s
|
| 980 |
+
Step 11450/50462 | Loss: 2.7466 | LR: 4.45e-04 | 1903.9ms/step | 31.5 samples/s
|
| 981 |
+
Step 11460/50462 | Loss: 2.7159 | LR: 4.45e-04 | 1906.5ms/step | 31.5 samples/s
|
| 982 |
+
Step 11470/50462 | Loss: 2.6956 | LR: 4.45e-04 | 1907.1ms/step | 31.5 samples/s
|
| 983 |
+
Step 11480/50462 | Loss: 2.6873 | LR: 4.45e-04 | 1907.1ms/step | 31.5 samples/s
|
| 984 |
+
Step 11490/50462 | Loss: 2.7066 | LR: 4.45e-04 | 1908.5ms/step | 31.4 samples/s
|
| 985 |
+
Step 11500/50462 | Loss: 2.7312 | LR: 4.44e-04 | 1907.1ms/step | 31.5 samples/s
|
| 986 |
+
Validation at step 11500: Loss = 2.7077, Perplexity = 15.00
|
| 987 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-11500
|
| 988 |
+
Step 11510/50462 | Loss: 2.7145 | LR: 4.44e-04 | 3248.7ms/step | 18.5 samples/s
|
| 989 |
+
Step 11520/50462 | Loss: 2.7501 | LR: 4.44e-04 | 1896.9ms/step | 31.6 samples/s
|
| 990 |
+
Step 11530/50462 | Loss: 2.7354 | LR: 4.44e-04 | 1902.0ms/step | 31.5 samples/s
|
| 991 |
+
Step 11540/50462 | Loss: 2.6794 | LR: 4.44e-04 | 1903.5ms/step | 31.5 samples/s
|
| 992 |
+
Step 11550/50462 | Loss: 2.6949 | LR: 4.44e-04 | 1903.2ms/step | 31.5 samples/s
|
| 993 |
+
Step 11560/50462 | Loss: 2.6975 | LR: 4.44e-04 | 1901.8ms/step | 31.5 samples/s
|
| 994 |
+
Step 11570/50462 | Loss: 2.7169 | LR: 4.44e-04 | 1906.2ms/step | 31.5 samples/s
|
| 995 |
+
Step 11580/50462 | Loss: 2.7052 | LR: 4.44e-04 | 1906.5ms/step | 31.5 samples/s
|
| 996 |
+
Step 11590/50462 | Loss: 2.7091 | LR: 4.44e-04 | 1909.5ms/step | 31.4 samples/s
|
| 997 |
+
Step 11600/50462 | Loss: 2.7195 | LR: 4.43e-04 | 1907.8ms/step | 31.5 samples/s
|
| 998 |
+
Step 11610/50462 | Loss: 2.6953 | LR: 4.43e-04 | 1903.3ms/step | 31.5 samples/s
|
| 999 |
+
Step 11620/50462 | Loss: 2.7133 | LR: 4.43e-04 | 1902.2ms/step | 31.5 samples/s
|
| 1000 |
+
Step 11630/50462 | Loss: 2.6737 | LR: 4.43e-04 | 1904.1ms/step | 31.5 samples/s
|
| 1001 |
+
Step 11640/50462 | Loss: 2.7434 | LR: 4.43e-04 | 1908.9ms/step | 31.4 samples/s
|
| 1002 |
+
Step 11650/50462 | Loss: 2.7158 | LR: 4.43e-04 | 1906.8ms/step | 31.5 samples/s
|
| 1003 |
+
Step 11660/50462 | Loss: 2.7090 | LR: 4.43e-04 | 1908.8ms/step | 31.4 samples/s
|
| 1004 |
+
Step 11670/50462 | Loss: 2.6766 | LR: 4.43e-04 | 1904.2ms/step | 31.5 samples/s
|
| 1005 |
+
Step 11680/50462 | Loss: 2.7149 | LR: 4.43e-04 | 1906.2ms/step | 31.5 samples/s
|
| 1006 |
+
Step 11690/50462 | Loss: 2.6769 | LR: 4.43e-04 | 1905.8ms/step | 31.5 samples/s
|
| 1007 |
+
Step 11700/50462 | Loss: 2.7220 | LR: 4.42e-04 | 1906.0ms/step | 31.5 samples/s
|
| 1008 |
+
Step 11710/50462 | Loss: 2.7158 | LR: 4.42e-04 | 1907.2ms/step | 31.5 samples/s
|
| 1009 |
+
Step 11720/50462 | Loss: 2.7109 | LR: 4.42e-04 | 1908.7ms/step | 31.4 samples/s
|
| 1010 |
+
Step 11730/50462 | Loss: 2.7142 | LR: 4.42e-04 | 1906.9ms/step | 31.5 samples/s
|
| 1011 |
+
Step 11740/50462 | Loss: 2.7105 | LR: 4.42e-04 | 1905.1ms/step | 31.5 samples/s
|
| 1012 |
+
Step 11750/50462 | Loss: 2.6731 | LR: 4.42e-04 | 1905.6ms/step | 31.5 samples/s
|
| 1013 |
+
Validation at step 11750: Loss = 2.7051, Perplexity = 14.96
|
| 1014 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-11750
|
| 1015 |
+
Step 11760/50462 | Loss: 2.7386 | LR: 4.42e-04 | 3255.6ms/step | 18.4 samples/s
|
| 1016 |
+
Step 11770/50462 | Loss: 2.6791 | LR: 4.42e-04 | 1897.2ms/step | 31.6 samples/s
|
| 1017 |
+
Step 11780/50462 | Loss: 2.6964 | LR: 4.42e-04 | 1901.1ms/step | 31.6 samples/s
|
| 1018 |
+
Step 11790/50462 | Loss: 2.7274 | LR: 4.42e-04 | 1903.4ms/step | 31.5 samples/s
|
| 1019 |
+
Step 11800/50462 | Loss: 2.7586 | LR: 4.41e-04 | 1903.7ms/step | 31.5 samples/s
|
| 1020 |
+
Step 11810/50462 | Loss: 2.7056 | LR: 4.41e-04 | 1903.3ms/step | 31.5 samples/s
|
| 1021 |
+
Step 11820/50462 | Loss: 2.7031 | LR: 4.41e-04 | 1908.0ms/step | 31.4 samples/s
|
| 1022 |
+
Step 11830/50462 | Loss: 2.6940 | LR: 4.41e-04 | 1905.4ms/step | 31.5 samples/s
|
| 1023 |
+
Step 11840/50462 | Loss: 2.7027 | LR: 4.41e-04 | 1905.2ms/step | 31.5 samples/s
|
| 1024 |
+
Step 11850/50462 | Loss: 2.6849 | LR: 4.41e-04 | 1906.8ms/step | 31.5 samples/s
|
| 1025 |
+
Step 11860/50462 | Loss: 2.7087 | LR: 4.41e-04 | 1902.5ms/step | 31.5 samples/s
|
| 1026 |
+
Step 11870/50462 | Loss: 2.7084 | LR: 4.41e-04 | 1903.3ms/step | 31.5 samples/s
|
| 1027 |
+
Step 11880/50462 | Loss: 2.6710 | LR: 4.41e-04 | 1904.9ms/step | 31.5 samples/s
|
| 1028 |
+
Step 11890/50462 | Loss: 2.7186 | LR: 4.41e-04 | 1907.2ms/step | 31.5 samples/s
|
| 1029 |
+
Step 11900/50462 | Loss: 2.7046 | LR: 4.40e-04 | 1906.9ms/step | 31.5 samples/s
|
| 1030 |
+
Step 11910/50462 | Loss: 2.6961 | LR: 4.40e-04 | 1909.0ms/step | 31.4 samples/s
|
| 1031 |
+
Step 11920/50462 | Loss: 2.7233 | LR: 4.40e-04 | 1904.9ms/step | 31.5 samples/s
|
| 1032 |
+
Step 11930/50462 | Loss: 2.7440 | LR: 4.40e-04 | 1906.6ms/step | 31.5 samples/s
|
| 1033 |
+
Step 11940/50462 | Loss: 2.7071 | LR: 4.40e-04 | 1906.6ms/step | 31.5 samples/s
|
| 1034 |
+
Step 11950/50462 | Loss: 2.7457 | LR: 4.40e-04 | 1907.3ms/step | 31.5 samples/s
|
| 1035 |
+
Step 11960/50462 | Loss: 2.7243 | LR: 4.40e-04 | 1907.6ms/step | 31.5 samples/s
|
| 1036 |
+
Step 11970/50462 | Loss: 2.6978 | LR: 4.40e-04 | 1908.6ms/step | 31.4 samples/s
|
| 1037 |
+
Step 11980/50462 | Loss: 2.7469 | LR: 4.40e-04 | 1902.2ms/step | 31.5 samples/s
|
| 1038 |
+
Step 11990/50462 | Loss: 2.7018 | LR: 4.40e-04 | 1904.8ms/step | 31.5 samples/s
|
| 1039 |
+
Step 12000/50462 | Loss: 2.7209 | LR: 4.39e-04 | 1905.8ms/step | 31.5 samples/s
|
| 1040 |
+
Validation at step 12000: Loss = 2.7016, Perplexity = 14.90
|
| 1041 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-12000
|
| 1042 |
+
Step 12010/50462 | Loss: 2.6758 | LR: 4.39e-04 | 3291.0ms/step | 18.2 samples/s
|
| 1043 |
+
Step 12020/50462 | Loss: 2.6987 | LR: 4.39e-04 | 1904.6ms/step | 31.5 samples/s
|
| 1044 |
+
Step 12030/50462 | Loss: 2.6817 | LR: 4.39e-04 | 1901.7ms/step | 31.6 samples/s
|
| 1045 |
+
Step 12040/50462 | Loss: 2.7505 | LR: 4.39e-04 | 1906.1ms/step | 31.5 samples/s
|
| 1046 |
+
Step 12050/50462 | Loss: 2.6904 | LR: 4.39e-04 | 1908.9ms/step | 31.4 samples/s
|
| 1047 |
+
Step 12060/50462 | Loss: 2.7068 | LR: 4.39e-04 | 1909.3ms/step | 31.4 samples/s
|
| 1048 |
+
Step 12070/50462 | Loss: 2.6976 | LR: 4.39e-04 | 1906.8ms/step | 31.5 samples/s
|
| 1049 |
+
Step 12080/50462 | Loss: 2.7214 | LR: 4.39e-04 | 1905.9ms/step | 31.5 samples/s
|
| 1050 |
+
Step 12090/50462 | Loss: 2.6625 | LR: 4.39e-04 | 1906.8ms/step | 31.5 samples/s
|
| 1051 |
+
Step 12100/50462 | Loss: 2.7012 | LR: 4.38e-04 | 1903.3ms/step | 31.5 samples/s
|
| 1052 |
+
Step 12110/50462 | Loss: 2.7230 | LR: 4.38e-04 | 1902.6ms/step | 31.5 samples/s
|
| 1053 |
+
Step 12120/50462 | Loss: 2.6712 | LR: 4.38e-04 | 1903.2ms/step | 31.5 samples/s
|
| 1054 |
+
Step 12130/50462 | Loss: 2.6770 | LR: 4.38e-04 | 1907.6ms/step | 31.5 samples/s
|
| 1055 |
+
Step 12140/50462 | Loss: 2.7039 | LR: 4.38e-04 | 1907.9ms/step | 31.4 samples/s
|
| 1056 |
+
Step 12150/50462 | Loss: 2.6892 | LR: 4.38e-04 | 1900.7ms/step | 31.6 samples/s
|
| 1057 |
+
Step 12160/50462 | Loss: 2.6452 | LR: 4.38e-04 | 1901.9ms/step | 31.5 samples/s
|
| 1058 |
+
Step 12170/50462 | Loss: 2.7328 | LR: 4.38e-04 | 1905.6ms/step | 31.5 samples/s
|
| 1059 |
+
Step 12180/50462 | Loss: 2.7158 | LR: 4.38e-04 | 1903.0ms/step | 31.5 samples/s
|
| 1060 |
+
Step 12190/50462 | Loss: 2.7543 | LR: 4.37e-04 | 1904.0ms/step | 31.5 samples/s
|
| 1061 |
+
Step 12200/50462 | Loss: 2.6931 | LR: 4.37e-04 | 1904.7ms/step | 31.5 samples/s
|
| 1062 |
+
Step 12210/50462 | Loss: 2.6960 | LR: 4.37e-04 | 1905.0ms/step | 31.5 samples/s
|
| 1063 |
+
Step 12220/50462 | Loss: 2.7311 | LR: 4.37e-04 | 1906.6ms/step | 31.5 samples/s
|
| 1064 |
+
Step 12230/50462 | Loss: 2.6934 | LR: 4.37e-04 | 1904.4ms/step | 31.5 samples/s
|
| 1065 |
+
Step 12240/50462 | Loss: 2.7507 | LR: 4.37e-04 | 1909.7ms/step | 31.4 samples/s
|
| 1066 |
+
Step 12250/50462 | Loss: 2.6643 | LR: 4.37e-04 | 1909.8ms/step | 31.4 samples/s
|
| 1067 |
+
Validation at step 12250: Loss = 2.6957, Perplexity = 14.82
|
| 1068 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-12250
|
| 1069 |
+
Step 12260/50462 | Loss: 2.6873 | LR: 4.37e-04 | 3241.1ms/step | 18.5 samples/s
|
| 1070 |
+
Step 12270/50462 | Loss: 2.6991 | LR: 4.37e-04 | 1897.5ms/step | 31.6 samples/s
|
| 1071 |
+
Step 12280/50462 | Loss: 2.7052 | LR: 4.37e-04 | 1902.0ms/step | 31.5 samples/s
|
| 1072 |
+
Step 12290/50462 | Loss: 2.6881 | LR: 4.36e-04 | 1903.6ms/step | 31.5 samples/s
|
| 1073 |
+
Step 12300/50462 | Loss: 2.6988 | LR: 4.36e-04 | 1907.7ms/step | 31.5 samples/s
|
| 1074 |
+
Step 12310/50462 | Loss: 2.6913 | LR: 4.36e-04 | 1906.1ms/step | 31.5 samples/s
|
| 1075 |
+
Step 12320/50462 | Loss: 2.6873 | LR: 4.36e-04 | 1908.0ms/step | 31.4 samples/s
|
| 1076 |
+
Step 12330/50462 | Loss: 2.7162 | LR: 4.36e-04 | 1901.6ms/step | 31.6 samples/s
|
| 1077 |
+
Step 12340/50462 | Loss: 2.7004 | LR: 4.36e-04 | 1900.5ms/step | 31.6 samples/s
|
| 1078 |
+
Step 12350/50462 | Loss: 2.6832 | LR: 4.36e-04 | 1903.8ms/step | 31.5 samples/s
|
| 1079 |
+
Step 12360/50462 | Loss: 2.7176 | LR: 4.36e-04 | 1906.0ms/step | 31.5 samples/s
|
| 1080 |
+
Step 12370/50462 | Loss: 2.6727 | LR: 4.36e-04 | 1905.8ms/step | 31.5 samples/s
|
| 1081 |
+
Step 12380/50462 | Loss: 2.6794 | LR: 4.35e-04 | 1908.4ms/step | 31.4 samples/s
|
| 1082 |
+
Step 12390/50462 | Loss: 2.7109 | LR: 4.35e-04 | 1907.7ms/step | 31.5 samples/s
|
| 1083 |
+
Step 12400/50462 | Loss: 2.7255 | LR: 4.35e-04 | 1909.6ms/step | 31.4 samples/s
|
| 1084 |
+
Step 12410/50462 | Loss: 2.7354 | LR: 4.35e-04 | 1908.9ms/step | 31.4 samples/s
|
| 1085 |
+
Step 12420/50462 | Loss: 2.7003 | LR: 4.35e-04 | 1907.2ms/step | 31.5 samples/s
|
| 1086 |
+
Step 12430/50462 | Loss: 2.7230 | LR: 4.35e-04 | 1908.0ms/step | 31.4 samples/s
|
| 1087 |
+
Step 12440/50462 | Loss: 2.7131 | LR: 4.35e-04 | 1906.1ms/step | 31.5 samples/s
|
| 1088 |
+
Step 12450/50462 | Loss: 2.7359 | LR: 4.35e-04 | 1909.2ms/step | 31.4 samples/s
|
| 1089 |
+
Step 12460/50462 | Loss: 2.7166 | LR: 4.35e-04 | 1908.0ms/step | 31.4 samples/s
|
| 1090 |
+
Step 12470/50462 | Loss: 2.7191 | LR: 4.35e-04 | 1907.0ms/step | 31.5 samples/s
|
| 1091 |
+
Step 12480/50462 | Loss: 2.7026 | LR: 4.34e-04 | 1904.2ms/step | 31.5 samples/s
|
| 1092 |
+
Step 12490/50462 | Loss: 2.6898 | LR: 4.34e-04 | 1906.9ms/step | 31.5 samples/s
|
| 1093 |
+
Step 12500/50462 | Loss: 2.6960 | LR: 4.34e-04 | 1905.7ms/step | 31.5 samples/s
|
| 1094 |
+
Validation at step 12500: Loss = 2.6927, Perplexity = 14.77
|
| 1095 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-12500
|
| 1096 |
+
Step 12510/50462 | Loss: 2.6836 | LR: 4.34e-04 | 3257.2ms/step | 18.4 samples/s
|
| 1097 |
+
Step 12520/50462 | Loss: 2.7257 | LR: 4.34e-04 | 1899.2ms/step | 31.6 samples/s
|
| 1098 |
+
Step 12530/50462 | Loss: 2.6820 | LR: 4.34e-04 | 1902.8ms/step | 31.5 samples/s
|
| 1099 |
+
Step 12540/50462 | Loss: 2.6964 | LR: 4.34e-04 | 1906.7ms/step | 31.5 samples/s
|
| 1100 |
+
Step 12550/50462 | Loss: 2.6979 | LR: 4.34e-04 | 1907.6ms/step | 31.5 samples/s
|
| 1101 |
+
Step 12560/50462 | Loss: 2.6855 | LR: 4.34e-04 | 1906.0ms/step | 31.5 samples/s
|
| 1102 |
+
Step 12570/50462 | Loss: 2.7298 | LR: 4.33e-04 | 1908.7ms/step | 31.4 samples/s
|
| 1103 |
+
Step 12580/50462 | Loss: 2.6988 | LR: 4.33e-04 | 1907.7ms/step | 31.5 samples/s
|
| 1104 |
+
Step 12590/50462 | Loss: 2.6999 | LR: 4.33e-04 | 1907.2ms/step | 31.5 samples/s
|
| 1105 |
+
Step 12600/50462 | Loss: 2.7541 | LR: 4.33e-04 | 1907.6ms/step | 31.5 samples/s
|
| 1106 |
+
Step 12610/50462 | Loss: 2.7157 | LR: 4.33e-04 | 1909.3ms/step | 31.4 samples/s
|
| 1107 |
+
Step 12620/50462 | Loss: 2.6870 | LR: 4.33e-04 | 1908.5ms/step | 31.4 samples/s
|
| 1108 |
+
Step 12630/50462 | Loss: 2.6573 | LR: 4.33e-04 | 1907.2ms/step | 31.5 samples/s
|
| 1109 |
+
Step 12640/50462 | Loss: 2.7045 | LR: 4.33e-04 | 1907.5ms/step | 31.5 samples/s
|
| 1110 |
+
Step 12650/50462 | Loss: 2.7262 | LR: 4.33e-04 | 1907.8ms/step | 31.5 samples/s
|
| 1111 |
+
Step 12660/50462 | Loss: 2.7201 | LR: 4.32e-04 | 1909.5ms/step | 31.4 samples/s
|
| 1112 |
+
Step 12670/50462 | Loss: 2.6852 | LR: 4.32e-04 | 1908.4ms/step | 31.4 samples/s
|
| 1113 |
+
Step 12680/50462 | Loss: 2.7066 | LR: 4.32e-04 | 1909.6ms/step | 31.4 samples/s
|
| 1114 |
+
Step 12690/50462 | Loss: 2.6689 | LR: 4.32e-04 | 1907.0ms/step | 31.5 samples/s
|
| 1115 |
+
Step 12700/50462 | Loss: 2.7451 | LR: 4.32e-04 | 1910.4ms/step | 31.4 samples/s
|
| 1116 |
+
Step 12710/50462 | Loss: 2.6870 | LR: 4.32e-04 | 1909.1ms/step | 31.4 samples/s
|
| 1117 |
+
Step 12720/50462 | Loss: 2.7038 | LR: 4.32e-04 | 1910.1ms/step | 31.4 samples/s
|
| 1118 |
+
Step 12730/50462 | Loss: 2.6886 | LR: 4.32e-04 | 1907.3ms/step | 31.5 samples/s
|
| 1119 |
+
Step 12740/50462 | Loss: 2.6828 | LR: 4.32e-04 | 1910.0ms/step | 31.4 samples/s
|
| 1120 |
+
Step 12750/50462 | Loss: 2.6637 | LR: 4.31e-04 | 1909.9ms/step | 31.4 samples/s
|
| 1121 |
+
Validation at step 12750: Loss = 2.6853, Perplexity = 14.66
|
| 1122 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-12750
|
| 1123 |
+
Step 12760/50462 | Loss: 2.6270 | LR: 4.31e-04 | 3254.7ms/step | 18.4 samples/s
|
| 1124 |
+
Step 12770/50462 | Loss: 2.6974 | LR: 4.31e-04 | 1902.0ms/step | 31.5 samples/s
|
| 1125 |
+
Step 12780/50462 | Loss: 2.6856 | LR: 4.31e-04 | 1903.4ms/step | 31.5 samples/s
|
| 1126 |
+
Step 12790/50462 | Loss: 2.6745 | LR: 4.31e-04 | 1907.4ms/step | 31.5 samples/s
|
| 1127 |
+
Step 12800/50462 | Loss: 2.6845 | LR: 4.31e-04 | 1907.3ms/step | 31.5 samples/s
|
| 1128 |
+
Step 12810/50462 | Loss: 2.7167 | LR: 4.31e-04 | 1909.2ms/step | 31.4 samples/s
|
| 1129 |
+
Step 12820/50462 | Loss: 2.6607 | LR: 4.31e-04 | 1907.6ms/step | 31.5 samples/s
|
| 1130 |
+
Step 12830/50462 | Loss: 2.6658 | LR: 4.31e-04 | 1905.5ms/step | 31.5 samples/s
|
| 1131 |
+
Step 12840/50462 | Loss: 2.6999 | LR: 4.31e-04 | 1903.9ms/step | 31.5 samples/s
|
| 1132 |
+
Step 12850/50462 | Loss: 2.7165 | LR: 4.30e-04 | 1906.8ms/step | 31.5 samples/s
|
| 1133 |
+
Step 12860/50462 | Loss: 2.7027 | LR: 4.30e-04 | 1908.5ms/step | 31.4 samples/s
|
| 1134 |
+
Step 12870/50462 | Loss: 2.7006 | LR: 4.30e-04 | 1908.7ms/step | 31.4 samples/s
|
| 1135 |
+
Step 12880/50462 | Loss: 2.6678 | LR: 4.30e-04 | 1909.8ms/step | 31.4 samples/s
|
| 1136 |
+
Step 12890/50462 | Loss: 2.6886 | LR: 4.30e-04 | 1907.9ms/step | 31.4 samples/s
|
| 1137 |
+
Step 12900/50462 | Loss: 2.7021 | LR: 4.30e-04 | 1907.1ms/step | 31.5 samples/s
|
| 1138 |
+
Step 12910/50462 | Loss: 2.6800 | LR: 4.30e-04 | 1910.0ms/step | 31.4 samples/s
|
| 1139 |
+
Step 12920/50462 | Loss: 2.6727 | LR: 4.30e-04 | 1910.9ms/step | 31.4 samples/s
|
| 1140 |
+
Step 12930/50462 | Loss: 2.6648 | LR: 4.30e-04 | 1912.0ms/step | 31.4 samples/s
|
| 1141 |
+
Step 12940/50462 | Loss: 2.7469 | LR: 4.29e-04 | 1909.0ms/step | 31.4 samples/s
|
| 1142 |
+
Step 12950/50462 | Loss: 2.6841 | LR: 4.29e-04 | 1909.1ms/step | 31.4 samples/s
|
| 1143 |
+
Step 12960/50462 | Loss: 2.7085 | LR: 4.29e-04 | 1902.6ms/step | 31.5 samples/s
|
| 1144 |
+
Step 12970/50462 | Loss: 2.6230 | LR: 4.29e-04 | 1899.3ms/step | 31.6 samples/s
|
| 1145 |
+
Step 12980/50462 | Loss: 2.7039 | LR: 4.29e-04 | 1902.3ms/step | 31.5 samples/s
|
| 1146 |
+
Step 12990/50462 | Loss: 2.6719 | LR: 4.29e-04 | 1905.5ms/step | 31.5 samples/s
|
| 1147 |
+
Step 13000/50462 | Loss: 2.6812 | LR: 4.29e-04 | 1906.2ms/step | 31.5 samples/s
|
| 1148 |
+
Validation at step 13000: Loss = 2.6839, Perplexity = 14.64
|
| 1149 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-13000
|
| 1150 |
+
Step 13010/50462 | Loss: 2.7142 | LR: 4.29e-04 | 3241.4ms/step | 18.5 samples/s
|
| 1151 |
+
Step 13020/50462 | Loss: 2.6871 | LR: 4.29e-04 | 1897.4ms/step | 31.6 samples/s
|
| 1152 |
+
Step 13030/50462 | Loss: 2.6725 | LR: 4.28e-04 | 1900.1ms/step | 31.6 samples/s
|
| 1153 |
+
Step 13040/50462 | Loss: 2.6713 | LR: 4.28e-04 | 1901.7ms/step | 31.6 samples/s
|
| 1154 |
+
Step 13050/50462 | Loss: 2.6566 | LR: 4.28e-04 | 1905.0ms/step | 31.5 samples/s
|
| 1155 |
+
Step 13060/50462 | Loss: 2.7241 | LR: 4.28e-04 | 1904.0ms/step | 31.5 samples/s
|
| 1156 |
+
Step 13070/50462 | Loss: 2.7281 | LR: 4.28e-04 | 1906.5ms/step | 31.5 samples/s
|
| 1157 |
+
Step 13080/50462 | Loss: 2.7306 | LR: 4.28e-04 | 1907.6ms/step | 31.5 samples/s
|
| 1158 |
+
Step 13090/50462 | Loss: 2.7062 | LR: 4.28e-04 | 1908.6ms/step | 31.4 samples/s
|
| 1159 |
+
Step 13100/50462 | Loss: 2.7008 | LR: 4.28e-04 | 1909.7ms/step | 31.4 samples/s
|
| 1160 |
+
Step 13110/50462 | Loss: 2.6811 | LR: 4.28e-04 | 1908.1ms/step | 31.4 samples/s
|
| 1161 |
+
Step 13120/50462 | Loss: 2.6920 | LR: 4.27e-04 | 1909.7ms/step | 31.4 samples/s
|
| 1162 |
+
Step 13130/50462 | Loss: 2.7042 | LR: 4.27e-04 | 1910.7ms/step | 31.4 samples/s
|
| 1163 |
+
Step 13140/50462 | Loss: 2.6688 | LR: 4.27e-04 | 1903.3ms/step | 31.5 samples/s
|
| 1164 |
+
Step 13150/50462 | Loss: 2.6924 | LR: 4.27e-04 | 1906.1ms/step | 31.5 samples/s
|
| 1165 |
+
Step 13160/50462 | Loss: 2.6956 | LR: 4.27e-04 | 1906.2ms/step | 31.5 samples/s
|
| 1166 |
+
Step 13170/50462 | Loss: 2.6461 | LR: 4.27e-04 | 1911.4ms/step | 31.4 samples/s
|
| 1167 |
+
Step 13180/50462 | Loss: 2.6686 | LR: 4.27e-04 | 1910.5ms/step | 31.4 samples/s
|
| 1168 |
+
Step 13190/50462 | Loss: 2.6776 | LR: 4.27e-04 | 1911.3ms/step | 31.4 samples/s
|
| 1169 |
+
Step 13200/50462 | Loss: 2.6869 | LR: 4.27e-04 | 1899.7ms/step | 31.6 samples/s
|
| 1170 |
+
Step 13210/50462 | Loss: 2.6889 | LR: 4.26e-04 | 1904.4ms/step | 31.5 samples/s
|
| 1171 |
+
Step 13220/50462 | Loss: 2.6768 | LR: 4.26e-04 | 1906.4ms/step | 31.5 samples/s
|
| 1172 |
+
Step 13230/50462 | Loss: 2.6647 | LR: 4.26e-04 | 1908.0ms/step | 31.4 samples/s
|
| 1173 |
+
Step 13240/50462 | Loss: 2.6874 | LR: 4.26e-04 | 1909.1ms/step | 31.4 samples/s
|
| 1174 |
+
Step 13250/50462 | Loss: 2.6569 | LR: 4.26e-04 | 1908.0ms/step | 31.4 samples/s
|
| 1175 |
+
Validation at step 13250: Loss = 2.6810, Perplexity = 14.60
|
| 1176 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-13250
|
| 1177 |
+
Step 13260/50462 | Loss: 2.6407 | LR: 4.26e-04 | 3237.1ms/step | 18.5 samples/s
|
| 1178 |
+
Step 13270/50462 | Loss: 2.7582 | LR: 4.26e-04 | 1899.0ms/step | 31.6 samples/s
|
| 1179 |
+
Step 13280/50462 | Loss: 2.6372 | LR: 4.26e-04 | 1902.4ms/step | 31.5 samples/s
|
| 1180 |
+
Step 13290/50462 | Loss: 2.6950 | LR: 4.26e-04 | 1905.4ms/step | 31.5 samples/s
|
| 1181 |
+
Step 13300/50462 | Loss: 2.6873 | LR: 4.25e-04 | 1907.4ms/step | 31.5 samples/s
|
| 1182 |
+
Step 13310/50462 | Loss: 2.6693 | LR: 4.25e-04 | 1899.8ms/step | 31.6 samples/s
|
| 1183 |
+
Step 13320/50462 | Loss: 2.6872 | LR: 4.25e-04 | 1901.5ms/step | 31.6 samples/s
|
| 1184 |
+
Step 13330/50462 | Loss: 2.7018 | LR: 4.25e-04 | 1904.2ms/step | 31.5 samples/s
|
| 1185 |
+
Step 13340/50462 | Loss: 2.7127 | LR: 4.25e-04 | 1908.5ms/step | 31.4 samples/s
|
| 1186 |
+
Step 13350/50462 | Loss: 2.6586 | LR: 4.25e-04 | 1907.3ms/step | 31.5 samples/s
|
| 1187 |
+
Step 13360/50462 | Loss: 2.7407 | LR: 4.25e-04 | 1908.1ms/step | 31.4 samples/s
|
| 1188 |
+
Step 13370/50462 | Loss: 2.6919 | LR: 4.25e-04 | 1906.5ms/step | 31.5 samples/s
|
epfl_final/SparseAtt/wandb/run-20260825_031743-qctui00g/files/requirements.txt
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
| 142 |
+
entmax==1.3
|
epfl_final/SparseAtt/wandb/run-20260825_031743-qctui00g/run-qctui00g.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3c2b2e18b7f645e9650547f2209eb5281b91bdb949d95b3c3f3871eaa38bd444
|
| 3 |
+
size 10289152
|
epfl_final/SparseAtt/wandb/run-20260825_031836-rgmkfpnl/files/output.log
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Starting training...
|
| 2 |
+
Validation at step 19000: Loss = 2.6393, Perplexity = 14.00
|
| 3 |
+
wandb: WARNING Tried to log to step 19000 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 4 |
+
Switching attention mode for step 19001: dense (dense step interval)
|
| 5 |
+
Step 19010/50462 | Loss: 2.6276 | LR: 3.51e-04 | 58825.0ms/step | 1.0 samples/s
|
| 6 |
+
wandb: WARNING Tried to log to step 19010 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 7 |
+
Step 19020/50462 | Loss: 2.6279 | LR: 3.51e-04 | 5183.6ms/step | 11.6 samples/s
|
| 8 |
+
wandb: WARNING Tried to log to step 19020 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 9 |
+
Step 19030/50462 | Loss: 2.6434 | LR: 3.51e-04 | 5177.4ms/step | 11.6 samples/s
|
| 10 |
+
wandb: WARNING Tried to log to step 19030 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 11 |
+
Step 19040/50462 | Loss: 2.6177 | LR: 3.51e-04 | 5186.6ms/step | 11.6 samples/s
|
| 12 |
+
wandb: WARNING Tried to log to step 19040 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 13 |
+
Step 19050/50462 | Loss: 2.6221 | LR: 3.51e-04 | 5186.3ms/step | 11.6 samples/s
|
| 14 |
+
wandb: WARNING Tried to log to step 19050 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 15 |
+
Step 19060/50462 | Loss: 2.6198 | LR: 3.50e-04 | 5184.0ms/step | 11.6 samples/s
|
| 16 |
+
wandb: WARNING Tried to log to step 19060 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 17 |
+
Step 19070/50462 | Loss: 2.6230 | LR: 3.50e-04 | 5177.5ms/step | 11.6 samples/s
|
| 18 |
+
wandb: WARNING Tried to log to step 19070 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 19 |
+
Step 19080/50462 | Loss: 2.5970 | LR: 3.50e-04 | 5177.9ms/step | 11.6 samples/s
|
| 20 |
+
wandb: WARNING Tried to log to step 19080 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 21 |
+
Step 19090/50462 | Loss: 2.6260 | LR: 3.50e-04 | 5197.3ms/step | 11.5 samples/s
|
| 22 |
+
wandb: WARNING Tried to log to step 19090 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 23 |
+
Step 19100/50462 | Loss: 2.6249 | LR: 3.50e-04 | 5206.8ms/step | 11.5 samples/s
|
| 24 |
+
wandb: WARNING Tried to log to step 19100 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 25 |
+
Step 19110/50462 | Loss: 2.6293 | LR: 3.50e-04 | 5206.9ms/step | 11.5 samples/s
|
| 26 |
+
wandb: WARNING Tried to log to step 19110 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 27 |
+
Step 19120/50462 | Loss: 2.6367 | LR: 3.50e-04 | 5205.9ms/step | 11.5 samples/s
|
| 28 |
+
wandb: WARNING Tried to log to step 19120 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 29 |
+
Step 19130/50462 | Loss: 2.6336 | LR: 3.49e-04 | 5207.3ms/step | 11.5 samples/s
|
| 30 |
+
wandb: WARNING Tried to log to step 19130 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 31 |
+
Step 19140/50462 | Loss: 2.6194 | LR: 3.49e-04 | 5209.7ms/step | 11.5 samples/s
|
| 32 |
+
wandb: WARNING Tried to log to step 19140 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 33 |
+
Step 19150/50462 | Loss: 2.6282 | LR: 3.49e-04 | 5211.6ms/step | 11.5 samples/s
|
| 34 |
+
wandb: WARNING Tried to log to step 19150 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 35 |
+
Step 19160/50462 | Loss: 2.6331 | LR: 3.49e-04 | 5213.4ms/step | 11.5 samples/s
|
| 36 |
+
wandb: WARNING Tried to log to step 19160 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 37 |
+
Step 19170/50462 | Loss: 2.6337 | LR: 3.49e-04 | 5207.5ms/step | 11.5 samples/s
|
| 38 |
+
Step 19180/50462 | Loss: 2.6268 | LR: 3.49e-04 | 5205.1ms/step | 11.5 samples/s
|
| 39 |
+
Step 19190/50462 | Loss: 2.6396 | LR: 3.49e-04 | 5209.5ms/step | 11.5 samples/s
|
| 40 |
+
Step 19200/50462 | Loss: 2.6277 | LR: 3.48e-04 | 5215.3ms/step | 11.5 samples/s
|
| 41 |
+
Step 19210/50462 | Loss: 2.6454 | LR: 3.48e-04 | 5209.6ms/step | 11.5 samples/s
|
| 42 |
+
Step 19220/50462 | Loss: 2.6127 | LR: 3.48e-04 | 5199.9ms/step | 11.5 samples/s
|
| 43 |
+
Step 19230/50462 | Loss: 2.6041 | LR: 3.48e-04 | 5211.5ms/step | 11.5 samples/s
|
| 44 |
+
Step 19240/50462 | Loss: 2.6336 | LR: 3.48e-04 | 5210.4ms/step | 11.5 samples/s
|
| 45 |
+
Step 19250/50462 | Loss: 2.6102 | LR: 3.48e-04 | 5209.3ms/step | 11.5 samples/s
|
| 46 |
+
Validation at step 19250: Loss = 2.6360, Perplexity = 13.96
|
| 47 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-19250
|
| 48 |
+
Step 19260/50462 | Loss: 2.6208 | LR: 3.48e-04 | 10086.1ms/step | 5.9 samples/s
|
| 49 |
+
Step 19270/50462 | Loss: 2.6216 | LR: 3.47e-04 | 5217.0ms/step | 11.5 samples/s
|
| 50 |
+
Step 19280/50462 | Loss: 2.6105 | LR: 3.47e-04 | 5209.9ms/step | 11.5 samples/s
|
| 51 |
+
Step 19290/50462 | Loss: 2.6180 | LR: 3.47e-04 | 5209.0ms/step | 11.5 samples/s
|
| 52 |
+
Step 19300/50462 | Loss: 2.6167 | LR: 3.47e-04 | 5206.7ms/step | 11.5 samples/s
|
| 53 |
+
Step 19310/50462 | Loss: 2.6291 | LR: 3.47e-04 | 5211.8ms/step | 11.5 samples/s
|
| 54 |
+
Step 19320/50462 | Loss: 2.6249 | LR: 3.47e-04 | 5213.9ms/step | 11.5 samples/s
|
| 55 |
+
Step 19330/50462 | Loss: 2.6130 | LR: 3.47e-04 | 5209.8ms/step | 11.5 samples/s
|
| 56 |
+
Step 19340/50462 | Loss: 2.6182 | LR: 3.46e-04 | 5217.0ms/step | 11.5 samples/s
|
| 57 |
+
Step 19350/50462 | Loss: 2.6240 | LR: 3.46e-04 | 5212.1ms/step | 11.5 samples/s
|
| 58 |
+
Step 19360/50462 | Loss: 2.6236 | LR: 3.46e-04 | 5212.8ms/step | 11.5 samples/s
|
| 59 |
+
Step 19370/50462 | Loss: 2.6154 | LR: 3.46e-04 | 5215.4ms/step | 11.5 samples/s
|
| 60 |
+
Step 19380/50462 | Loss: 2.5828 | LR: 3.46e-04 | 5209.3ms/step | 11.5 samples/s
|
| 61 |
+
Step 19390/50462 | Loss: 2.6033 | LR: 3.46e-04 | 5211.8ms/step | 11.5 samples/s
|
| 62 |
+
Step 19400/50462 | Loss: 2.6069 | LR: 3.46e-04 | 5213.9ms/step | 11.5 samples/s
|
| 63 |
+
Step 19410/50462 | Loss: 2.6311 | LR: 3.45e-04 | 5213.7ms/step | 11.5 samples/s
|
| 64 |
+
Step 19420/50462 | Loss: 2.6305 | LR: 3.45e-04 | 5215.1ms/step | 11.5 samples/s
|
| 65 |
+
Step 19430/50462 | Loss: 2.6104 | LR: 3.45e-04 | 5210.1ms/step | 11.5 samples/s
|
| 66 |
+
Step 19440/50462 | Loss: 2.6210 | LR: 3.45e-04 | 5213.0ms/step | 11.5 samples/s
|
| 67 |
+
Step 19450/50462 | Loss: 2.6450 | LR: 3.45e-04 | 5208.8ms/step | 11.5 samples/s
|
| 68 |
+
Step 19460/50462 | Loss: 2.5854 | LR: 3.45e-04 | 5212.8ms/step | 11.5 samples/s
|
| 69 |
+
Step 19470/50462 | Loss: 2.6128 | LR: 3.45e-04 | 5211.7ms/step | 11.5 samples/s
|
| 70 |
+
Step 19480/50462 | Loss: 2.6226 | LR: 3.44e-04 | 5216.4ms/step | 11.5 samples/s
|
| 71 |
+
Step 19490/50462 | Loss: 2.6375 | LR: 3.44e-04 | 5213.7ms/step | 11.5 samples/s
|
| 72 |
+
Step 19500/50462 | Loss: 2.6199 | LR: 3.44e-04 | 5210.8ms/step | 11.5 samples/s
|
| 73 |
+
Validation at step 19500: Loss = 2.6327, Perplexity = 13.91
|
| 74 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-19500
|
| 75 |
+
Step 19510/50462 | Loss: 2.6441 | LR: 3.44e-04 | 10084.1ms/step | 5.9 samples/s
|
| 76 |
+
Step 19520/50462 | Loss: 2.6217 | LR: 3.44e-04 | 5188.9ms/step | 11.6 samples/s
|
| 77 |
+
Step 19530/50462 | Loss: 2.6357 | LR: 3.44e-04 | 5207.1ms/step | 11.5 samples/s
|
| 78 |
+
Step 19540/50462 | Loss: 2.6183 | LR: 3.44e-04 | 5215.2ms/step | 11.5 samples/s
|
| 79 |
+
Step 19550/50462 | Loss: 2.6490 | LR: 3.43e-04 | 5214.6ms/step | 11.5 samples/s
|
| 80 |
+
Step 19560/50462 | Loss: 2.6155 | LR: 3.43e-04 | 5209.2ms/step | 11.5 samples/s
|
| 81 |
+
Step 19570/50462 | Loss: 2.6192 | LR: 3.43e-04 | 5216.5ms/step | 11.5 samples/s
|
| 82 |
+
Step 19580/50462 | Loss: 2.6125 | LR: 3.43e-04 | 5218.2ms/step | 11.5 samples/s
|
| 83 |
+
Step 19590/50462 | Loss: 2.6193 | LR: 3.43e-04 | 5213.1ms/step | 11.5 samples/s
|
| 84 |
+
Step 19600/50462 | Loss: 2.6108 | LR: 3.43e-04 | 5209.8ms/step | 11.5 samples/s
|
| 85 |
+
Step 19610/50462 | Loss: 2.6287 | LR: 3.42e-04 | 5210.3ms/step | 11.5 samples/s
|
| 86 |
+
Step 19620/50462 | Loss: 2.6168 | LR: 3.42e-04 | 5210.2ms/step | 11.5 samples/s
|
| 87 |
+
Step 19630/50462 | Loss: 2.6554 | LR: 3.42e-04 | 5212.8ms/step | 11.5 samples/s
|
| 88 |
+
Step 19640/50462 | Loss: 2.6127 | LR: 3.42e-04 | 5210.0ms/step | 11.5 samples/s
|
| 89 |
+
Step 19650/50462 | Loss: 2.6324 | LR: 3.42e-04 | 5206.1ms/step | 11.5 samples/s
|
| 90 |
+
Step 19660/50462 | Loss: 2.6299 | LR: 3.42e-04 | 5211.5ms/step | 11.5 samples/s
|
| 91 |
+
Step 19670/50462 | Loss: 2.6102 | LR: 3.42e-04 | 5210.8ms/step | 11.5 samples/s
|
| 92 |
+
Step 19680/50462 | Loss: 2.5796 | LR: 3.41e-04 | 5210.2ms/step | 11.5 samples/s
|
| 93 |
+
Step 19690/50462 | Loss: 2.6139 | LR: 3.41e-04 | 5205.2ms/step | 11.5 samples/s
|
| 94 |
+
Step 19700/50462 | Loss: 2.6395 | LR: 3.41e-04 | 5212.5ms/step | 11.5 samples/s
|
| 95 |
+
Step 19710/50462 | Loss: 2.6274 | LR: 3.41e-04 | 5211.4ms/step | 11.5 samples/s
|
| 96 |
+
Step 19720/50462 | Loss: 2.6226 | LR: 3.41e-04 | 5214.9ms/step | 11.5 samples/s
|
| 97 |
+
Step 19730/50462 | Loss: 2.6307 | LR: 3.41e-04 | 5214.8ms/step | 11.5 samples/s
|
| 98 |
+
Step 19740/50462 | Loss: 2.6085 | LR: 3.41e-04 | 5216.6ms/step | 11.5 samples/s
|
| 99 |
+
Step 19750/50462 | Loss: 2.5990 | LR: 3.40e-04 | 5218.6ms/step | 11.5 samples/s
|
| 100 |
+
Validation at step 19750: Loss = 2.6302, Perplexity = 13.88
|
| 101 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-19750
|
| 102 |
+
Step 19760/50462 | Loss: 2.6374 | LR: 3.40e-04 | 10133.4ms/step | 5.9 samples/s
|
| 103 |
+
Step 19770/50462 | Loss: 2.6072 | LR: 3.40e-04 | 5201.9ms/step | 11.5 samples/s
|
| 104 |
+
Step 19780/50462 | Loss: 2.6165 | LR: 3.40e-04 | 5192.3ms/step | 11.6 samples/s
|
| 105 |
+
Step 19790/50462 | Loss: 2.6049 | LR: 3.40e-04 | 5201.5ms/step | 11.5 samples/s
|
| 106 |
+
Step 19800/50462 | Loss: 2.6400 | LR: 3.40e-04 | 5208.4ms/step | 11.5 samples/s
|
| 107 |
+
Step 19810/50462 | Loss: 2.6358 | LR: 3.40e-04 | 5208.6ms/step | 11.5 samples/s
|
| 108 |
+
Step 19820/50462 | Loss: 2.6181 | LR: 3.39e-04 | 5211.8ms/step | 11.5 samples/s
|
| 109 |
+
Step 19830/50462 | Loss: 2.6115 | LR: 3.39e-04 | 5210.6ms/step | 11.5 samples/s
|
| 110 |
+
Step 19840/50462 | Loss: 2.6226 | LR: 3.39e-04 | 5208.6ms/step | 11.5 samples/s
|
| 111 |
+
Step 19850/50462 | Loss: 2.6281 | LR: 3.39e-04 | 5216.2ms/step | 11.5 samples/s
|
| 112 |
+
Step 19860/50462 | Loss: 2.5926 | LR: 3.39e-04 | 5211.7ms/step | 11.5 samples/s
|
| 113 |
+
Step 19870/50462 | Loss: 2.6337 | LR: 3.39e-04 | 5213.9ms/step | 11.5 samples/s
|
| 114 |
+
Step 19880/50462 | Loss: 2.6258 | LR: 3.39e-04 | 5203.8ms/step | 11.5 samples/s
|
| 115 |
+
Step 19890/50462 | Loss: 2.6196 | LR: 3.38e-04 | 5206.4ms/step | 11.5 samples/s
|
| 116 |
+
Step 19900/50462 | Loss: 2.6335 | LR: 3.38e-04 | 5211.7ms/step | 11.5 samples/s
|
| 117 |
+
Step 19910/50462 | Loss: 2.6116 | LR: 3.38e-04 | 5212.3ms/step | 11.5 samples/s
|
| 118 |
+
Step 19920/50462 | Loss: 2.6139 | LR: 3.38e-04 | 5217.4ms/step | 11.5 samples/s
|
| 119 |
+
Step 19930/50462 | Loss: 2.6116 | LR: 3.38e-04 | 5213.0ms/step | 11.5 samples/s
|
| 120 |
+
Step 19940/50462 | Loss: 2.6022 | LR: 3.38e-04 | 5214.0ms/step | 11.5 samples/s
|
| 121 |
+
Step 19950/50462 | Loss: 2.6203 | LR: 3.37e-04 | 5216.3ms/step | 11.5 samples/s
|
| 122 |
+
Step 19960/50462 | Loss: 2.6059 | LR: 3.37e-04 | 5213.4ms/step | 11.5 samples/s
|
| 123 |
+
Step 19970/50462 | Loss: 2.6136 | LR: 3.37e-04 | 5215.7ms/step | 11.5 samples/s
|
| 124 |
+
Step 19980/50462 | Loss: 2.6060 | LR: 3.37e-04 | 5211.4ms/step | 11.5 samples/s
|
| 125 |
+
Step 19990/50462 | Loss: 2.6302 | LR: 3.37e-04 | 5219.2ms/step | 11.5 samples/s
|
| 126 |
+
Step 20000/50462 | Loss: 2.6289 | LR: 3.37e-04 | 5215.4ms/step | 11.5 samples/s
|
| 127 |
+
Validation at step 20000: Loss = 2.6269, Perplexity = 13.83
|
| 128 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-20000
|
| 129 |
+
Step 20010/50462 | Loss: 2.6132 | LR: 3.37e-04 | 10096.1ms/step | 5.9 samples/s
|
| 130 |
+
Step 20020/50462 | Loss: 2.6169 | LR: 3.36e-04 | 5209.0ms/step | 11.5 samples/s
|
| 131 |
+
Step 20030/50462 | Loss: 2.6383 | LR: 3.36e-04 | 5209.3ms/step | 11.5 samples/s
|
| 132 |
+
Step 20040/50462 | Loss: 2.5934 | LR: 3.36e-04 | 5209.4ms/step | 11.5 samples/s
|
| 133 |
+
Step 20050/50462 | Loss: 2.6217 | LR: 3.36e-04 | 5217.0ms/step | 11.5 samples/s
|
| 134 |
+
Step 20060/50462 | Loss: 2.6177 | LR: 3.36e-04 | 5201.9ms/step | 11.5 samples/s
|
| 135 |
+
Step 20070/50462 | Loss: 2.6223 | LR: 3.36e-04 | 5210.9ms/step | 11.5 samples/s
|
| 136 |
+
Step 20080/50462 | Loss: 2.6101 | LR: 3.36e-04 | 5212.8ms/step | 11.5 samples/s
|
| 137 |
+
Step 20090/50462 | Loss: 2.6152 | LR: 3.35e-04 | 5211.5ms/step | 11.5 samples/s
|
| 138 |
+
Step 20100/50462 | Loss: 2.6061 | LR: 3.35e-04 | 5217.2ms/step | 11.5 samples/s
|
| 139 |
+
Step 20110/50462 | Loss: 2.6024 | LR: 3.35e-04 | 5207.9ms/step | 11.5 samples/s
|
| 140 |
+
Step 20120/50462 | Loss: 2.6510 | LR: 3.35e-04 | 5214.3ms/step | 11.5 samples/s
|
| 141 |
+
Step 20130/50462 | Loss: 2.6158 | LR: 3.35e-04 | 5211.2ms/step | 11.5 samples/s
|
| 142 |
+
Step 20140/50462 | Loss: 2.6115 | LR: 3.35e-04 | 5210.2ms/step | 11.5 samples/s
|
| 143 |
+
Step 20150/50462 | Loss: 2.6288 | LR: 3.35e-04 | 5207.9ms/step | 11.5 samples/s
|
| 144 |
+
Step 20160/50462 | Loss: 2.6027 | LR: 3.34e-04 | 5213.5ms/step | 11.5 samples/s
|
| 145 |
+
Step 20170/50462 | Loss: 2.6014 | LR: 3.34e-04 | 5210.2ms/step | 11.5 samples/s
|
| 146 |
+
Step 20180/50462 | Loss: 2.6079 | LR: 3.34e-04 | 5211.8ms/step | 11.5 samples/s
|
| 147 |
+
Step 20190/50462 | Loss: 2.5964 | LR: 3.34e-04 | 5217.5ms/step | 11.5 samples/s
|
| 148 |
+
Step 20200/50462 | Loss: 2.6267 | LR: 3.34e-04 | 5212.1ms/step | 11.5 samples/s
|
| 149 |
+
Step 20210/50462 | Loss: 2.6348 | LR: 3.34e-04 | 5214.7ms/step | 11.5 samples/s
|
| 150 |
+
Step 20220/50462 | Loss: 2.6126 | LR: 3.33e-04 | 5213.1ms/step | 11.5 samples/s
|
| 151 |
+
Step 20230/50462 | Loss: 2.6279 | LR: 3.33e-04 | 5216.2ms/step | 11.5 samples/s
|
| 152 |
+
Step 20240/50462 | Loss: 2.6147 | LR: 3.33e-04 | 5210.9ms/step | 11.5 samples/s
|
| 153 |
+
Step 20250/50462 | Loss: 2.6301 | LR: 3.33e-04 | 5212.2ms/step | 11.5 samples/s
|
| 154 |
+
Validation at step 20250: Loss = 2.6241, Perplexity = 13.79
|
| 155 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-20250
|
| 156 |
+
Step 20260/50462 | Loss: 2.6042 | LR: 3.33e-04 | 10053.0ms/step | 6.0 samples/s
|
| 157 |
+
Step 20270/50462 | Loss: 2.6282 | LR: 3.33e-04 | 5213.1ms/step | 11.5 samples/s
|
| 158 |
+
Step 20280/50462 | Loss: 2.6116 | LR: 3.33e-04 | 5205.2ms/step | 11.5 samples/s
|
| 159 |
+
Step 20290/50462 | Loss: 2.6401 | LR: 3.32e-04 | 5210.7ms/step | 11.5 samples/s
|
| 160 |
+
Step 20300/50462 | Loss: 2.6155 | LR: 3.32e-04 | 5212.4ms/step | 11.5 samples/s
|
| 161 |
+
Step 20310/50462 | Loss: 2.6005 | LR: 3.32e-04 | 5198.0ms/step | 11.5 samples/s
|
| 162 |
+
Step 20320/50462 | Loss: 2.6076 | LR: 3.32e-04 | 5206.6ms/step | 11.5 samples/s
|
| 163 |
+
Step 20330/50462 | Loss: 2.6374 | LR: 3.32e-04 | 5211.5ms/step | 11.5 samples/s
|
| 164 |
+
Step 20340/50462 | Loss: 2.6162 | LR: 3.32e-04 | 5205.3ms/step | 11.5 samples/s
|
| 165 |
+
Step 20350/50462 | Loss: 2.6025 | LR: 3.32e-04 | 5208.6ms/step | 11.5 samples/s
|
| 166 |
+
Step 20360/50462 | Loss: 2.5897 | LR: 3.31e-04 | 5210.7ms/step | 11.5 samples/s
|
| 167 |
+
Step 20370/50462 | Loss: 2.6137 | LR: 3.31e-04 | 5207.8ms/step | 11.5 samples/s
|
| 168 |
+
Step 20380/50462 | Loss: 2.6348 | LR: 3.31e-04 | 5209.4ms/step | 11.5 samples/s
|
| 169 |
+
Step 20390/50462 | Loss: 2.6204 | LR: 3.31e-04 | 5203.2ms/step | 11.5 samples/s
|
| 170 |
+
Step 20400/50462 | Loss: 2.6244 | LR: 3.31e-04 | 5211.3ms/step | 11.5 samples/s
|
| 171 |
+
Step 20410/50462 | Loss: 2.5975 | LR: 3.31e-04 | 5212.8ms/step | 11.5 samples/s
|
| 172 |
+
Step 20420/50462 | Loss: 2.5898 | LR: 3.30e-04 | 5221.2ms/step | 11.5 samples/s
|
| 173 |
+
Step 20430/50462 | Loss: 2.5838 | LR: 3.30e-04 | 5208.3ms/step | 11.5 samples/s
|
| 174 |
+
Step 20440/50462 | Loss: 2.6201 | LR: 3.30e-04 | 5215.3ms/step | 11.5 samples/s
|
| 175 |
+
Step 20450/50462 | Loss: 2.5900 | LR: 3.30e-04 | 5205.1ms/step | 11.5 samples/s
|
| 176 |
+
Step 20460/50462 | Loss: 2.6250 | LR: 3.30e-04 | 5212.1ms/step | 11.5 samples/s
|
| 177 |
+
Step 20470/50462 | Loss: 2.6140 | LR: 3.30e-04 | 5210.3ms/step | 11.5 samples/s
|
| 178 |
+
Step 20480/50462 | Loss: 2.5986 | LR: 3.30e-04 | 5205.8ms/step | 11.5 samples/s
|
| 179 |
+
Step 20490/50462 | Loss: 2.6229 | LR: 3.29e-04 | 5211.0ms/step | 11.5 samples/s
|
| 180 |
+
Step 20500/50462 | Loss: 2.6161 | LR: 3.29e-04 | 5213.9ms/step | 11.5 samples/s
|
| 181 |
+
Validation at step 20500: Loss = 2.6218, Perplexity = 13.76
|
| 182 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-20500
|
| 183 |
+
Step 20510/50462 | Loss: 2.6112 | LR: 3.29e-04 | 10051.6ms/step | 6.0 samples/s
|
| 184 |
+
Step 20520/50462 | Loss: 2.6233 | LR: 3.29e-04 | 5211.8ms/step | 11.5 samples/s
|
| 185 |
+
Step 20530/50462 | Loss: 2.6291 | LR: 3.29e-04 | 5208.1ms/step | 11.5 samples/s
|
| 186 |
+
Step 20540/50462 | Loss: 2.6162 | LR: 3.29e-04 | 5209.2ms/step | 11.5 samples/s
|
| 187 |
+
Step 20550/50462 | Loss: 2.6237 | LR: 3.29e-04 | 5205.4ms/step | 11.5 samples/s
|
| 188 |
+
Step 20560/50462 | Loss: 2.5945 | LR: 3.28e-04 | 5207.2ms/step | 11.5 samples/s
|
| 189 |
+
Step 20570/50462 | Loss: 2.5894 | LR: 3.28e-04 | 5208.5ms/step | 11.5 samples/s
|
| 190 |
+
Step 20580/50462 | Loss: 2.6280 | LR: 3.28e-04 | 5209.3ms/step | 11.5 samples/s
|
| 191 |
+
Step 20590/50462 | Loss: 2.6126 | LR: 3.28e-04 | 5204.8ms/step | 11.5 samples/s
|
| 192 |
+
Step 20600/50462 | Loss: 2.6064 | LR: 3.28e-04 | 5209.0ms/step | 11.5 samples/s
|
| 193 |
+
Step 20610/50462 | Loss: 2.6273 | LR: 3.28e-04 | 5213.3ms/step | 11.5 samples/s
|
| 194 |
+
Step 20620/50462 | Loss: 2.5983 | LR: 3.27e-04 | 5212.2ms/step | 11.5 samples/s
|
| 195 |
+
Step 20630/50462 | Loss: 2.6366 | LR: 3.27e-04 | 5206.8ms/step | 11.5 samples/s
|
| 196 |
+
Step 20640/50462 | Loss: 2.6148 | LR: 3.27e-04 | 5211.1ms/step | 11.5 samples/s
|
| 197 |
+
Step 20650/50462 | Loss: 2.6098 | LR: 3.27e-04 | 5210.6ms/step | 11.5 samples/s
|
| 198 |
+
Step 20660/50462 | Loss: 2.6029 | LR: 3.27e-04 | 5210.3ms/step | 11.5 samples/s
|
epfl_final/SparseAtt/wandb/run-20260825_031836-rgmkfpnl/files/requirements.txt
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
| 142 |
+
entmax==1.3
|
epfl_final/SparseAtt/wandb/run-20260825_031836-rgmkfpnl/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-08-25T03:18:36.775278Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"configs/p2_1p2b_dense_seed70_2gpu.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/TokenPrune.git",
|
| 14 |
+
"commit": "5637886f0cc760009989b14e68c6ae06d4827418"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "h100-workspace-2-gpu-0-12",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 23 |
+
"gpu_count": 2,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "9374931091456",
|
| 27 |
+
"used": "4850446520320"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1623005732864"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-7004b604-21a2-890e-0312-3eee1eb93c67"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 43 |
+
"memoryTotal": "85520809984",
|
| 44 |
+
"cudaCores": 16896,
|
| 45 |
+
"architecture": "Hopper",
|
| 46 |
+
"uuid": "GPU-3f5bace4-0066-71eb-863d-73a3a6d79e3b"
|
| 47 |
+
}
|
| 48 |
+
],
|
| 49 |
+
"cudaVersion": "13.0",
|
| 50 |
+
"writerId": "vf2fdaq6haciwtllnlhxis2b82cwh92w"
|
| 51 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260825_031836-rgmkfpnl/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260825_031836-rgmkfpnl/logs/debug.log
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-08-25 03:18:36,796 INFO MainThread:229 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-08-25 03:18:36,797 INFO MainThread:229 [wandb_setup.py:_flush():81] Configure stats pid to 229
|
| 3 |
+
2026-08-25 03:18:36,797 INFO MainThread:229 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-08-25 03:18:36,797 INFO MainThread:229 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260825_031836-rgmkfpnl/logs/debug.log
|
| 5 |
+
2026-08-25 03:18:36,797 INFO MainThread:229 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260825_031836-rgmkfpnl/logs/debug-internal.log
|
| 6 |
+
2026-08-25 03:18:36,797 INFO MainThread:229 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-08-25 03:18:36,797 INFO MainThread:229 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-19000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 15, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 70, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b dense 8k seed70 2gpu resume', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'dense', 'seed70'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-08-25 03:18:36,797 INFO MainThread:229 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-08-25 03:18:37,559 INFO MainThread:229 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-08-25 03:18:37,988 INFO MainThread:229 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-08-25 03:18:37,991 INFO MainThread:229 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-08-25 03:18:38,030 INFO MainThread:229 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-08-25 03:18:38,552 INFO MainThread:229 [wandb_init.py:init():1051] run resumed
|
| 15 |
+
2026-08-25 03:18:38,553 INFO MainThread:229 [wandb_init.py:init():1056] starting run threads in backend
|
| 16 |
+
2026-08-25 03:18:38,724 INFO MainThread:229 [wandb_run.py:_console_start():2554] atexit reg
|
| 17 |
+
2026-08-25 03:18:38,724 INFO MainThread:229 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 18 |
+
2026-08-25 03:18:38,724 INFO MainThread:229 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 19 |
+
2026-08-25 03:18:38,724 INFO MainThread:229 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 20 |
+
2026-08-25 03:18:38,731 INFO MainThread:229 [wandb_init.py:init():1094] run started, returning control to user process
|
epfl_final/SparseAtt/wandb/run-20260825_031836-rgmkfpnl/run-rgmkfpnl.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:395a08b21b720ed1277d2c93f24bba8e5db2eeff6b9ea93467608fe024e13eec
|
| 3 |
+
size 1933312
|
epfl_final/SparseAtt/wandb/run-20260825_134623-qctui00g/files/output.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260825_134623-qctui00g/files/requirements.txt
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
| 142 |
+
entmax==1.3
|
epfl_final/SparseAtt/wandb/run-20260825_134623-qctui00g/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-08-25T13:46:23.233653Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"configs/p2_1p2b_dense_seed71_6gpu.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/TokenPrune.git",
|
| 14 |
+
"commit": "5637886f0cc760009989b14e68c6ae06d4827418"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "h100-train-6-gpu-0-36",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 23 |
+
"gpu_count": 6,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "9371709865984",
|
| 27 |
+
"used": "6628745785344"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1623005732864"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-98029aa4-c589-b65b-3ab7-6495256f31ed"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 43 |
+
"memoryTotal": "85520809984",
|
| 44 |
+
"cudaCores": 16896,
|
| 45 |
+
"architecture": "Hopper",
|
| 46 |
+
"uuid": "GPU-37a1c6c2-d3df-3064-44df-33fa82304bd0"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 50 |
+
"memoryTotal": "85520809984",
|
| 51 |
+
"cudaCores": 16896,
|
| 52 |
+
"architecture": "Hopper",
|
| 53 |
+
"uuid": "GPU-9a903f43-9348-6fa6-bcf7-e12bd01803a7"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 57 |
+
"memoryTotal": "85520809984",
|
| 58 |
+
"cudaCores": 16896,
|
| 59 |
+
"architecture": "Hopper",
|
| 60 |
+
"uuid": "GPU-24b8d952-dc08-33b9-d5df-54ce871fd288"
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 64 |
+
"memoryTotal": "85520809984",
|
| 65 |
+
"cudaCores": 16896,
|
| 66 |
+
"architecture": "Hopper",
|
| 67 |
+
"uuid": "GPU-b87e8111-63b6-b364-28b9-c4ce747aec08"
|
| 68 |
+
},
|
| 69 |
+
{
|
| 70 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 71 |
+
"memoryTotal": "85520809984",
|
| 72 |
+
"cudaCores": 16896,
|
| 73 |
+
"architecture": "Hopper",
|
| 74 |
+
"uuid": "GPU-bbbaaf81-9234-a1c6-0093-6d87b4e6af4c"
|
| 75 |
+
}
|
| 76 |
+
],
|
| 77 |
+
"cudaVersion": "13.0",
|
| 78 |
+
"writerId": "gddirbrgmxpp0auwnnxbbrtphub8t2o7"
|
| 79 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260825_134623-qctui00g/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260825_134623-qctui00g/logs/debug.log
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-08-25 13:46:23,256 INFO MainThread:231 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-08-25 13:46:23,256 INFO MainThread:231 [wandb_setup.py:_flush():81] Configure stats pid to 231
|
| 3 |
+
2026-08-25 13:46:23,256 INFO MainThread:231 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-08-25 13:46:23,256 INFO MainThread:231 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260825_134623-qctui00g/logs/debug.log
|
| 5 |
+
2026-08-25 13:46:23,256 INFO MainThread:231 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260825_134623-qctui00g/logs/debug-internal.log
|
| 6 |
+
2026-08-25 13:46:23,256 INFO MainThread:231 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-08-25 13:46:23,257 INFO MainThread:231 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-13250', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 71, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b dense 8k seed71 6gpu', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'dense', 'seed71'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-08-25 13:46:23,257 INFO MainThread:231 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-08-25 13:46:24,026 INFO MainThread:231 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-08-25 13:46:24,569 INFO MainThread:231 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-08-25 13:46:24,575 INFO MainThread:231 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-08-25 13:46:24,607 INFO MainThread:231 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-08-25 13:46:25,231 INFO MainThread:231 [wandb_init.py:init():1051] run resumed
|
| 15 |
+
2026-08-25 13:46:25,233 INFO MainThread:231 [wandb_init.py:init():1056] starting run threads in backend
|
| 16 |
+
2026-08-25 13:46:25,677 INFO MainThread:231 [wandb_run.py:_console_start():2554] atexit reg
|
| 17 |
+
2026-08-25 13:46:25,677 INFO MainThread:231 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 18 |
+
2026-08-25 13:46:25,677 INFO MainThread:231 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 19 |
+
2026-08-25 13:46:25,677 INFO MainThread:231 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 20 |
+
2026-08-25 13:46:25,686 INFO MainThread:231 [wandb_init.py:init():1094] run started, returning control to user process
|
epfl_final/SparseAtt/wandb/run-20260825_134623-qctui00g/run-qctui00g.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d8ce713b4fd0ca06350e870fecb068905dcd0896af1f6d929081250d12089cd3
|
| 3 |
+
size 11665408
|
epfl_final/SparseAtt/wandb/run-20260825_135931-rgmkfpnl/files/output.log
ADDED
|
@@ -0,0 +1,501 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Starting training...
|
| 2 |
+
Validation at step 20500: Loss = 2.6218, Perplexity = 13.76
|
| 3 |
+
wandb: WARNING Tried to log to step 20500 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 4 |
+
Switching attention mode for step 20501: dense (dense step interval)
|
| 5 |
+
Step 20510/50462 | Loss: 2.6112 | LR: 3.29e-04 | 73487.6ms/step | 0.8 samples/s
|
| 6 |
+
wandb: WARNING Tried to log to step 20510 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 7 |
+
Step 20520/50462 | Loss: 2.6233 | LR: 3.29e-04 | 5205.4ms/step | 11.5 samples/s
|
| 8 |
+
wandb: WARNING Tried to log to step 20520 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 9 |
+
Step 20530/50462 | Loss: 2.6291 | LR: 3.29e-04 | 5229.9ms/step | 11.5 samples/s
|
| 10 |
+
wandb: WARNING Tried to log to step 20530 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 11 |
+
Step 20540/50462 | Loss: 2.6162 | LR: 3.29e-04 | 5238.7ms/step | 11.5 samples/s
|
| 12 |
+
wandb: WARNING Tried to log to step 20540 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 13 |
+
Step 20550/50462 | Loss: 2.6236 | LR: 3.29e-04 | 5243.9ms/step | 11.4 samples/s
|
| 14 |
+
wandb: WARNING Tried to log to step 20550 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 15 |
+
Step 20560/50462 | Loss: 2.5945 | LR: 3.28e-04 | 5245.3ms/step | 11.4 samples/s
|
| 16 |
+
wandb: WARNING Tried to log to step 20560 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 17 |
+
Step 20570/50462 | Loss: 2.5894 | LR: 3.28e-04 | 5247.0ms/step | 11.4 samples/s
|
| 18 |
+
wandb: WARNING Tried to log to step 20570 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 19 |
+
Step 20580/50462 | Loss: 2.6280 | LR: 3.28e-04 | 5253.8ms/step | 11.4 samples/s
|
| 20 |
+
wandb: WARNING Tried to log to step 20580 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 21 |
+
Step 20590/50462 | Loss: 2.6127 | LR: 3.28e-04 | 5246.6ms/step | 11.4 samples/s
|
| 22 |
+
wandb: WARNING Tried to log to step 20590 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 23 |
+
Step 20600/50462 | Loss: 2.6063 | LR: 3.28e-04 | 5249.4ms/step | 11.4 samples/s
|
| 24 |
+
wandb: WARNING Tried to log to step 20600 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 25 |
+
Step 20610/50462 | Loss: 2.6273 | LR: 3.28e-04 | 5231.0ms/step | 11.5 samples/s
|
| 26 |
+
wandb: WARNING Tried to log to step 20610 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 27 |
+
Step 20620/50462 | Loss: 2.5983 | LR: 3.27e-04 | 5223.2ms/step | 11.5 samples/s
|
| 28 |
+
wandb: WARNING Tried to log to step 20620 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 29 |
+
Step 20630/50462 | Loss: 2.6366 | LR: 3.27e-04 | 5231.8ms/step | 11.5 samples/s
|
| 30 |
+
wandb: WARNING Tried to log to step 20630 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 31 |
+
Step 20640/50462 | Loss: 2.6148 | LR: 3.27e-04 | 5241.1ms/step | 11.4 samples/s
|
| 32 |
+
wandb: WARNING Tried to log to step 20640 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 33 |
+
Step 20650/50462 | Loss: 2.6097 | LR: 3.27e-04 | 5239.8ms/step | 11.5 samples/s
|
| 34 |
+
wandb: WARNING Tried to log to step 20650 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 35 |
+
Step 20660/50462 | Loss: 2.6029 | LR: 3.27e-04 | 5236.5ms/step | 11.5 samples/s
|
| 36 |
+
Step 20670/50462 | Loss: 2.6037 | LR: 3.27e-04 | 5234.5ms/step | 11.5 samples/s
|
| 37 |
+
Step 20680/50462 | Loss: 2.5932 | LR: 3.27e-04 | 5247.3ms/step | 11.4 samples/s
|
| 38 |
+
Step 20690/50462 | Loss: 2.5839 | LR: 3.26e-04 | 5250.9ms/step | 11.4 samples/s
|
| 39 |
+
Step 20700/50462 | Loss: 2.6363 | LR: 3.26e-04 | 5250.7ms/step | 11.4 samples/s
|
| 40 |
+
Step 20710/50462 | Loss: 2.6164 | LR: 3.26e-04 | 5253.2ms/step | 11.4 samples/s
|
| 41 |
+
Step 20720/50462 | Loss: 2.6060 | LR: 3.26e-04 | 5254.0ms/step | 11.4 samples/s
|
| 42 |
+
Step 20730/50462 | Loss: 2.6053 | LR: 3.26e-04 | 5247.7ms/step | 11.4 samples/s
|
| 43 |
+
Step 20740/50462 | Loss: 2.6442 | LR: 3.26e-04 | 5249.8ms/step | 11.4 samples/s
|
| 44 |
+
Step 20750/50462 | Loss: 2.6224 | LR: 3.26e-04 | 5249.3ms/step | 11.4 samples/s
|
| 45 |
+
Validation at step 20750: Loss = 2.6191, Perplexity = 13.72
|
| 46 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-20750
|
| 47 |
+
Step 20760/50462 | Loss: 2.6160 | LR: 3.25e-04 | 10161.8ms/step | 5.9 samples/s
|
| 48 |
+
Step 20770/50462 | Loss: 2.6229 | LR: 3.25e-04 | 5253.7ms/step | 11.4 samples/s
|
| 49 |
+
Step 20780/50462 | Loss: 2.6009 | LR: 3.25e-04 | 5252.2ms/step | 11.4 samples/s
|
| 50 |
+
Step 20790/50462 | Loss: 2.6164 | LR: 3.25e-04 | 5254.2ms/step | 11.4 samples/s
|
| 51 |
+
Step 20800/50462 | Loss: 2.5992 | LR: 3.25e-04 | 5258.3ms/step | 11.4 samples/s
|
| 52 |
+
Step 20810/50462 | Loss: 2.5972 | LR: 3.25e-04 | 5258.7ms/step | 11.4 samples/s
|
| 53 |
+
Step 20820/50462 | Loss: 2.6085 | LR: 3.24e-04 | 5261.7ms/step | 11.4 samples/s
|
| 54 |
+
Step 20830/50462 | Loss: 2.5909 | LR: 3.24e-04 | 5256.6ms/step | 11.4 samples/s
|
| 55 |
+
Step 20840/50462 | Loss: 2.6176 | LR: 3.24e-04 | 5260.6ms/step | 11.4 samples/s
|
| 56 |
+
Step 20850/50462 | Loss: 2.6036 | LR: 3.24e-04 | 5261.2ms/step | 11.4 samples/s
|
| 57 |
+
Step 20860/50462 | Loss: 2.6108 | LR: 3.24e-04 | 5253.8ms/step | 11.4 samples/s
|
| 58 |
+
Step 20870/50462 | Loss: 2.5858 | LR: 3.24e-04 | 5254.0ms/step | 11.4 samples/s
|
| 59 |
+
Step 20880/50462 | Loss: 2.6262 | LR: 3.24e-04 | 5263.0ms/step | 11.4 samples/s
|
| 60 |
+
Step 20890/50462 | Loss: 2.5930 | LR: 3.23e-04 | 5259.3ms/step | 11.4 samples/s
|
| 61 |
+
Step 20900/50462 | Loss: 2.6070 | LR: 3.23e-04 | 5258.7ms/step | 11.4 samples/s
|
| 62 |
+
Step 20910/50462 | Loss: 2.6087 | LR: 3.23e-04 | 5256.9ms/step | 11.4 samples/s
|
| 63 |
+
Step 20920/50462 | Loss: 2.6149 | LR: 3.23e-04 | 5237.1ms/step | 11.5 samples/s
|
| 64 |
+
Step 20930/50462 | Loss: 2.6183 | LR: 3.23e-04 | 5257.2ms/step | 11.4 samples/s
|
| 65 |
+
Step 20940/50462 | Loss: 2.6188 | LR: 3.23e-04 | 5266.6ms/step | 11.4 samples/s
|
| 66 |
+
Step 20950/50462 | Loss: 2.5984 | LR: 3.23e-04 | 5267.3ms/step | 11.4 samples/s
|
| 67 |
+
Step 20960/50462 | Loss: 2.6118 | LR: 3.22e-04 | 5268.8ms/step | 11.4 samples/s
|
| 68 |
+
Step 20970/50462 | Loss: 2.5773 | LR: 3.22e-04 | 5263.5ms/step | 11.4 samples/s
|
| 69 |
+
Step 20980/50462 | Loss: 2.5792 | LR: 3.22e-04 | 5265.1ms/step | 11.4 samples/s
|
| 70 |
+
Step 20990/50462 | Loss: 2.6189 | LR: 3.22e-04 | 5269.0ms/step | 11.4 samples/s
|
| 71 |
+
Step 21000/50462 | Loss: 2.6017 | LR: 3.22e-04 | 5265.6ms/step | 11.4 samples/s
|
| 72 |
+
Validation at step 21000: Loss = 2.6156, Perplexity = 13.68
|
| 73 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-21000
|
| 74 |
+
Step 21010/50462 | Loss: 2.6160 | LR: 3.22e-04 | 10149.1ms/step | 5.9 samples/s
|
| 75 |
+
Step 21020/50462 | Loss: 2.6125 | LR: 3.21e-04 | 5251.7ms/step | 11.4 samples/s
|
| 76 |
+
Step 21030/50462 | Loss: 2.6044 | LR: 3.21e-04 | 5260.8ms/step | 11.4 samples/s
|
| 77 |
+
Step 21040/50462 | Loss: 2.5867 | LR: 3.21e-04 | 5267.4ms/step | 11.4 samples/s
|
| 78 |
+
Step 21050/50462 | Loss: 2.5965 | LR: 3.21e-04 | 5265.4ms/step | 11.4 samples/s
|
| 79 |
+
Step 21060/50462 | Loss: 2.6080 | LR: 3.21e-04 | 5270.1ms/step | 11.4 samples/s
|
| 80 |
+
Step 21070/50462 | Loss: 2.6120 | LR: 3.21e-04 | 5270.5ms/step | 11.4 samples/s
|
| 81 |
+
Step 21080/50462 | Loss: 2.6176 | LR: 3.21e-04 | 5242.5ms/step | 11.4 samples/s
|
| 82 |
+
Step 21090/50462 | Loss: 2.6208 | LR: 3.20e-04 | 5256.4ms/step | 11.4 samples/s
|
| 83 |
+
Step 21100/50462 | Loss: 2.6153 | LR: 3.20e-04 | 5231.1ms/step | 11.5 samples/s
|
| 84 |
+
Step 21110/50462 | Loss: 2.6228 | LR: 3.20e-04 | 5257.0ms/step | 11.4 samples/s
|
| 85 |
+
Step 21120/50462 | Loss: 2.6257 | LR: 3.20e-04 | 5265.2ms/step | 11.4 samples/s
|
| 86 |
+
Step 21130/50462 | Loss: 2.5953 | LR: 3.20e-04 | 5256.5ms/step | 11.4 samples/s
|
| 87 |
+
Step 21140/50462 | Loss: 2.6149 | LR: 3.20e-04 | 5261.0ms/step | 11.4 samples/s
|
| 88 |
+
Step 21150/50462 | Loss: 2.5738 | LR: 3.19e-04 | 5260.1ms/step | 11.4 samples/s
|
| 89 |
+
Step 21160/50462 | Loss: 2.6027 | LR: 3.19e-04 | 5265.8ms/step | 11.4 samples/s
|
| 90 |
+
Step 21170/50462 | Loss: 2.6093 | LR: 3.19e-04 | 5265.0ms/step | 11.4 samples/s
|
| 91 |
+
Step 21180/50462 | Loss: 2.5972 | LR: 3.19e-04 | 5264.6ms/step | 11.4 samples/s
|
| 92 |
+
Step 21190/50462 | Loss: 2.5984 | LR: 3.19e-04 | 5261.6ms/step | 11.4 samples/s
|
| 93 |
+
Step 21200/50462 | Loss: 2.6017 | LR: 3.19e-04 | 5259.2ms/step | 11.4 samples/s
|
| 94 |
+
Step 21210/50462 | Loss: 2.6179 | LR: 3.19e-04 | 5268.5ms/step | 11.4 samples/s
|
| 95 |
+
Step 21220/50462 | Loss: 2.5977 | LR: 3.18e-04 | 5262.3ms/step | 11.4 samples/s
|
| 96 |
+
Step 21230/50462 | Loss: 2.5941 | LR: 3.18e-04 | 5252.0ms/step | 11.4 samples/s
|
| 97 |
+
Step 21240/50462 | Loss: 2.6092 | LR: 3.18e-04 | 5244.4ms/step | 11.4 samples/s
|
| 98 |
+
Step 21250/50462 | Loss: 2.6074 | LR: 3.18e-04 | 5259.3ms/step | 11.4 samples/s
|
| 99 |
+
Validation at step 21250: Loss = 2.6139, Perplexity = 13.65
|
| 100 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-21250
|
| 101 |
+
Step 21260/50462 | Loss: 2.6159 | LR: 3.18e-04 | 10146.0ms/step | 5.9 samples/s
|
| 102 |
+
Step 21270/50462 | Loss: 2.6063 | LR: 3.18e-04 | 5234.7ms/step | 11.5 samples/s
|
| 103 |
+
Step 21280/50462 | Loss: 2.6003 | LR: 3.17e-04 | 5245.1ms/step | 11.4 samples/s
|
| 104 |
+
Step 21290/50462 | Loss: 2.5630 | LR: 3.17e-04 | 5259.0ms/step | 11.4 samples/s
|
| 105 |
+
Step 21300/50462 | Loss: 2.6113 | LR: 3.17e-04 | 5255.7ms/step | 11.4 samples/s
|
| 106 |
+
Step 21310/50462 | Loss: 2.6041 | LR: 3.17e-04 | 5258.0ms/step | 11.4 samples/s
|
| 107 |
+
Step 21320/50462 | Loss: 2.6098 | LR: 3.17e-04 | 5254.2ms/step | 11.4 samples/s
|
| 108 |
+
Step 21330/50462 | Loss: 2.5988 | LR: 3.17e-04 | 5255.0ms/step | 11.4 samples/s
|
| 109 |
+
Step 21340/50462 | Loss: 2.5928 | LR: 3.17e-04 | 5259.6ms/step | 11.4 samples/s
|
| 110 |
+
Step 21350/50462 | Loss: 2.6224 | LR: 3.16e-04 | 5268.1ms/step | 11.4 samples/s
|
| 111 |
+
Step 21360/50462 | Loss: 2.6198 | LR: 3.16e-04 | 5260.9ms/step | 11.4 samples/s
|
| 112 |
+
Step 21370/50462 | Loss: 2.6144 | LR: 3.16e-04 | 5254.4ms/step | 11.4 samples/s
|
| 113 |
+
Step 21380/50462 | Loss: 2.6049 | LR: 3.16e-04 | 5247.8ms/step | 11.4 samples/s
|
| 114 |
+
Step 21390/50462 | Loss: 2.6159 | LR: 3.16e-04 | 5218.8ms/step | 11.5 samples/s
|
| 115 |
+
Step 21400/50462 | Loss: 2.6273 | LR: 3.16e-04 | 5217.5ms/step | 11.5 samples/s
|
| 116 |
+
Step 21410/50462 | Loss: 2.5950 | LR: 3.16e-04 | 5215.5ms/step | 11.5 samples/s
|
| 117 |
+
Step 21420/50462 | Loss: 2.6088 | LR: 3.15e-04 | 5210.9ms/step | 11.5 samples/s
|
| 118 |
+
Step 21430/50462 | Loss: 2.5994 | LR: 3.15e-04 | 5213.2ms/step | 11.5 samples/s
|
| 119 |
+
Step 21440/50462 | Loss: 2.5965 | LR: 3.15e-04 | 5213.2ms/step | 11.5 samples/s
|
| 120 |
+
Step 21450/50462 | Loss: 2.5874 | LR: 3.15e-04 | 5242.4ms/step | 11.4 samples/s
|
| 121 |
+
Step 21460/50462 | Loss: 2.6141 | LR: 3.15e-04 | 5248.5ms/step | 11.4 samples/s
|
| 122 |
+
Step 21470/50462 | Loss: 2.6077 | LR: 3.15e-04 | 5252.6ms/step | 11.4 samples/s
|
| 123 |
+
Step 21480/50462 | Loss: 2.5992 | LR: 3.14e-04 | 5259.2ms/step | 11.4 samples/s
|
| 124 |
+
Step 21490/50462 | Loss: 2.6112 | LR: 3.14e-04 | 5260.9ms/step | 11.4 samples/s
|
| 125 |
+
Step 21500/50462 | Loss: 2.6164 | LR: 3.14e-04 | 5266.0ms/step | 11.4 samples/s
|
| 126 |
+
Validation at step 21500: Loss = 2.6104, Perplexity = 13.61
|
| 127 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-21500
|
| 128 |
+
Step 21510/50462 | Loss: 2.5989 | LR: 3.14e-04 | 10179.1ms/step | 5.9 samples/s
|
| 129 |
+
Step 21520/50462 | Loss: 2.5899 | LR: 3.14e-04 | 5253.1ms/step | 11.4 samples/s
|
| 130 |
+
Step 21530/50462 | Loss: 2.6009 | LR: 3.14e-04 | 5264.4ms/step | 11.4 samples/s
|
| 131 |
+
Step 21540/50462 | Loss: 2.5802 | LR: 3.14e-04 | 5268.1ms/step | 11.4 samples/s
|
| 132 |
+
Step 21550/50462 | Loss: 2.6108 | LR: 3.13e-04 | 5267.1ms/step | 11.4 samples/s
|
| 133 |
+
Step 21560/50462 | Loss: 2.5729 | LR: 3.13e-04 | 5266.3ms/step | 11.4 samples/s
|
| 134 |
+
Step 21570/50462 | Loss: 2.5971 | LR: 3.13e-04 | 5268.0ms/step | 11.4 samples/s
|
| 135 |
+
Step 21580/50462 | Loss: 2.5911 | LR: 3.13e-04 | 5268.1ms/step | 11.4 samples/s
|
| 136 |
+
Step 21590/50462 | Loss: 2.6012 | LR: 3.13e-04 | 5268.5ms/step | 11.4 samples/s
|
| 137 |
+
Step 21600/50462 | Loss: 2.6141 | LR: 3.13e-04 | 5262.9ms/step | 11.4 samples/s
|
| 138 |
+
Step 21610/50462 | Loss: 2.5978 | LR: 3.12e-04 | 5265.1ms/step | 11.4 samples/s
|
| 139 |
+
Step 21620/50462 | Loss: 2.6168 | LR: 3.12e-04 | 5262.3ms/step | 11.4 samples/s
|
| 140 |
+
Step 21630/50462 | Loss: 2.6069 | LR: 3.12e-04 | 5254.3ms/step | 11.4 samples/s
|
| 141 |
+
Step 21640/50462 | Loss: 2.5944 | LR: 3.12e-04 | 5261.4ms/step | 11.4 samples/s
|
| 142 |
+
Step 21650/50462 | Loss: 2.5747 | LR: 3.12e-04 | 5263.6ms/step | 11.4 samples/s
|
| 143 |
+
Step 21660/50462 | Loss: 2.5932 | LR: 3.12e-04 | 5261.5ms/step | 11.4 samples/s
|
| 144 |
+
Step 21670/50462 | Loss: 2.6050 | LR: 3.12e-04 | 5267.1ms/step | 11.4 samples/s
|
| 145 |
+
Step 21680/50462 | Loss: 2.6057 | LR: 3.11e-04 | 5253.2ms/step | 11.4 samples/s
|
| 146 |
+
Step 21690/50462 | Loss: 2.5947 | LR: 3.11e-04 | 5259.4ms/step | 11.4 samples/s
|
| 147 |
+
Step 21700/50462 | Loss: 2.5844 | LR: 3.11e-04 | 5264.8ms/step | 11.4 samples/s
|
| 148 |
+
Step 21710/50462 | Loss: 2.5913 | LR: 3.11e-04 | 5264.9ms/step | 11.4 samples/s
|
| 149 |
+
Step 21720/50462 | Loss: 2.5991 | LR: 3.11e-04 | 5268.6ms/step | 11.4 samples/s
|
| 150 |
+
Step 21730/50462 | Loss: 2.6112 | LR: 3.11e-04 | 5269.3ms/step | 11.4 samples/s
|
| 151 |
+
Step 21740/50462 | Loss: 2.6083 | LR: 3.10e-04 | 5268.4ms/step | 11.4 samples/s
|
| 152 |
+
Step 21750/50462 | Loss: 2.6004 | LR: 3.10e-04 | 5273.4ms/step | 11.4 samples/s
|
| 153 |
+
Validation at step 21750: Loss = 2.6080, Perplexity = 13.57
|
| 154 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-21750
|
| 155 |
+
Step 21760/50462 | Loss: 2.6074 | LR: 3.10e-04 | 10161.4ms/step | 5.9 samples/s
|
| 156 |
+
Step 21770/50462 | Loss: 2.6016 | LR: 3.10e-04 | 5244.2ms/step | 11.4 samples/s
|
| 157 |
+
Step 21780/50462 | Loss: 2.5773 | LR: 3.10e-04 | 5259.1ms/step | 11.4 samples/s
|
| 158 |
+
Step 21790/50462 | Loss: 2.6089 | LR: 3.10e-04 | 5268.0ms/step | 11.4 samples/s
|
| 159 |
+
Step 21800/50462 | Loss: 2.5855 | LR: 3.10e-04 | 5270.6ms/step | 11.4 samples/s
|
| 160 |
+
Step 21810/50462 | Loss: 2.5769 | LR: 3.09e-04 | 5272.1ms/step | 11.4 samples/s
|
| 161 |
+
Step 21820/50462 | Loss: 2.6056 | LR: 3.09e-04 | 5265.1ms/step | 11.4 samples/s
|
| 162 |
+
Step 21830/50462 | Loss: 2.6061 | LR: 3.09e-04 | 5265.0ms/step | 11.4 samples/s
|
| 163 |
+
Step 21840/50462 | Loss: 2.5733 | LR: 3.09e-04 | 5260.3ms/step | 11.4 samples/s
|
| 164 |
+
Step 21850/50462 | Loss: 2.5809 | LR: 3.09e-04 | 5264.9ms/step | 11.4 samples/s
|
| 165 |
+
Step 21860/50462 | Loss: 2.5939 | LR: 3.09e-04 | 5269.7ms/step | 11.4 samples/s
|
| 166 |
+
Step 21870/50462 | Loss: 2.5784 | LR: 3.08e-04 | 5267.7ms/step | 11.4 samples/s
|
| 167 |
+
Step 21880/50462 | Loss: 2.5816 | LR: 3.08e-04 | 5269.5ms/step | 11.4 samples/s
|
| 168 |
+
Step 21890/50462 | Loss: 2.6046 | LR: 3.08e-04 | 5271.2ms/step | 11.4 samples/s
|
| 169 |
+
Step 21900/50462 | Loss: 2.5729 | LR: 3.08e-04 | 5274.1ms/step | 11.4 samples/s
|
| 170 |
+
Step 21910/50462 | Loss: 2.5749 | LR: 3.08e-04 | 5267.8ms/step | 11.4 samples/s
|
| 171 |
+
Step 21920/50462 | Loss: 2.5677 | LR: 3.08e-04 | 5257.8ms/step | 11.4 samples/s
|
| 172 |
+
Step 21930/50462 | Loss: 2.5807 | LR: 3.08e-04 | 5259.3ms/step | 11.4 samples/s
|
| 173 |
+
Step 21940/50462 | Loss: 2.5948 | LR: 3.07e-04 | 5263.4ms/step | 11.4 samples/s
|
| 174 |
+
Step 21950/50462 | Loss: 2.5698 | LR: 3.07e-04 | 5259.1ms/step | 11.4 samples/s
|
| 175 |
+
Step 21960/50462 | Loss: 2.5905 | LR: 3.07e-04 | 5264.5ms/step | 11.4 samples/s
|
| 176 |
+
Step 21970/50462 | Loss: 2.5827 | LR: 3.07e-04 | 5274.1ms/step | 11.4 samples/s
|
| 177 |
+
Step 21980/50462 | Loss: 2.5938 | LR: 3.07e-04 | 5261.7ms/step | 11.4 samples/s
|
| 178 |
+
Step 21990/50462 | Loss: 2.5951 | LR: 3.07e-04 | 5252.5ms/step | 11.4 samples/s
|
| 179 |
+
Step 22000/50462 | Loss: 2.5888 | LR: 3.06e-04 | 5257.4ms/step | 11.4 samples/s
|
| 180 |
+
Validation at step 22000: Loss = 2.6049, Perplexity = 13.53
|
| 181 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-22000
|
| 182 |
+
Step 22010/50462 | Loss: 2.5863 | LR: 3.06e-04 | 10145.8ms/step | 5.9 samples/s
|
| 183 |
+
Step 22020/50462 | Loss: 2.5894 | LR: 3.06e-04 | 5251.8ms/step | 11.4 samples/s
|
| 184 |
+
Step 22030/50462 | Loss: 2.5922 | LR: 3.06e-04 | 5261.6ms/step | 11.4 samples/s
|
| 185 |
+
Step 22040/50462 | Loss: 2.6266 | LR: 3.06e-04 | 5264.4ms/step | 11.4 samples/s
|
| 186 |
+
Step 22050/50462 | Loss: 2.5755 | LR: 3.06e-04 | 5235.4ms/step | 11.5 samples/s
|
| 187 |
+
Step 22060/50462 | Loss: 2.5925 | LR: 3.06e-04 | 5252.3ms/step | 11.4 samples/s
|
| 188 |
+
Step 22070/50462 | Loss: 2.6067 | LR: 3.05e-04 | 5258.2ms/step | 11.4 samples/s
|
| 189 |
+
Step 22080/50462 | Loss: 2.6158 | LR: 3.05e-04 | 5250.7ms/step | 11.4 samples/s
|
| 190 |
+
Step 22090/50462 | Loss: 2.6080 | LR: 3.05e-04 | 5264.0ms/step | 11.4 samples/s
|
| 191 |
+
Step 22100/50462 | Loss: 2.6147 | LR: 3.05e-04 | 5258.5ms/step | 11.4 samples/s
|
| 192 |
+
Step 22110/50462 | Loss: 2.5835 | LR: 3.05e-04 | 5258.6ms/step | 11.4 samples/s
|
| 193 |
+
Step 22120/50462 | Loss: 2.6326 | LR: 3.05e-04 | 5260.8ms/step | 11.4 samples/s
|
| 194 |
+
Step 22130/50462 | Loss: 2.5781 | LR: 3.04e-04 | 5260.7ms/step | 11.4 samples/s
|
| 195 |
+
Step 22140/50462 | Loss: 2.6273 | LR: 3.04e-04 | 5265.4ms/step | 11.4 samples/s
|
| 196 |
+
Step 22150/50462 | Loss: 2.5844 | LR: 3.04e-04 | 5261.1ms/step | 11.4 samples/s
|
| 197 |
+
Step 22160/50462 | Loss: 2.5692 | LR: 3.04e-04 | 5253.6ms/step | 11.4 samples/s
|
| 198 |
+
Step 22170/50462 | Loss: 2.5957 | LR: 3.04e-04 | 5267.5ms/step | 11.4 samples/s
|
| 199 |
+
Step 22180/50462 | Loss: 2.5708 | LR: 3.04e-04 | 5258.5ms/step | 11.4 samples/s
|
| 200 |
+
Step 22190/50462 | Loss: 2.6149 | LR: 3.04e-04 | 5268.5ms/step | 11.4 samples/s
|
| 201 |
+
Step 22200/50462 | Loss: 2.5919 | LR: 3.03e-04 | 5265.0ms/step | 11.4 samples/s
|
| 202 |
+
Step 22210/50462 | Loss: 2.5919 | LR: 3.03e-04 | 5258.6ms/step | 11.4 samples/s
|
| 203 |
+
Step 22220/50462 | Loss: 2.5884 | LR: 3.03e-04 | 5262.2ms/step | 11.4 samples/s
|
| 204 |
+
Step 22230/50462 | Loss: 2.5622 | LR: 3.03e-04 | 5257.7ms/step | 11.4 samples/s
|
| 205 |
+
Step 22240/50462 | Loss: 2.6225 | LR: 3.03e-04 | 5244.9ms/step | 11.4 samples/s
|
| 206 |
+
Step 22250/50462 | Loss: 2.5995 | LR: 3.03e-04 | 5256.9ms/step | 11.4 samples/s
|
| 207 |
+
Validation at step 22250: Loss = 2.6026, Perplexity = 13.50
|
| 208 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-22250
|
| 209 |
+
Step 22260/50462 | Loss: 2.5734 | LR: 3.02e-04 | 10160.4ms/step | 5.9 samples/s
|
| 210 |
+
Step 22270/50462 | Loss: 2.5848 | LR: 3.02e-04 | 5263.0ms/step | 11.4 samples/s
|
| 211 |
+
Step 22280/50462 | Loss: 2.5922 | LR: 3.02e-04 | 5266.6ms/step | 11.4 samples/s
|
| 212 |
+
Step 22290/50462 | Loss: 2.5892 | LR: 3.02e-04 | 5259.5ms/step | 11.4 samples/s
|
| 213 |
+
Step 22300/50462 | Loss: 2.5944 | LR: 3.02e-04 | 5256.5ms/step | 11.4 samples/s
|
| 214 |
+
Step 22310/50462 | Loss: 2.5769 | LR: 3.02e-04 | 5264.8ms/step | 11.4 samples/s
|
| 215 |
+
Step 22320/50462 | Loss: 2.6190 | LR: 3.02e-04 | 5261.1ms/step | 11.4 samples/s
|
| 216 |
+
Step 22330/50462 | Loss: 2.5938 | LR: 3.01e-04 | 5263.9ms/step | 11.4 samples/s
|
| 217 |
+
Step 22340/50462 | Loss: 2.6030 | LR: 3.01e-04 | 5268.6ms/step | 11.4 samples/s
|
| 218 |
+
Step 22350/50462 | Loss: 2.5909 | LR: 3.01e-04 | 5255.3ms/step | 11.4 samples/s
|
| 219 |
+
Step 22360/50462 | Loss: 2.5801 | LR: 3.01e-04 | 5248.6ms/step | 11.4 samples/s
|
| 220 |
+
Step 22370/50462 | Loss: 2.5950 | LR: 3.01e-04 | 5269.2ms/step | 11.4 samples/s
|
| 221 |
+
Step 22380/50462 | Loss: 2.5749 | LR: 3.01e-04 | 5263.6ms/step | 11.4 samples/s
|
| 222 |
+
Step 22390/50462 | Loss: 2.5847 | LR: 3.00e-04 | 5258.8ms/step | 11.4 samples/s
|
| 223 |
+
Step 22400/50462 | Loss: 2.5919 | LR: 3.00e-04 | 5264.2ms/step | 11.4 samples/s
|
| 224 |
+
Step 22410/50462 | Loss: 2.6246 | LR: 3.00e-04 | 5259.5ms/step | 11.4 samples/s
|
| 225 |
+
Step 22420/50462 | Loss: 2.5804 | LR: 3.00e-04 | 5271.5ms/step | 11.4 samples/s
|
| 226 |
+
Step 22430/50462 | Loss: 2.5943 | LR: 3.00e-04 | 5272.9ms/step | 11.4 samples/s
|
| 227 |
+
Step 22440/50462 | Loss: 2.5676 | LR: 3.00e-04 | 5266.9ms/step | 11.4 samples/s
|
| 228 |
+
Step 22450/50462 | Loss: 2.6003 | LR: 3.00e-04 | 5265.2ms/step | 11.4 samples/s
|
| 229 |
+
Step 22460/50462 | Loss: 2.5658 | LR: 2.99e-04 | 5267.0ms/step | 11.4 samples/s
|
| 230 |
+
Step 22470/50462 | Loss: 2.5626 | LR: 2.99e-04 | 5259.8ms/step | 11.4 samples/s
|
| 231 |
+
Step 22480/50462 | Loss: 2.5766 | LR: 2.99e-04 | 5251.7ms/step | 11.4 samples/s
|
| 232 |
+
Step 22490/50462 | Loss: 2.5852 | LR: 2.99e-04 | 5259.2ms/step | 11.4 samples/s
|
| 233 |
+
Step 22500/50462 | Loss: 2.5788 | LR: 2.99e-04 | 5263.5ms/step | 11.4 samples/s
|
| 234 |
+
Validation at step 22500: Loss = 2.5996, Perplexity = 13.46
|
| 235 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-22500
|
| 236 |
+
Step 22510/50462 | Loss: 2.5989 | LR: 2.99e-04 | 10144.9ms/step | 5.9 samples/s
|
| 237 |
+
Step 22520/50462 | Loss: 2.5642 | LR: 2.98e-04 | 5244.6ms/step | 11.4 samples/s
|
| 238 |
+
Step 22530/50462 | Loss: 2.5969 | LR: 2.98e-04 | 5258.5ms/step | 11.4 samples/s
|
| 239 |
+
Step 22540/50462 | Loss: 2.5846 | LR: 2.98e-04 | 5259.1ms/step | 11.4 samples/s
|
| 240 |
+
Step 22550/50462 | Loss: 2.5898 | LR: 2.98e-04 | 5264.6ms/step | 11.4 samples/s
|
| 241 |
+
Step 22560/50462 | Loss: 2.6055 | LR: 2.98e-04 | 5264.6ms/step | 11.4 samples/s
|
| 242 |
+
Step 22570/50462 | Loss: 2.6023 | LR: 2.98e-04 | 5262.5ms/step | 11.4 samples/s
|
| 243 |
+
Step 22580/50462 | Loss: 2.6035 | LR: 2.98e-04 | 5255.6ms/step | 11.4 samples/s
|
| 244 |
+
Step 22590/50462 | Loss: 2.6083 | LR: 2.97e-04 | 5255.7ms/step | 11.4 samples/s
|
| 245 |
+
Step 22600/50462 | Loss: 2.5752 | LR: 2.97e-04 | 5252.8ms/step | 11.4 samples/s
|
| 246 |
+
Step 22610/50462 | Loss: 2.5696 | LR: 2.97e-04 | 5262.5ms/step | 11.4 samples/s
|
| 247 |
+
Step 22620/50462 | Loss: 2.6038 | LR: 2.97e-04 | 5266.1ms/step | 11.4 samples/s
|
| 248 |
+
Step 22630/50462 | Loss: 2.5854 | LR: 2.97e-04 | 5264.2ms/step | 11.4 samples/s
|
| 249 |
+
Step 22640/50462 | Loss: 2.5761 | LR: 2.97e-04 | 5251.2ms/step | 11.4 samples/s
|
| 250 |
+
Step 22650/50462 | Loss: 2.5773 | LR: 2.96e-04 | 5267.6ms/step | 11.4 samples/s
|
| 251 |
+
Step 22660/50462 | Loss: 2.5884 | LR: 2.96e-04 | 5269.1ms/step | 11.4 samples/s
|
| 252 |
+
Step 22670/50462 | Loss: 2.5719 | LR: 2.96e-04 | 5260.9ms/step | 11.4 samples/s
|
| 253 |
+
Step 22680/50462 | Loss: 2.5738 | LR: 2.96e-04 | 5262.2ms/step | 11.4 samples/s
|
| 254 |
+
Step 22690/50462 | Loss: 2.5776 | LR: 2.96e-04 | 5269.4ms/step | 11.4 samples/s
|
| 255 |
+
Step 22700/50462 | Loss: 2.5859 | LR: 2.96e-04 | 5262.4ms/step | 11.4 samples/s
|
| 256 |
+
Step 22710/50462 | Loss: 2.5766 | LR: 2.95e-04 | 5262.4ms/step | 11.4 samples/s
|
| 257 |
+
Step 22720/50462 | Loss: 2.5836 | LR: 2.95e-04 | 5259.3ms/step | 11.4 samples/s
|
| 258 |
+
Step 22730/50462 | Loss: 2.6038 | LR: 2.95e-04 | 5249.5ms/step | 11.4 samples/s
|
| 259 |
+
Step 22740/50462 | Loss: 2.6143 | LR: 2.95e-04 | 5254.4ms/step | 11.4 samples/s
|
| 260 |
+
Step 22750/50462 | Loss: 2.5793 | LR: 2.95e-04 | 5261.4ms/step | 11.4 samples/s
|
| 261 |
+
Validation at step 22750: Loss = 2.5974, Perplexity = 13.43
|
| 262 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-22750
|
| 263 |
+
Step 22760/50462 | Loss: 2.6008 | LR: 2.95e-04 | 10178.3ms/step | 5.9 samples/s
|
| 264 |
+
Step 22770/50462 | Loss: 2.5736 | LR: 2.95e-04 | 5246.7ms/step | 11.4 samples/s
|
| 265 |
+
Step 22780/50462 | Loss: 2.5999 | LR: 2.94e-04 | 5250.2ms/step | 11.4 samples/s
|
| 266 |
+
Step 22790/50462 | Loss: 2.5771 | LR: 2.94e-04 | 5244.4ms/step | 11.4 samples/s
|
| 267 |
+
Step 22800/50462 | Loss: 2.5982 | LR: 2.94e-04 | 5248.5ms/step | 11.4 samples/s
|
| 268 |
+
Step 22810/50462 | Loss: 2.5987 | LR: 2.94e-04 | 5251.3ms/step | 11.4 samples/s
|
| 269 |
+
Step 22820/50462 | Loss: 2.5649 | LR: 2.94e-04 | 5249.1ms/step | 11.4 samples/s
|
| 270 |
+
Step 22830/50462 | Loss: 2.6157 | LR: 2.94e-04 | 5249.5ms/step | 11.4 samples/s
|
| 271 |
+
Step 22840/50462 | Loss: 2.5676 | LR: 2.93e-04 | 5256.7ms/step | 11.4 samples/s
|
| 272 |
+
Step 22850/50462 | Loss: 2.5994 | LR: 2.93e-04 | 5248.7ms/step | 11.4 samples/s
|
| 273 |
+
Step 22860/50462 | Loss: 2.5954 | LR: 2.93e-04 | 5253.9ms/step | 11.4 samples/s
|
| 274 |
+
Step 22870/50462 | Loss: 2.5975 | LR: 2.93e-04 | 5250.7ms/step | 11.4 samples/s
|
| 275 |
+
Step 22880/50462 | Loss: 2.5859 | LR: 2.93e-04 | 5250.8ms/step | 11.4 samples/s
|
| 276 |
+
Step 22890/50462 | Loss: 2.5689 | LR: 2.93e-04 | 5254.3ms/step | 11.4 samples/s
|
| 277 |
+
Step 22900/50462 | Loss: 2.6003 | LR: 2.93e-04 | 5254.2ms/step | 11.4 samples/s
|
| 278 |
+
Step 22910/50462 | Loss: 2.5851 | LR: 2.92e-04 | 5258.0ms/step | 11.4 samples/s
|
| 279 |
+
Step 22920/50462 | Loss: 2.5803 | LR: 2.92e-04 | 5251.7ms/step | 11.4 samples/s
|
| 280 |
+
Step 22930/50462 | Loss: 2.5798 | LR: 2.92e-04 | 5246.7ms/step | 11.4 samples/s
|
| 281 |
+
Step 22940/50462 | Loss: 2.5810 | LR: 2.92e-04 | 5243.6ms/step | 11.4 samples/s
|
| 282 |
+
Step 22950/50462 | Loss: 2.5955 | LR: 2.92e-04 | 5246.8ms/step | 11.4 samples/s
|
| 283 |
+
Step 22960/50462 | Loss: 2.5917 | LR: 2.92e-04 | 5259.0ms/step | 11.4 samples/s
|
| 284 |
+
Step 22970/50462 | Loss: 2.6244 | LR: 2.91e-04 | 5253.0ms/step | 11.4 samples/s
|
| 285 |
+
Step 22980/50462 | Loss: 2.5927 | LR: 2.91e-04 | 5255.1ms/step | 11.4 samples/s
|
| 286 |
+
Step 22990/50462 | Loss: 2.5821 | LR: 2.91e-04 | 5254.7ms/step | 11.4 samples/s
|
| 287 |
+
Step 23000/50462 | Loss: 2.6071 | LR: 2.91e-04 | 5252.9ms/step | 11.4 samples/s
|
| 288 |
+
Validation at step 23000: Loss = 2.5949, Perplexity = 13.40
|
| 289 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-23000
|
| 290 |
+
Step 23010/50462 | Loss: 2.5853 | LR: 2.91e-04 | 10141.2ms/step | 5.9 samples/s
|
| 291 |
+
Step 23020/50462 | Loss: 2.6143 | LR: 2.91e-04 | 5228.8ms/step | 11.5 samples/s
|
| 292 |
+
Step 23030/50462 | Loss: 2.5960 | LR: 2.91e-04 | 5209.4ms/step | 11.5 samples/s
|
| 293 |
+
Step 23040/50462 | Loss: 2.5961 | LR: 2.90e-04 | 5236.3ms/step | 11.5 samples/s
|
| 294 |
+
Step 23050/50462 | Loss: 2.5690 | LR: 2.90e-04 | 5235.8ms/step | 11.5 samples/s
|
| 295 |
+
Step 23060/50462 | Loss: 2.5766 | LR: 2.90e-04 | 5232.8ms/step | 11.5 samples/s
|
| 296 |
+
Step 23070/50462 | Loss: 2.5979 | LR: 2.90e-04 | 5253.4ms/step | 11.4 samples/s
|
| 297 |
+
Step 23080/50462 | Loss: 2.5925 | LR: 2.90e-04 | 5234.8ms/step | 11.5 samples/s
|
| 298 |
+
Step 23090/50462 | Loss: 2.5660 | LR: 2.90e-04 | 5240.0ms/step | 11.5 samples/s
|
| 299 |
+
Step 23100/50462 | Loss: 2.5961 | LR: 2.89e-04 | 5257.7ms/step | 11.4 samples/s
|
| 300 |
+
Step 23110/50462 | Loss: 2.5747 | LR: 2.89e-04 | 5246.4ms/step | 11.4 samples/s
|
| 301 |
+
Step 23120/50462 | Loss: 2.5819 | LR: 2.89e-04 | 5243.8ms/step | 11.4 samples/s
|
| 302 |
+
Step 23130/50462 | Loss: 2.5663 | LR: 2.89e-04 | 5246.9ms/step | 11.4 samples/s
|
| 303 |
+
Step 23140/50462 | Loss: 2.5955 | LR: 2.89e-04 | 5247.0ms/step | 11.4 samples/s
|
| 304 |
+
Step 23150/50462 | Loss: 2.5604 | LR: 2.89e-04 | 5251.1ms/step | 11.4 samples/s
|
| 305 |
+
Step 23160/50462 | Loss: 2.5950 | LR: 2.88e-04 | 5259.8ms/step | 11.4 samples/s
|
| 306 |
+
Step 23170/50462 | Loss: 2.6173 | LR: 2.88e-04 | 5256.4ms/step | 11.4 samples/s
|
| 307 |
+
Step 23180/50462 | Loss: 2.5632 | LR: 2.88e-04 | 5246.2ms/step | 11.4 samples/s
|
| 308 |
+
Step 23190/50462 | Loss: 2.5776 | LR: 2.88e-04 | 5250.4ms/step | 11.4 samples/s
|
| 309 |
+
Step 23200/50462 | Loss: 2.5839 | LR: 2.88e-04 | 5241.4ms/step | 11.4 samples/s
|
| 310 |
+
Step 23210/50462 | Loss: 2.5610 | LR: 2.88e-04 | 5256.8ms/step | 11.4 samples/s
|
| 311 |
+
Step 23220/50462 | Loss: 2.5682 | LR: 2.88e-04 | 5219.3ms/step | 11.5 samples/s
|
| 312 |
+
Step 23230/50462 | Loss: 2.5712 | LR: 2.87e-04 | 5245.6ms/step | 11.4 samples/s
|
| 313 |
+
Step 23240/50462 | Loss: 2.5851 | LR: 2.87e-04 | 5250.9ms/step | 11.4 samples/s
|
| 314 |
+
Step 23250/50462 | Loss: 2.5637 | LR: 2.87e-04 | 5253.8ms/step | 11.4 samples/s
|
| 315 |
+
Validation at step 23250: Loss = 2.5926, Perplexity = 13.37
|
| 316 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-23250
|
| 317 |
+
Step 23260/50462 | Loss: 2.5726 | LR: 2.87e-04 | 10160.1ms/step | 5.9 samples/s
|
| 318 |
+
Step 23270/50462 | Loss: 2.5895 | LR: 2.87e-04 | 5244.6ms/step | 11.4 samples/s
|
| 319 |
+
Step 23280/50462 | Loss: 2.5851 | LR: 2.87e-04 | 5255.9ms/step | 11.4 samples/s
|
| 320 |
+
Step 23290/50462 | Loss: 2.5927 | LR: 2.86e-04 | 5264.9ms/step | 11.4 samples/s
|
| 321 |
+
Step 23300/50462 | Loss: 2.5873 | LR: 2.86e-04 | 5260.0ms/step | 11.4 samples/s
|
| 322 |
+
Step 23310/50462 | Loss: 2.5930 | LR: 2.86e-04 | 5267.4ms/step | 11.4 samples/s
|
| 323 |
+
Step 23320/50462 | Loss: 2.5865 | LR: 2.86e-04 | 5250.2ms/step | 11.4 samples/s
|
| 324 |
+
Step 23330/50462 | Loss: 2.5710 | LR: 2.86e-04 | 5259.0ms/step | 11.4 samples/s
|
| 325 |
+
Step 23340/50462 | Loss: 2.5898 | LR: 2.86e-04 | 5255.3ms/step | 11.4 samples/s
|
| 326 |
+
Step 23350/50462 | Loss: 2.5587 | LR: 2.86e-04 | 5257.3ms/step | 11.4 samples/s
|
| 327 |
+
Step 23360/50462 | Loss: 2.5872 | LR: 2.85e-04 | 5260.8ms/step | 11.4 samples/s
|
| 328 |
+
Step 23370/50462 | Loss: 2.5741 | LR: 2.85e-04 | 5256.2ms/step | 11.4 samples/s
|
| 329 |
+
Step 23380/50462 | Loss: 2.5798 | LR: 2.85e-04 | 5261.4ms/step | 11.4 samples/s
|
| 330 |
+
Step 23390/50462 | Loss: 2.5625 | LR: 2.85e-04 | 5261.8ms/step | 11.4 samples/s
|
| 331 |
+
Step 23400/50462 | Loss: 2.5762 | LR: 2.85e-04 | 5262.8ms/step | 11.4 samples/s
|
| 332 |
+
Step 23410/50462 | Loss: 2.5800 | LR: 2.85e-04 | 5261.9ms/step | 11.4 samples/s
|
| 333 |
+
Step 23420/50462 | Loss: 2.5713 | LR: 2.84e-04 | 5257.3ms/step | 11.4 samples/s
|
| 334 |
+
Step 23430/50462 | Loss: 2.5580 | LR: 2.84e-04 | 5262.1ms/step | 11.4 samples/s
|
| 335 |
+
Step 23440/50462 | Loss: 2.5798 | LR: 2.84e-04 | 5265.6ms/step | 11.4 samples/s
|
| 336 |
+
Step 23450/50462 | Loss: 2.5629 | LR: 2.84e-04 | 5261.8ms/step | 11.4 samples/s
|
| 337 |
+
Step 23460/50462 | Loss: 2.5688 | LR: 2.84e-04 | 5258.5ms/step | 11.4 samples/s
|
| 338 |
+
Step 23470/50462 | Loss: 2.5779 | LR: 2.84e-04 | 5257.0ms/step | 11.4 samples/s
|
| 339 |
+
Step 23480/50462 | Loss: 2.6059 | LR: 2.83e-04 | 5259.7ms/step | 11.4 samples/s
|
| 340 |
+
Step 23490/50462 | Loss: 2.6297 | LR: 2.83e-04 | 5259.3ms/step | 11.4 samples/s
|
| 341 |
+
Step 23500/50462 | Loss: 2.5668 | LR: 2.83e-04 | 5256.0ms/step | 11.4 samples/s
|
| 342 |
+
Validation at step 23500: Loss = 2.5905, Perplexity = 13.34
|
| 343 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-23500
|
| 344 |
+
Step 23510/50462 | Loss: 2.5825 | LR: 2.83e-04 | 10187.1ms/step | 5.9 samples/s
|
| 345 |
+
Step 23520/50462 | Loss: 2.5825 | LR: 2.83e-04 | 5246.0ms/step | 11.4 samples/s
|
| 346 |
+
Step 23530/50462 | Loss: 2.5526 | LR: 2.83e-04 | 5254.3ms/step | 11.4 samples/s
|
| 347 |
+
Step 23540/50462 | Loss: 2.5740 | LR: 2.83e-04 | 5256.9ms/step | 11.4 samples/s
|
| 348 |
+
Step 23550/50462 | Loss: 2.5652 | LR: 2.82e-04 | 5249.0ms/step | 11.4 samples/s
|
| 349 |
+
Step 23560/50462 | Loss: 2.5853 | LR: 2.82e-04 | 5239.3ms/step | 11.5 samples/s
|
| 350 |
+
Step 23570/50462 | Loss: 2.5812 | LR: 2.82e-04 | 5235.9ms/step | 11.5 samples/s
|
| 351 |
+
Step 23580/50462 | Loss: 2.5890 | LR: 2.82e-04 | 5241.9ms/step | 11.4 samples/s
|
| 352 |
+
Step 23590/50462 | Loss: 2.5674 | LR: 2.82e-04 | 5227.8ms/step | 11.5 samples/s
|
| 353 |
+
Step 23600/50462 | Loss: 2.5863 | LR: 2.82e-04 | 5217.1ms/step | 11.5 samples/s
|
| 354 |
+
Step 23610/50462 | Loss: 2.5677 | LR: 2.81e-04 | 5201.2ms/step | 11.5 samples/s
|
| 355 |
+
Step 23620/50462 | Loss: 2.5653 | LR: 2.81e-04 | 5206.2ms/step | 11.5 samples/s
|
| 356 |
+
Step 23630/50462 | Loss: 2.5869 | LR: 2.81e-04 | 5232.3ms/step | 11.5 samples/s
|
| 357 |
+
Step 23640/50462 | Loss: 2.5539 | LR: 2.81e-04 | 5234.4ms/step | 11.5 samples/s
|
| 358 |
+
Step 23650/50462 | Loss: 2.5493 | LR: 2.81e-04 | 5227.7ms/step | 11.5 samples/s
|
| 359 |
+
Step 23660/50462 | Loss: 2.5773 | LR: 2.81e-04 | 5227.7ms/step | 11.5 samples/s
|
| 360 |
+
Step 23670/50462 | Loss: 2.5690 | LR: 2.81e-04 | 5214.0ms/step | 11.5 samples/s
|
| 361 |
+
Step 23680/50462 | Loss: 2.5809 | LR: 2.80e-04 | 5230.4ms/step | 11.5 samples/s
|
| 362 |
+
Step 23690/50462 | Loss: 2.5686 | LR: 2.80e-04 | 5230.8ms/step | 11.5 samples/s
|
| 363 |
+
Step 23700/50462 | Loss: 2.5698 | LR: 2.80e-04 | 5226.8ms/step | 11.5 samples/s
|
| 364 |
+
Step 23710/50462 | Loss: 2.5800 | LR: 2.80e-04 | 5211.2ms/step | 11.5 samples/s
|
| 365 |
+
Step 23720/50462 | Loss: 2.5895 | LR: 2.80e-04 | 5233.0ms/step | 11.5 samples/s
|
| 366 |
+
Step 23730/50462 | Loss: 2.5735 | LR: 2.80e-04 | 5223.8ms/step | 11.5 samples/s
|
| 367 |
+
Step 23740/50462 | Loss: 2.5926 | LR: 2.79e-04 | 5203.3ms/step | 11.5 samples/s
|
| 368 |
+
Step 23750/50462 | Loss: 2.5848 | LR: 2.79e-04 | 5205.7ms/step | 11.5 samples/s
|
| 369 |
+
Validation at step 23750: Loss = 2.5878, Perplexity = 13.30
|
| 370 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-23750
|
| 371 |
+
Step 23760/50462 | Loss: 2.5821 | LR: 2.79e-04 | 10137.3ms/step | 5.9 samples/s
|
| 372 |
+
Step 23770/50462 | Loss: 2.5880 | LR: 2.79e-04 | 5211.2ms/step | 11.5 samples/s
|
| 373 |
+
Step 23780/50462 | Loss: 2.5421 | LR: 2.79e-04 | 5219.2ms/step | 11.5 samples/s
|
| 374 |
+
Step 23790/50462 | Loss: 2.5625 | LR: 2.79e-04 | 5222.5ms/step | 11.5 samples/s
|
| 375 |
+
Step 23800/50462 | Loss: 2.5682 | LR: 2.78e-04 | 5225.2ms/step | 11.5 samples/s
|
| 376 |
+
Step 23810/50462 | Loss: 2.5905 | LR: 2.78e-04 | 5216.4ms/step | 11.5 samples/s
|
| 377 |
+
Step 23820/50462 | Loss: 2.5850 | LR: 2.78e-04 | 5213.3ms/step | 11.5 samples/s
|
| 378 |
+
Step 23830/50462 | Loss: 2.5642 | LR: 2.78e-04 | 5215.4ms/step | 11.5 samples/s
|
| 379 |
+
Step 23840/50462 | Loss: 2.5845 | LR: 2.78e-04 | 5233.8ms/step | 11.5 samples/s
|
| 380 |
+
Step 23850/50462 | Loss: 2.5567 | LR: 2.78e-04 | 5230.0ms/step | 11.5 samples/s
|
| 381 |
+
Step 23860/50462 | Loss: 2.5698 | LR: 2.78e-04 | 5218.1ms/step | 11.5 samples/s
|
| 382 |
+
Step 23870/50462 | Loss: 2.5532 | LR: 2.77e-04 | 5211.5ms/step | 11.5 samples/s
|
| 383 |
+
Step 23880/50462 | Loss: 2.5835 | LR: 2.77e-04 | 5201.9ms/step | 11.5 samples/s
|
| 384 |
+
Step 23890/50462 | Loss: 2.5565 | LR: 2.77e-04 | 5216.8ms/step | 11.5 samples/s
|
| 385 |
+
Step 23900/50462 | Loss: 2.5736 | LR: 2.77e-04 | 5221.0ms/step | 11.5 samples/s
|
| 386 |
+
Step 23910/50462 | Loss: 2.5594 | LR: 2.77e-04 | 5214.7ms/step | 11.5 samples/s
|
| 387 |
+
Step 23920/50462 | Loss: 2.5899 | LR: 2.77e-04 | 5223.6ms/step | 11.5 samples/s
|
| 388 |
+
Step 23930/50462 | Loss: 2.5588 | LR: 2.76e-04 | 5229.7ms/step | 11.5 samples/s
|
| 389 |
+
Step 23940/50462 | Loss: 2.5760 | LR: 2.76e-04 | 5235.1ms/step | 11.5 samples/s
|
| 390 |
+
Step 23950/50462 | Loss: 2.5849 | LR: 2.76e-04 | 5223.4ms/step | 11.5 samples/s
|
| 391 |
+
Step 23960/50462 | Loss: 2.5932 | LR: 2.76e-04 | 5221.1ms/step | 11.5 samples/s
|
| 392 |
+
Step 23970/50462 | Loss: 2.5874 | LR: 2.76e-04 | 5225.1ms/step | 11.5 samples/s
|
| 393 |
+
Step 23980/50462 | Loss: 2.5517 | LR: 2.76e-04 | 5222.5ms/step | 11.5 samples/s
|
| 394 |
+
Step 23990/50462 | Loss: 2.5754 | LR: 2.75e-04 | 5228.4ms/step | 11.5 samples/s
|
| 395 |
+
Step 24000/50462 | Loss: 2.5795 | LR: 2.75e-04 | 5224.6ms/step | 11.5 samples/s
|
| 396 |
+
Validation at step 24000: Loss = 2.5853, Perplexity = 13.27
|
| 397 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-24000
|
| 398 |
+
Step 24010/50462 | Loss: 2.5826 | LR: 2.75e-04 | 10097.1ms/step | 5.9 samples/s
|
| 399 |
+
Step 24020/50462 | Loss: 2.5752 | LR: 2.75e-04 | 5210.7ms/step | 11.5 samples/s
|
| 400 |
+
Step 24030/50462 | Loss: 2.5895 | LR: 2.75e-04 | 5210.5ms/step | 11.5 samples/s
|
| 401 |
+
Step 24040/50462 | Loss: 2.5644 | LR: 2.75e-04 | 5218.1ms/step | 11.5 samples/s
|
| 402 |
+
Step 24050/50462 | Loss: 2.5780 | LR: 2.75e-04 | 5219.7ms/step | 11.5 samples/s
|
| 403 |
+
Step 24060/50462 | Loss: 2.5559 | LR: 2.74e-04 | 5225.6ms/step | 11.5 samples/s
|
| 404 |
+
Step 24070/50462 | Loss: 2.5547 | LR: 2.74e-04 | 5230.0ms/step | 11.5 samples/s
|
| 405 |
+
Step 24080/50462 | Loss: 2.5815 | LR: 2.74e-04 | 5232.6ms/step | 11.5 samples/s
|
| 406 |
+
Step 24090/50462 | Loss: 2.5782 | LR: 2.74e-04 | 5221.1ms/step | 11.5 samples/s
|
| 407 |
+
Step 24100/50462 | Loss: 2.5803 | LR: 2.74e-04 | 5212.7ms/step | 11.5 samples/s
|
| 408 |
+
Step 24110/50462 | Loss: 2.5711 | LR: 2.74e-04 | 5217.0ms/step | 11.5 samples/s
|
| 409 |
+
Step 24120/50462 | Loss: 2.5890 | LR: 2.73e-04 | 5216.1ms/step | 11.5 samples/s
|
| 410 |
+
Step 24130/50462 | Loss: 2.5679 | LR: 2.73e-04 | 5219.3ms/step | 11.5 samples/s
|
| 411 |
+
Step 24140/50462 | Loss: 2.5718 | LR: 2.73e-04 | 5211.2ms/step | 11.5 samples/s
|
| 412 |
+
Step 24150/50462 | Loss: 2.5741 | LR: 2.73e-04 | 5212.5ms/step | 11.5 samples/s
|
| 413 |
+
Step 24160/50462 | Loss: 2.5990 | LR: 2.73e-04 | 5219.9ms/step | 11.5 samples/s
|
| 414 |
+
Step 24170/50462 | Loss: 2.5876 | LR: 2.73e-04 | 5232.2ms/step | 11.5 samples/s
|
| 415 |
+
Step 24180/50462 | Loss: 2.5972 | LR: 2.72e-04 | 5232.9ms/step | 11.5 samples/s
|
| 416 |
+
Step 24190/50462 | Loss: 2.5532 | LR: 2.72e-04 | 5229.8ms/step | 11.5 samples/s
|
| 417 |
+
Step 24200/50462 | Loss: 2.5618 | LR: 2.72e-04 | 5223.7ms/step | 11.5 samples/s
|
| 418 |
+
Step 24210/50462 | Loss: 2.5991 | LR: 2.72e-04 | 5239.3ms/step | 11.5 samples/s
|
| 419 |
+
Step 24220/50462 | Loss: 2.5619 | LR: 2.72e-04 | 5230.2ms/step | 11.5 samples/s
|
| 420 |
+
Step 24230/50462 | Loss: 2.5719 | LR: 2.72e-04 | 5217.9ms/step | 11.5 samples/s
|
| 421 |
+
Step 24240/50462 | Loss: 2.5832 | LR: 2.72e-04 | 5236.8ms/step | 11.5 samples/s
|
| 422 |
+
Step 24250/50462 | Loss: 2.5773 | LR: 2.71e-04 | 5218.7ms/step | 11.5 samples/s
|
| 423 |
+
Validation at step 24250: Loss = 2.5833, Perplexity = 13.24
|
| 424 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-24250
|
| 425 |
+
Step 24260/50462 | Loss: 2.5640 | LR: 2.71e-04 | 10099.3ms/step | 5.9 samples/s
|
| 426 |
+
Step 24270/50462 | Loss: 2.5928 | LR: 2.71e-04 | 5225.3ms/step | 11.5 samples/s
|
| 427 |
+
Step 24280/50462 | Loss: 2.5830 | LR: 2.71e-04 | 5233.7ms/step | 11.5 samples/s
|
| 428 |
+
Step 24290/50462 | Loss: 2.5623 | LR: 2.71e-04 | 5236.4ms/step | 11.5 samples/s
|
| 429 |
+
Step 24300/50462 | Loss: 2.5790 | LR: 2.71e-04 | 5217.7ms/step | 11.5 samples/s
|
| 430 |
+
Step 24310/50462 | Loss: 2.5583 | LR: 2.70e-04 | 5234.3ms/step | 11.5 samples/s
|
| 431 |
+
Step 24320/50462 | Loss: 2.5886 | LR: 2.70e-04 | 5238.7ms/step | 11.5 samples/s
|
| 432 |
+
Step 24330/50462 | Loss: 2.5694 | LR: 2.70e-04 | 5232.7ms/step | 11.5 samples/s
|
| 433 |
+
Step 24340/50462 | Loss: 2.5625 | LR: 2.70e-04 | 5219.6ms/step | 11.5 samples/s
|
| 434 |
+
Step 24350/50462 | Loss: 2.5720 | LR: 2.70e-04 | 5233.9ms/step | 11.5 samples/s
|
| 435 |
+
Step 24360/50462 | Loss: 2.5505 | LR: 2.70e-04 | 5239.9ms/step | 11.5 samples/s
|
| 436 |
+
Step 24370/50462 | Loss: 2.5561 | LR: 2.70e-04 | 5240.6ms/step | 11.4 samples/s
|
| 437 |
+
Step 24380/50462 | Loss: 2.5929 | LR: 2.69e-04 | 5238.4ms/step | 11.5 samples/s
|
| 438 |
+
Step 24390/50462 | Loss: 2.5895 | LR: 2.69e-04 | 5233.0ms/step | 11.5 samples/s
|
| 439 |
+
Step 24400/50462 | Loss: 2.5693 | LR: 2.69e-04 | 5243.4ms/step | 11.4 samples/s
|
| 440 |
+
Step 24410/50462 | Loss: 2.5606 | LR: 2.69e-04 | 5239.7ms/step | 11.5 samples/s
|
| 441 |
+
Step 24420/50462 | Loss: 2.5741 | LR: 2.69e-04 | 5212.3ms/step | 11.5 samples/s
|
| 442 |
+
Step 24430/50462 | Loss: 2.5902 | LR: 2.69e-04 | 5227.4ms/step | 11.5 samples/s
|
| 443 |
+
Step 24440/50462 | Loss: 2.5794 | LR: 2.68e-04 | 5214.8ms/step | 11.5 samples/s
|
| 444 |
+
Step 24450/50462 | Loss: 2.5438 | LR: 2.68e-04 | 5217.5ms/step | 11.5 samples/s
|
| 445 |
+
Step 24460/50462 | Loss: 2.5742 | LR: 2.68e-04 | 5240.0ms/step | 11.5 samples/s
|
| 446 |
+
Step 24470/50462 | Loss: 2.5748 | LR: 2.68e-04 | 5227.5ms/step | 11.5 samples/s
|
| 447 |
+
Step 24480/50462 | Loss: 2.5773 | LR: 2.68e-04 | 5233.1ms/step | 11.5 samples/s
|
| 448 |
+
Step 24490/50462 | Loss: 2.5644 | LR: 2.68e-04 | 5239.5ms/step | 11.5 samples/s
|
| 449 |
+
Step 24500/50462 | Loss: 2.5876 | LR: 2.67e-04 | 5228.4ms/step | 11.5 samples/s
|
| 450 |
+
Validation at step 24500: Loss = 2.5811, Perplexity = 13.21
|
| 451 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-24500
|
| 452 |
+
Step 24510/50462 | Loss: 2.5880 | LR: 2.67e-04 | 10141.1ms/step | 5.9 samples/s
|
| 453 |
+
Step 24520/50462 | Loss: 2.5981 | LR: 2.67e-04 | 5225.8ms/step | 11.5 samples/s
|
| 454 |
+
Step 24530/50462 | Loss: 2.5603 | LR: 2.67e-04 | 5225.4ms/step | 11.5 samples/s
|
| 455 |
+
Step 24540/50462 | Loss: 2.5961 | LR: 2.67e-04 | 5241.4ms/step | 11.4 samples/s
|
| 456 |
+
Step 24550/50462 | Loss: 2.5534 | LR: 2.67e-04 | 5236.2ms/step | 11.5 samples/s
|
| 457 |
+
Step 24560/50462 | Loss: 2.5905 | LR: 2.67e-04 | 5234.0ms/step | 11.5 samples/s
|
| 458 |
+
Step 24570/50462 | Loss: 2.5791 | LR: 2.66e-04 | 5238.8ms/step | 11.5 samples/s
|
| 459 |
+
Step 24580/50462 | Loss: 2.5699 | LR: 2.66e-04 | 5225.2ms/step | 11.5 samples/s
|
| 460 |
+
Step 24590/50462 | Loss: 2.5891 | LR: 2.66e-04 | 5215.5ms/step | 11.5 samples/s
|
| 461 |
+
Step 24600/50462 | Loss: 2.5722 | LR: 2.66e-04 | 5214.2ms/step | 11.5 samples/s
|
| 462 |
+
Step 24610/50462 | Loss: 2.5772 | LR: 2.66e-04 | 5229.8ms/step | 11.5 samples/s
|
| 463 |
+
Step 24620/50462 | Loss: 2.5845 | LR: 2.66e-04 | 5221.0ms/step | 11.5 samples/s
|
| 464 |
+
Step 24630/50462 | Loss: 2.5606 | LR: 2.65e-04 | 5229.8ms/step | 11.5 samples/s
|
| 465 |
+
Step 24640/50462 | Loss: 2.5854 | LR: 2.65e-04 | 5242.8ms/step | 11.4 samples/s
|
| 466 |
+
Step 24650/50462 | Loss: 2.5641 | LR: 2.65e-04 | 5244.9ms/step | 11.4 samples/s
|
| 467 |
+
Step 24660/50462 | Loss: 2.5776 | LR: 2.65e-04 | 5221.2ms/step | 11.5 samples/s
|
| 468 |
+
Step 24670/50462 | Loss: 2.5805 | LR: 2.65e-04 | 5202.6ms/step | 11.5 samples/s
|
| 469 |
+
Step 24680/50462 | Loss: 2.5672 | LR: 2.65e-04 | 5240.9ms/step | 11.4 samples/s
|
| 470 |
+
Step 24690/50462 | Loss: 2.5786 | LR: 2.64e-04 | 5244.2ms/step | 11.4 samples/s
|
| 471 |
+
Step 24700/50462 | Loss: 2.5703 | LR: 2.64e-04 | 5240.5ms/step | 11.4 samples/s
|
| 472 |
+
Step 24710/50462 | Loss: 2.5919 | LR: 2.64e-04 | 5231.2ms/step | 11.5 samples/s
|
| 473 |
+
Step 24720/50462 | Loss: 2.5526 | LR: 2.64e-04 | 5239.2ms/step | 11.5 samples/s
|
| 474 |
+
Step 24730/50462 | Loss: 2.5615 | LR: 2.64e-04 | 5251.0ms/step | 11.4 samples/s
|
| 475 |
+
Step 24740/50462 | Loss: 2.5601 | LR: 2.64e-04 | 5247.9ms/step | 11.4 samples/s
|
| 476 |
+
Step 24750/50462 | Loss: 2.5584 | LR: 2.64e-04 | 5254.1ms/step | 11.4 samples/s
|
| 477 |
+
Validation at step 24750: Loss = 2.5786, Perplexity = 13.18
|
| 478 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-24750
|
| 479 |
+
Step 24760/50462 | Loss: 2.5698 | LR: 2.63e-04 | 10136.6ms/step | 5.9 samples/s
|
| 480 |
+
Step 24770/50462 | Loss: 2.5647 | LR: 2.63e-04 | 5229.3ms/step | 11.5 samples/s
|
| 481 |
+
Step 24780/50462 | Loss: 2.5581 | LR: 2.63e-04 | 5243.3ms/step | 11.4 samples/s
|
| 482 |
+
Step 24790/50462 | Loss: 2.5728 | LR: 2.63e-04 | 5239.2ms/step | 11.5 samples/s
|
| 483 |
+
Step 24800/50462 | Loss: 2.5682 | LR: 2.63e-04 | 5206.8ms/step | 11.5 samples/s
|
| 484 |
+
Step 24810/50462 | Loss: 2.5696 | LR: 2.63e-04 | 5236.0ms/step | 11.5 samples/s
|
| 485 |
+
Step 24820/50462 | Loss: 2.5859 | LR: 2.62e-04 | 5239.8ms/step | 11.5 samples/s
|
| 486 |
+
Step 24830/50462 | Loss: 2.5520 | LR: 2.62e-04 | 5223.5ms/step | 11.5 samples/s
|
| 487 |
+
Step 24840/50462 | Loss: 2.5834 | LR: 2.62e-04 | 5243.3ms/step | 11.4 samples/s
|
| 488 |
+
Step 24850/50462 | Loss: 2.5677 | LR: 2.62e-04 | 5229.6ms/step | 11.5 samples/s
|
| 489 |
+
Step 24860/50462 | Loss: 2.5612 | LR: 2.62e-04 | 5246.4ms/step | 11.4 samples/s
|
| 490 |
+
Step 24870/50462 | Loss: 2.5683 | LR: 2.62e-04 | 5242.4ms/step | 11.4 samples/s
|
| 491 |
+
Step 24880/50462 | Loss: 2.5503 | LR: 2.61e-04 | 5245.5ms/step | 11.4 samples/s
|
| 492 |
+
Step 24890/50462 | Loss: 2.5759 | LR: 2.61e-04 | 5213.3ms/step | 11.5 samples/s
|
| 493 |
+
Step 24900/50462 | Loss: 2.5666 | LR: 2.61e-04 | 5221.0ms/step | 11.5 samples/s
|
| 494 |
+
Step 24910/50462 | Loss: 2.5689 | LR: 2.61e-04 | 5242.4ms/step | 11.4 samples/s
|
| 495 |
+
Step 24920/50462 | Loss: 2.5463 | LR: 2.61e-04 | 5220.0ms/step | 11.5 samples/s
|
| 496 |
+
Step 24930/50462 | Loss: 2.6253 | LR: 2.61e-04 | 5207.7ms/step | 11.5 samples/s
|
| 497 |
+
Step 24940/50462 | Loss: 2.5709 | LR: 2.61e-04 | 5225.0ms/step | 11.5 samples/s
|
| 498 |
+
Step 24950/50462 | Loss: 2.5711 | LR: 2.60e-04 | 5219.2ms/step | 11.5 samples/s
|
| 499 |
+
Step 24960/50462 | Loss: 2.5634 | LR: 2.60e-04 | 5228.8ms/step | 11.5 samples/s
|
| 500 |
+
Step 24970/50462 | Loss: 2.5673 | LR: 2.60e-04 | 5235.6ms/step | 11.5 samples/s
|
| 501 |
+
Step 24980/50462 | Loss: 2.5
|
epfl_final/SparseAtt/wandb/run-20260825_135931-rgmkfpnl/files/requirements.txt
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
| 142 |
+
entmax==1.3
|
epfl_final/SparseAtt/wandb/run-20260825_135931-rgmkfpnl/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-08-25T13:59:31.094761Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"configs/p2_1p2b_dense_seed70_2gpu.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/TokenPrune.git",
|
| 14 |
+
"commit": "5637886f0cc760009989b14e68c6ae06d4827418"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "h100-workspace-2-gpu-0-17",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 23 |
+
"gpu_count": 2,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "9374931091456",
|
| 27 |
+
"used": "6938253725696"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1623005732864"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-cd5c7d97-0f62-2f41-36c8-233c8f57b814"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 43 |
+
"memoryTotal": "85520809984",
|
| 44 |
+
"cudaCores": 16896,
|
| 45 |
+
"architecture": "Hopper",
|
| 46 |
+
"uuid": "GPU-b97c2df9-b38b-c4e9-4d9d-780e91b62380"
|
| 47 |
+
}
|
| 48 |
+
],
|
| 49 |
+
"cudaVersion": "13.0",
|
| 50 |
+
"writerId": "jtwqh5baamrs691a5ddzsd1dud9jlgwh"
|
| 51 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260825_135931-rgmkfpnl/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260825_135931-rgmkfpnl/logs/debug.log
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-08-25 13:59:31,119 INFO MainThread:219 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-08-25 13:59:31,120 INFO MainThread:219 [wandb_setup.py:_flush():81] Configure stats pid to 219
|
| 3 |
+
2026-08-25 13:59:31,120 INFO MainThread:219 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-08-25 13:59:31,120 INFO MainThread:219 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260825_135931-rgmkfpnl/logs/debug.log
|
| 5 |
+
2026-08-25 13:59:31,120 INFO MainThread:219 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260825_135931-rgmkfpnl/logs/debug-internal.log
|
| 6 |
+
2026-08-25 13:59:31,120 INFO MainThread:219 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-08-25 13:59:31,120 INFO MainThread:219 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-20500', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 15, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 70, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b dense 8k seed70 2gpu resume', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'dense', 'seed70'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-08-25 13:59:31,120 INFO MainThread:219 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-08-25 13:59:32,007 INFO MainThread:219 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-08-25 13:59:32,508 INFO MainThread:219 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-08-25 13:59:32,518 INFO MainThread:219 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-08-25 13:59:32,573 INFO MainThread:219 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-08-25 13:59:33,258 INFO MainThread:219 [wandb_init.py:init():1051] run resumed
|
| 15 |
+
2026-08-25 13:59:33,261 INFO MainThread:219 [wandb_init.py:init():1056] starting run threads in backend
|
| 16 |
+
2026-08-25 13:59:33,648 INFO MainThread:219 [wandb_run.py:_console_start():2554] atexit reg
|
| 17 |
+
2026-08-25 13:59:33,648 INFO MainThread:219 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 18 |
+
2026-08-25 13:59:33,648 INFO MainThread:219 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 19 |
+
2026-08-25 13:59:33,649 INFO MainThread:219 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 20 |
+
2026-08-25 13:59:33,673 INFO MainThread:219 [wandb_init.py:init():1094] run started, returning control to user process
|
epfl_final/SparseAtt/wandb/run-20260825_135931-rgmkfpnl/run-rgmkfpnl.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:75705185b1d388dd662b00c42869e222169adfa36e19d0e8a4082bd987d87cc4
|
| 3 |
+
size 5242880
|
epfl_final/SparseAtt/wandb/run-20260825_192830-6sx9f1kf/files/output.log
ADDED
|
@@ -0,0 +1,309 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Starting training...
|
| 2 |
+
Switching attention mode for step 1: sparse_stochastic (configured sparse attention)
|
| 3 |
+
Step 10/44000 | Loss: 12.0711 | LR: 6.67e-06 | 7337.1ms/step | 2.7 samples/s
|
| 4 |
+
Step 20/44000 | Loss: 11.9717 | LR: 1.33e-05 | 1348.8ms/step | 14.8 samples/s
|
| 5 |
+
Step 30/44000 | Loss: 11.6495 | LR: 2.00e-05 | 1349.9ms/step | 14.8 samples/s
|
| 6 |
+
Step 40/44000 | Loss: 11.1050 | LR: 2.67e-05 | 1348.7ms/step | 14.8 samples/s
|
| 7 |
+
Step 50/44000 | Loss: 10.7634 | LR: 3.33e-05 | 1345.1ms/step | 14.9 samples/s
|
| 8 |
+
Step 60/44000 | Loss: 10.5279 | LR: 4.00e-05 | 1349.4ms/step | 14.8 samples/s
|
| 9 |
+
Step 70/44000 | Loss: 10.2709 | LR: 4.67e-05 | 1347.0ms/step | 14.8 samples/s
|
| 10 |
+
Step 80/44000 | Loss: 9.9987 | LR: 5.33e-05 | 1346.2ms/step | 14.9 samples/s
|
| 11 |
+
Step 90/44000 | Loss: 9.6598 | LR: 6.00e-05 | 1351.3ms/step | 14.8 samples/s
|
| 12 |
+
Step 100/44000 | Loss: 9.1523 | LR: 6.67e-05 | 1354.3ms/step | 14.8 samples/s
|
| 13 |
+
Step 110/44000 | Loss: 8.6439 | LR: 7.33e-05 | 1350.7ms/step | 14.8 samples/s
|
| 14 |
+
Step 120/44000 | Loss: 8.1440 | LR: 8.00e-05 | 1349.4ms/step | 14.8 samples/s
|
| 15 |
+
Step 130/44000 | Loss: 7.7750 | LR: 8.67e-05 | 1349.2ms/step | 14.8 samples/s
|
| 16 |
+
Step 140/44000 | Loss: 7.5434 | LR: 9.33e-05 | 1349.3ms/step | 14.8 samples/s
|
| 17 |
+
Step 150/44000 | Loss: 7.3708 | LR: 1.00e-04 | 1351.9ms/step | 14.8 samples/s
|
| 18 |
+
Step 160/44000 | Loss: 7.2375 | LR: 1.07e-04 | 1352.3ms/step | 14.8 samples/s
|
| 19 |
+
Step 170/44000 | Loss: 7.1023 | LR: 1.13e-04 | 1352.8ms/step | 14.8 samples/s
|
| 20 |
+
Step 180/44000 | Loss: 7.0167 | LR: 1.20e-04 | 1358.7ms/step | 14.7 samples/s
|
| 21 |
+
Step 190/44000 | Loss: 6.9254 | LR: 1.27e-04 | 1357.2ms/step | 14.7 samples/s
|
| 22 |
+
Step 200/44000 | Loss: 6.8553 | LR: 1.33e-04 | 1360.8ms/step | 14.7 samples/s
|
| 23 |
+
Step 210/44000 | Loss: 6.7191 | LR: 1.40e-04 | 1361.9ms/step | 14.7 samples/s
|
| 24 |
+
Step 220/44000 | Loss: 6.6928 | LR: 1.47e-04 | 1368.2ms/step | 14.6 samples/s
|
| 25 |
+
Step 230/44000 | Loss: 6.5972 | LR: 1.53e-04 | 1361.7ms/step | 14.7 samples/s
|
| 26 |
+
Step 240/44000 | Loss: 6.5085 | LR: 1.60e-04 | 1359.7ms/step | 14.7 samples/s
|
| 27 |
+
Step 250/44000 | Loss: 6.4586 | LR: 1.67e-04 | 1366.0ms/step | 14.6 samples/s
|
| 28 |
+
Validation at step 250: Loss = 6.4323, Perplexity = 621.61
|
| 29 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-250
|
| 30 |
+
Step 260/44000 | Loss: 6.4339 | LR: 1.73e-04 | 6337.9ms/step | 3.2 samples/s
|
| 31 |
+
Step 270/44000 | Loss: 6.3622 | LR: 1.80e-04 | 1361.0ms/step | 14.7 samples/s
|
| 32 |
+
Step 280/44000 | Loss: 6.3080 | LR: 1.87e-04 | 1368.4ms/step | 14.6 samples/s
|
| 33 |
+
Step 290/44000 | Loss: 6.2640 | LR: 1.93e-04 | 1363.1ms/step | 14.7 samples/s
|
| 34 |
+
Step 300/44000 | Loss: 6.2404 | LR: 2.00e-04 | 1361.7ms/step | 14.7 samples/s
|
| 35 |
+
Step 310/44000 | Loss: 6.1606 | LR: 2.07e-04 | 1362.2ms/step | 14.7 samples/s
|
| 36 |
+
Step 320/44000 | Loss: 6.1321 | LR: 2.13e-04 | 1357.4ms/step | 14.7 samples/s
|
| 37 |
+
Step 330/44000 | Loss: 6.0755 | LR: 2.20e-04 | 1366.1ms/step | 14.6 samples/s
|
| 38 |
+
Step 340/44000 | Loss: 6.0630 | LR: 2.27e-04 | 1360.5ms/step | 14.7 samples/s
|
| 39 |
+
Step 350/44000 | Loss: 6.0028 | LR: 2.33e-04 | 1362.8ms/step | 14.7 samples/s
|
| 40 |
+
Step 360/44000 | Loss: 5.9657 | LR: 2.40e-04 | 1368.0ms/step | 14.6 samples/s
|
| 41 |
+
Step 370/44000 | Loss: 5.9471 | LR: 2.47e-04 | 1363.4ms/step | 14.7 samples/s
|
| 42 |
+
Step 380/44000 | Loss: 5.9143 | LR: 2.53e-04 | 1364.3ms/step | 14.7 samples/s
|
| 43 |
+
Step 390/44000 | Loss: 5.8743 | LR: 2.60e-04 | 1363.4ms/step | 14.7 samples/s
|
| 44 |
+
Step 400/44000 | Loss: 5.8099 | LR: 2.67e-04 | 1360.2ms/step | 14.7 samples/s
|
| 45 |
+
Step 410/44000 | Loss: 5.7998 | LR: 2.73e-04 | 1364.9ms/step | 14.7 samples/s
|
| 46 |
+
Step 420/44000 | Loss: 5.7829 | LR: 2.80e-04 | 1365.5ms/step | 14.6 samples/s
|
| 47 |
+
Step 430/44000 | Loss: 5.7467 | LR: 2.87e-04 | 1368.5ms/step | 14.6 samples/s
|
| 48 |
+
Step 440/44000 | Loss: 5.7170 | LR: 2.93e-04 | 1364.4ms/step | 14.7 samples/s
|
| 49 |
+
Step 450/44000 | Loss: 5.6652 | LR: 3.00e-04 | 1361.5ms/step | 14.7 samples/s
|
| 50 |
+
Step 460/44000 | Loss: 5.6324 | LR: 3.07e-04 | 1361.8ms/step | 14.7 samples/s
|
| 51 |
+
Step 470/44000 | Loss: 5.6002 | LR: 3.13e-04 | 1356.9ms/step | 14.7 samples/s
|
| 52 |
+
Step 480/44000 | Loss: 5.5505 | LR: 3.20e-04 | 1357.6ms/step | 14.7 samples/s
|
| 53 |
+
Step 490/44000 | Loss: 5.5154 | LR: 3.27e-04 | 1357.4ms/step | 14.7 samples/s
|
| 54 |
+
Step 500/44000 | Loss: 5.5163 | LR: 3.33e-04 | 1357.4ms/step | 14.7 samples/s
|
| 55 |
+
Validation at step 500: Loss = 5.4815, Perplexity = 240.20
|
| 56 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-500
|
| 57 |
+
Step 510/44000 | Loss: 5.4842 | LR: 3.40e-04 | 3893.2ms/step | 5.1 samples/s
|
| 58 |
+
Step 520/44000 | Loss: 5.4256 | LR: 3.47e-04 | 1367.2ms/step | 14.6 samples/s
|
| 59 |
+
Step 530/44000 | Loss: 5.4154 | LR: 3.53e-04 | 1368.3ms/step | 14.6 samples/s
|
| 60 |
+
Step 540/44000 | Loss: 5.3837 | LR: 3.60e-04 | 1362.0ms/step | 14.7 samples/s
|
| 61 |
+
Step 550/44000 | Loss: 5.3384 | LR: 3.67e-04 | 1363.4ms/step | 14.7 samples/s
|
| 62 |
+
Step 560/44000 | Loss: 5.3207 | LR: 3.73e-04 | 1365.5ms/step | 14.6 samples/s
|
| 63 |
+
Step 570/44000 | Loss: 5.2742 | LR: 3.80e-04 | 1367.1ms/step | 14.6 samples/s
|
| 64 |
+
Step 580/44000 | Loss: 5.2714 | LR: 3.87e-04 | 1362.7ms/step | 14.7 samples/s
|
| 65 |
+
Step 590/44000 | Loss: 5.2158 | LR: 3.93e-04 | 1362.5ms/step | 14.7 samples/s
|
| 66 |
+
Step 600/44000 | Loss: 5.2123 | LR: 4.00e-04 | 1366.8ms/step | 14.6 samples/s
|
| 67 |
+
Step 610/44000 | Loss: 5.1481 | LR: 4.07e-04 | 1363.8ms/step | 14.7 samples/s
|
| 68 |
+
Step 620/44000 | Loss: 5.1340 | LR: 4.13e-04 | 1364.8ms/step | 14.7 samples/s
|
| 69 |
+
Step 630/44000 | Loss: 5.1076 | LR: 4.20e-04 | 1361.0ms/step | 14.7 samples/s
|
| 70 |
+
Step 640/44000 | Loss: 5.1054 | LR: 4.27e-04 | 1359.0ms/step | 14.7 samples/s
|
| 71 |
+
Step 650/44000 | Loss: 5.0491 | LR: 4.33e-04 | 1364.2ms/step | 14.7 samples/s
|
| 72 |
+
Step 660/44000 | Loss: 5.0562 | LR: 4.40e-04 | 1362.1ms/step | 14.7 samples/s
|
| 73 |
+
Step 670/44000 | Loss: 5.0276 | LR: 4.47e-04 | 1362.4ms/step | 14.7 samples/s
|
| 74 |
+
Step 680/44000 | Loss: 4.9559 | LR: 4.53e-04 | 1363.4ms/step | 14.7 samples/s
|
| 75 |
+
Step 690/44000 | Loss: 4.9932 | LR: 4.60e-04 | 1365.8ms/step | 14.6 samples/s
|
| 76 |
+
Step 700/44000 | Loss: 4.9010 | LR: 4.67e-04 | 1366.0ms/step | 14.6 samples/s
|
| 77 |
+
Step 710/44000 | Loss: 4.9001 | LR: 4.73e-04 | 1364.0ms/step | 14.7 samples/s
|
| 78 |
+
Step 720/44000 | Loss: 4.8877 | LR: 4.80e-04 | 1358.9ms/step | 14.7 samples/s
|
| 79 |
+
Step 730/44000 | Loss: 4.8307 | LR: 4.87e-04 | 1360.0ms/step | 14.7 samples/s
|
| 80 |
+
Step 740/44000 | Loss: 4.8304 | LR: 4.93e-04 | 1360.6ms/step | 14.7 samples/s
|
| 81 |
+
Step 750/44000 | Loss: 4.7977 | LR: 5.00e-04 | 1364.2ms/step | 14.7 samples/s
|
| 82 |
+
Validation at step 750: Loss = 4.7705, Perplexity = 117.98
|
| 83 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-750
|
| 84 |
+
Step 760/44000 | Loss: 4.7924 | LR: 5.00e-04 | 3881.2ms/step | 5.2 samples/s
|
| 85 |
+
Step 770/44000 | Loss: 4.7358 | LR: 5.00e-04 | 1360.7ms/step | 14.7 samples/s
|
| 86 |
+
Step 780/44000 | Loss: 4.6945 | LR: 5.00e-04 | 1362.2ms/step | 14.7 samples/s
|
| 87 |
+
Step 790/44000 | Loss: 4.6635 | LR: 5.00e-04 | 1358.9ms/step | 14.7 samples/s
|
| 88 |
+
Step 800/44000 | Loss: 4.6523 | LR: 5.00e-04 | 1362.2ms/step | 14.7 samples/s
|
| 89 |
+
Step 810/44000 | Loss: 4.6363 | LR: 5.00e-04 | 1364.8ms/step | 14.7 samples/s
|
| 90 |
+
Step 820/44000 | Loss: 4.6026 | LR: 5.00e-04 | 1364.5ms/step | 14.7 samples/s
|
| 91 |
+
Step 830/44000 | Loss: 4.5735 | LR: 5.00e-04 | 1363.0ms/step | 14.7 samples/s
|
| 92 |
+
Step 840/44000 | Loss: 4.5417 | LR: 5.00e-04 | 1369.5ms/step | 14.6 samples/s
|
| 93 |
+
Step 850/44000 | Loss: 4.5144 | LR: 5.00e-04 | 1363.5ms/step | 14.7 samples/s
|
| 94 |
+
Step 860/44000 | Loss: 4.4845 | LR: 5.00e-04 | 1361.9ms/step | 14.7 samples/s
|
| 95 |
+
Step 870/44000 | Loss: 4.4991 | LR: 5.00e-04 | 1361.9ms/step | 14.7 samples/s
|
| 96 |
+
Step 880/44000 | Loss: 4.4861 | LR: 5.00e-04 | 1364.0ms/step | 14.7 samples/s
|
| 97 |
+
Step 890/44000 | Loss: 4.4595 | LR: 5.00e-04 | 1362.2ms/step | 14.7 samples/s
|
| 98 |
+
Step 900/44000 | Loss: 4.4329 | LR: 5.00e-04 | 1360.2ms/step | 14.7 samples/s
|
| 99 |
+
Step 910/44000 | Loss: 4.4364 | LR: 5.00e-04 | 1361.7ms/step | 14.7 samples/s
|
| 100 |
+
Step 920/44000 | Loss: 4.3790 | LR: 5.00e-04 | 1365.4ms/step | 14.6 samples/s
|
| 101 |
+
Step 930/44000 | Loss: 4.3906 | LR: 5.00e-04 | 1364.6ms/step | 14.7 samples/s
|
| 102 |
+
Step 940/44000 | Loss: 4.3651 | LR: 5.00e-04 | 1364.3ms/step | 14.7 samples/s
|
| 103 |
+
Step 950/44000 | Loss: 4.3562 | LR: 5.00e-04 | 1367.6ms/step | 14.6 samples/s
|
| 104 |
+
Step 960/44000 | Loss: 4.3581 | LR: 5.00e-04 | 1362.6ms/step | 14.7 samples/s
|
| 105 |
+
Step 970/44000 | Loss: 4.3182 | LR: 5.00e-04 | 1362.7ms/step | 14.7 samples/s
|
| 106 |
+
Step 980/44000 | Loss: 4.3303 | LR: 5.00e-04 | 1364.4ms/step | 14.7 samples/s
|
| 107 |
+
Step 990/44000 | Loss: 4.2891 | LR: 5.00e-04 | 1366.8ms/step | 14.6 samples/s
|
| 108 |
+
Step 1000/44000 | Loss: 4.2516 | LR: 5.00e-04 | 1364.5ms/step | 14.7 samples/s
|
| 109 |
+
Validation at step 1000: Loss = 4.2733, Perplexity = 71.76
|
| 110 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-1000
|
| 111 |
+
Step 1010/44000 | Loss: 4.2714 | LR: 5.00e-04 | 3858.3ms/step | 5.2 samples/s
|
| 112 |
+
Step 1020/44000 | Loss: 4.2798 | LR: 5.00e-04 | 1362.7ms/step | 14.7 samples/s
|
| 113 |
+
Step 1030/44000 | Loss: 4.2469 | LR: 5.00e-04 | 1359.1ms/step | 14.7 samples/s
|
| 114 |
+
Step 1040/44000 | Loss: 4.2219 | LR: 5.00e-04 | 1367.3ms/step | 14.6 samples/s
|
| 115 |
+
Step 1050/44000 | Loss: 4.2022 | LR: 5.00e-04 | 1366.8ms/step | 14.6 samples/s
|
| 116 |
+
Step 1060/44000 | Loss: 4.2040 | LR: 5.00e-04 | 1361.2ms/step | 14.7 samples/s
|
| 117 |
+
Step 1070/44000 | Loss: 4.1619 | LR: 5.00e-04 | 1364.8ms/step | 14.7 samples/s
|
| 118 |
+
Step 1080/44000 | Loss: 4.1578 | LR: 5.00e-04 | 1365.3ms/step | 14.6 samples/s
|
| 119 |
+
Step 1090/44000 | Loss: 4.2069 | LR: 5.00e-04 | 1362.6ms/step | 14.7 samples/s
|
| 120 |
+
Step 1100/44000 | Loss: 4.1523 | LR: 5.00e-04 | 1361.9ms/step | 14.7 samples/s
|
| 121 |
+
Step 1110/44000 | Loss: 4.1572 | LR: 5.00e-04 | 1361.0ms/step | 14.7 samples/s
|
| 122 |
+
Step 1120/44000 | Loss: 4.1286 | LR: 5.00e-04 | 1361.2ms/step | 14.7 samples/s
|
| 123 |
+
Step 1130/44000 | Loss: 4.1544 | LR: 5.00e-04 | 1364.0ms/step | 14.7 samples/s
|
| 124 |
+
Step 1140/44000 | Loss: 4.0852 | LR: 5.00e-04 | 1365.0ms/step | 14.7 samples/s
|
| 125 |
+
Step 1150/44000 | Loss: 4.1095 | LR: 5.00e-04 | 1368.8ms/step | 14.6 samples/s
|
| 126 |
+
Step 1160/44000 | Loss: 4.0869 | LR: 5.00e-04 | 1365.0ms/step | 14.7 samples/s
|
| 127 |
+
Step 1170/44000 | Loss: 4.1058 | LR: 5.00e-04 | 1361.2ms/step | 14.7 samples/s
|
| 128 |
+
Step 1180/44000 | Loss: 4.0906 | LR: 5.00e-04 | 1365.3ms/step | 14.6 samples/s
|
| 129 |
+
Step 1190/44000 | Loss: 4.0867 | LR: 5.00e-04 | 1359.5ms/step | 14.7 samples/s
|
| 130 |
+
Step 1200/44000 | Loss: 4.0732 | LR: 5.00e-04 | 1359.0ms/step | 14.7 samples/s
|
| 131 |
+
Step 1210/44000 | Loss: 4.0530 | LR: 5.00e-04 | 1356.4ms/step | 14.7 samples/s
|
| 132 |
+
Step 1220/44000 | Loss: 4.0610 | LR: 5.00e-04 | 1357.3ms/step | 14.7 samples/s
|
| 133 |
+
Step 1230/44000 | Loss: 4.0312 | LR: 5.00e-04 | 1352.6ms/step | 14.8 samples/s
|
| 134 |
+
Step 1240/44000 | Loss: 4.0391 | LR: 5.00e-04 | 1357.0ms/step | 14.7 samples/s
|
| 135 |
+
Step 1250/44000 | Loss: 3.9951 | LR: 5.00e-04 | 1356.5ms/step | 14.7 samples/s
|
| 136 |
+
Validation at step 1250: Loss = 4.0120, Perplexity = 55.26
|
| 137 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-1250
|
| 138 |
+
Step 1260/44000 | Loss: 4.0145 | LR: 5.00e-04 | 3857.9ms/step | 5.2 samples/s
|
| 139 |
+
Step 1270/44000 | Loss: 4.0223 | LR: 5.00e-04 | 1354.0ms/step | 14.8 samples/s
|
| 140 |
+
Step 1280/44000 | Loss: 4.0026 | LR: 5.00e-04 | 1355.7ms/step | 14.8 samples/s
|
| 141 |
+
Step 1290/44000 | Loss: 3.9988 | LR: 5.00e-04 | 1355.0ms/step | 14.8 samples/s
|
| 142 |
+
Step 1300/44000 | Loss: 3.9714 | LR: 5.00e-04 | 1353.9ms/step | 14.8 samples/s
|
| 143 |
+
Step 1310/44000 | Loss: 3.9568 | LR: 5.00e-04 | 1353.0ms/step | 14.8 samples/s
|
| 144 |
+
Step 1320/44000 | Loss: 3.9634 | LR: 5.00e-04 | 1353.1ms/step | 14.8 samples/s
|
| 145 |
+
Step 1330/44000 | Loss: 3.9521 | LR: 5.00e-04 | 1354.6ms/step | 14.8 samples/s
|
| 146 |
+
Step 1340/44000 | Loss: 3.9543 | LR: 5.00e-04 | 1356.6ms/step | 14.7 samples/s
|
| 147 |
+
Step 1350/44000 | Loss: 3.9717 | LR: 5.00e-04 | 1352.0ms/step | 14.8 samples/s
|
| 148 |
+
Step 1360/44000 | Loss: 3.9713 | LR: 5.00e-04 | 1352.8ms/step | 14.8 samples/s
|
| 149 |
+
Step 1370/44000 | Loss: 3.9478 | LR: 5.00e-04 | 1356.2ms/step | 14.7 samples/s
|
| 150 |
+
Step 1380/44000 | Loss: 3.9247 | LR: 5.00e-04 | 1353.5ms/step | 14.8 samples/s
|
| 151 |
+
Step 1390/44000 | Loss: 3.9088 | LR: 5.00e-04 | 1355.6ms/step | 14.8 samples/s
|
| 152 |
+
Step 1400/44000 | Loss: 3.9230 | LR: 5.00e-04 | 1355.7ms/step | 14.8 samples/s
|
| 153 |
+
Step 1410/44000 | Loss: 3.9060 | LR: 5.00e-04 | 1353.5ms/step | 14.8 samples/s
|
| 154 |
+
Step 1420/44000 | Loss: 3.9135 | LR: 5.00e-04 | 1353.1ms/step | 14.8 samples/s
|
| 155 |
+
Step 1430/44000 | Loss: 3.8995 | LR: 5.00e-04 | 1350.9ms/step | 14.8 samples/s
|
| 156 |
+
Step 1440/44000 | Loss: 3.8967 | LR: 5.00e-04 | 1352.4ms/step | 14.8 samples/s
|
| 157 |
+
Step 1450/44000 | Loss: 3.8772 | LR: 5.00e-04 | 1351.6ms/step | 14.8 samples/s
|
| 158 |
+
Step 1460/44000 | Loss: 3.8708 | LR: 5.00e-04 | 1351.6ms/step | 14.8 samples/s
|
| 159 |
+
Step 1470/44000 | Loss: 3.8727 | LR: 5.00e-04 | 1351.7ms/step | 14.8 samples/s
|
| 160 |
+
Step 1480/44000 | Loss: 3.8852 | LR: 5.00e-04 | 1350.0ms/step | 14.8 samples/s
|
| 161 |
+
Step 1490/44000 | Loss: 3.8835 | LR: 5.00e-04 | 1350.0ms/step | 14.8 samples/s
|
| 162 |
+
Step 1500/44000 | Loss: 3.8918 | LR: 5.00e-04 | 1353.4ms/step | 14.8 samples/s
|
| 163 |
+
Validation at step 1500: Loss = 3.8619, Perplexity = 47.55
|
| 164 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-1500
|
| 165 |
+
Step 1510/44000 | Loss: 3.8767 | LR: 5.00e-04 | 3844.1ms/step | 5.2 samples/s
|
| 166 |
+
Step 1520/44000 | Loss: 3.8616 | LR: 5.00e-04 | 1353.3ms/step | 14.8 samples/s
|
| 167 |
+
Step 1530/44000 | Loss: 3.8317 | LR: 5.00e-04 | 1349.9ms/step | 14.8 samples/s
|
| 168 |
+
Step 1540/44000 | Loss: 3.8673 | LR: 5.00e-04 | 1352.6ms/step | 14.8 samples/s
|
| 169 |
+
Step 1550/44000 | Loss: 3.8586 | LR: 5.00e-04 | 1360.5ms/step | 14.7 samples/s
|
| 170 |
+
Step 1560/44000 | Loss: 3.8401 | LR: 5.00e-04 | 1355.9ms/step | 14.7 samples/s
|
| 171 |
+
Step 1570/44000 | Loss: 3.8208 | LR: 5.00e-04 | 1357.2ms/step | 14.7 samples/s
|
| 172 |
+
Step 1580/44000 | Loss: 3.8636 | LR: 5.00e-04 | 1352.6ms/step | 14.8 samples/s
|
| 173 |
+
Step 1590/44000 | Loss: 3.8059 | LR: 5.00e-04 | 1354.6ms/step | 14.8 samples/s
|
| 174 |
+
Step 1600/44000 | Loss: 3.8345 | LR: 5.00e-04 | 1353.4ms/step | 14.8 samples/s
|
| 175 |
+
Step 1610/44000 | Loss: 3.8351 | LR: 5.00e-04 | 1352.3ms/step | 14.8 samples/s
|
| 176 |
+
Step 1620/44000 | Loss: 3.7816 | LR: 5.00e-04 | 1354.8ms/step | 14.8 samples/s
|
| 177 |
+
Step 1630/44000 | Loss: 3.7856 | LR: 4.99e-04 | 1354.5ms/step | 14.8 samples/s
|
| 178 |
+
Step 1640/44000 | Loss: 3.7963 | LR: 4.99e-04 | 1352.8ms/step | 14.8 samples/s
|
| 179 |
+
Step 1650/44000 | Loss: 3.7829 | LR: 4.99e-04 | 1351.4ms/step | 14.8 samples/s
|
| 180 |
+
Step 1660/44000 | Loss: 3.7934 | LR: 4.99e-04 | 1357.3ms/step | 14.7 samples/s
|
| 181 |
+
Step 1670/44000 | Loss: 3.8035 | LR: 4.99e-04 | 1351.3ms/step | 14.8 samples/s
|
| 182 |
+
Step 1680/44000 | Loss: 3.7785 | LR: 4.99e-04 | 1354.9ms/step | 14.8 samples/s
|
| 183 |
+
Step 1690/44000 | Loss: 3.7695 | LR: 4.99e-04 | 1351.1ms/step | 14.8 samples/s
|
| 184 |
+
Step 1700/44000 | Loss: 3.7499 | LR: 4.99e-04 | 1352.3ms/step | 14.8 samples/s
|
| 185 |
+
Step 1710/44000 | Loss: 3.7566 | LR: 4.99e-04 | 1350.2ms/step | 14.8 samples/s
|
| 186 |
+
Step 1720/44000 | Loss: 3.7609 | LR: 4.99e-04 | 1350.5ms/step | 14.8 samples/s
|
| 187 |
+
Step 1730/44000 | Loss: 3.7503 | LR: 4.99e-04 | 1358.4ms/step | 14.7 samples/s
|
| 188 |
+
Step 1740/44000 | Loss: 3.7779 | LR: 4.99e-04 | 1352.1ms/step | 14.8 samples/s
|
| 189 |
+
Step 1750/44000 | Loss: 3.7930 | LR: 4.99e-04 | 1355.4ms/step | 14.8 samples/s
|
| 190 |
+
Validation at step 1750: Loss = 3.7526, Perplexity = 42.63
|
| 191 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-1750
|
| 192 |
+
Step 1760/44000 | Loss: 3.7621 | LR: 4.99e-04 | 3886.9ms/step | 5.1 samples/s
|
| 193 |
+
Step 1770/44000 | Loss: 3.7366 | LR: 4.99e-04 | 1358.5ms/step | 14.7 samples/s
|
| 194 |
+
Step 1780/44000 | Loss: 3.7455 | LR: 4.99e-04 | 1361.8ms/step | 14.7 samples/s
|
| 195 |
+
Step 1790/44000 | Loss: 3.7231 | LR: 4.99e-04 | 1365.1ms/step | 14.7 samples/s
|
| 196 |
+
Step 1800/44000 | Loss: 3.7170 | LR: 4.99e-04 | 1363.6ms/step | 14.7 samples/s
|
| 197 |
+
Step 1810/44000 | Loss: 3.7430 | LR: 4.99e-04 | 1360.7ms/step | 14.7 samples/s
|
| 198 |
+
Step 1820/44000 | Loss: 3.7280 | LR: 4.99e-04 | 1359.1ms/step | 14.7 samples/s
|
| 199 |
+
Step 1830/44000 | Loss: 3.7129 | LR: 4.99e-04 | 1356.7ms/step | 14.7 samples/s
|
| 200 |
+
Step 1840/44000 | Loss: 3.7172 | LR: 4.99e-04 | 1360.2ms/step | 14.7 samples/s
|
| 201 |
+
Step 1850/44000 | Loss: 3.7115 | LR: 4.99e-04 | 1358.9ms/step | 14.7 samples/s
|
| 202 |
+
Step 1860/44000 | Loss: 3.7145 | LR: 4.99e-04 | 1357.2ms/step | 14.7 samples/s
|
| 203 |
+
Step 1870/44000 | Loss: 3.7432 | LR: 4.99e-04 | 1361.3ms/step | 14.7 samples/s
|
| 204 |
+
Step 1880/44000 | Loss: 3.7112 | LR: 4.99e-04 | 1359.7ms/step | 14.7 samples/s
|
| 205 |
+
Step 1890/44000 | Loss: 3.6872 | LR: 4.99e-04 | 1360.2ms/step | 14.7 samples/s
|
| 206 |
+
Step 1900/44000 | Loss: 3.7188 | LR: 4.99e-04 | 1361.2ms/step | 14.7 samples/s
|
| 207 |
+
Step 1910/44000 | Loss: 3.6731 | LR: 4.99e-04 | 1361.8ms/step | 14.7 samples/s
|
| 208 |
+
Step 1920/44000 | Loss: 3.6885 | LR: 4.99e-04 | 1358.6ms/step | 14.7 samples/s
|
| 209 |
+
Step 1930/44000 | Loss: 3.6980 | LR: 4.99e-04 | 1357.0ms/step | 14.7 samples/s
|
| 210 |
+
Step 1940/44000 | Loss: 3.6770 | LR: 4.99e-04 | 1356.6ms/step | 14.7 samples/s
|
| 211 |
+
Step 1950/44000 | Loss: 3.6759 | LR: 4.99e-04 | 1358.3ms/step | 14.7 samples/s
|
| 212 |
+
Step 1960/44000 | Loss: 3.6877 | LR: 4.99e-04 | 1361.2ms/step | 14.7 samples/s
|
| 213 |
+
Step 1970/44000 | Loss: 3.6887 | LR: 4.99e-04 | 1362.6ms/step | 14.7 samples/s
|
| 214 |
+
Step 1980/44000 | Loss: 3.6511 | LR: 4.99e-04 | 1360.4ms/step | 14.7 samples/s
|
| 215 |
+
Step 1990/44000 | Loss: 3.6831 | LR: 4.99e-04 | 1359.3ms/step | 14.7 samples/s
|
| 216 |
+
Step 2000/44000 | Loss: 3.6669 | LR: 4.99e-04 | 1356.7ms/step | 14.7 samples/s
|
| 217 |
+
Validation at step 2000: Loss = 3.6727, Perplexity = 39.36
|
| 218 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-2000
|
| 219 |
+
Step 2010/44000 | Loss: 3.6578 | LR: 4.99e-04 | 3858.1ms/step | 5.2 samples/s
|
| 220 |
+
Step 2020/44000 | Loss: 3.6638 | LR: 4.99e-04 | 1359.7ms/step | 14.7 samples/s
|
| 221 |
+
Step 2030/44000 | Loss: 3.6607 | LR: 4.99e-04 | 1360.6ms/step | 14.7 samples/s
|
| 222 |
+
Step 2040/44000 | Loss: 3.6861 | LR: 4.99e-04 | 1366.5ms/step | 14.6 samples/s
|
| 223 |
+
Step 2050/44000 | Loss: 3.6554 | LR: 4.99e-04 | 1366.1ms/step | 14.6 samples/s
|
| 224 |
+
Step 2060/44000 | Loss: 3.6552 | LR: 4.99e-04 | 1357.6ms/step | 14.7 samples/s
|
| 225 |
+
Step 2070/44000 | Loss: 3.6447 | LR: 4.99e-04 | 1361.8ms/step | 14.7 samples/s
|
| 226 |
+
Step 2080/44000 | Loss: 3.6344 | LR: 4.99e-04 | 1362.7ms/step | 14.7 samples/s
|
| 227 |
+
Step 2090/44000 | Loss: 3.6519 | LR: 4.99e-04 | 1364.7ms/step | 14.7 samples/s
|
| 228 |
+
Step 2100/44000 | Loss: 3.6354 | LR: 4.99e-04 | 1363.7ms/step | 14.7 samples/s
|
| 229 |
+
Step 2110/44000 | Loss: 3.6871 | LR: 4.99e-04 | 1363.4ms/step | 14.7 samples/s
|
| 230 |
+
Step 2120/44000 | Loss: 3.6250 | LR: 4.99e-04 | 1365.0ms/step | 14.7 samples/s
|
| 231 |
+
Step 2130/44000 | Loss: 3.6637 | LR: 4.99e-04 | 1362.6ms/step | 14.7 samples/s
|
| 232 |
+
Step 2140/44000 | Loss: 3.6473 | LR: 4.99e-04 | 1363.6ms/step | 14.7 samples/s
|
| 233 |
+
Step 2150/44000 | Loss: 3.6361 | LR: 4.99e-04 | 1358.8ms/step | 14.7 samples/s
|
| 234 |
+
Step 2160/44000 | Loss: 3.6796 | LR: 4.99e-04 | 1360.7ms/step | 14.7 samples/s
|
| 235 |
+
Step 2170/44000 | Loss: 3.6585 | LR: 4.99e-04 | 1361.2ms/step | 14.7 samples/s
|
| 236 |
+
Step 2180/44000 | Loss: 3.6202 | LR: 4.99e-04 | 1364.7ms/step | 14.7 samples/s
|
| 237 |
+
Step 2190/44000 | Loss: 3.5909 | LR: 4.99e-04 | 1362.8ms/step | 14.7 samples/s
|
| 238 |
+
Step 2200/44000 | Loss: 3.6456 | LR: 4.99e-04 | 1364.9ms/step | 14.7 samples/s
|
| 239 |
+
Step 2210/44000 | Loss: 3.6462 | LR: 4.99e-04 | 1362.4ms/step | 14.7 samples/s
|
| 240 |
+
Step 2220/44000 | Loss: 3.6216 | LR: 4.99e-04 | 1360.7ms/step | 14.7 samples/s
|
| 241 |
+
Step 2230/44000 | Loss: 3.5979 | LR: 4.99e-04 | 1355.0ms/step | 14.8 samples/s
|
| 242 |
+
Step 2240/44000 | Loss: 3.5852 | LR: 4.99e-04 | 1358.8ms/step | 14.7 samples/s
|
| 243 |
+
Step 2250/44000 | Loss: 3.6189 | LR: 4.99e-04 | 1356.9ms/step | 14.7 samples/s
|
| 244 |
+
Validation at step 2250: Loss = 3.6066, Perplexity = 36.84
|
| 245 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-2250
|
| 246 |
+
Step 2260/44000 | Loss: 3.6230 | LR: 4.98e-04 | 3865.2ms/step | 5.2 samples/s
|
| 247 |
+
Step 2270/44000 | Loss: 3.6361 | LR: 4.98e-04 | 1362.6ms/step | 14.7 samples/s
|
| 248 |
+
Step 2280/44000 | Loss: 3.6061 | LR: 4.98e-04 | 1362.9ms/step | 14.7 samples/s
|
| 249 |
+
Step 2290/44000 | Loss: 3.6187 | LR: 4.98e-04 | 1359.5ms/step | 14.7 samples/s
|
| 250 |
+
Step 2300/44000 | Loss: 3.6018 | LR: 4.98e-04 | 1362.9ms/step | 14.7 samples/s
|
| 251 |
+
Step 2310/44000 | Loss: 3.5893 | LR: 4.98e-04 | 1360.6ms/step | 14.7 samples/s
|
| 252 |
+
Step 2320/44000 | Loss: 3.5825 | LR: 4.98e-04 | 1361.1ms/step | 14.7 samples/s
|
| 253 |
+
Step 2330/44000 | Loss: 3.5693 | LR: 4.98e-04 | 1362.1ms/step | 14.7 samples/s
|
| 254 |
+
Step 2340/44000 | Loss: 3.6163 | LR: 4.98e-04 | 1359.4ms/step | 14.7 samples/s
|
| 255 |
+
Step 2350/44000 | Loss: 3.6004 | LR: 4.98e-04 | 1360.6ms/step | 14.7 samples/s
|
| 256 |
+
Step 2360/44000 | Loss: 3.5961 | LR: 4.98e-04 | 1362.2ms/step | 14.7 samples/s
|
| 257 |
+
Step 2370/44000 | Loss: 3.5617 | LR: 4.98e-04 | 1362.2ms/step | 14.7 samples/s
|
| 258 |
+
Step 2380/44000 | Loss: 3.5395 | LR: 4.98e-04 | 1359.6ms/step | 14.7 samples/s
|
| 259 |
+
Step 2390/44000 | Loss: 3.5808 | LR: 4.98e-04 | 1362.8ms/step | 14.7 samples/s
|
| 260 |
+
Step 2400/44000 | Loss: 3.5630 | LR: 4.98e-04 | 1363.6ms/step | 14.7 samples/s
|
| 261 |
+
Step 2410/44000 | Loss: 3.5863 | LR: 4.98e-04 | 1365.5ms/step | 14.6 samples/s
|
| 262 |
+
Step 2420/44000 | Loss: 3.5684 | LR: 4.98e-04 | 1364.4ms/step | 14.7 samples/s
|
| 263 |
+
Step 2430/44000 | Loss: 3.5871 | LR: 4.98e-04 | 1365.0ms/step | 14.7 samples/s
|
| 264 |
+
Step 2440/44000 | Loss: 3.5826 | LR: 4.98e-04 | 1363.2ms/step | 14.7 samples/s
|
| 265 |
+
Step 2450/44000 | Loss: 3.6135 | LR: 4.98e-04 | 1359.5ms/step | 14.7 samples/s
|
| 266 |
+
Step 2460/44000 | Loss: 3.5397 | LR: 4.98e-04 | 1362.1ms/step | 14.7 samples/s
|
| 267 |
+
Step 2470/44000 | Loss: 3.5845 | LR: 4.98e-04 | 1361.4ms/step | 14.7 samples/s
|
| 268 |
+
Step 2480/44000 | Loss: 3.5921 | LR: 4.98e-04 | 1363.6ms/step | 14.7 samples/s
|
| 269 |
+
Step 2490/44000 | Loss: 3.5702 | LR: 4.98e-04 | 1364.9ms/step | 14.7 samples/s
|
| 270 |
+
Step 2500/44000 | Loss: 3.5535 | LR: 4.98e-04 | 1362.9ms/step | 14.7 samples/s
|
| 271 |
+
Validation at step 2500: Loss = 3.5555, Perplexity = 35.00
|
| 272 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-2500
|
| 273 |
+
Step 2510/44000 | Loss: 3.5582 | LR: 4.98e-04 | 3873.3ms/step | 5.2 samples/s
|
| 274 |
+
Step 2520/44000 | Loss: 3.5496 | LR: 4.98e-04 | 1356.9ms/step | 14.7 samples/s
|
| 275 |
+
Step 2530/44000 | Loss: 3.5313 | LR: 4.98e-04 | 1356.0ms/step | 14.7 samples/s
|
| 276 |
+
Step 2540/44000 | Loss: 3.5369 | LR: 4.98e-04 | 1358.0ms/step | 14.7 samples/s
|
| 277 |
+
Step 2550/44000 | Loss: 3.5082 | LR: 4.98e-04 | 1360.9ms/step | 14.7 samples/s
|
| 278 |
+
Step 2560/44000 | Loss: 3.5580 | LR: 4.98e-04 | 1364.1ms/step | 14.7 samples/s
|
| 279 |
+
Step 2570/44000 | Loss: 3.5531 | LR: 4.98e-04 | 1363.7ms/step | 14.7 samples/s
|
| 280 |
+
Step 2580/44000 | Loss: 3.5283 | LR: 4.98e-04 | 1364.4ms/step | 14.7 samples/s
|
| 281 |
+
Step 2590/44000 | Loss: 3.5522 | LR: 4.98e-04 | 1362.0ms/step | 14.7 samples/s
|
| 282 |
+
Step 2600/44000 | Loss: 3.5163 | LR: 4.98e-04 | 1360.8ms/step | 14.7 samples/s
|
| 283 |
+
Step 2610/44000 | Loss: 3.5404 | LR: 4.98e-04 | 1359.1ms/step | 14.7 samples/s
|
| 284 |
+
Step 2620/44000 | Loss: 3.5333 | LR: 4.98e-04 | 1359.1ms/step | 14.7 samples/s
|
| 285 |
+
Step 2630/44000 | Loss: 3.5329 | LR: 4.98e-04 | 1367.4ms/step | 14.6 samples/s
|
| 286 |
+
Step 2640/44000 | Loss: 3.5292 | LR: 4.98e-04 | 1357.1ms/step | 14.7 samples/s
|
| 287 |
+
Step 2650/44000 | Loss: 3.5057 | LR: 4.98e-04 | 1361.4ms/step | 14.7 samples/s
|
| 288 |
+
Step 2660/44000 | Loss: 3.5152 | LR: 4.98e-04 | 1361.8ms/step | 14.7 samples/s
|
| 289 |
+
Step 2670/44000 | Loss: 3.5575 | LR: 4.98e-04 | 1363.0ms/step | 14.7 samples/s
|
| 290 |
+
Step 2680/44000 | Loss: 3.5442 | LR: 4.98e-04 | 1358.0ms/step | 14.7 samples/s
|
| 291 |
+
Step 2690/44000 | Loss: 3.5181 | LR: 4.98e-04 | 1360.0ms/step | 14.7 samples/s
|
| 292 |
+
Step 2700/44000 | Loss: 3.5460 | LR: 4.97e-04 | 1360.9ms/step | 14.7 samples/s
|
| 293 |
+
Step 2710/44000 | Loss: 3.5127 | LR: 4.97e-04 | 1359.6ms/step | 14.7 samples/s
|
| 294 |
+
Step 2720/44000 | Loss: 3.5330 | LR: 4.97e-04 | 1360.0ms/step | 14.7 samples/s
|
| 295 |
+
Step 2730/44000 | Loss: 3.4644 | LR: 4.97e-04 | 1357.0ms/step | 14.7 samples/s
|
| 296 |
+
Step 2740/44000 | Loss: 3.5057 | LR: 4.97e-04 | 1353.9ms/step | 14.8 samples/s
|
| 297 |
+
Step 2750/44000 | Loss: 3.5142 | LR: 4.97e-04 | 1361.5ms/step | 14.7 samples/s
|
| 298 |
+
Validation at step 2750: Loss = 3.5116, Perplexity = 33.50
|
| 299 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-2750
|
| 300 |
+
Step 2760/44000 | Loss: 3.5153 | LR: 4.97e-04 | 3857.2ms/step | 5.2 samples/s
|
| 301 |
+
Step 2770/44000 | Loss: 3.5200 | LR: 4.97e-04 | 1355.1ms/step | 14.8 samples/s
|
| 302 |
+
Step 2780/44000 | Loss: 3.4972 | LR: 4.97e-04 | 1351.8ms/step | 14.8 samples/s
|
| 303 |
+
Step 2790/44000 | Loss: 3.4920 | LR: 4.97e-04 | 1355.4ms/step | 14.8 samples/s
|
| 304 |
+
Step 2800/44000 | Loss: 3.5118 | LR: 4.97e-04 | 1359.8ms/step | 14.7 samples/s
|
| 305 |
+
Step 2810/44000 | Loss: 3.4875 | LR: 4.97e-04 | 1358.3ms/step | 14.7 samples/s
|
| 306 |
+
Step 2820/44000 | Loss: 3.4969 | LR: 4.97e-04 | 1357.8ms/step | 14.7 samples/s
|
| 307 |
+
Step 2830/44000 | Loss: 3.4963 | LR: 4.97e-04 | 1359.5ms/step | 14.7 samples/s
|
| 308 |
+
Step 2840/44000 | Loss: 3.4830 | LR: 4.97e-04 | 1360.5ms/step | 14.7 samples/s
|
| 309 |
+
Step 2850/44000 | Loss: 3.4863 | LR: 4.97e-04 | 1359.6ms/step | 14.7 samples/s
|
epfl_final/SparseAtt/wandb/run-20260825_192830-6sx9f1kf/files/requirements.txt
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
| 142 |
+
entmax==1.3
|
epfl_final/SparseAtt/wandb/run-20260825_192830-6sx9f1kf/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-08-25T19:28:30.746726Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"configs/ladder8k/365m_edge_dense_altmixed_seed49.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/TokenPrune.git",
|
| 14 |
+
"commit": "5637886f0cc760009989b14e68c6ae06d4827418"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "h100-workspace-0-1",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 23 |
+
"gpu_count": 1,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "9374931091456",
|
| 27 |
+
"used": "5419094904832"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1623005626368"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-3ea6c840-350f-c68c-7562-6e0affa4e153"
|
| 40 |
+
}
|
| 41 |
+
],
|
| 42 |
+
"cudaVersion": "13.0",
|
| 43 |
+
"writerId": "ocdr6tbrrd1vrsa9lko7wu6lyesmzbyu"
|
| 44 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260825_192830-6sx9f1kf/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,573 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-08-25T19:28:31.387776511Z","level":"INFO","msg":"wandb-core"}
|
| 2 |
+
{"time":"2026-08-25T19:28:31.393175851Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
|
| 3 |
+
{"time":"2026-08-25T19:28:31.78548368Z","level":"INFO","msg":"stream: created new stream","id":"6sx9f1kf"}
|
| 4 |
+
{"time":"2026-08-25T19:28:31.785570039Z","level":"INFO","msg":"handler: started"}
|
| 5 |
+
{"time":"2026-08-25T19:28:31.787634925Z","level":"INFO","msg":"stream: started"}
|
| 6 |
+
{"time":"2026-08-25T19:28:31.788736091Z","level":"INFO","msg":"sender: started"}
|
| 7 |
+
{"time":"2026-08-25T19:28:31.788754409Z","level":"INFO","msg":"writer: started","stream_id":"6sx9f1kf"}
|
| 8 |
+
{"time":"2026-08-25T19:28:32.492279025Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
|
| 9 |
+
{"time":"2026-08-25T19:28:32.749660066Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 10 |
+
{"time":"2026-08-25T19:28:47.492711347Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"console_offset":1,"console_lines":1,"uploaded_len":2}
|
| 11 |
+
{"time":"2026-08-25T19:28:47.851574348Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 12 |
+
{"time":"2026-08-25T19:29:02.493318882Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
|
| 13 |
+
{"time":"2026-08-25T19:29:02.745114956Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 14 |
+
{"time":"2026-08-25T19:29:17.493192619Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":4,"events_lines":2}
|
| 15 |
+
{"time":"2026-08-25T19:29:17.808625258Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 16 |
+
{"time":"2026-08-25T19:29:32.492689544Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":6,"events_lines":2}
|
| 17 |
+
{"time":"2026-08-25T19:29:32.785872767Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 18 |
+
{"time":"2026-08-25T19:29:47.492456171Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":8,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 19 |
+
{"time":"2026-08-25T19:29:47.804299014Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 20 |
+
{"time":"2026-08-25T19:30:02.493061092Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":10,"events_lines":2,"console_offset":3,"console_lines":1}
|
| 21 |
+
{"time":"2026-08-25T19:30:02.809268869Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 22 |
+
{"time":"2026-08-25T19:30:17.492877223Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":12,"events_lines":2,"console_offset":4,"console_lines":1}
|
| 23 |
+
{"time":"2026-08-25T19:30:17.823231885Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 24 |
+
{"time":"2026-08-25T19:30:32.492344614Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":1,"events_offset":14,"events_lines":2,"console_offset":5,"console_lines":1}
|
| 25 |
+
{"time":"2026-08-25T19:30:32.791599152Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 26 |
+
{"time":"2026-08-25T19:30:47.492359482Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":3,"history_lines":1,"events_offset":16,"events_lines":2,"console_offset":6,"console_lines":1}
|
| 27 |
+
{"time":"2026-08-25T19:30:47.893426793Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 28 |
+
{"time":"2026-08-25T19:31:02.492831294Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":4,"history_lines":1,"events_offset":18,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 29 |
+
{"time":"2026-08-25T19:31:02.788156148Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 30 |
+
{"time":"2026-08-25T19:31:17.492341058Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":1,"events_offset":20,"events_lines":2,"console_offset":8,"console_lines":1}
|
| 31 |
+
{"time":"2026-08-25T19:31:17.800147948Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 32 |
+
{"time":"2026-08-25T19:31:32.493235611Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":1,"events_offset":22,"events_lines":2,"console_offset":9,"console_lines":1}
|
| 33 |
+
{"time":"2026-08-25T19:31:32.736760523Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 34 |
+
{"time":"2026-08-25T19:31:47.492688239Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":7,"history_lines":2,"events_offset":24,"events_lines":2,"console_offset":10,"console_lines":2}
|
| 35 |
+
{"time":"2026-08-25T19:31:47.834310972Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 36 |
+
{"time":"2026-08-25T19:32:02.492467085Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":1,"events_offset":26,"events_lines":2,"console_offset":12,"console_lines":1}
|
| 37 |
+
{"time":"2026-08-25T19:32:02.838678957Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 38 |
+
{"time":"2026-08-25T19:32:17.492951385Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":10,"history_lines":1,"events_offset":28,"events_lines":2,"console_offset":13,"console_lines":1}
|
| 39 |
+
{"time":"2026-08-25T19:32:17.827761786Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 40 |
+
{"time":"2026-08-25T19:32:32.492647668Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":11,"history_lines":1,"events_offset":30,"events_lines":2,"console_offset":14,"console_lines":1}
|
| 41 |
+
{"time":"2026-08-25T19:32:32.76648745Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 42 |
+
{"time":"2026-08-25T19:32:47.493080571Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":12,"history_lines":1,"events_offset":32,"events_lines":2,"console_offset":15,"console_lines":1}
|
| 43 |
+
{"time":"2026-08-25T19:32:47.943112287Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 44 |
+
{"time":"2026-08-25T19:33:02.493374697Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":13,"history_lines":1,"events_offset":34,"events_lines":2,"console_offset":16,"console_lines":1}
|
| 45 |
+
{"time":"2026-08-25T19:33:03.029510097Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 46 |
+
{"time":"2026-08-25T19:33:17.492830621Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":14,"history_lines":1,"events_offset":36,"events_lines":2,"console_offset":17,"console_lines":1}
|
| 47 |
+
{"time":"2026-08-25T19:33:17.98572527Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 48 |
+
{"time":"2026-08-25T19:33:32.494084411Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":15,"history_lines":1,"events_offset":38,"events_lines":2,"console_offset":18,"console_lines":1}
|
| 49 |
+
{"time":"2026-08-25T19:33:32.808639908Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 50 |
+
{"time":"2026-08-25T19:33:47.493027508Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":16,"history_lines":1,"events_offset":40,"events_lines":2,"console_offset":19,"console_lines":1}
|
| 51 |
+
{"time":"2026-08-25T19:33:47.759067071Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 52 |
+
{"time":"2026-08-25T19:34:02.492900804Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":17,"history_lines":1,"events_offset":42,"events_lines":2,"console_offset":20,"console_lines":1}
|
| 53 |
+
{"time":"2026-08-25T19:34:02.958282522Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 54 |
+
{"time":"2026-08-25T19:34:17.49245198Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":18,"history_lines":2,"events_offset":44,"events_lines":2,"console_offset":21,"console_lines":2}
|
| 55 |
+
{"time":"2026-08-25T19:34:17.805169921Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 56 |
+
{"time":"2026-08-25T19:34:32.493093115Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":20,"history_lines":1,"events_offset":46,"events_lines":2,"console_offset":23,"console_lines":1}
|
| 57 |
+
{"time":"2026-08-25T19:34:32.834743289Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 58 |
+
{"time":"2026-08-25T19:34:47.492689628Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":21,"history_lines":1,"events_offset":48,"events_lines":2,"console_offset":24,"console_lines":1}
|
| 59 |
+
{"time":"2026-08-25T19:34:47.873764658Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 60 |
+
{"time":"2026-08-25T19:35:02.492868562Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":22,"history_lines":1,"events_offset":50,"events_lines":2,"console_offset":25,"console_lines":1}
|
| 61 |
+
{"time":"2026-08-25T19:35:02.820153174Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 62 |
+
{"time":"2026-08-25T19:35:17.493223629Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":23,"history_lines":1,"events_offset":52,"events_lines":2,"console_offset":26,"console_lines":1}
|
| 63 |
+
{"time":"2026-08-25T19:35:17.872165721Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 64 |
+
{"time":"2026-08-25T19:35:32.493215373Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":54,"events_lines":2}
|
| 65 |
+
{"time":"2026-08-25T19:35:32.77455871Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 66 |
+
{"time":"2026-08-25T19:35:47.493122861Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":56,"events_lines":2}
|
| 67 |
+
{"time":"2026-08-25T19:35:47.767092667Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 68 |
+
{"time":"2026-08-25T19:36:02.492781697Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":58,"events_lines":2,"console_offset":27,"console_lines":2}
|
| 69 |
+
{"time":"2026-08-25T19:36:02.815259596Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 70 |
+
{"time":"2026-08-25T19:36:17.493159254Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":24,"history_lines":1,"events_offset":60,"events_lines":2,"console_offset":29,"console_lines":1}
|
| 71 |
+
{"time":"2026-08-25T19:36:17.845041835Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 72 |
+
{"time":"2026-08-25T19:36:32.492783842Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":25,"history_lines":1,"events_offset":62,"events_lines":2,"console_offset":30,"console_lines":1}
|
| 73 |
+
{"time":"2026-08-25T19:36:32.897157877Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 74 |
+
{"time":"2026-08-25T19:36:47.493006314Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":26,"history_lines":1,"events_offset":64,"events_lines":2,"console_offset":31,"console_lines":1}
|
| 75 |
+
{"time":"2026-08-25T19:36:47.819417471Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 76 |
+
{"time":"2026-08-25T19:37:02.493205782Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":27,"history_lines":1,"events_offset":66,"events_lines":2,"console_offset":32,"console_lines":1}
|
| 77 |
+
{"time":"2026-08-25T19:37:02.86947106Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 78 |
+
{"time":"2026-08-25T19:37:17.493037149Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":28,"history_lines":1,"events_offset":68,"events_lines":2,"console_offset":33,"console_lines":1}
|
| 79 |
+
{"time":"2026-08-25T19:37:17.838761615Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 80 |
+
{"time":"2026-08-25T19:37:32.492401959Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":29,"history_lines":1,"events_offset":70,"events_lines":2,"console_offset":34,"console_lines":1}
|
| 81 |
+
{"time":"2026-08-25T19:37:32.814231327Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 82 |
+
{"time":"2026-08-25T19:37:47.492391222Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":30,"history_lines":1,"events_offset":72,"events_lines":2,"console_offset":35,"console_lines":1}
|
| 83 |
+
{"time":"2026-08-25T19:37:47.788521655Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 84 |
+
{"time":"2026-08-25T19:38:02.492588707Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":31,"history_lines":1,"events_offset":74,"events_lines":2,"console_offset":36,"console_lines":1}
|
| 85 |
+
{"time":"2026-08-25T19:38:02.818520727Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 86 |
+
{"time":"2026-08-25T19:38:17.492615266Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":32,"history_lines":2,"events_offset":76,"events_lines":2,"console_offset":37,"console_lines":2}
|
| 87 |
+
{"time":"2026-08-25T19:38:17.776716648Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 88 |
+
{"time":"2026-08-25T19:38:32.492764218Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":34,"history_lines":1,"events_offset":78,"events_lines":2,"console_offset":39,"console_lines":1}
|
| 89 |
+
{"time":"2026-08-25T19:38:32.749533304Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 90 |
+
{"time":"2026-08-25T19:38:47.492915874Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":35,"history_lines":1,"events_offset":80,"events_lines":2,"console_offset":40,"console_lines":1}
|
| 91 |
+
{"time":"2026-08-25T19:38:47.867230887Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 92 |
+
{"time":"2026-08-25T19:39:02.493023064Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":36,"history_lines":1,"events_offset":82,"events_lines":2,"console_offset":41,"console_lines":1}
|
| 93 |
+
{"time":"2026-08-25T19:39:02.877661728Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 94 |
+
{"time":"2026-08-25T19:39:17.493076744Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":37,"history_lines":1,"events_offset":84,"events_lines":2,"console_offset":42,"console_lines":1}
|
| 95 |
+
{"time":"2026-08-25T19:39:18.149536243Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 96 |
+
{"time":"2026-08-25T19:39:32.493320958Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":38,"history_lines":1,"events_offset":86,"events_lines":2,"console_offset":43,"console_lines":1}
|
| 97 |
+
{"time":"2026-08-25T19:39:32.827277433Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 98 |
+
{"time":"2026-08-25T19:39:47.493330141Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":39,"history_lines":1,"events_offset":88,"events_lines":2,"console_offset":44,"console_lines":1}
|
| 99 |
+
{"time":"2026-08-25T19:39:47.827660035Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 100 |
+
{"time":"2026-08-25T19:40:02.494688962Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":40,"history_lines":1,"events_offset":90,"events_lines":2,"console_offset":45,"console_lines":1}
|
| 101 |
+
{"time":"2026-08-25T19:40:02.95861905Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 102 |
+
{"time":"2026-08-25T19:40:17.492745266Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":41,"history_lines":1,"events_offset":92,"events_lines":2,"console_offset":46,"console_lines":1}
|
| 103 |
+
{"time":"2026-08-25T19:40:17.765424654Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 104 |
+
{"time":"2026-08-25T19:40:32.492903592Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":42,"history_lines":1,"events_offset":94,"events_lines":2,"console_offset":47,"console_lines":1}
|
| 105 |
+
{"time":"2026-08-25T19:40:32.7880806Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 106 |
+
{"time":"2026-08-25T19:40:47.492911293Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":43,"history_lines":2,"events_offset":96,"events_lines":2,"console_offset":48,"console_lines":2}
|
| 107 |
+
{"time":"2026-08-25T19:40:47.769510852Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 108 |
+
{"time":"2026-08-25T19:41:02.493056349Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":45,"history_lines":1,"events_offset":98,"events_lines":2,"console_offset":50,"console_lines":1}
|
| 109 |
+
{"time":"2026-08-25T19:41:02.814274233Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 110 |
+
{"time":"2026-08-25T19:41:17.493044712Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":46,"history_lines":1,"events_offset":100,"events_lines":2,"console_offset":51,"console_lines":1}
|
| 111 |
+
{"time":"2026-08-25T19:41:17.754576176Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 112 |
+
{"time":"2026-08-25T19:41:32.493118141Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":47,"history_lines":1,"events_offset":102,"events_lines":2,"console_offset":52,"console_lines":1}
|
| 113 |
+
{"time":"2026-08-25T19:41:32.755760006Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 114 |
+
{"time":"2026-08-25T19:41:47.492392265Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":48,"history_lines":1,"events_offset":104,"events_lines":2,"console_offset":53,"console_lines":1}
|
| 115 |
+
{"time":"2026-08-25T19:41:47.849048712Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 116 |
+
{"time":"2026-08-25T19:42:02.492284914Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":106,"events_lines":2}
|
| 117 |
+
{"time":"2026-08-25T19:42:02.747466327Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 118 |
+
{"time":"2026-08-25T19:42:17.492786984Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":108,"events_lines":2,"console_offset":54,"console_lines":2}
|
| 119 |
+
{"time":"2026-08-25T19:42:17.768132879Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 120 |
+
{"time":"2026-08-25T19:42:32.492843178Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":49,"history_lines":1,"events_offset":110,"events_lines":2,"console_offset":56,"console_lines":1}
|
| 121 |
+
{"time":"2026-08-25T19:42:32.756577275Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 122 |
+
{"time":"2026-08-25T19:42:47.49304594Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":50,"history_lines":2,"events_offset":112,"events_lines":2,"console_offset":57,"console_lines":2}
|
| 123 |
+
{"time":"2026-08-25T19:42:47.805131172Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 124 |
+
{"time":"2026-08-25T19:43:02.493198848Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":52,"history_lines":1,"events_offset":114,"events_lines":2,"console_offset":59,"console_lines":1}
|
| 125 |
+
{"time":"2026-08-25T19:43:02.983722078Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 126 |
+
{"time":"2026-08-25T19:43:17.492803177Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":53,"history_lines":1,"events_offset":116,"events_lines":2,"console_offset":60,"console_lines":1}
|
| 127 |
+
{"time":"2026-08-25T19:43:17.806110336Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 128 |
+
{"time":"2026-08-25T19:43:32.492882395Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":54,"history_lines":1,"events_offset":118,"events_lines":2,"console_offset":61,"console_lines":1}
|
| 129 |
+
{"time":"2026-08-25T19:43:32.824192274Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 130 |
+
{"time":"2026-08-25T19:43:47.493196734Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":55,"history_lines":1,"events_offset":120,"events_lines":2,"console_offset":62,"console_lines":1}
|
| 131 |
+
{"time":"2026-08-25T19:43:47.798136736Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 132 |
+
{"time":"2026-08-25T19:44:02.493048992Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":56,"history_lines":1,"events_offset":122,"events_lines":2,"console_offset":63,"console_lines":1}
|
| 133 |
+
{"time":"2026-08-25T19:44:03.027123462Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 134 |
+
{"time":"2026-08-25T19:44:17.493156002Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":57,"history_lines":1,"events_offset":124,"events_lines":2,"console_offset":64,"console_lines":1}
|
| 135 |
+
{"time":"2026-08-25T19:44:17.859159441Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 136 |
+
{"time":"2026-08-25T19:44:32.492829193Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":58,"history_lines":1,"events_offset":126,"events_lines":2,"console_offset":65,"console_lines":1}
|
| 137 |
+
{"time":"2026-08-25T19:44:32.777555239Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 138 |
+
{"time":"2026-08-25T19:44:47.492504427Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":59,"history_lines":1,"events_offset":128,"events_lines":2,"console_offset":66,"console_lines":1}
|
| 139 |
+
{"time":"2026-08-25T19:44:47.755567942Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 140 |
+
{"time":"2026-08-25T19:45:02.492751546Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":60,"history_lines":1,"events_offset":130,"events_lines":2,"console_offset":67,"console_lines":1}
|
| 141 |
+
{"time":"2026-08-25T19:45:02.823194389Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 142 |
+
{"time":"2026-08-25T19:45:17.49252051Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":61,"history_lines":2,"events_offset":132,"events_lines":2,"console_offset":68,"console_lines":2}
|
| 143 |
+
{"time":"2026-08-25T19:45:17.781579172Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 144 |
+
{"time":"2026-08-25T19:45:32.492817143Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":63,"history_lines":1,"events_offset":134,"events_lines":2,"console_offset":70,"console_lines":1}
|
| 145 |
+
{"time":"2026-08-25T19:45:32.866501242Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 146 |
+
{"time":"2026-08-25T19:45:47.493069708Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":64,"history_lines":1,"events_offset":136,"events_lines":2,"console_offset":71,"console_lines":1}
|
| 147 |
+
{"time":"2026-08-25T19:45:47.897938513Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 148 |
+
{"time":"2026-08-25T19:46:02.492992302Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":65,"history_lines":1,"events_offset":138,"events_lines":2,"console_offset":72,"console_lines":1}
|
| 149 |
+
{"time":"2026-08-25T19:46:02.743862701Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 150 |
+
{"time":"2026-08-25T19:46:17.493066973Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":66,"history_lines":1,"events_offset":140,"events_lines":2,"console_offset":73,"console_lines":1}
|
| 151 |
+
{"time":"2026-08-25T19:46:17.827751727Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 152 |
+
{"time":"2026-08-25T19:46:32.492345584Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":67,"history_lines":1,"events_offset":142,"events_lines":2,"console_offset":74,"console_lines":1}
|
| 153 |
+
{"time":"2026-08-25T19:46:32.859894783Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 154 |
+
{"time":"2026-08-25T19:46:47.49328653Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":68,"history_lines":1,"events_offset":144,"events_lines":2,"console_offset":75,"console_lines":1}
|
| 155 |
+
{"time":"2026-08-25T19:46:47.829790764Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 156 |
+
{"time":"2026-08-25T19:47:02.492786683Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":69,"history_lines":1,"events_offset":146,"events_lines":2,"console_offset":76,"console_lines":1}
|
| 157 |
+
{"time":"2026-08-25T19:47:03.278853308Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 158 |
+
{"time":"2026-08-25T19:47:17.492697609Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":70,"history_lines":1,"events_offset":148,"events_lines":2,"console_offset":77,"console_lines":1}
|
| 159 |
+
{"time":"2026-08-25T19:47:17.853739651Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 160 |
+
{"time":"2026-08-25T19:47:32.492595405Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":71,"history_lines":1,"events_offset":150,"events_lines":2,"console_offset":78,"console_lines":1}
|
| 161 |
+
{"time":"2026-08-25T19:47:32.854846235Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 162 |
+
{"time":"2026-08-25T19:47:47.493051186Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":72,"history_lines":2,"events_offset":152,"events_lines":2,"console_offset":79,"console_lines":2}
|
| 163 |
+
{"time":"2026-08-25T19:47:47.856322421Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 164 |
+
{"time":"2026-08-25T19:48:02.493015632Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":154,"events_lines":2}
|
| 165 |
+
{"time":"2026-08-25T19:48:02.844922169Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 166 |
+
{"time":"2026-08-25T19:48:17.493005577Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":156,"events_lines":2,"console_offset":81,"console_lines":2}
|
| 167 |
+
{"time":"2026-08-25T19:48:17.739923147Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 168 |
+
{"time":"2026-08-25T19:48:32.492323757Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":74,"history_lines":1,"events_offset":158,"events_lines":2,"console_offset":83,"console_lines":1}
|
| 169 |
+
{"time":"2026-08-25T19:48:32.78851378Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 170 |
+
{"time":"2026-08-25T19:48:47.493384032Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":75,"history_lines":1,"events_offset":160,"events_lines":2,"console_offset":84,"console_lines":1}
|
| 171 |
+
{"time":"2026-08-25T19:48:47.802160518Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 172 |
+
{"time":"2026-08-25T19:49:02.492498678Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":76,"history_lines":1,"events_offset":162,"events_lines":2,"console_offset":85,"console_lines":1}
|
| 173 |
+
{"time":"2026-08-25T19:49:02.822242146Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 174 |
+
{"time":"2026-08-25T19:49:17.492632538Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":77,"history_lines":1,"events_offset":164,"events_lines":2,"console_offset":86,"console_lines":1}
|
| 175 |
+
{"time":"2026-08-25T19:49:17.766056479Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 176 |
+
{"time":"2026-08-25T19:49:32.492867879Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":78,"history_lines":1,"events_offset":166,"events_lines":2,"console_offset":87,"console_lines":1}
|
| 177 |
+
{"time":"2026-08-25T19:49:32.731195899Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 178 |
+
{"time":"2026-08-25T19:49:47.493130801Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":79,"history_lines":1,"events_offset":168,"events_lines":2,"console_offset":88,"console_lines":1}
|
| 179 |
+
{"time":"2026-08-25T19:49:47.91186396Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 180 |
+
{"time":"2026-08-25T19:50:02.493006545Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":80,"history_lines":2,"events_offset":170,"events_lines":2,"console_offset":89,"console_lines":2}
|
| 181 |
+
{"time":"2026-08-25T19:50:02.891014894Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 182 |
+
{"time":"2026-08-25T19:50:17.492991512Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":82,"history_lines":1,"events_offset":172,"events_lines":2,"console_offset":91,"console_lines":1}
|
| 183 |
+
{"time":"2026-08-25T19:50:17.904627562Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 184 |
+
{"time":"2026-08-25T19:50:32.493058191Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":83,"history_lines":1,"events_offset":174,"events_lines":2,"console_offset":92,"console_lines":1}
|
| 185 |
+
{"time":"2026-08-25T19:50:32.807713094Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 186 |
+
{"time":"2026-08-25T19:50:47.493260162Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":84,"history_lines":1,"events_offset":176,"events_lines":2,"console_offset":93,"console_lines":1}
|
| 187 |
+
{"time":"2026-08-25T19:50:47.890746371Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 188 |
+
{"time":"2026-08-25T19:51:02.492665142Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":85,"history_lines":1,"events_offset":178,"events_lines":2,"console_offset":94,"console_lines":1}
|
| 189 |
+
{"time":"2026-08-25T19:51:02.820531204Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 190 |
+
{"time":"2026-08-25T19:51:17.492549018Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":86,"history_lines":1,"events_offset":180,"events_lines":2,"console_offset":95,"console_lines":1}
|
| 191 |
+
{"time":"2026-08-25T19:51:17.945372575Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 192 |
+
{"time":"2026-08-25T19:51:32.492516179Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":87,"history_lines":1,"events_offset":182,"events_lines":2,"console_offset":96,"console_lines":1}
|
| 193 |
+
{"time":"2026-08-25T19:51:32.979191837Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 194 |
+
{"time":"2026-08-25T19:51:47.492724329Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":88,"history_lines":1,"events_offset":184,"events_lines":2,"console_offset":97,"console_lines":1}
|
| 195 |
+
{"time":"2026-08-25T19:51:47.972588736Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 196 |
+
{"time":"2026-08-25T19:52:02.492871829Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":89,"history_lines":1,"events_offset":186,"events_lines":2,"console_offset":98,"console_lines":1}
|
| 197 |
+
{"time":"2026-08-25T19:52:03.290607357Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 198 |
+
{"time":"2026-08-25T19:52:17.492842926Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":90,"history_lines":1,"events_offset":188,"events_lines":2,"console_offset":99,"console_lines":1}
|
| 199 |
+
{"time":"2026-08-25T19:52:17.891526915Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 200 |
+
{"time":"2026-08-25T19:52:32.492895804Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":91,"history_lines":2,"events_offset":190,"events_lines":2,"console_offset":100,"console_lines":2}
|
| 201 |
+
{"time":"2026-08-25T19:52:32.790554872Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 202 |
+
{"time":"2026-08-25T19:52:47.493203053Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":93,"history_lines":1,"events_offset":192,"events_lines":2,"console_offset":102,"console_lines":1}
|
| 203 |
+
{"time":"2026-08-25T19:52:47.835107218Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 204 |
+
{"time":"2026-08-25T19:53:02.493286357Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":94,"history_lines":1,"events_offset":194,"events_lines":2,"console_offset":103,"console_lines":1}
|
| 205 |
+
{"time":"2026-08-25T19:53:02.815387541Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 206 |
+
{"time":"2026-08-25T19:53:17.492464396Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":95,"history_lines":1,"events_offset":196,"events_lines":2,"console_offset":104,"console_lines":1}
|
| 207 |
+
{"time":"2026-08-25T19:53:17.880180542Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 208 |
+
{"time":"2026-08-25T19:53:32.492412894Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":96,"history_lines":1,"events_offset":198,"events_lines":2,"console_offset":105,"console_lines":1}
|
| 209 |
+
{"time":"2026-08-25T19:53:32.932805114Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 210 |
+
{"time":"2026-08-25T19:53:47.492524245Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":97,"history_lines":1,"events_offset":200,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 211 |
+
{"time":"2026-08-25T19:53:47.797170665Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 212 |
+
{"time":"2026-08-25T19:54:02.492841223Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":98,"history_lines":1,"events_offset":202,"events_lines":2,"console_offset":107,"console_lines":1}
|
| 213 |
+
{"time":"2026-08-25T19:54:02.830282807Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 214 |
+
{"time":"2026-08-25T19:54:17.492784994Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":204,"events_lines":2,"console_offset":108,"console_lines":1}
|
| 215 |
+
{"time":"2026-08-25T19:54:17.846451541Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 216 |
+
{"time":"2026-08-25T19:54:32.493221541Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":99,"history_lines":1,"events_offset":206,"events_lines":2,"console_offset":109,"console_lines":2}
|
| 217 |
+
{"time":"2026-08-25T19:54:32.828556224Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 218 |
+
{"time":"2026-08-25T19:54:47.49240161Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":100,"history_lines":1,"events_offset":208,"events_lines":2,"console_offset":111,"console_lines":1}
|
| 219 |
+
{"time":"2026-08-25T19:54:47.867254219Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 220 |
+
{"time":"2026-08-25T19:55:02.492642294Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":101,"history_lines":1,"events_offset":210,"events_lines":2,"console_offset":112,"console_lines":1}
|
| 221 |
+
{"time":"2026-08-25T19:55:02.860059755Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 222 |
+
{"time":"2026-08-25T19:55:17.492582139Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":102,"history_lines":1,"events_offset":212,"events_lines":2,"console_offset":113,"console_lines":1}
|
| 223 |
+
{"time":"2026-08-25T19:55:17.904333528Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 224 |
+
{"time":"2026-08-25T19:55:32.492540201Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":103,"history_lines":1,"events_offset":214,"events_lines":2,"console_offset":114,"console_lines":1}
|
| 225 |
+
{"time":"2026-08-25T19:55:32.757132172Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 226 |
+
{"time":"2026-08-25T19:55:47.492396601Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":104,"history_lines":1,"events_offset":216,"events_lines":2,"console_offset":115,"console_lines":1}
|
| 227 |
+
{"time":"2026-08-25T19:55:47.806334175Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 228 |
+
{"time":"2026-08-25T19:56:02.49290781Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":105,"history_lines":1,"events_offset":218,"events_lines":2,"console_offset":116,"console_lines":1}
|
| 229 |
+
{"time":"2026-08-25T19:56:02.9370645Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 230 |
+
{"time":"2026-08-25T19:56:17.492421417Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":106,"history_lines":1,"events_offset":220,"events_lines":2,"console_offset":117,"console_lines":1}
|
| 231 |
+
{"time":"2026-08-25T19:56:17.787868941Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 232 |
+
{"time":"2026-08-25T19:56:32.492899356Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":107,"history_lines":1,"events_offset":222,"events_lines":2,"console_offset":118,"console_lines":1}
|
| 233 |
+
{"time":"2026-08-25T19:56:32.770529926Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 234 |
+
{"time":"2026-08-25T19:56:47.493066951Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":108,"history_lines":1,"events_offset":224,"events_lines":2,"console_offset":119,"console_lines":1}
|
| 235 |
+
{"time":"2026-08-25T19:56:47.827914025Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 236 |
+
{"time":"2026-08-25T19:57:02.49311626Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":109,"history_lines":2,"events_offset":226,"events_lines":2,"console_offset":120,"console_lines":2}
|
| 237 |
+
{"time":"2026-08-25T19:57:02.818174445Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 238 |
+
{"time":"2026-08-25T19:57:17.492351711Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":111,"history_lines":1,"events_offset":228,"events_lines":2,"console_offset":122,"console_lines":1}
|
| 239 |
+
{"time":"2026-08-25T19:57:17.776717944Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 240 |
+
{"time":"2026-08-25T19:57:32.492511585Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":112,"history_lines":1,"events_offset":230,"events_lines":2,"console_offset":123,"console_lines":1}
|
| 241 |
+
{"time":"2026-08-25T19:57:32.767576287Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 242 |
+
{"time":"2026-08-25T19:57:47.492669776Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":113,"history_lines":1,"events_offset":232,"events_lines":2,"console_offset":124,"console_lines":1}
|
| 243 |
+
{"time":"2026-08-25T19:57:47.80145104Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 244 |
+
{"time":"2026-08-25T19:58:02.492638944Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":114,"history_lines":1,"events_offset":234,"events_lines":2,"console_offset":125,"console_lines":1}
|
| 245 |
+
{"time":"2026-08-25T19:58:02.778369437Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 246 |
+
{"time":"2026-08-25T19:58:17.493038416Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":115,"history_lines":1,"events_offset":236,"events_lines":2,"console_offset":126,"console_lines":1}
|
| 247 |
+
{"time":"2026-08-25T19:58:17.829146025Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 248 |
+
{"time":"2026-08-25T19:58:32.493045061Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":116,"history_lines":1,"events_offset":238,"events_lines":2,"console_offset":127,"console_lines":1}
|
| 249 |
+
{"time":"2026-08-25T19:58:32.962128607Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 250 |
+
{"time":"2026-08-25T19:58:47.493037244Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":117,"history_lines":1,"events_offset":240,"events_lines":2,"console_offset":128,"console_lines":1}
|
| 251 |
+
{"time":"2026-08-25T19:58:48.075335416Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 252 |
+
{"time":"2026-08-25T19:59:02.492332424Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":118,"history_lines":1,"events_offset":242,"events_lines":2,"console_offset":129,"console_lines":1}
|
| 253 |
+
{"time":"2026-08-25T19:59:02.952084316Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 254 |
+
{"time":"2026-08-25T19:59:17.492336916Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":119,"history_lines":1,"events_offset":244,"events_lines":2,"console_offset":130,"console_lines":1}
|
| 255 |
+
{"time":"2026-08-25T19:59:17.795510704Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 256 |
+
{"time":"2026-08-25T19:59:32.493323315Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":120,"history_lines":2,"events_offset":246,"events_lines":2,"console_offset":131,"console_lines":2}
|
| 257 |
+
{"time":"2026-08-25T19:59:32.927709895Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 258 |
+
{"time":"2026-08-25T19:59:47.493860964Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":122,"history_lines":1,"events_offset":248,"events_lines":2,"console_offset":133,"console_lines":1}
|
| 259 |
+
{"time":"2026-08-25T19:59:47.860312551Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 260 |
+
{"time":"2026-08-25T20:00:02.492606973Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":123,"history_lines":1,"events_offset":250,"events_lines":2,"console_offset":134,"console_lines":1}
|
| 261 |
+
{"time":"2026-08-25T20:00:02.855501991Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 262 |
+
{"time":"2026-08-25T20:00:17.493005975Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":252,"events_lines":2}
|
| 263 |
+
{"time":"2026-08-25T20:00:17.920238305Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 264 |
+
{"time":"2026-08-25T20:00:32.492809466Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":254,"events_lines":2,"console_offset":135,"console_lines":2}
|
| 265 |
+
{"time":"2026-08-25T20:00:33.016261888Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 266 |
+
{"time":"2026-08-25T20:00:47.49291631Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":124,"history_lines":1,"events_offset":256,"events_lines":2,"console_offset":137,"console_lines":1}
|
| 267 |
+
{"time":"2026-08-25T20:00:47.957310374Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 268 |
+
{"time":"2026-08-25T20:01:02.492541875Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":125,"history_lines":1,"events_offset":258,"events_lines":2,"console_offset":138,"console_lines":1}
|
| 269 |
+
{"time":"2026-08-25T20:01:02.925565386Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 270 |
+
{"time":"2026-08-25T20:01:17.49290984Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":126,"history_lines":1,"events_offset":260,"events_lines":2,"console_offset":139,"console_lines":1}
|
| 271 |
+
{"time":"2026-08-25T20:01:17.94749182Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 272 |
+
{"time":"2026-08-25T20:01:32.493098216Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":127,"history_lines":2,"events_offset":262,"events_lines":2,"console_offset":140,"console_lines":2}
|
| 273 |
+
{"time":"2026-08-25T20:01:32.71857948Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 274 |
+
{"time":"2026-08-25T20:01:47.492987254Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":129,"history_lines":1,"events_offset":264,"events_lines":2,"console_offset":142,"console_lines":1}
|
| 275 |
+
{"time":"2026-08-25T20:01:47.856049382Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 276 |
+
{"time":"2026-08-25T20:02:02.493115671Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":130,"history_lines":1,"events_offset":266,"events_lines":2,"console_offset":143,"console_lines":1}
|
| 277 |
+
{"time":"2026-08-25T20:02:02.963827953Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 278 |
+
{"time":"2026-08-25T20:02:17.492617871Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":131,"history_lines":1,"events_offset":268,"events_lines":2,"console_offset":144,"console_lines":1}
|
| 279 |
+
{"time":"2026-08-25T20:02:18.032573142Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 280 |
+
{"time":"2026-08-25T20:02:32.492662721Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":132,"history_lines":1,"events_offset":270,"events_lines":2,"console_offset":145,"console_lines":1}
|
| 281 |
+
{"time":"2026-08-25T20:02:32.973675266Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 282 |
+
{"time":"2026-08-25T20:02:47.492683367Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":133,"history_lines":1,"events_offset":272,"events_lines":2,"console_offset":146,"console_lines":1}
|
| 283 |
+
{"time":"2026-08-25T20:02:47.895598712Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 284 |
+
{"time":"2026-08-25T20:03:02.492856089Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":134,"history_lines":1,"events_offset":274,"events_lines":2,"console_offset":147,"console_lines":1}
|
| 285 |
+
{"time":"2026-08-25T20:03:02.877559931Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 286 |
+
{"time":"2026-08-25T20:03:17.49304088Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":135,"history_lines":1,"events_offset":276,"events_lines":2,"console_offset":148,"console_lines":1}
|
| 287 |
+
{"time":"2026-08-25T20:03:17.814882976Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 288 |
+
{"time":"2026-08-25T20:03:32.493311229Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":136,"history_lines":1,"events_offset":278,"events_lines":2,"console_offset":149,"console_lines":1}
|
| 289 |
+
{"time":"2026-08-25T20:03:32.81240841Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 290 |
+
{"time":"2026-08-25T20:03:47.493192216Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":137,"history_lines":2,"events_offset":280,"events_lines":2,"console_offset":150,"console_lines":2}
|
| 291 |
+
{"time":"2026-08-25T20:03:47.765971371Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 292 |
+
{"time":"2026-08-25T20:04:02.493249837Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":139,"history_lines":1,"events_offset":282,"events_lines":2,"console_offset":152,"console_lines":1}
|
| 293 |
+
{"time":"2026-08-25T20:04:02.785611276Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 294 |
+
{"time":"2026-08-25T20:04:17.492644025Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":140,"history_lines":1,"events_offset":284,"events_lines":2,"console_offset":153,"console_lines":1}
|
| 295 |
+
{"time":"2026-08-25T20:04:17.833139155Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 296 |
+
{"time":"2026-08-25T20:04:32.492842066Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":141,"history_lines":1,"events_offset":286,"events_lines":2,"console_offset":154,"console_lines":1}
|
| 297 |
+
{"time":"2026-08-25T20:04:33.031693406Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 298 |
+
{"time":"2026-08-25T20:04:47.492889091Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":142,"history_lines":1,"events_offset":288,"events_lines":2,"console_offset":155,"console_lines":1}
|
| 299 |
+
{"time":"2026-08-25T20:04:47.73721145Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 300 |
+
{"time":"2026-08-25T20:05:02.492709928Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":143,"history_lines":1,"events_offset":290,"events_lines":2,"console_offset":156,"console_lines":1}
|
| 301 |
+
{"time":"2026-08-25T20:05:02.737627277Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 302 |
+
{"time":"2026-08-25T20:05:17.49283555Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":144,"history_lines":1,"events_offset":292,"events_lines":2,"console_offset":157,"console_lines":1}
|
| 303 |
+
{"time":"2026-08-25T20:05:17.87132603Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 304 |
+
{"time":"2026-08-25T20:05:32.492841652Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":145,"history_lines":1,"events_offset":294,"events_lines":2,"console_offset":158,"console_lines":1}
|
| 305 |
+
{"time":"2026-08-25T20:05:32.929509002Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 306 |
+
{"time":"2026-08-25T20:05:47.492927765Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":146,"history_lines":1,"events_offset":296,"events_lines":2,"console_offset":159,"console_lines":1}
|
| 307 |
+
{"time":"2026-08-25T20:05:47.776793069Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 308 |
+
{"time":"2026-08-25T20:06:02.492738467Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":147,"history_lines":2,"events_offset":298,"events_lines":2,"console_offset":160,"console_lines":2}
|
| 309 |
+
{"time":"2026-08-25T20:06:03.044159394Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 310 |
+
{"time":"2026-08-25T20:06:17.4932726Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":300,"events_lines":2}
|
| 311 |
+
{"time":"2026-08-25T20:06:17.755428449Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 312 |
+
{"time":"2026-08-25T20:06:32.493326215Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":302,"events_lines":2,"console_offset":162,"console_lines":2}
|
| 313 |
+
{"time":"2026-08-25T20:06:32.936099045Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 314 |
+
{"time":"2026-08-25T20:06:47.493090637Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":149,"history_lines":1,"events_offset":304,"events_lines":2,"console_offset":164,"console_lines":1}
|
| 315 |
+
{"time":"2026-08-25T20:06:47.744833494Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 316 |
+
{"time":"2026-08-25T20:07:02.492595091Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":150,"history_lines":1,"events_offset":306,"events_lines":2,"console_offset":165,"console_lines":1}
|
| 317 |
+
{"time":"2026-08-25T20:07:02.840644298Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 318 |
+
{"time":"2026-08-25T20:07:17.493095173Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":151,"history_lines":1,"events_offset":308,"events_lines":2,"console_offset":166,"console_lines":1}
|
| 319 |
+
{"time":"2026-08-25T20:07:18.11117847Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 320 |
+
{"time":"2026-08-25T20:07:32.492767658Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":152,"history_lines":1,"events_offset":310,"events_lines":2,"console_offset":167,"console_lines":1}
|
| 321 |
+
{"time":"2026-08-25T20:07:32.802968735Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 322 |
+
{"time":"2026-08-25T20:07:47.492847312Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":153,"history_lines":1,"events_offset":312,"events_lines":2,"console_offset":168,"console_lines":1}
|
| 323 |
+
{"time":"2026-08-25T20:07:48.01489284Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 324 |
+
{"time":"2026-08-25T20:08:02.49277601Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":154,"history_lines":2,"events_offset":314,"events_lines":2,"console_offset":169,"console_lines":2}
|
| 325 |
+
{"time":"2026-08-25T20:08:03.008344221Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 326 |
+
{"time":"2026-08-25T20:08:17.492764147Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":156,"history_lines":1,"events_offset":316,"events_lines":2,"console_offset":171,"console_lines":1}
|
| 327 |
+
{"time":"2026-08-25T20:08:17.97510607Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 328 |
+
{"time":"2026-08-25T20:08:32.49281653Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":157,"history_lines":1,"events_offset":318,"events_lines":2,"console_offset":172,"console_lines":1}
|
| 329 |
+
{"time":"2026-08-25T20:08:32.989770735Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 330 |
+
{"time":"2026-08-25T20:08:47.49242588Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":158,"history_lines":1,"events_offset":320,"events_lines":2,"console_offset":173,"console_lines":1}
|
| 331 |
+
{"time":"2026-08-25T20:08:47.966655022Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 332 |
+
{"time":"2026-08-25T20:09:02.492771522Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":159,"history_lines":1,"events_offset":322,"events_lines":2,"console_offset":174,"console_lines":1}
|
| 333 |
+
{"time":"2026-08-25T20:09:02.991570406Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 334 |
+
{"time":"2026-08-25T20:09:17.49340265Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":160,"history_lines":1,"events_offset":324,"events_lines":2,"console_offset":175,"console_lines":1}
|
| 335 |
+
{"time":"2026-08-25T20:09:17.814714723Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 336 |
+
{"time":"2026-08-25T20:09:32.492861465Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":161,"history_lines":1,"events_offset":326,"events_lines":2,"console_offset":176,"console_lines":1}
|
| 337 |
+
{"time":"2026-08-25T20:09:32.733360047Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 338 |
+
{"time":"2026-08-25T20:09:47.493359104Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":162,"history_lines":1,"events_offset":328,"events_lines":2,"console_offset":177,"console_lines":1}
|
| 339 |
+
{"time":"2026-08-25T20:09:47.809950798Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 340 |
+
{"time":"2026-08-25T20:10:02.492753263Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":163,"history_lines":1,"events_offset":330,"events_lines":2,"console_offset":178,"console_lines":1}
|
| 341 |
+
{"time":"2026-08-25T20:10:03.05170365Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 342 |
+
{"time":"2026-08-25T20:10:17.492598817Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":164,"history_lines":2,"events_offset":332,"events_lines":2,"console_offset":179,"console_lines":2}
|
| 343 |
+
{"time":"2026-08-25T20:10:17.98149389Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 344 |
+
{"time":"2026-08-25T20:10:32.493050914Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":166,"history_lines":1,"events_offset":334,"events_lines":2,"console_offset":181,"console_lines":1}
|
| 345 |
+
{"time":"2026-08-25T20:10:32.818318065Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 346 |
+
{"time":"2026-08-25T20:10:47.492338112Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":167,"history_lines":1,"events_offset":336,"events_lines":2,"console_offset":182,"console_lines":1}
|
| 347 |
+
{"time":"2026-08-25T20:10:47.766176546Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 348 |
+
{"time":"2026-08-25T20:11:02.492701719Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":168,"history_lines":1,"events_offset":338,"events_lines":2,"console_offset":183,"console_lines":1}
|
| 349 |
+
{"time":"2026-08-25T20:11:03.100762083Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 350 |
+
{"time":"2026-08-25T20:11:17.4925793Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":169,"history_lines":1,"events_offset":340,"events_lines":2,"console_offset":184,"console_lines":1}
|
| 351 |
+
{"time":"2026-08-25T20:11:18.009791647Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 352 |
+
{"time":"2026-08-25T20:11:32.492323441Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":170,"history_lines":1,"events_offset":342,"events_lines":2,"console_offset":185,"console_lines":1}
|
| 353 |
+
{"time":"2026-08-25T20:11:33.029176505Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 354 |
+
{"time":"2026-08-25T20:11:47.49337057Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":171,"history_lines":1,"events_offset":344,"events_lines":2,"console_offset":186,"console_lines":1}
|
| 355 |
+
{"time":"2026-08-25T20:11:47.942998525Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 356 |
+
{"time":"2026-08-25T20:12:02.493255982Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":172,"history_lines":1,"events_offset":346,"events_lines":2,"console_offset":187,"console_lines":1}
|
| 357 |
+
{"time":"2026-08-25T20:12:02.823024804Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 358 |
+
{"time":"2026-08-25T20:12:17.492733234Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":173,"history_lines":1,"events_offset":348,"events_lines":2,"console_offset":188,"console_lines":1}
|
| 359 |
+
{"time":"2026-08-25T20:12:17.817107669Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 360 |
+
{"time":"2026-08-25T20:12:32.492602512Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":350,"events_lines":2,"console_offset":189,"console_lines":2}
|
| 361 |
+
{"time":"2026-08-25T20:12:32.777864527Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 362 |
+
{"time":"2026-08-25T20:12:47.492960741Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":174,"history_lines":1,"events_offset":352,"events_lines":2,"console_offset":191,"console_lines":1}
|
| 363 |
+
{"time":"2026-08-25T20:12:47.763771969Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 364 |
+
{"time":"2026-08-25T20:13:02.49306475Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":175,"history_lines":1,"events_offset":354,"events_lines":2,"console_offset":192,"console_lines":1}
|
| 365 |
+
{"time":"2026-08-25T20:13:02.739350137Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 366 |
+
{"time":"2026-08-25T20:13:17.493231543Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":176,"history_lines":1,"events_offset":356,"events_lines":2,"console_offset":193,"console_lines":1}
|
| 367 |
+
{"time":"2026-08-25T20:13:17.791609028Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 368 |
+
{"time":"2026-08-25T20:13:32.49273231Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":177,"history_lines":1,"events_offset":358,"events_lines":2,"console_offset":194,"console_lines":1}
|
| 369 |
+
{"time":"2026-08-25T20:13:33.051821091Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 370 |
+
{"time":"2026-08-25T20:13:47.492598243Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":178,"history_lines":1,"events_offset":360,"events_lines":2,"console_offset":195,"console_lines":1}
|
| 371 |
+
{"time":"2026-08-25T20:13:47.923463164Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 372 |
+
{"time":"2026-08-25T20:14:02.492768741Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":179,"history_lines":1,"events_offset":362,"events_lines":2,"console_offset":196,"console_lines":1}
|
| 373 |
+
{"time":"2026-08-25T20:14:03.22291645Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 374 |
+
{"time":"2026-08-25T20:14:17.493330024Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":180,"history_lines":1,"events_offset":364,"events_lines":2,"console_offset":197,"console_lines":1}
|
| 375 |
+
{"time":"2026-08-25T20:14:17.985483083Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 376 |
+
{"time":"2026-08-25T20:14:32.492912073Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":181,"history_lines":1,"events_offset":366,"events_lines":2,"console_offset":198,"console_lines":1}
|
| 377 |
+
{"time":"2026-08-25T20:14:32.938731593Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 378 |
+
{"time":"2026-08-25T20:14:47.49233282Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":182,"history_lines":2,"events_offset":368,"events_lines":2,"console_offset":199,"console_lines":2}
|
| 379 |
+
{"time":"2026-08-25T20:14:48.09674535Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 380 |
+
{"time":"2026-08-25T20:15:02.493205519Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":184,"history_lines":1,"events_offset":370,"events_lines":2,"console_offset":201,"console_lines":1}
|
| 381 |
+
{"time":"2026-08-25T20:15:02.79897764Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 382 |
+
{"time":"2026-08-25T20:15:17.493219072Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":185,"history_lines":1,"events_offset":372,"events_lines":2,"console_offset":202,"console_lines":1}
|
| 383 |
+
{"time":"2026-08-25T20:15:18.067344967Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 384 |
+
{"time":"2026-08-25T20:15:32.493006578Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":186,"history_lines":1,"events_offset":374,"events_lines":2,"console_offset":203,"console_lines":1}
|
| 385 |
+
{"time":"2026-08-25T20:15:32.732138048Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 386 |
+
{"time":"2026-08-25T20:15:47.492669648Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":187,"history_lines":1,"events_offset":376,"events_lines":2,"console_offset":204,"console_lines":1}
|
| 387 |
+
{"time":"2026-08-25T20:15:47.80040209Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 388 |
+
{"time":"2026-08-25T20:16:02.493412191Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":188,"history_lines":1,"events_offset":378,"events_lines":2,"console_offset":205,"console_lines":1}
|
| 389 |
+
{"time":"2026-08-25T20:16:02.890816321Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 390 |
+
{"time":"2026-08-25T20:16:17.492959749Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":189,"history_lines":1,"events_offset":380,"events_lines":2,"console_offset":206,"console_lines":1}
|
| 391 |
+
{"time":"2026-08-25T20:16:17.728054092Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 392 |
+
{"time":"2026-08-25T20:16:32.492741826Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":190,"history_lines":1,"events_offset":382,"events_lines":2,"console_offset":207,"console_lines":1}
|
| 393 |
+
{"time":"2026-08-25T20:16:32.918087444Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 394 |
+
{"time":"2026-08-25T20:16:47.49256172Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":191,"history_lines":1,"events_offset":384,"events_lines":2,"console_offset":208,"console_lines":1}
|
| 395 |
+
{"time":"2026-08-25T20:16:47.736456192Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 396 |
+
{"time":"2026-08-25T20:17:02.492595193Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":192,"history_lines":1,"events_offset":386,"events_lines":2,"console_offset":209,"console_lines":1}
|
| 397 |
+
{"time":"2026-08-25T20:17:02.732712226Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 398 |
+
{"time":"2026-08-25T20:17:17.492411391Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":193,"history_lines":2,"events_offset":388,"events_lines":2,"console_offset":210,"console_lines":2}
|
| 399 |
+
{"time":"2026-08-25T20:17:17.955543538Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 400 |
+
{"time":"2026-08-25T20:17:32.493009389Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":195,"history_lines":1,"events_offset":390,"events_lines":2,"console_offset":212,"console_lines":1}
|
| 401 |
+
{"time":"2026-08-25T20:17:32.725101033Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 402 |
+
{"time":"2026-08-25T20:17:47.493329581Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":196,"history_lines":1,"events_offset":392,"events_lines":2,"console_offset":213,"console_lines":1}
|
| 403 |
+
{"time":"2026-08-25T20:17:47.741658724Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 404 |
+
{"time":"2026-08-25T20:18:02.493349625Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":197,"history_lines":1,"events_offset":394,"events_lines":2,"console_offset":214,"console_lines":1}
|
| 405 |
+
{"time":"2026-08-25T20:18:03.001777317Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 406 |
+
{"time":"2026-08-25T20:18:17.493149429Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":198,"history_lines":1,"events_offset":396,"events_lines":2,"console_offset":215,"console_lines":1}
|
| 407 |
+
{"time":"2026-08-25T20:18:17.737353774Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 408 |
+
{"time":"2026-08-25T20:18:32.492416687Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":398,"events_lines":2}
|
| 409 |
+
{"time":"2026-08-25T20:18:32.743103124Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 410 |
+
{"time":"2026-08-25T20:18:47.492856408Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":400,"events_lines":2,"console_offset":216,"console_lines":2}
|
| 411 |
+
{"time":"2026-08-25T20:18:47.743552349Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 412 |
+
{"time":"2026-08-25T20:19:02.492925921Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":199,"history_lines":1,"events_offset":402,"events_lines":2,"console_offset":218,"console_lines":1}
|
| 413 |
+
{"time":"2026-08-25T20:19:03.053653049Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 414 |
+
{"time":"2026-08-25T20:19:17.492989982Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":200,"history_lines":2,"events_offset":404,"events_lines":2,"console_offset":219,"console_lines":2}
|
| 415 |
+
{"time":"2026-08-25T20:19:17.956341328Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 416 |
+
{"time":"2026-08-25T20:19:32.493321914Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":202,"history_lines":1,"events_offset":406,"events_lines":2,"console_offset":221,"console_lines":1}
|
| 417 |
+
{"time":"2026-08-25T20:19:32.894701261Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 418 |
+
{"time":"2026-08-25T20:19:47.492528174Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":203,"history_lines":1,"events_offset":408,"events_lines":2,"console_offset":222,"console_lines":1}
|
| 419 |
+
{"time":"2026-08-25T20:19:47.750599443Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 420 |
+
{"time":"2026-08-25T20:20:02.492529311Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":204,"history_lines":1,"events_offset":410,"events_lines":2,"console_offset":223,"console_lines":1}
|
| 421 |
+
{"time":"2026-08-25T20:20:02.754571587Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 422 |
+
{"time":"2026-08-25T20:20:17.49267868Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":205,"history_lines":1,"events_offset":412,"events_lines":2,"console_offset":224,"console_lines":1}
|
| 423 |
+
{"time":"2026-08-25T20:20:17.752057684Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 424 |
+
{"time":"2026-08-25T20:20:32.492573248Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":206,"history_lines":1,"events_offset":414,"events_lines":2,"console_offset":225,"console_lines":1}
|
| 425 |
+
{"time":"2026-08-25T20:20:32.722551422Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 426 |
+
{"time":"2026-08-25T20:20:47.492663378Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":207,"history_lines":1,"events_offset":416,"events_lines":2,"console_offset":226,"console_lines":1}
|
| 427 |
+
{"time":"2026-08-25T20:20:48.490199216Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 428 |
+
{"time":"2026-08-25T20:21:02.492760198Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":208,"history_lines":1,"events_offset":418,"events_lines":2,"console_offset":227,"console_lines":1}
|
| 429 |
+
{"time":"2026-08-25T20:21:02.745315719Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 430 |
+
{"time":"2026-08-25T20:21:17.492810158Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":209,"history_lines":1,"events_offset":420,"events_lines":2,"console_offset":228,"console_lines":1}
|
| 431 |
+
{"time":"2026-08-25T20:21:18.157112178Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 432 |
+
{"time":"2026-08-25T20:21:32.492868731Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":210,"history_lines":1,"events_offset":422,"events_lines":2,"console_offset":229,"console_lines":1}
|
| 433 |
+
{"time":"2026-08-25T20:21:32.756108707Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 434 |
+
{"time":"2026-08-25T20:21:47.492380527Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":211,"history_lines":2,"events_offset":424,"events_lines":2,"console_offset":230,"console_lines":2}
|
| 435 |
+
{"time":"2026-08-25T20:21:47.752712655Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 436 |
+
{"time":"2026-08-25T20:22:02.493012417Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":213,"history_lines":1,"events_offset":426,"events_lines":2,"console_offset":232,"console_lines":1}
|
| 437 |
+
{"time":"2026-08-25T20:22:03.104728041Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 438 |
+
{"time":"2026-08-25T20:22:17.492413346Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":214,"history_lines":1,"events_offset":428,"events_lines":2,"console_offset":233,"console_lines":1}
|
| 439 |
+
{"time":"2026-08-25T20:22:18.110648919Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 440 |
+
{"time":"2026-08-25T20:22:32.492510117Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":215,"history_lines":1,"events_offset":430,"events_lines":2,"console_offset":234,"console_lines":1}
|
| 441 |
+
{"time":"2026-08-25T20:22:32.736087545Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 442 |
+
{"time":"2026-08-25T20:22:47.492343744Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":216,"history_lines":1,"events_offset":432,"events_lines":2,"console_offset":235,"console_lines":1}
|
| 443 |
+
{"time":"2026-08-25T20:22:47.726919764Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 444 |
+
{"time":"2026-08-25T20:23:02.493223224Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":217,"history_lines":1,"events_offset":434,"events_lines":2,"console_offset":236,"console_lines":1}
|
| 445 |
+
{"time":"2026-08-25T20:23:02.761153905Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 446 |
+
{"time":"2026-08-25T20:23:17.492920367Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":218,"history_lines":1,"events_offset":436,"events_lines":2,"console_offset":237,"console_lines":1}
|
| 447 |
+
{"time":"2026-08-25T20:23:17.756080625Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 448 |
+
{"time":"2026-08-25T20:23:32.493158819Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":219,"history_lines":1,"events_offset":438,"events_lines":2,"console_offset":238,"console_lines":1}
|
| 449 |
+
{"time":"2026-08-25T20:23:32.740369489Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 450 |
+
{"time":"2026-08-25T20:23:47.493092195Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":220,"history_lines":1,"events_offset":440,"events_lines":2,"console_offset":239,"console_lines":1}
|
| 451 |
+
{"time":"2026-08-25T20:23:47.724524033Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 452 |
+
{"time":"2026-08-25T20:24:02.492818982Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":221,"history_lines":1,"events_offset":442,"events_lines":2,"console_offset":240,"console_lines":1}
|
| 453 |
+
{"time":"2026-08-25T20:24:02.741031961Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 454 |
+
{"time":"2026-08-25T20:24:17.493145435Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":222,"history_lines":2,"events_offset":444,"events_lines":2,"console_offset":241,"console_lines":2}
|
| 455 |
+
{"time":"2026-08-25T20:24:17.725060236Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 456 |
+
{"time":"2026-08-25T20:24:32.493304258Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":446,"events_lines":2}
|
| 457 |
+
{"time":"2026-08-25T20:24:32.719800677Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 458 |
+
{"time":"2026-08-25T20:24:47.493372125Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":448,"events_lines":2,"console_offset":243,"console_lines":2}
|
| 459 |
+
{"time":"2026-08-25T20:24:47.754527174Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 460 |
+
{"time":"2026-08-25T20:25:02.492717892Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":224,"history_lines":1,"events_offset":450,"events_lines":2,"console_offset":245,"console_lines":1}
|
| 461 |
+
{"time":"2026-08-25T20:25:02.717493999Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 462 |
+
{"time":"2026-08-25T20:25:17.492918888Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":225,"history_lines":1,"events_offset":452,"events_lines":2,"console_offset":246,"console_lines":1}
|
| 463 |
+
{"time":"2026-08-25T20:25:17.793177634Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 464 |
+
{"time":"2026-08-25T20:25:32.492487428Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":226,"history_lines":1,"events_offset":454,"events_lines":2,"console_offset":247,"console_lines":1}
|
| 465 |
+
{"time":"2026-08-25T20:25:32.72346027Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 466 |
+
{"time":"2026-08-25T20:25:47.49300639Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":227,"history_lines":1,"events_offset":456,"events_lines":2,"console_offset":248,"console_lines":1}
|
| 467 |
+
{"time":"2026-08-25T20:25:47.914737456Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 468 |
+
{"time":"2026-08-25T20:26:02.492542672Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":228,"history_lines":1,"events_offset":458,"events_lines":2,"console_offset":249,"console_lines":1}
|
| 469 |
+
{"time":"2026-08-25T20:26:02.801095238Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 470 |
+
{"time":"2026-08-25T20:26:17.493084707Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":229,"history_lines":2,"events_offset":460,"events_lines":2,"console_offset":250,"console_lines":2}
|
| 471 |
+
{"time":"2026-08-25T20:26:17.740416559Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 472 |
+
{"time":"2026-08-25T20:26:32.49299596Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":231,"history_lines":1,"events_offset":462,"events_lines":2,"console_offset":252,"console_lines":1}
|
| 473 |
+
{"time":"2026-08-25T20:26:32.736680949Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 474 |
+
{"time":"2026-08-25T20:26:47.492528757Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":232,"history_lines":1,"events_offset":464,"events_lines":2,"console_offset":253,"console_lines":1}
|
| 475 |
+
{"time":"2026-08-25T20:26:47.772688061Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 476 |
+
{"time":"2026-08-25T20:27:02.492984484Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":233,"history_lines":1,"events_offset":466,"events_lines":2,"console_offset":254,"console_lines":1}
|
| 477 |
+
{"time":"2026-08-25T20:27:02.70409634Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 478 |
+
{"time":"2026-08-25T20:27:17.492372023Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":234,"history_lines":1,"events_offset":468,"events_lines":2,"console_offset":255,"console_lines":1}
|
| 479 |
+
{"time":"2026-08-25T20:27:17.744190243Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 480 |
+
{"time":"2026-08-25T20:27:32.493312886Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":235,"history_lines":1,"events_offset":470,"events_lines":2,"console_offset":256,"console_lines":1}
|
| 481 |
+
{"time":"2026-08-25T20:27:32.758211392Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 482 |
+
{"time":"2026-08-25T20:27:47.492991931Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":236,"history_lines":1,"events_offset":472,"events_lines":2,"console_offset":257,"console_lines":1}
|
| 483 |
+
{"time":"2026-08-25T20:27:47.724739857Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 484 |
+
{"time":"2026-08-25T20:28:02.492661557Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":237,"history_lines":1,"events_offset":474,"events_lines":2,"console_offset":258,"console_lines":1}
|
| 485 |
+
{"time":"2026-08-25T20:28:02.735331453Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 486 |
+
{"time":"2026-08-25T20:28:17.493121466Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":238,"history_lines":1,"events_offset":476,"events_lines":2,"console_offset":259,"console_lines":1}
|
| 487 |
+
{"time":"2026-08-25T20:28:17.757077728Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 488 |
+
{"time":"2026-08-25T20:28:32.493131291Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":239,"history_lines":1,"events_offset":478,"events_lines":2,"console_offset":260,"console_lines":1}
|
| 489 |
+
{"time":"2026-08-25T20:28:32.795774686Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 490 |
+
{"time":"2026-08-25T20:28:47.493013234Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":240,"history_lines":2,"events_offset":480,"events_lines":2,"console_offset":261,"console_lines":2}
|
| 491 |
+
{"time":"2026-08-25T20:28:48.416665016Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 492 |
+
{"time":"2026-08-25T20:29:02.49317719Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":242,"history_lines":1,"events_offset":482,"events_lines":2,"console_offset":263,"console_lines":1}
|
| 493 |
+
{"time":"2026-08-25T20:29:02.725323071Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 494 |
+
{"time":"2026-08-25T20:29:17.492459838Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":243,"history_lines":1,"events_offset":484,"events_lines":2,"console_offset":264,"console_lines":1}
|
| 495 |
+
{"time":"2026-08-25T20:29:18.38720502Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 496 |
+
{"time":"2026-08-25T20:29:32.493251111Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":244,"history_lines":1,"events_offset":486,"events_lines":2,"console_offset":265,"console_lines":1}
|
| 497 |
+
{"time":"2026-08-25T20:29:32.740381827Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 498 |
+
{"time":"2026-08-25T20:29:47.492813056Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":245,"history_lines":1,"events_offset":488,"events_lines":2,"console_offset":266,"console_lines":1}
|
| 499 |
+
{"time":"2026-08-25T20:29:47.711185094Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 500 |
+
{"time":"2026-08-25T20:30:02.493407135Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":246,"history_lines":1,"events_offset":490,"events_lines":2,"console_offset":267,"console_lines":1}
|
| 501 |
+
{"time":"2026-08-25T20:30:02.974592426Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 502 |
+
{"time":"2026-08-25T20:30:17.493039746Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":247,"history_lines":1,"events_offset":492,"events_lines":2,"console_offset":268,"console_lines":1}
|
| 503 |
+
{"time":"2026-08-25T20:30:17.753747371Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 504 |
+
{"time":"2026-08-25T20:30:32.492652998Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":248,"history_lines":1,"events_offset":494,"events_lines":2,"console_offset":269,"console_lines":1}
|
| 505 |
+
{"time":"2026-08-25T20:30:32.722194806Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 506 |
+
{"time":"2026-08-25T20:30:47.493070873Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":496,"events_lines":2,"console_offset":270,"console_lines":2}
|
| 507 |
+
{"time":"2026-08-25T20:30:47.713425886Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 508 |
+
{"time":"2026-08-25T20:31:02.492346601Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":249,"history_lines":1,"events_offset":498,"events_lines":2,"console_offset":272,"console_lines":1}
|
| 509 |
+
{"time":"2026-08-25T20:31:02.703515478Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 510 |
+
{"time":"2026-08-25T20:31:17.493374428Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":250,"history_lines":1,"events_offset":500,"events_lines":2,"console_offset":273,"console_lines":1}
|
| 511 |
+
{"time":"2026-08-25T20:31:17.704374774Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 512 |
+
{"time":"2026-08-25T20:31:32.49327544Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":251,"history_lines":1,"events_offset":502,"events_lines":2,"console_offset":274,"console_lines":1}
|
| 513 |
+
{"time":"2026-08-25T20:31:32.7000584Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 514 |
+
{"time":"2026-08-25T20:31:47.492407733Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":252,"history_lines":1,"events_offset":504,"events_lines":2,"console_offset":275,"console_lines":1}
|
| 515 |
+
{"time":"2026-08-25T20:31:47.806987014Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 516 |
+
{"time":"2026-08-25T20:32:02.492804166Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":253,"history_lines":1,"events_offset":506,"events_lines":2,"console_offset":276,"console_lines":1}
|
| 517 |
+
{"time":"2026-08-25T20:32:02.777560618Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 518 |
+
{"time":"2026-08-25T20:32:17.493268701Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":254,"history_lines":1,"events_offset":508,"events_lines":2,"console_offset":277,"console_lines":1}
|
| 519 |
+
{"time":"2026-08-25T20:32:17.721621721Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 520 |
+
{"time":"2026-08-25T20:32:32.492351881Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":255,"history_lines":1,"events_offset":510,"events_lines":2,"console_offset":278,"console_lines":1}
|
| 521 |
+
{"time":"2026-08-25T20:32:32.731336206Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 522 |
+
{"time":"2026-08-25T20:32:47.49304703Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":256,"history_lines":1,"events_offset":512,"events_lines":2,"console_offset":279,"console_lines":1}
|
| 523 |
+
{"time":"2026-08-25T20:32:47.709650333Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 524 |
+
{"time":"2026-08-25T20:33:02.492707723Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":257,"history_lines":1,"events_offset":514,"events_lines":2,"console_offset":280,"console_lines":1}
|
| 525 |
+
{"time":"2026-08-25T20:33:02.747466815Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 526 |
+
{"time":"2026-08-25T20:33:17.493082744Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":258,"history_lines":2,"events_offset":516,"events_lines":2,"console_offset":281,"console_lines":2}
|
| 527 |
+
{"time":"2026-08-25T20:33:17.730429251Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 528 |
+
{"time":"2026-08-25T20:33:32.492898529Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":260,"history_lines":1,"events_offset":518,"events_lines":2,"console_offset":283,"console_lines":1}
|
| 529 |
+
{"time":"2026-08-25T20:33:32.713745668Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 530 |
+
{"time":"2026-08-25T20:33:47.492490289Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":261,"history_lines":1,"events_offset":520,"events_lines":2,"console_offset":284,"console_lines":1}
|
| 531 |
+
{"time":"2026-08-25T20:33:47.740314732Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 532 |
+
{"time":"2026-08-25T20:34:02.492804009Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":262,"history_lines":1,"events_offset":522,"events_lines":2,"console_offset":285,"console_lines":1}
|
| 533 |
+
{"time":"2026-08-25T20:34:02.797285762Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 534 |
+
{"time":"2026-08-25T20:34:17.493051038Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":263,"history_lines":1,"events_offset":524,"events_lines":2,"console_offset":286,"console_lines":1}
|
| 535 |
+
{"time":"2026-08-25T20:34:18.163084264Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 536 |
+
{"time":"2026-08-25T20:34:32.492920954Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":264,"history_lines":1,"events_offset":526,"events_lines":2,"console_offset":287,"console_lines":1}
|
| 537 |
+
{"time":"2026-08-25T20:34:32.725810117Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 538 |
+
{"time":"2026-08-25T20:34:47.493288023Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":265,"history_lines":1,"events_offset":528,"events_lines":2,"console_offset":288,"console_lines":1}
|
| 539 |
+
{"time":"2026-08-25T20:34:47.751619255Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 540 |
+
{"time":"2026-08-25T20:35:02.493164268Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":266,"history_lines":1,"events_offset":530,"events_lines":2,"console_offset":289,"console_lines":1}
|
| 541 |
+
{"time":"2026-08-25T20:35:02.736612568Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 542 |
+
{"time":"2026-08-25T20:35:17.493324168Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":267,"history_lines":1,"events_offset":532,"events_lines":2,"console_offset":290,"console_lines":1}
|
| 543 |
+
{"time":"2026-08-25T20:35:17.721060537Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 544 |
+
{"time":"2026-08-25T20:35:32.492452886Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":268,"history_lines":1,"events_offset":534,"events_lines":2,"console_offset":291,"console_lines":1}
|
| 545 |
+
{"time":"2026-08-25T20:35:32.796778366Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 546 |
+
{"time":"2026-08-25T20:35:47.493016899Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":269,"history_lines":2,"events_offset":536,"events_lines":2,"console_offset":292,"console_lines":2}
|
| 547 |
+
{"time":"2026-08-25T20:35:47.839082587Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 548 |
+
{"time":"2026-08-25T20:36:02.493113955Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":271,"history_lines":1,"events_offset":538,"events_lines":2,"console_offset":294,"console_lines":1}
|
| 549 |
+
{"time":"2026-08-25T20:36:02.97728482Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 550 |
+
{"time":"2026-08-25T20:36:17.493236819Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":272,"history_lines":1,"events_offset":540,"events_lines":2,"console_offset":295,"console_lines":1}
|
| 551 |
+
{"time":"2026-08-25T20:36:17.746438204Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 552 |
+
{"time":"2026-08-25T20:36:32.493129032Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":273,"history_lines":1,"events_offset":542,"events_lines":2,"console_offset":296,"console_lines":1}
|
| 553 |
+
{"time":"2026-08-25T20:36:32.826431264Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 554 |
+
{"time":"2026-08-25T20:36:47.493008095Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":544,"events_lines":2}
|
| 555 |
+
{"time":"2026-08-25T20:36:47.724540103Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 556 |
+
{"time":"2026-08-25T20:37:02.493029542Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":546,"events_lines":2,"console_offset":297,"console_lines":2}
|
| 557 |
+
{"time":"2026-08-25T20:37:02.725954182Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 558 |
+
{"time":"2026-08-25T20:37:17.493098129Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":274,"history_lines":1,"events_offset":548,"events_lines":2,"console_offset":299,"console_lines":1}
|
| 559 |
+
{"time":"2026-08-25T20:37:17.785856245Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 560 |
+
{"time":"2026-08-25T20:37:32.492686468Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":275,"history_lines":1,"events_offset":550,"events_lines":2,"console_offset":300,"console_lines":1}
|
| 561 |
+
{"time":"2026-08-25T20:37:32.755396279Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 562 |
+
{"time":"2026-08-25T20:37:47.492466604Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":276,"history_lines":2,"events_offset":552,"events_lines":2,"console_offset":301,"console_lines":2}
|
| 563 |
+
{"time":"2026-08-25T20:37:47.769672798Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 564 |
+
{"time":"2026-08-25T20:38:02.492987757Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":278,"history_lines":1,"events_offset":554,"events_lines":2,"console_offset":303,"console_lines":1}
|
| 565 |
+
{"time":"2026-08-25T20:38:02.801718874Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 566 |
+
{"time":"2026-08-25T20:38:17.493076344Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":279,"history_lines":1,"events_offset":556,"events_lines":2,"console_offset":304,"console_lines":1}
|
| 567 |
+
{"time":"2026-08-25T20:38:17.750250429Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 568 |
+
{"time":"2026-08-25T20:38:32.493039574Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":280,"history_lines":1,"events_offset":558,"events_lines":2,"console_offset":305,"console_lines":1}
|
| 569 |
+
{"time":"2026-08-25T20:38:32.713695601Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 570 |
+
{"time":"2026-08-25T20:38:47.492580271Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":281,"history_lines":1,"events_offset":560,"events_lines":2,"console_offset":306,"console_lines":1}
|
| 571 |
+
{"time":"2026-08-25T20:38:47.743374308Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 572 |
+
{"time":"2026-08-25T20:39:02.49322521Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":282,"history_lines":1,"events_offset":562,"events_lines":2,"console_offset":307,"console_lines":1}
|
| 573 |
+
{"time":"2026-08-25T20:39:02.822488233Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
epfl_final/SparseAtt/wandb/run-20260825_192830-6sx9f1kf/logs/debug.log
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-08-25 19:28:30,763 INFO MainThread:830 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-08-25 19:28:30,764 INFO MainThread:830 [wandb_setup.py:_flush():81] Configure stats pid to 830
|
| 3 |
+
2026-08-25 19:28:30,764 INFO MainThread:830 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-08-25 19:28:30,764 INFO MainThread:830 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260825_192830-6sx9f1kf/logs/debug.log
|
| 5 |
+
2026-08-25 19:28:30,764 INFO MainThread:830 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260825_192830-6sx9f1kf/logs/debug-internal.log
|
| 6 |
+
2026-08-25 19:28:30,764 INFO MainThread:830 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-08-25 19:28:30,764 INFO MainThread:830 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 49, 'verbose': False, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m edge-dense alternating-mixed seed49', 'wandb_tags': ['qwen3', '365m', '8k', 'edge-dense', 'alternating-mixed', 'seed49'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': [3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23], 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed', 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': [63, 47, 31, 21, 11], 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-08-25 19:28:30,764 INFO MainThread:830 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-08-25 19:28:31,373 INFO MainThread:830 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-08-25 19:28:31,788 INFO MainThread:830 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-08-25 19:28:31,790 INFO MainThread:830 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-08-25 19:28:31,817 INFO MainThread:830 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-08-25 19:28:32,219 INFO MainThread:830 [wandb_init.py:init():1056] starting run threads in backend
|
| 15 |
+
2026-08-25 19:28:32,450 INFO MainThread:830 [wandb_run.py:_console_start():2554] atexit reg
|
| 16 |
+
2026-08-25 19:28:32,450 INFO MainThread:830 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 17 |
+
2026-08-25 19:28:32,450 INFO MainThread:830 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 18 |
+
2026-08-25 19:28:32,450 INFO MainThread:830 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 19 |
+
2026-08-25 19:28:32,487 INFO MainThread:830 [wandb_init.py:init():1094] run started, returning control to user process
|
epfl_final/SparseAtt/wandb/run-20260825_192830-6sx9f1kf/run-6sx9f1kf.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1f7f3acb3207fd9b1311ad00dbee918d4fd0d7c862a87b8e9a19f5b2b5ae36c2
|
| 3 |
+
size 819200
|
epfl_final/SparseAtt/wandb/run-20260825_204951-qctui00g/files/output.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260825_204951-qctui00g/files/requirements.txt
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
| 142 |
+
entmax==1.3
|
epfl_final/SparseAtt/wandb/run-20260825_204951-qctui00g/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-08-25T20:49:51.973492Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"configs/p2_1p2b_dense_seed71_6gpu.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/SparseAtt.git",
|
| 14 |
+
"commit": "8f862f749eb4d5c17f688eb1562235e3fe5b2550"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "h100-train-6-gpu-0-37",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 23 |
+
"gpu_count": 6,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "9374931091456",
|
| 27 |
+
"used": "4880294522880"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1623005732864"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-99f99f92-e501-0b15-5782-6beb2c34fa1a"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 43 |
+
"memoryTotal": "85520809984",
|
| 44 |
+
"cudaCores": 16896,
|
| 45 |
+
"architecture": "Hopper",
|
| 46 |
+
"uuid": "GPU-7004b604-21a2-890e-0312-3eee1eb93c67"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 50 |
+
"memoryTotal": "85520809984",
|
| 51 |
+
"cudaCores": 16896,
|
| 52 |
+
"architecture": "Hopper",
|
| 53 |
+
"uuid": "GPU-1aef7113-cfe5-3f49-53bb-a84f3c5abfa0"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 57 |
+
"memoryTotal": "85520809984",
|
| 58 |
+
"cudaCores": 16896,
|
| 59 |
+
"architecture": "Hopper",
|
| 60 |
+
"uuid": "GPU-5e17ddff-2e95-8c68-b074-b0189417e834"
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 64 |
+
"memoryTotal": "85520809984",
|
| 65 |
+
"cudaCores": 16896,
|
| 66 |
+
"architecture": "Hopper",
|
| 67 |
+
"uuid": "GPU-61e64705-1798-f2b5-568f-877549238703"
|
| 68 |
+
},
|
| 69 |
+
{
|
| 70 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 71 |
+
"memoryTotal": "85520809984",
|
| 72 |
+
"cudaCores": 16896,
|
| 73 |
+
"architecture": "Hopper",
|
| 74 |
+
"uuid": "GPU-9cd8a6e5-05e9-cfde-0a28-d8058797b3bf"
|
| 75 |
+
}
|
| 76 |
+
],
|
| 77 |
+
"cudaVersion": "13.0",
|
| 78 |
+
"writerId": "agl1rby47fuai4zgn2lrgv0yd6bz05ao"
|
| 79 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260825_204951-qctui00g/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260825_204951-qctui00g/logs/debug.log
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-08-25 20:49:52,006 INFO MainThread:302 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-08-25 20:49:52,006 INFO MainThread:302 [wandb_setup.py:_flush():81] Configure stats pid to 302
|
| 3 |
+
2026-08-25 20:49:52,006 INFO MainThread:302 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-08-25 20:49:52,006 INFO MainThread:302 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260825_204951-qctui00g/logs/debug.log
|
| 5 |
+
2026-08-25 20:49:52,006 INFO MainThread:302 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260825_204951-qctui00g/logs/debug-internal.log
|
| 6 |
+
2026-08-25 20:49:52,006 INFO MainThread:302 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-08-25 20:49:52,006 INFO MainThread:302 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-25250', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 71, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b dense 8k seed71 6gpu', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'dense', 'seed71'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-08-25 20:49:52,006 INFO MainThread:302 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-08-25 20:49:53,220 INFO MainThread:302 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-08-25 20:49:53,797 INFO MainThread:302 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-08-25 20:49:53,801 INFO MainThread:302 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-08-25 20:49:53,845 INFO MainThread:302 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-08-25 20:49:54,543 INFO MainThread:302 [wandb_init.py:init():1051] run resumed
|
| 15 |
+
2026-08-25 20:49:54,546 INFO MainThread:302 [wandb_init.py:init():1056] starting run threads in backend
|
| 16 |
+
2026-08-25 20:49:54,843 INFO MainThread:302 [wandb_run.py:_console_start():2554] atexit reg
|
| 17 |
+
2026-08-25 20:49:54,843 INFO MainThread:302 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 18 |
+
2026-08-25 20:49:54,843 INFO MainThread:302 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 19 |
+
2026-08-25 20:49:54,844 INFO MainThread:302 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 20 |
+
2026-08-25 20:49:54,853 INFO MainThread:302 [wandb_init.py:init():1094] run started, returning control to user process
|
epfl_final/SparseAtt/wandb/run-20260825_204951-qctui00g/run-qctui00g.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:56d899e0de1e36818c10de751aebdcc1ded3a2358c818caca3a198faf859f85e
|
| 3 |
+
size 10551296
|
epfl_final/SparseAtt/wandb/run-20260825_205215-yl9g6w7y/files/output.log
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Starting training...
|
| 2 |
+
Validation at step 2750: Loss = 3.5117, Perplexity = 33.50
|
| 3 |
+
Switching attention mode for step 2751: sparse_stochastic (configured sparse attention)
|
| 4 |
+
Step 2760/44000 | Loss: 3.4835 | LR: 4.97e-04 | 19957.9ms/step | 1.0 samples/s
|
| 5 |
+
Step 2770/44000 | Loss: 3.5085 | LR: 4.97e-04 | 1379.4ms/step | 14.5 samples/s
|
| 6 |
+
Step 2780/44000 | Loss: 3.5160 | LR: 4.97e-04 | 1383.5ms/step | 14.5 samples/s
|
| 7 |
+
Step 2790/44000 | Loss: 3.5128 | LR: 4.97e-04 | 1386.7ms/step | 14.4 samples/s
|
epfl_final/SparseAtt/wandb/run-20260825_205215-yl9g6w7y/files/requirements.txt
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
| 142 |
+
entmax==1.3
|
epfl_final/SparseAtt/wandb/run-20260825_205215-yl9g6w7y/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-08-25T20:52:15.913270Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"configs/ladder8k/365m_edge_dense_altmixed_seed49.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/SparseAtt.git",
|
| 14 |
+
"commit": "8f862f749eb4d5c17f688eb1562235e3fe5b2550"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "h100-workspace-0-2",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 23 |
+
"gpu_count": 1,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "9374931091456",
|
| 27 |
+
"used": "6330140106752"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1623005732864"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-a2122da0-1020-9f78-7e6f-a1d5011f694b"
|
| 40 |
+
}
|
| 41 |
+
],
|
| 42 |
+
"cudaVersion": "13.0",
|
| 43 |
+
"writerId": "8kf7ide60n5km8vqubgj8tuk45nyktq9"
|
| 44 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260825_205215-yl9g6w7y/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-08-25T20:52:17.238890082Z","level":"INFO","msg":"wandb-core"}
|
| 2 |
+
{"time":"2026-08-25T20:52:17.281493412Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
|
| 3 |
+
{"time":"2026-08-25T20:52:17.81396823Z","level":"INFO","msg":"stream: created new stream","id":"yl9g6w7y"}
|
| 4 |
+
{"time":"2026-08-25T20:52:17.814064364Z","level":"INFO","msg":"handler: started"}
|
| 5 |
+
{"time":"2026-08-25T20:52:17.824537576Z","level":"INFO","msg":"stream: started"}
|
| 6 |
+
{"time":"2026-08-25T20:52:17.824606489Z","level":"INFO","msg":"writer: started","stream_id":"yl9g6w7y"}
|
| 7 |
+
{"time":"2026-08-25T20:52:17.824653239Z","level":"INFO","msg":"sender: started"}
|
| 8 |
+
{"time":"2026-08-25T20:52:18.944600546Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
|
| 9 |
+
{"time":"2026-08-25T20:52:19.20716495Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 10 |
+
{"time":"2026-08-25T20:52:33.945122514Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":0,"events_lines":2,"uploaded_len":2}
|
| 11 |
+
{"time":"2026-08-25T20:52:34.362201633Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 12 |
+
{"time":"2026-08-25T20:52:48.944626932Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
|
| 13 |
+
{"time":"2026-08-25T20:52:49.246788375Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 14 |
+
{"time":"2026-08-25T20:53:03.945170211Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":4,"events_lines":2}
|
| 15 |
+
{"time":"2026-08-25T20:53:04.288537317Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 16 |
+
{"time":"2026-08-25T20:53:18.945554834Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":6,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 17 |
+
{"time":"2026-08-25T20:53:19.373436486Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 18 |
+
{"time":"2026-08-25T20:53:33.945606139Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":8,"events_lines":2}
|
| 19 |
+
{"time":"2026-08-25T20:53:34.336107372Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 20 |
+
{"time":"2026-08-25T20:53:48.944868105Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":10,"events_lines":2}
|
| 21 |
+
{"time":"2026-08-25T20:53:49.277042774Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 22 |
+
{"time":"2026-08-25T20:54:03.945650331Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":12,"events_lines":2}
|
| 23 |
+
{"time":"2026-08-25T20:54:04.295019071Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 24 |
+
{"time":"2026-08-25T20:54:18.945153247Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":14,"events_lines":2}
|
| 25 |
+
{"time":"2026-08-25T20:54:19.251778888Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 26 |
+
{"time":"2026-08-25T20:54:33.945557759Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":16,"events_lines":2}
|
| 27 |
+
{"time":"2026-08-25T20:54:34.213762816Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 28 |
+
{"time":"2026-08-25T20:54:48.945345089Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":18,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 29 |
+
{"time":"2026-08-25T20:54:49.229636012Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 30 |
+
{"time":"2026-08-25T20:55:03.945099791Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":20,"events_lines":2}
|
| 31 |
+
{"time":"2026-08-25T20:55:04.341342118Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 32 |
+
{"time":"2026-08-25T20:55:18.945210233Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":22,"events_lines":2}
|
| 33 |
+
{"time":"2026-08-25T20:55:19.219561501Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 34 |
+
{"time":"2026-08-25T20:55:33.944954981Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":24,"events_lines":2}
|
| 35 |
+
{"time":"2026-08-25T20:55:34.273513149Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 36 |
+
{"time":"2026-08-25T20:55:48.944965264Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":26,"events_lines":2,"console_offset":3,"console_lines":1}
|
| 37 |
+
{"time":"2026-08-25T20:55:49.211387859Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 38 |
+
{"time":"2026-08-25T20:56:03.944877707Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":28,"events_lines":2,"console_offset":4,"console_lines":1}
|
| 39 |
+
{"time":"2026-08-25T20:56:04.25373275Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 40 |
+
{"time":"2026-08-25T20:56:18.945452279Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":1,"events_offset":30,"events_lines":2,"console_offset":5,"console_lines":1}
|
| 41 |
+
{"time":"2026-08-25T20:56:19.277663345Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
epfl_final/SparseAtt/wandb/run-20260825_205215-yl9g6w7y/logs/debug.log
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-08-25 20:52:15,953 INFO MainThread:210 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-08-25 20:52:15,954 INFO MainThread:210 [wandb_setup.py:_flush():81] Configure stats pid to 210
|
| 3 |
+
2026-08-25 20:52:15,954 INFO MainThread:210 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-08-25 20:52:15,954 INFO MainThread:210 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260825_205215-yl9g6w7y/logs/debug.log
|
| 5 |
+
2026-08-25 20:52:15,954 INFO MainThread:210 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260825_205215-yl9g6w7y/logs/debug-internal.log
|
| 6 |
+
2026-08-25 20:52:15,954 INFO MainThread:210 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-08-25 20:52:15,954 INFO MainThread:210 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_205024', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-2750', 'force_new_run': True, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 49, 'verbose': False, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m edge-dense alternating-mixed seed49', 'wandb_tags': ['qwen3', '365m', '8k', 'edge-dense', 'alternating-mixed', 'seed49'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': [3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23], 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed', 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': [63, 47, 31, 21, 11], 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-08-25 20:52:15,954 INFO MainThread:210 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-08-25 20:52:17,208 INFO MainThread:210 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-08-25 20:52:17,825 INFO MainThread:210 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-08-25 20:52:17,828 INFO MainThread:210 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-08-25 20:52:17,863 INFO MainThread:210 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-08-25 20:52:18,568 INFO MainThread:210 [wandb_init.py:init():1056] starting run threads in backend
|
| 15 |
+
2026-08-25 20:52:18,924 INFO MainThread:210 [wandb_run.py:_console_start():2554] atexit reg
|
| 16 |
+
2026-08-25 20:52:18,925 INFO MainThread:210 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 17 |
+
2026-08-25 20:52:18,925 INFO MainThread:210 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 18 |
+
2026-08-25 20:52:18,925 INFO MainThread:210 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 19 |
+
2026-08-25 20:52:18,939 INFO MainThread:210 [wandb_init.py:init():1094] run started, returning control to user process
|
epfl_final/SparseAtt/wandb/run-20260825_205215-yl9g6w7y/run-yl9g6w7y.wandb
ADDED
|
Binary file (32.8 kB). View file
|
|
|
epfl_final/SparseAtt/wandb/run-20260825_211848-rgmkfpnl/files/output.log
ADDED
|
@@ -0,0 +1,411 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Starting training...
|
| 2 |
+
Validation at step 24750: Loss = 2.5786, Perplexity = 13.18
|
| 3 |
+
wandb: WARNING Tried to log to step 24750 that is less than the current step 24961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 4 |
+
Switching attention mode for step 24751: dense (dense step interval)
|
| 5 |
+
Step 24760/50462 | Loss: 2.5698 | LR: 2.63e-04 | 87198.7ms/step | 0.7 samples/s
|
| 6 |
+
wandb: WARNING Tried to log to step 24760 that is less than the current step 24961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 7 |
+
Step 24770/50462 | Loss: 2.5647 | LR: 2.63e-04 | 5198.7ms/step | 11.5 samples/s
|
| 8 |
+
wandb: WARNING Tried to log to step 24770 that is less than the current step 24961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 9 |
+
Step 24780/50462 | Loss: 2.5581 | LR: 2.63e-04 | 5228.5ms/step | 11.5 samples/s
|
| 10 |
+
wandb: WARNING Tried to log to step 24780 that is less than the current step 24961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 11 |
+
Step 24790/50462 | Loss: 2.5728 | LR: 2.63e-04 | 5231.6ms/step | 11.5 samples/s
|
| 12 |
+
wandb: WARNING Tried to log to step 24790 that is less than the current step 24961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 13 |
+
Step 24800/50462 | Loss: 2.5682 | LR: 2.63e-04 | 5232.8ms/step | 11.5 samples/s
|
| 14 |
+
wandb: WARNING Tried to log to step 24800 that is less than the current step 24961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 15 |
+
Step 24810/50462 | Loss: 2.5696 | LR: 2.63e-04 | 5221.4ms/step | 11.5 samples/s
|
| 16 |
+
wandb: WARNING Tried to log to step 24810 that is less than the current step 24961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 17 |
+
Step 24820/50462 | Loss: 2.5859 | LR: 2.62e-04 | 5229.9ms/step | 11.5 samples/s
|
| 18 |
+
wandb: WARNING Tried to log to step 24820 that is less than the current step 24961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 19 |
+
Step 24830/50462 | Loss: 2.5520 | LR: 2.62e-04 | 5231.1ms/step | 11.5 samples/s
|
| 20 |
+
wandb: WARNING Tried to log to step 24830 that is less than the current step 24961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 21 |
+
Step 24840/50462 | Loss: 2.5834 | LR: 2.62e-04 | 5238.5ms/step | 11.5 samples/s
|
| 22 |
+
wandb: WARNING Tried to log to step 24840 that is less than the current step 24961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 23 |
+
Step 24850/50462 | Loss: 2.5677 | LR: 2.62e-04 | 5241.1ms/step | 11.4 samples/s
|
| 24 |
+
wandb: WARNING Tried to log to step 24850 that is less than the current step 24961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 25 |
+
Step 24860/50462 | Loss: 2.5612 | LR: 2.62e-04 | 5238.2ms/step | 11.5 samples/s
|
| 26 |
+
wandb: WARNING Tried to log to step 24860 that is less than the current step 24961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 27 |
+
Step 24870/50462 | Loss: 2.5683 | LR: 2.62e-04 | 5237.2ms/step | 11.5 samples/s
|
| 28 |
+
wandb: WARNING Tried to log to step 24870 that is less than the current step 24961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 29 |
+
Step 24880/50462 | Loss: 2.5503 | LR: 2.61e-04 | 5240.6ms/step | 11.4 samples/s
|
| 30 |
+
wandb: WARNING Tried to log to step 24880 that is less than the current step 24961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 31 |
+
Step 24890/50462 | Loss: 2.5759 | LR: 2.61e-04 | 5241.1ms/step | 11.4 samples/s
|
| 32 |
+
wandb: WARNING Tried to log to step 24890 that is less than the current step 24961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 33 |
+
Step 24900/50462 | Loss: 2.5666 | LR: 2.61e-04 | 5249.8ms/step | 11.4 samples/s
|
| 34 |
+
wandb: WARNING Tried to log to step 24900 that is less than the current step 24961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 35 |
+
Step 24910/50462 | Loss: 2.5689 | LR: 2.61e-04 | 5248.8ms/step | 11.4 samples/s
|
| 36 |
+
wandb: WARNING Tried to log to step 24910 that is less than the current step 24961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 37 |
+
Step 24920/50462 | Loss: 2.5462 | LR: 2.61e-04 | 5246.7ms/step | 11.4 samples/s
|
| 38 |
+
wandb: WARNING Tried to log to step 24920 that is less than the current step 24961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 39 |
+
Step 24930/50462 | Loss: 2.6253 | LR: 2.61e-04 | 5231.7ms/step | 11.5 samples/s
|
| 40 |
+
wandb: WARNING Tried to log to step 24930 that is less than the current step 24961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 41 |
+
Step 24940/50462 | Loss: 2.5710 | LR: 2.61e-04 | 5235.6ms/step | 11.5 samples/s
|
| 42 |
+
wandb: WARNING Tried to log to step 24940 that is less than the current step 24961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 43 |
+
Step 24950/50462 | Loss: 2.5711 | LR: 2.60e-04 | 5234.9ms/step | 11.5 samples/s
|
| 44 |
+
wandb: WARNING Tried to log to step 24950 that is less than the current step 24961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 45 |
+
Step 24960/50462 | Loss: 2.5634 | LR: 2.60e-04 | 5237.7ms/step | 11.5 samples/s
|
| 46 |
+
wandb: WARNING Tried to log to step 24960 that is less than the current step 24961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 47 |
+
Step 24970/50462 | Loss: 2.5673 | LR: 2.60e-04 | 5239.5ms/step | 11.5 samples/s
|
| 48 |
+
Step 24980/50462 | Loss: 2.5675 | LR: 2.60e-04 | 5238.8ms/step | 11.5 samples/s
|
| 49 |
+
Step 24990/50462 | Loss: 2.5440 | LR: 2.60e-04 | 5241.4ms/step | 11.4 samples/s
|
| 50 |
+
Step 25000/50462 | Loss: 2.5817 | LR: 2.60e-04 | 5233.2ms/step | 11.5 samples/s
|
| 51 |
+
Validation at step 25000: Loss = 2.5765, Perplexity = 13.15
|
| 52 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-25000
|
| 53 |
+
Step 25010/50462 | Loss: 2.5603 | LR: 2.59e-04 | 10089.0ms/step | 5.9 samples/s
|
| 54 |
+
Step 25020/50462 | Loss: 2.5626 | LR: 2.59e-04 | 5218.1ms/step | 11.5 samples/s
|
| 55 |
+
Step 25030/50462 | Loss: 2.5706 | LR: 2.59e-04 | 5244.3ms/step | 11.4 samples/s
|
| 56 |
+
Step 25040/50462 | Loss: 2.5667 | LR: 2.59e-04 | 5247.5ms/step | 11.4 samples/s
|
| 57 |
+
Step 25050/50462 | Loss: 2.5594 | LR: 2.59e-04 | 5244.5ms/step | 11.4 samples/s
|
| 58 |
+
Step 25060/50462 | Loss: 2.5511 | LR: 2.59e-04 | 5237.7ms/step | 11.5 samples/s
|
| 59 |
+
Step 25070/50462 | Loss: 2.5564 | LR: 2.58e-04 | 5244.6ms/step | 11.4 samples/s
|
| 60 |
+
Step 25080/50462 | Loss: 2.5498 | LR: 2.58e-04 | 5249.2ms/step | 11.4 samples/s
|
| 61 |
+
Step 25090/50462 | Loss: 2.5951 | LR: 2.58e-04 | 5255.7ms/step | 11.4 samples/s
|
| 62 |
+
Step 25100/50462 | Loss: 2.5621 | LR: 2.58e-04 | 5250.3ms/step | 11.4 samples/s
|
| 63 |
+
Step 25110/50462 | Loss: 2.5585 | LR: 2.58e-04 | 5249.9ms/step | 11.4 samples/s
|
| 64 |
+
Step 25120/50462 | Loss: 2.5777 | LR: 2.58e-04 | 5247.0ms/step | 11.4 samples/s
|
| 65 |
+
Step 25130/50462 | Loss: 2.5772 | LR: 2.58e-04 | 5257.5ms/step | 11.4 samples/s
|
| 66 |
+
Step 25140/50462 | Loss: 2.5667 | LR: 2.57e-04 | 5246.1ms/step | 11.4 samples/s
|
| 67 |
+
Step 25150/50462 | Loss: 2.5317 | LR: 2.57e-04 | 5244.1ms/step | 11.4 samples/s
|
| 68 |
+
Step 25160/50462 | Loss: 2.5566 | LR: 2.57e-04 | 5246.8ms/step | 11.4 samples/s
|
| 69 |
+
Step 25170/50462 | Loss: 2.5529 | LR: 2.57e-04 | 5245.9ms/step | 11.4 samples/s
|
| 70 |
+
Step 25180/50462 | Loss: 2.5437 | LR: 2.57e-04 | 5246.1ms/step | 11.4 samples/s
|
| 71 |
+
Step 25190/50462 | Loss: 2.5755 | LR: 2.57e-04 | 5234.8ms/step | 11.5 samples/s
|
| 72 |
+
Step 25200/50462 | Loss: 2.5538 | LR: 2.56e-04 | 5247.4ms/step | 11.4 samples/s
|
| 73 |
+
Step 25210/50462 | Loss: 2.5657 | LR: 2.56e-04 | 5243.8ms/step | 11.4 samples/s
|
| 74 |
+
Step 25220/50462 | Loss: 2.5711 | LR: 2.56e-04 | 5250.9ms/step | 11.4 samples/s
|
| 75 |
+
Step 25230/50462 | Loss: 2.5683 | LR: 2.56e-04 | 5243.1ms/step | 11.4 samples/s
|
| 76 |
+
Step 25240/50462 | Loss: 2.5718 | LR: 2.56e-04 | 5234.8ms/step | 11.5 samples/s
|
| 77 |
+
Step 25250/50462 | Loss: 2.5813 | LR: 2.56e-04 | 5232.1ms/step | 11.5 samples/s
|
| 78 |
+
Validation at step 25250: Loss = 2.5751, Perplexity = 13.13
|
| 79 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-25250
|
| 80 |
+
Step 25260/50462 | Loss: 2.5679 | LR: 2.55e-04 | 10147.5ms/step | 5.9 samples/s
|
| 81 |
+
Step 25270/50462 | Loss: 2.5392 | LR: 2.55e-04 | 5239.2ms/step | 11.5 samples/s
|
| 82 |
+
Step 25280/50462 | Loss: 2.5464 | LR: 2.55e-04 | 5243.3ms/step | 11.4 samples/s
|
| 83 |
+
Step 25290/50462 | Loss: 2.5557 | LR: 2.55e-04 | 5243.5ms/step | 11.4 samples/s
|
| 84 |
+
Step 25300/50462 | Loss: 2.5736 | LR: 2.55e-04 | 5244.2ms/step | 11.4 samples/s
|
| 85 |
+
Step 25310/50462 | Loss: 2.5468 | LR: 2.55e-04 | 5247.4ms/step | 11.4 samples/s
|
| 86 |
+
Step 25320/50462 | Loss: 2.5674 | LR: 2.55e-04 | 5245.6ms/step | 11.4 samples/s
|
| 87 |
+
Step 25330/50462 | Loss: 2.5654 | LR: 2.54e-04 | 5252.3ms/step | 11.4 samples/s
|
| 88 |
+
Step 25340/50462 | Loss: 2.5937 | LR: 2.54e-04 | 5256.6ms/step | 11.4 samples/s
|
| 89 |
+
Step 25350/50462 | Loss: 2.5529 | LR: 2.54e-04 | 5253.9ms/step | 11.4 samples/s
|
| 90 |
+
Step 25360/50462 | Loss: 2.5608 | LR: 2.54e-04 | 5250.4ms/step | 11.4 samples/s
|
| 91 |
+
Step 25370/50462 | Loss: 2.5447 | LR: 2.54e-04 | 5239.9ms/step | 11.5 samples/s
|
| 92 |
+
Step 25380/50462 | Loss: 2.5710 | LR: 2.54e-04 | 5246.5ms/step | 11.4 samples/s
|
| 93 |
+
Step 25390/50462 | Loss: 2.5595 | LR: 2.53e-04 | 5241.8ms/step | 11.4 samples/s
|
| 94 |
+
Step 25400/50462 | Loss: 2.5480 | LR: 2.53e-04 | 5246.2ms/step | 11.4 samples/s
|
| 95 |
+
Step 25410/50462 | Loss: 2.5704 | LR: 2.53e-04 | 5251.6ms/step | 11.4 samples/s
|
| 96 |
+
Step 25420/50462 | Loss: 2.5576 | LR: 2.53e-04 | 5240.9ms/step | 11.4 samples/s
|
| 97 |
+
Step 25430/50462 | Loss: 2.5458 | LR: 2.53e-04 | 5208.2ms/step | 11.5 samples/s
|
| 98 |
+
Step 25440/50462 | Loss: 2.5514 | LR: 2.53e-04 | 5242.1ms/step | 11.4 samples/s
|
| 99 |
+
Step 25450/50462 | Loss: 2.5468 | LR: 2.52e-04 | 5231.2ms/step | 11.5 samples/s
|
| 100 |
+
Step 25460/50462 | Loss: 2.5491 | LR: 2.52e-04 | 5253.1ms/step | 11.4 samples/s
|
| 101 |
+
Step 25470/50462 | Loss: 2.5684 | LR: 2.52e-04 | 5248.1ms/step | 11.4 samples/s
|
| 102 |
+
Step 25480/50462 | Loss: 2.5679 | LR: 2.52e-04 | 5240.8ms/step | 11.4 samples/s
|
| 103 |
+
Step 25490/50462 | Loss: 2.5754 | LR: 2.52e-04 | 5265.9ms/step | 11.4 samples/s
|
| 104 |
+
Step 25500/50462 | Loss: 2.5480 | LR: 2.52e-04 | 5263.0ms/step | 11.4 samples/s
|
| 105 |
+
Validation at step 25500: Loss = 2.5727, Perplexity = 13.10
|
| 106 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-25500
|
| 107 |
+
Step 25510/50462 | Loss: 2.5342 | LR: 2.52e-04 | 10137.5ms/step | 5.9 samples/s
|
| 108 |
+
Step 25520/50462 | Loss: 2.5660 | LR: 2.51e-04 | 5234.5ms/step | 11.5 samples/s
|
| 109 |
+
Step 25530/50462 | Loss: 2.5908 | LR: 2.51e-04 | 5258.6ms/step | 11.4 samples/s
|
| 110 |
+
Step 25540/50462 | Loss: 2.5507 | LR: 2.51e-04 | 5267.2ms/step | 11.4 samples/s
|
| 111 |
+
Step 25550/50462 | Loss: 2.5496 | LR: 2.51e-04 | 5260.8ms/step | 11.4 samples/s
|
| 112 |
+
Step 25560/50462 | Loss: 2.5512 | LR: 2.51e-04 | 5254.4ms/step | 11.4 samples/s
|
| 113 |
+
Step 25570/50462 | Loss: 2.5648 | LR: 2.51e-04 | 5257.1ms/step | 11.4 samples/s
|
| 114 |
+
Step 25580/50462 | Loss: 2.5713 | LR: 2.50e-04 | 5262.1ms/step | 11.4 samples/s
|
| 115 |
+
Step 25590/50462 | Loss: 2.5526 | LR: 2.50e-04 | 5265.7ms/step | 11.4 samples/s
|
| 116 |
+
Step 25600/50462 | Loss: 2.5434 | LR: 2.50e-04 | 5263.9ms/step | 11.4 samples/s
|
| 117 |
+
Step 25610/50462 | Loss: 2.5641 | LR: 2.50e-04 | 5265.4ms/step | 11.4 samples/s
|
| 118 |
+
Step 25620/50462 | Loss: 2.5592 | LR: 2.50e-04 | 5266.6ms/step | 11.4 samples/s
|
| 119 |
+
Step 25630/50462 | Loss: 2.5550 | LR: 2.50e-04 | 5265.8ms/step | 11.4 samples/s
|
| 120 |
+
Step 25640/50462 | Loss: 2.5517 | LR: 2.49e-04 | 5263.8ms/step | 11.4 samples/s
|
| 121 |
+
Step 25650/50462 | Loss: 2.5657 | LR: 2.49e-04 | 5254.7ms/step | 11.4 samples/s
|
| 122 |
+
Step 25660/50462 | Loss: 2.5508 | LR: 2.49e-04 | 5259.5ms/step | 11.4 samples/s
|
| 123 |
+
Step 25670/50462 | Loss: 2.5697 | LR: 2.49e-04 | 5265.9ms/step | 11.4 samples/s
|
| 124 |
+
Step 25680/50462 | Loss: 2.5912 | LR: 2.49e-04 | 5253.0ms/step | 11.4 samples/s
|
| 125 |
+
Step 25690/50462 | Loss: 2.5754 | LR: 2.49e-04 | 5255.5ms/step | 11.4 samples/s
|
| 126 |
+
Step 25700/50462 | Loss: 2.5734 | LR: 2.49e-04 | 5258.6ms/step | 11.4 samples/s
|
| 127 |
+
Step 25710/50462 | Loss: 2.5733 | LR: 2.48e-04 | 5264.6ms/step | 11.4 samples/s
|
| 128 |
+
Step 25720/50462 | Loss: 2.5567 | LR: 2.48e-04 | 5264.9ms/step | 11.4 samples/s
|
| 129 |
+
Step 25730/50462 | Loss: 2.5616 | LR: 2.48e-04 | 5265.2ms/step | 11.4 samples/s
|
| 130 |
+
Step 25740/50462 | Loss: 2.5615 | LR: 2.48e-04 | 5261.5ms/step | 11.4 samples/s
|
| 131 |
+
Step 25750/50462 | Loss: 2.5651 | LR: 2.48e-04 | 5260.4ms/step | 11.4 samples/s
|
| 132 |
+
Validation at step 25750: Loss = 2.5703, Perplexity = 13.07
|
| 133 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-25750
|
| 134 |
+
Step 25760/50462 | Loss: 2.5394 | LR: 2.48e-04 | 10137.3ms/step | 5.9 samples/s
|
| 135 |
+
Step 25770/50462 | Loss: 2.5517 | LR: 2.47e-04 | 5241.8ms/step | 11.4 samples/s
|
| 136 |
+
Step 25780/50462 | Loss: 2.5598 | LR: 2.47e-04 | 5251.3ms/step | 11.4 samples/s
|
| 137 |
+
Step 25790/50462 | Loss: 2.5514 | LR: 2.47e-04 | 5260.9ms/step | 11.4 samples/s
|
| 138 |
+
Step 25800/50462 | Loss: 2.5604 | LR: 2.47e-04 | 5254.6ms/step | 11.4 samples/s
|
| 139 |
+
Step 25810/50462 | Loss: 2.5578 | LR: 2.47e-04 | 5257.5ms/step | 11.4 samples/s
|
| 140 |
+
Step 25820/50462 | Loss: 2.5348 | LR: 2.47e-04 | 5254.5ms/step | 11.4 samples/s
|
| 141 |
+
Step 25830/50462 | Loss: 2.5306 | LR: 2.46e-04 | 5251.0ms/step | 11.4 samples/s
|
| 142 |
+
Step 25840/50462 | Loss: 2.5602 | LR: 2.46e-04 | 5255.2ms/step | 11.4 samples/s
|
| 143 |
+
Step 25850/50462 | Loss: 2.5506 | LR: 2.46e-04 | 5254.0ms/step | 11.4 samples/s
|
| 144 |
+
Step 25860/50462 | Loss: 2.5401 | LR: 2.46e-04 | 5257.0ms/step | 11.4 samples/s
|
| 145 |
+
Step 25870/50462 | Loss: 2.5615 | LR: 2.46e-04 | 5256.1ms/step | 11.4 samples/s
|
| 146 |
+
Step 25880/50462 | Loss: 2.5588 | LR: 2.46e-04 | 5258.7ms/step | 11.4 samples/s
|
| 147 |
+
Step 25890/50462 | Loss: 2.5490 | LR: 2.46e-04 | 5258.8ms/step | 11.4 samples/s
|
| 148 |
+
Step 25900/50462 | Loss: 2.5480 | LR: 2.45e-04 | 5253.7ms/step | 11.4 samples/s
|
| 149 |
+
Step 25910/50462 | Loss: 2.5357 | LR: 2.45e-04 | 5247.1ms/step | 11.4 samples/s
|
| 150 |
+
Step 25920/50462 | Loss: 2.5553 | LR: 2.45e-04 | 5242.5ms/step | 11.4 samples/s
|
| 151 |
+
Step 25930/50462 | Loss: 2.5529 | LR: 2.45e-04 | 5248.5ms/step | 11.4 samples/s
|
| 152 |
+
Step 25940/50462 | Loss: 2.5688 | LR: 2.45e-04 | 5249.5ms/step | 11.4 samples/s
|
| 153 |
+
Step 25950/50462 | Loss: 2.5556 | LR: 2.45e-04 | 5244.2ms/step | 11.4 samples/s
|
| 154 |
+
Step 25960/50462 | Loss: 2.5499 | LR: 2.44e-04 | 5233.5ms/step | 11.5 samples/s
|
| 155 |
+
Step 25970/50462 | Loss: 2.5684 | LR: 2.44e-04 | 5250.4ms/step | 11.4 samples/s
|
| 156 |
+
Step 25980/50462 | Loss: 2.5751 | LR: 2.44e-04 | 5251.2ms/step | 11.4 samples/s
|
| 157 |
+
Step 25990/50462 | Loss: 2.5509 | LR: 2.44e-04 | 5250.8ms/step | 11.4 samples/s
|
| 158 |
+
Step 26000/50462 | Loss: 2.5291 | LR: 2.44e-04 | 5248.0ms/step | 11.4 samples/s
|
| 159 |
+
Validation at step 26000: Loss = 2.5678, Perplexity = 13.04
|
| 160 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-26000
|
| 161 |
+
Step 26010/50462 | Loss: 2.5436 | LR: 2.44e-04 | 10121.8ms/step | 5.9 samples/s
|
| 162 |
+
Step 26020/50462 | Loss: 2.5766 | LR: 2.43e-04 | 5244.5ms/step | 11.4 samples/s
|
| 163 |
+
Step 26030/50462 | Loss: 2.5447 | LR: 2.43e-04 | 5254.1ms/step | 11.4 samples/s
|
| 164 |
+
Step 26040/50462 | Loss: 2.5890 | LR: 2.43e-04 | 5246.0ms/step | 11.4 samples/s
|
| 165 |
+
Step 26050/50462 | Loss: 2.5612 | LR: 2.43e-04 | 5246.9ms/step | 11.4 samples/s
|
| 166 |
+
Step 26060/50462 | Loss: 2.5462 | LR: 2.43e-04 | 5257.8ms/step | 11.4 samples/s
|
| 167 |
+
Step 26070/50462 | Loss: 2.5728 | LR: 2.43e-04 | 5263.5ms/step | 11.4 samples/s
|
| 168 |
+
Step 26080/50462 | Loss: 2.5466 | LR: 2.43e-04 | 5256.6ms/step | 11.4 samples/s
|
| 169 |
+
Step 26090/50462 | Loss: 2.5420 | LR: 2.42e-04 | 5241.6ms/step | 11.4 samples/s
|
| 170 |
+
Step 26100/50462 | Loss: 2.5545 | LR: 2.42e-04 | 5251.8ms/step | 11.4 samples/s
|
| 171 |
+
Step 26110/50462 | Loss: 2.5604 | LR: 2.42e-04 | 5261.0ms/step | 11.4 samples/s
|
| 172 |
+
Step 26120/50462 | Loss: 2.5527 | LR: 2.42e-04 | 5256.8ms/step | 11.4 samples/s
|
| 173 |
+
Step 26130/50462 | Loss: 2.5625 | LR: 2.42e-04 | 5256.6ms/step | 11.4 samples/s
|
| 174 |
+
Step 26140/50462 | Loss: 2.5650 | LR: 2.42e-04 | 5230.9ms/step | 11.5 samples/s
|
| 175 |
+
Step 26150/50462 | Loss: 2.5512 | LR: 2.41e-04 | 5247.7ms/step | 11.4 samples/s
|
| 176 |
+
Step 26160/50462 | Loss: 2.5464 | LR: 2.41e-04 | 5254.0ms/step | 11.4 samples/s
|
| 177 |
+
Step 26170/50462 | Loss: 2.5605 | LR: 2.41e-04 | 5261.0ms/step | 11.4 samples/s
|
| 178 |
+
Step 26180/50462 | Loss: 2.5583 | LR: 2.41e-04 | 5256.0ms/step | 11.4 samples/s
|
| 179 |
+
Step 26190/50462 | Loss: 2.5554 | LR: 2.41e-04 | 5247.0ms/step | 11.4 samples/s
|
| 180 |
+
Step 26200/50462 | Loss: 2.5303 | LR: 2.41e-04 | 5259.7ms/step | 11.4 samples/s
|
| 181 |
+
Step 26210/50462 | Loss: 2.5637 | LR: 2.40e-04 | 5262.3ms/step | 11.4 samples/s
|
| 182 |
+
Step 26220/50462 | Loss: 2.5616 | LR: 2.40e-04 | 5260.7ms/step | 11.4 samples/s
|
| 183 |
+
Step 26230/50462 | Loss: 2.5193 | LR: 2.40e-04 | 5259.6ms/step | 11.4 samples/s
|
| 184 |
+
Step 26240/50462 | Loss: 2.5459 | LR: 2.40e-04 | 5264.1ms/step | 11.4 samples/s
|
| 185 |
+
Step 26250/50462 | Loss: 2.5540 | LR: 2.40e-04 | 5264.1ms/step | 11.4 samples/s
|
| 186 |
+
Validation at step 26250: Loss = 2.5660, Perplexity = 13.01
|
| 187 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-26250
|
| 188 |
+
Step 26260/50462 | Loss: 2.5554 | LR: 2.40e-04 | 10141.9ms/step | 5.9 samples/s
|
| 189 |
+
Step 26270/50462 | Loss: 2.5579 | LR: 2.40e-04 | 5241.4ms/step | 11.4 samples/s
|
| 190 |
+
Step 26280/50462 | Loss: 2.5722 | LR: 2.39e-04 | 5257.5ms/step | 11.4 samples/s
|
| 191 |
+
Step 26290/50462 | Loss: 2.5593 | LR: 2.39e-04 | 5255.5ms/step | 11.4 samples/s
|
| 192 |
+
Step 26300/50462 | Loss: 2.5582 | LR: 2.39e-04 | 5256.9ms/step | 11.4 samples/s
|
| 193 |
+
Step 26310/50462 | Loss: 2.5663 | LR: 2.39e-04 | 5257.0ms/step | 11.4 samples/s
|
| 194 |
+
Step 26320/50462 | Loss: 2.5650 | LR: 2.39e-04 | 5252.8ms/step | 11.4 samples/s
|
| 195 |
+
Step 26330/50462 | Loss: 2.5675 | LR: 2.39e-04 | 5255.6ms/step | 11.4 samples/s
|
| 196 |
+
Step 26340/50462 | Loss: 2.5409 | LR: 2.38e-04 | 5259.3ms/step | 11.4 samples/s
|
| 197 |
+
Step 26350/50462 | Loss: 2.5726 | LR: 2.38e-04 | 5252.9ms/step | 11.4 samples/s
|
| 198 |
+
Step 26360/50462 | Loss: 2.5599 | LR: 2.38e-04 | 5249.4ms/step | 11.4 samples/s
|
| 199 |
+
Step 26370/50462 | Loss: 2.5465 | LR: 2.38e-04 | 5250.0ms/step | 11.4 samples/s
|
| 200 |
+
Step 26380/50462 | Loss: 2.5752 | LR: 2.38e-04 | 5250.2ms/step | 11.4 samples/s
|
| 201 |
+
Step 26390/50462 | Loss: 2.5342 | LR: 2.38e-04 | 5254.0ms/step | 11.4 samples/s
|
| 202 |
+
Step 26400/50462 | Loss: 2.5464 | LR: 2.37e-04 | 5256.8ms/step | 11.4 samples/s
|
| 203 |
+
Step 26410/50462 | Loss: 2.5385 | LR: 2.37e-04 | 5255.8ms/step | 11.4 samples/s
|
| 204 |
+
Step 26420/50462 | Loss: 2.5684 | LR: 2.37e-04 | 5254.2ms/step | 11.4 samples/s
|
| 205 |
+
Step 26430/50462 | Loss: 2.5531 | LR: 2.37e-04 | 5253.1ms/step | 11.4 samples/s
|
| 206 |
+
Step 26440/50462 | Loss: 2.5595 | LR: 2.37e-04 | 5258.2ms/step | 11.4 samples/s
|
| 207 |
+
Step 26450/50462 | Loss: 2.5574 | LR: 2.37e-04 | 5261.5ms/step | 11.4 samples/s
|
| 208 |
+
Step 26460/50462 | Loss: 2.5660 | LR: 2.37e-04 | 5254.0ms/step | 11.4 samples/s
|
| 209 |
+
Step 26470/50462 | Loss: 2.5401 | LR: 2.36e-04 | 5254.0ms/step | 11.4 samples/s
|
| 210 |
+
Step 26480/50462 | Loss: 2.5565 | LR: 2.36e-04 | 5255.8ms/step | 11.4 samples/s
|
| 211 |
+
Step 26490/50462 | Loss: 2.5629 | LR: 2.36e-04 | 5256.4ms/step | 11.4 samples/s
|
| 212 |
+
Step 26500/50462 | Loss: 2.5464 | LR: 2.36e-04 | 5255.8ms/step | 11.4 samples/s
|
| 213 |
+
Validation at step 26500: Loss = 2.5639, Perplexity = 12.99
|
| 214 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-26500
|
| 215 |
+
Step 26510/50462 | Loss: 2.5298 | LR: 2.36e-04 | 10103.5ms/step | 5.9 samples/s
|
| 216 |
+
Step 26520/50462 | Loss: 2.5407 | LR: 2.36e-04 | 5254.4ms/step | 11.4 samples/s
|
| 217 |
+
Step 26530/50462 | Loss: 2.5572 | LR: 2.35e-04 | 5256.7ms/step | 11.4 samples/s
|
| 218 |
+
Step 26540/50462 | Loss: 2.5473 | LR: 2.35e-04 | 5257.1ms/step | 11.4 samples/s
|
| 219 |
+
Step 26550/50462 | Loss: 2.5329 | LR: 2.35e-04 | 5257.2ms/step | 11.4 samples/s
|
| 220 |
+
Step 26560/50462 | Loss: 2.5507 | LR: 2.35e-04 | 5255.8ms/step | 11.4 samples/s
|
| 221 |
+
Step 26570/50462 | Loss: 2.5298 | LR: 2.35e-04 | 5257.6ms/step | 11.4 samples/s
|
| 222 |
+
Step 26580/50462 | Loss: 2.5342 | LR: 2.35e-04 | 5262.0ms/step | 11.4 samples/s
|
| 223 |
+
Step 26590/50462 | Loss: 2.5534 | LR: 2.34e-04 | 5253.6ms/step | 11.4 samples/s
|
| 224 |
+
Step 26600/50462 | Loss: 2.5821 | LR: 2.34e-04 | 5253.5ms/step | 11.4 samples/s
|
| 225 |
+
Step 26610/50462 | Loss: 2.5572 | LR: 2.34e-04 | 5259.7ms/step | 11.4 samples/s
|
| 226 |
+
Step 26620/50462 | Loss: 2.5449 | LR: 2.34e-04 | 5260.9ms/step | 11.4 samples/s
|
| 227 |
+
Step 26630/50462 | Loss: 2.5508 | LR: 2.34e-04 | 5267.9ms/step | 11.4 samples/s
|
| 228 |
+
Step 26640/50462 | Loss: 2.5472 | LR: 2.34e-04 | 5262.1ms/step | 11.4 samples/s
|
| 229 |
+
Step 26650/50462 | Loss: 2.5687 | LR: 2.34e-04 | 5260.7ms/step | 11.4 samples/s
|
| 230 |
+
Step 26660/50462 | Loss: 2.5616 | LR: 2.33e-04 | 5256.5ms/step | 11.4 samples/s
|
| 231 |
+
Step 26670/50462 | Loss: 2.5563 | LR: 2.33e-04 | 5253.1ms/step | 11.4 samples/s
|
| 232 |
+
Step 26680/50462 | Loss: 2.5221 | LR: 2.33e-04 | 5256.9ms/step | 11.4 samples/s
|
| 233 |
+
Step 26690/50462 | Loss: 2.5739 | LR: 2.33e-04 | 5262.1ms/step | 11.4 samples/s
|
| 234 |
+
Step 26700/50462 | Loss: 2.5242 | LR: 2.33e-04 | 5253.5ms/step | 11.4 samples/s
|
| 235 |
+
Step 26710/50462 | Loss: 2.5569 | LR: 2.33e-04 | 5252.4ms/step | 11.4 samples/s
|
| 236 |
+
Step 26720/50462 | Loss: 2.5416 | LR: 2.32e-04 | 5258.6ms/step | 11.4 samples/s
|
| 237 |
+
Step 26730/50462 | Loss: 2.5616 | LR: 2.32e-04 | 5256.1ms/step | 11.4 samples/s
|
| 238 |
+
Step 26740/50462 | Loss: 2.5649 | LR: 2.32e-04 | 5258.7ms/step | 11.4 samples/s
|
| 239 |
+
Step 26750/50462 | Loss: 2.5317 | LR: 2.32e-04 | 5258.8ms/step | 11.4 samples/s
|
| 240 |
+
Validation at step 26750: Loss = 2.5616, Perplexity = 12.96
|
| 241 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-26750
|
| 242 |
+
Step 26760/50462 | Loss: 2.5485 | LR: 2.32e-04 | 10159.3ms/step | 5.9 samples/s
|
| 243 |
+
Step 26770/50462 | Loss: 2.5715 | LR: 2.32e-04 | 5235.1ms/step | 11.5 samples/s
|
| 244 |
+
Step 26780/50462 | Loss: 2.5430 | LR: 2.31e-04 | 5246.7ms/step | 11.4 samples/s
|
| 245 |
+
Step 26790/50462 | Loss: 2.5436 | LR: 2.31e-04 | 5251.0ms/step | 11.4 samples/s
|
| 246 |
+
Step 26800/50462 | Loss: 2.5563 | LR: 2.31e-04 | 5254.5ms/step | 11.4 samples/s
|
| 247 |
+
Step 26810/50462 | Loss: 2.5560 | LR: 2.31e-04 | 5257.8ms/step | 11.4 samples/s
|
| 248 |
+
Step 26820/50462 | Loss: 2.5127 | LR: 2.31e-04 | 5255.4ms/step | 11.4 samples/s
|
| 249 |
+
Step 26830/50462 | Loss: 2.5543 | LR: 2.31e-04 | 5262.1ms/step | 11.4 samples/s
|
| 250 |
+
Step 26840/50462 | Loss: 2.5544 | LR: 2.31e-04 | 5259.2ms/step | 11.4 samples/s
|
| 251 |
+
Step 26850/50462 | Loss: 2.5588 | LR: 2.30e-04 | 5254.3ms/step | 11.4 samples/s
|
| 252 |
+
Step 26860/50462 | Loss: 2.5412 | LR: 2.30e-04 | 5260.6ms/step | 11.4 samples/s
|
| 253 |
+
Step 26870/50462 | Loss: 2.5278 | LR: 2.30e-04 | 5260.5ms/step | 11.4 samples/s
|
| 254 |
+
Step 26880/50462 | Loss: 2.5520 | LR: 2.30e-04 | 5257.7ms/step | 11.4 samples/s
|
| 255 |
+
Step 26890/50462 | Loss: 2.5566 | LR: 2.30e-04 | 5252.0ms/step | 11.4 samples/s
|
| 256 |
+
Step 26900/50462 | Loss: 2.5488 | LR: 2.30e-04 | 5258.7ms/step | 11.4 samples/s
|
| 257 |
+
Step 26910/50462 | Loss: 2.5826 | LR: 2.29e-04 | 5261.6ms/step | 11.4 samples/s
|
| 258 |
+
Step 26920/50462 | Loss: 2.5512 | LR: 2.29e-04 | 5260.7ms/step | 11.4 samples/s
|
| 259 |
+
Step 26930/50462 | Loss: 2.5493 | LR: 2.29e-04 | 5261.4ms/step | 11.4 samples/s
|
| 260 |
+
Step 26940/50462 | Loss: 2.5482 | LR: 2.29e-04 | 5258.7ms/step | 11.4 samples/s
|
| 261 |
+
Step 26950/50462 | Loss: 2.5398 | LR: 2.29e-04 | 5263.7ms/step | 11.4 samples/s
|
| 262 |
+
Step 26960/50462 | Loss: 2.5737 | LR: 2.29e-04 | 5257.9ms/step | 11.4 samples/s
|
| 263 |
+
Step 26970/50462 | Loss: 2.5404 | LR: 2.28e-04 | 5264.3ms/step | 11.4 samples/s
|
| 264 |
+
Step 26980/50462 | Loss: 2.5507 | LR: 2.28e-04 | 5263.3ms/step | 11.4 samples/s
|
| 265 |
+
Step 26990/50462 | Loss: 2.5544 | LR: 2.28e-04 | 5256.3ms/step | 11.4 samples/s
|
| 266 |
+
Step 27000/50462 | Loss: 2.5469 | LR: 2.28e-04 | 5257.0ms/step | 11.4 samples/s
|
| 267 |
+
Validation at step 27000: Loss = 2.5602, Perplexity = 12.94
|
| 268 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-27000
|
| 269 |
+
Step 27010/50462 | Loss: 2.5522 | LR: 2.28e-04 | 10135.3ms/step | 5.9 samples/s
|
| 270 |
+
Step 27020/50462 | Loss: 2.5338 | LR: 2.28e-04 | 5251.6ms/step | 11.4 samples/s
|
| 271 |
+
Step 27030/50462 | Loss: 2.5631 | LR: 2.28e-04 | 5256.1ms/step | 11.4 samples/s
|
| 272 |
+
Step 27040/50462 | Loss: 2.5417 | LR: 2.27e-04 | 5264.8ms/step | 11.4 samples/s
|
| 273 |
+
Step 27050/50462 | Loss: 2.5429 | LR: 2.27e-04 | 5263.9ms/step | 11.4 samples/s
|
| 274 |
+
Step 27060/50462 | Loss: 2.5310 | LR: 2.27e-04 | 5259.9ms/step | 11.4 samples/s
|
| 275 |
+
Step 27070/50462 | Loss: 2.5241 | LR: 2.27e-04 | 5250.3ms/step | 11.4 samples/s
|
| 276 |
+
Step 27080/50462 | Loss: 2.5489 | LR: 2.27e-04 | 5259.6ms/step | 11.4 samples/s
|
| 277 |
+
Step 27090/50462 | Loss: 2.5665 | LR: 2.27e-04 | 5259.3ms/step | 11.4 samples/s
|
| 278 |
+
Step 27100/50462 | Loss: 2.5524 | LR: 2.26e-04 | 5261.7ms/step | 11.4 samples/s
|
| 279 |
+
Step 27110/50462 | Loss: 2.5511 | LR: 2.26e-04 | 5258.3ms/step | 11.4 samples/s
|
| 280 |
+
Step 27120/50462 | Loss: 2.5439 | LR: 2.26e-04 | 5262.8ms/step | 11.4 samples/s
|
| 281 |
+
Step 27130/50462 | Loss: 2.5308 | LR: 2.26e-04 | 5257.1ms/step | 11.4 samples/s
|
| 282 |
+
Step 27140/50462 | Loss: 2.5517 | LR: 2.26e-04 | 5251.9ms/step | 11.4 samples/s
|
| 283 |
+
Step 27150/50462 | Loss: 2.5360 | LR: 2.26e-04 | 5256.2ms/step | 11.4 samples/s
|
| 284 |
+
Step 27160/50462 | Loss: 2.5471 | LR: 2.25e-04 | 5258.7ms/step | 11.4 samples/s
|
| 285 |
+
Step 27170/50462 | Loss: 2.5520 | LR: 2.25e-04 | 5258.6ms/step | 11.4 samples/s
|
| 286 |
+
Step 27180/50462 | Loss: 2.5487 | LR: 2.25e-04 | 5255.7ms/step | 11.4 samples/s
|
| 287 |
+
Step 27190/50462 | Loss: 2.5513 | LR: 2.25e-04 | 5259.4ms/step | 11.4 samples/s
|
| 288 |
+
Step 27200/50462 | Loss: 2.5424 | LR: 2.25e-04 | 5262.5ms/step | 11.4 samples/s
|
| 289 |
+
Step 27210/50462 | Loss: 2.5416 | LR: 2.25e-04 | 5260.5ms/step | 11.4 samples/s
|
| 290 |
+
Step 27220/50462 | Loss: 2.5302 | LR: 2.25e-04 | 5260.9ms/step | 11.4 samples/s
|
| 291 |
+
Step 27230/50462 | Loss: 2.5520 | LR: 2.24e-04 | 5261.9ms/step | 11.4 samples/s
|
| 292 |
+
Step 27240/50462 | Loss: 2.5634 | LR: 2.24e-04 | 5254.5ms/step | 11.4 samples/s
|
| 293 |
+
Step 27250/50462 | Loss: 2.5306 | LR: 2.24e-04 | 5257.4ms/step | 11.4 samples/s
|
| 294 |
+
Validation at step 27250: Loss = 2.5583, Perplexity = 12.91
|
| 295 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-27250
|
| 296 |
+
Step 27260/50462 | Loss: 2.5551 | LR: 2.24e-04 | 10187.5ms/step | 5.9 samples/s
|
| 297 |
+
Step 27270/50462 | Loss: 2.5729 | LR: 2.24e-04 | 5256.5ms/step | 11.4 samples/s
|
| 298 |
+
Step 27280/50462 | Loss: 2.5839 | LR: 2.24e-04 | 5263.4ms/step | 11.4 samples/s
|
| 299 |
+
Step 27290/50462 | Loss: 2.5326 | LR: 2.23e-04 | 5264.9ms/step | 11.4 samples/s
|
| 300 |
+
Step 27300/50462 | Loss: 2.5496 | LR: 2.23e-04 | 5266.5ms/step | 11.4 samples/s
|
| 301 |
+
Step 27310/50462 | Loss: 2.5701 | LR: 2.23e-04 | 5266.9ms/step | 11.4 samples/s
|
| 302 |
+
Step 27320/50462 | Loss: 2.5553 | LR: 2.23e-04 | 5264.2ms/step | 11.4 samples/s
|
| 303 |
+
Step 27330/50462 | Loss: 2.5670 | LR: 2.23e-04 | 5259.7ms/step | 11.4 samples/s
|
| 304 |
+
Step 27340/50462 | Loss: 2.5194 | LR: 2.23e-04 | 5267.5ms/step | 11.4 samples/s
|
| 305 |
+
Step 27350/50462 | Loss: 2.5437 | LR: 2.23e-04 | 5263.0ms/step | 11.4 samples/s
|
| 306 |
+
Step 27360/50462 | Loss: 2.5266 | LR: 2.22e-04 | 5265.7ms/step | 11.4 samples/s
|
| 307 |
+
Step 27370/50462 | Loss: 2.5356 | LR: 2.22e-04 | 5262.9ms/step | 11.4 samples/s
|
| 308 |
+
Step 27380/50462 | Loss: 2.5331 | LR: 2.22e-04 | 5265.5ms/step | 11.4 samples/s
|
| 309 |
+
Step 27390/50462 | Loss: 2.5479 | LR: 2.22e-04 | 5264.2ms/step | 11.4 samples/s
|
| 310 |
+
Step 27400/50462 | Loss: 2.5478 | LR: 2.22e-04 | 5266.9ms/step | 11.4 samples/s
|
| 311 |
+
Step 27410/50462 | Loss: 2.5555 | LR: 2.22e-04 | 5267.3ms/step | 11.4 samples/s
|
| 312 |
+
Step 27420/50462 | Loss: 2.5394 | LR: 2.21e-04 | 5263.9ms/step | 11.4 samples/s
|
| 313 |
+
Step 27430/50462 | Loss: 2.5522 | LR: 2.21e-04 | 5268.5ms/step | 11.4 samples/s
|
| 314 |
+
Step 27440/50462 | Loss: 2.5588 | LR: 2.21e-04 | 5269.8ms/step | 11.4 samples/s
|
| 315 |
+
Step 27450/50462 | Loss: 2.5416 | LR: 2.21e-04 | 5271.3ms/step | 11.4 samples/s
|
| 316 |
+
Step 27460/50462 | Loss: 2.5404 | LR: 2.21e-04 | 5257.9ms/step | 11.4 samples/s
|
| 317 |
+
Step 27470/50462 | Loss: 2.5315 | LR: 2.21e-04 | 5267.7ms/step | 11.4 samples/s
|
| 318 |
+
Step 27480/50462 | Loss: 2.5513 | LR: 2.20e-04 | 5268.0ms/step | 11.4 samples/s
|
| 319 |
+
Step 27490/50462 | Loss: 2.5399 | LR: 2.20e-04 | 5262.2ms/step | 11.4 samples/s
|
| 320 |
+
Step 27500/50462 | Loss: 2.5737 | LR: 2.20e-04 | 5261.6ms/step | 11.4 samples/s
|
| 321 |
+
Validation at step 27500: Loss = 2.5566, Perplexity = 12.89
|
| 322 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-27500
|
| 323 |
+
Step 27510/50462 | Loss: 2.5543 | LR: 2.20e-04 | 10188.2ms/step | 5.9 samples/s
|
| 324 |
+
Step 27520/50462 | Loss: 2.5230 | LR: 2.20e-04 | 5250.6ms/step | 11.4 samples/s
|
| 325 |
+
Step 27530/50462 | Loss: 2.5456 | LR: 2.20e-04 | 5265.3ms/step | 11.4 samples/s
|
| 326 |
+
Step 27540/50462 | Loss: 2.5183 | LR: 2.20e-04 | 5265.4ms/step | 11.4 samples/s
|
| 327 |
+
Step 27550/50462 | Loss: 2.5393 | LR: 2.19e-04 | 5265.4ms/step | 11.4 samples/s
|
| 328 |
+
Step 27560/50462 | Loss: 2.5420 | LR: 2.19e-04 | 5263.4ms/step | 11.4 samples/s
|
| 329 |
+
Step 27570/50462 | Loss: 2.5377 | LR: 2.19e-04 | 5265.7ms/step | 11.4 samples/s
|
| 330 |
+
Step 27580/50462 | Loss: 2.5277 | LR: 2.19e-04 | 5264.9ms/step | 11.4 samples/s
|
| 331 |
+
Step 27590/50462 | Loss: 2.5272 | LR: 2.19e-04 | 5258.7ms/step | 11.4 samples/s
|
| 332 |
+
Step 27600/50462 | Loss: 2.5710 | LR: 2.19e-04 | 5264.2ms/step | 11.4 samples/s
|
| 333 |
+
Step 27610/50462 | Loss: 2.5337 | LR: 2.18e-04 | 5268.2ms/step | 11.4 samples/s
|
| 334 |
+
Step 27620/50462 | Loss: 2.5532 | LR: 2.18e-04 | 5269.2ms/step | 11.4 samples/s
|
| 335 |
+
Step 27630/50462 | Loss: 2.5491 | LR: 2.18e-04 | 5267.2ms/step | 11.4 samples/s
|
| 336 |
+
Step 27640/50462 | Loss: 2.5527 | LR: 2.18e-04 | 5270.9ms/step | 11.4 samples/s
|
| 337 |
+
Step 27650/50462 | Loss: 2.5575 | LR: 2.18e-04 | 5264.6ms/step | 11.4 samples/s
|
| 338 |
+
Step 27660/50462 | Loss: 2.5284 | LR: 2.18e-04 | 5270.9ms/step | 11.4 samples/s
|
| 339 |
+
Step 27670/50462 | Loss: 2.5542 | LR: 2.17e-04 | 5257.6ms/step | 11.4 samples/s
|
| 340 |
+
Step 27680/50462 | Loss: 2.5464 | LR: 2.17e-04 | 5263.7ms/step | 11.4 samples/s
|
| 341 |
+
Step 27690/50462 | Loss: 2.5485 | LR: 2.17e-04 | 5258.9ms/step | 11.4 samples/s
|
| 342 |
+
Step 27700/50462 | Loss: 2.5395 | LR: 2.17e-04 | 5255.9ms/step | 11.4 samples/s
|
| 343 |
+
Step 27710/50462 | Loss: 2.5392 | LR: 2.17e-04 | 5265.2ms/step | 11.4 samples/s
|
| 344 |
+
Step 27720/50462 | Loss: 2.5337 | LR: 2.17e-04 | 5260.7ms/step | 11.4 samples/s
|
| 345 |
+
Step 27730/50462 | Loss: 2.5611 | LR: 2.17e-04 | 5255.3ms/step | 11.4 samples/s
|
| 346 |
+
Step 27740/50462 | Loss: 2.5562 | LR: 2.16e-04 | 5259.9ms/step | 11.4 samples/s
|
| 347 |
+
Step 27750/50462 | Loss: 2.5304 | LR: 2.16e-04 | 5261.2ms/step | 11.4 samples/s
|
| 348 |
+
Validation at step 27750: Loss = 2.5544, Perplexity = 12.86
|
| 349 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-27750
|
| 350 |
+
Step 27760/50462 | Loss: 2.5423 | LR: 2.16e-04 | 10131.2ms/step | 5.9 samples/s
|
| 351 |
+
Step 27770/50462 | Loss: 2.5413 | LR: 2.16e-04 | 5253.4ms/step | 11.4 samples/s
|
| 352 |
+
Step 27780/50462 | Loss: 2.5463 | LR: 2.16e-04 | 5258.1ms/step | 11.4 samples/s
|
| 353 |
+
Step 27790/50462 | Loss: 2.5480 | LR: 2.16e-04 | 5262.4ms/step | 11.4 samples/s
|
| 354 |
+
Step 27800/50462 | Loss: 2.5220 | LR: 2.15e-04 | 5266.4ms/step | 11.4 samples/s
|
| 355 |
+
Step 27810/50462 | Loss: 2.5394 | LR: 2.15e-04 | 5263.1ms/step | 11.4 samples/s
|
| 356 |
+
Step 27820/50462 | Loss: 2.5454 | LR: 2.15e-04 | 5258.1ms/step | 11.4 samples/s
|
| 357 |
+
Step 27830/50462 | Loss: 2.5460 | LR: 2.15e-04 | 5262.4ms/step | 11.4 samples/s
|
| 358 |
+
Step 27840/50462 | Loss: 2.5400 | LR: 2.15e-04 | 5258.7ms/step | 11.4 samples/s
|
| 359 |
+
Step 27850/50462 | Loss: 2.5068 | LR: 2.15e-04 | 5259.7ms/step | 11.4 samples/s
|
| 360 |
+
Step 27860/50462 | Loss: 2.5375 | LR: 2.15e-04 | 5258.9ms/step | 11.4 samples/s
|
| 361 |
+
Step 27870/50462 | Loss: 2.5526 | LR: 2.14e-04 | 5259.5ms/step | 11.4 samples/s
|
| 362 |
+
Step 27880/50462 | Loss: 2.5222 | LR: 2.14e-04 | 5251.8ms/step | 11.4 samples/s
|
| 363 |
+
Step 27890/50462 | Loss: 2.5494 | LR: 2.14e-04 | 5258.5ms/step | 11.4 samples/s
|
| 364 |
+
Step 27900/50462 | Loss: 2.5263 | LR: 2.14e-04 | 5253.9ms/step | 11.4 samples/s
|
| 365 |
+
Step 27910/50462 | Loss: 2.5251 | LR: 2.14e-04 | 5251.5ms/step | 11.4 samples/s
|
| 366 |
+
Step 27920/50462 | Loss: 2.5378 | LR: 2.14e-04 | 5252.6ms/step | 11.4 samples/s
|
| 367 |
+
Step 27930/50462 | Loss: 2.5478 | LR: 2.13e-04 | 5246.8ms/step | 11.4 samples/s
|
| 368 |
+
Step 27940/50462 | Loss: 2.5401 | LR: 2.13e-04 | 5248.0ms/step | 11.4 samples/s
|
| 369 |
+
Step 27950/50462 | Loss: 2.5372 | LR: 2.13e-04 | 5245.2ms/step | 11.4 samples/s
|
| 370 |
+
Step 27960/50462 | Loss: 2.5426 | LR: 2.13e-04 | 5256.1ms/step | 11.4 samples/s
|
| 371 |
+
Step 27970/50462 | Loss: 2.5752 | LR: 2.13e-04 | 5253.6ms/step | 11.4 samples/s
|
| 372 |
+
Step 27980/50462 | Loss: 2.5507 | LR: 2.13e-04 | 5252.0ms/step | 11.4 samples/s
|
| 373 |
+
Step 27990/50462 | Loss: 2.5516 | LR: 2.12e-04 | 5255.1ms/step | 11.4 samples/s
|
| 374 |
+
Step 28000/50462 | Loss: 2.5391 | LR: 2.12e-04 | 5258.8ms/step | 11.4 samples/s
|
| 375 |
+
Validation at step 28000: Loss = 2.5527, Perplexity = 12.84
|
| 376 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-28000
|
| 377 |
+
Step 28010/50462 | Loss: 2.5627 | LR: 2.12e-04 | 10195.1ms/step | 5.9 samples/s
|
| 378 |
+
Step 28020/50462 | Loss: 2.5472 | LR: 2.12e-04 | 5242.4ms/step | 11.4 samples/s
|
| 379 |
+
Step 28030/50462 | Loss: 2.5410 | LR: 2.12e-04 | 5258.4ms/step | 11.4 samples/s
|
| 380 |
+
Step 28040/50462 | Loss: 2.5492 | LR: 2.12e-04 | 5261.7ms/step | 11.4 samples/s
|
| 381 |
+
Step 28050/50462 | Loss: 2.5236 | LR: 2.12e-04 | 5262.7ms/step | 11.4 samples/s
|
| 382 |
+
Step 28060/50462 | Loss: 2.5275 | LR: 2.11e-04 | 5262.0ms/step | 11.4 samples/s
|
| 383 |
+
Step 28070/50462 | Loss: 2.5235 | LR: 2.11e-04 | 5265.7ms/step | 11.4 samples/s
|
| 384 |
+
Step 28080/50462 | Loss: 2.5544 | LR: 2.11e-04 | 5257.6ms/step | 11.4 samples/s
|
| 385 |
+
Step 28090/50462 | Loss: 2.5210 | LR: 2.11e-04 | 5257.9ms/step | 11.4 samples/s
|
| 386 |
+
Step 28100/50462 | Loss: 2.5418 | LR: 2.11e-04 | 5263.2ms/step | 11.4 samples/s
|
| 387 |
+
Step 28110/50462 | Loss: 2.5275 | LR: 2.11e-04 | 5256.4ms/step | 11.4 samples/s
|
| 388 |
+
Step 28120/50462 | Loss: 2.5278 | LR: 2.10e-04 | 5265.4ms/step | 11.4 samples/s
|
| 389 |
+
Step 28130/50462 | Loss: 2.5235 | LR: 2.10e-04 | 5263.9ms/step | 11.4 samples/s
|
| 390 |
+
Step 28140/50462 | Loss: 2.5428 | LR: 2.10e-04 | 5261.0ms/step | 11.4 samples/s
|
| 391 |
+
Step 28150/50462 | Loss: 2.5422 | LR: 2.10e-04 | 5262.3ms/step | 11.4 samples/s
|
| 392 |
+
Step 28160/50462 | Loss: 2.5169 | LR: 2.10e-04 | 5263.3ms/step | 11.4 samples/s
|
| 393 |
+
Step 28170/50462 | Loss: 2.5341 | LR: 2.10e-04 | 5262.7ms/step | 11.4 samples/s
|
| 394 |
+
Step 28180/50462 | Loss: 2.5402 | LR: 2.10e-04 | 5262.2ms/step | 11.4 samples/s
|
| 395 |
+
Step 28190/50462 | Loss: 2.5424 | LR: 2.09e-04 | 5265.1ms/step | 11.4 samples/s
|
| 396 |
+
Step 28200/50462 | Loss: 2.5146 | LR: 2.09e-04 | 5266.3ms/step | 11.4 samples/s
|
| 397 |
+
Step 28210/50462 | Loss: 2.5325 | LR: 2.09e-04 | 5256.8ms/step | 11.4 samples/s
|
| 398 |
+
Step 28220/50462 | Loss: 2.5630 | LR: 2.09e-04 | 5259.7ms/step | 11.4 samples/s
|
| 399 |
+
Step 28230/50462 | Loss: 2.5407 | LR: 2.09e-04 | 5266.9ms/step | 11.4 samples/s
|
| 400 |
+
Step 28240/50462 | Loss: 2.5368 | LR: 2.09e-04 | 5262.7ms/step | 11.4 samples/s
|
| 401 |
+
Step 28250/50462 | Loss: 2.5530 | LR: 2.08e-04 | 5261.8ms/step | 11.4 samples/s
|
| 402 |
+
Validation at step 28250: Loss = 2.5511, Perplexity = 12.82
|
| 403 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-28250
|
| 404 |
+
Step 28260/50462 | Loss: 2.5321 | LR: 2.08e-04 | 10167.7ms/step | 5.9 samples/s
|
| 405 |
+
Step 28270/50462 | Loss: 2.5436 | LR: 2.08e-04 | 5249.7ms/step | 11.4 samples/s
|
| 406 |
+
Step 28280/50462 | Loss: 2.5219 | LR: 2.08e-04 | 5261.0ms/step | 11.4 samples/s
|
| 407 |
+
Step 28290/50462 | Loss: 2.5453 | LR: 2.08e-04 | 5266.5ms/step | 11.4 samples/s
|
| 408 |
+
Step 28300/50462 | Loss: 2.5472 | LR: 2.08e-04 | 5266.3ms/step | 11.4 samples/s
|
| 409 |
+
Step 28310/50462 | Loss: 2.5197 | LR: 2.07e-04 | 5269.2ms/step | 11.4 samples/s
|
| 410 |
+
Step 28320/50462 | Loss: 2.5512 | LR: 2.07e-04 | 5262.5ms/step | 11.4 samples/s
|
| 411 |
+
Step 28330/50462 | Loss: 2.5459 | LR: 2.07e-04 | 5261.9ms/step | 11.4 samples/s
|
epfl_final/SparseAtt/wandb/run-20260825_211848-rgmkfpnl/files/requirements.txt
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
| 142 |
+
entmax==1.3
|
epfl_final/SparseAtt/wandb/run-20260825_211848-rgmkfpnl/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-08-25T21:18:48.108361Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"configs/p2_1p2b_dense_seed70_2gpu.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/SparseAtt.git",
|
| 14 |
+
"commit": "8f862f749eb4d5c17f688eb1562235e3fe5b2550"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "h100-workspace-2-gpu-0-18",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 23 |
+
"gpu_count": 2,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "9374931091456",
|
| 27 |
+
"used": "6969043943424"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1623005732864"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-cd5c7d97-0f62-2f41-36c8-233c8f57b814"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 43 |
+
"memoryTotal": "85520809984",
|
| 44 |
+
"cudaCores": 16896,
|
| 45 |
+
"architecture": "Hopper",
|
| 46 |
+
"uuid": "GPU-b97c2df9-b38b-c4e9-4d9d-780e91b62380"
|
| 47 |
+
}
|
| 48 |
+
],
|
| 49 |
+
"cudaVersion": "13.0",
|
| 50 |
+
"writerId": "nuwd316huzbbqyzdlgs1q4u9oj6wbi3f"
|
| 51 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260825_211848-rgmkfpnl/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/SparseAtt/wandb/run-20260825_211848-rgmkfpnl/logs/debug.log
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-08-25 21:18:48,138 INFO MainThread:244 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-08-25 21:18:48,139 INFO MainThread:244 [wandb_setup.py:_flush():81] Configure stats pid to 244
|
| 3 |
+
2026-08-25 21:18:48,139 INFO MainThread:244 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-08-25 21:18:48,139 INFO MainThread:244 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260825_211848-rgmkfpnl/logs/debug.log
|
| 5 |
+
2026-08-25 21:18:48,139 INFO MainThread:244 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260825_211848-rgmkfpnl/logs/debug-internal.log
|
| 6 |
+
2026-08-25 21:18:48,139 INFO MainThread:244 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-08-25 21:18:48,139 INFO MainThread:244 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-24750', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 15, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 70, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b dense 8k seed70 2gpu resume', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'dense', 'seed70'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
|
| 9 |
+
2026-08-25 21:18:48,139 INFO MainThread:244 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-08-25 21:18:49,141 INFO MainThread:244 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-08-25 21:18:49,714 INFO MainThread:244 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-08-25 21:18:49,721 INFO MainThread:244 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-08-25 21:18:49,746 INFO MainThread:244 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-08-25 21:18:50,343 INFO MainThread:244 [wandb_init.py:init():1051] run resumed
|
| 15 |
+
2026-08-25 21:18:50,346 INFO MainThread:244 [wandb_init.py:init():1056] starting run threads in backend
|
| 16 |
+
2026-08-25 21:18:50,635 INFO MainThread:244 [wandb_run.py:_console_start():2554] atexit reg
|
| 17 |
+
2026-08-25 21:18:50,635 INFO MainThread:244 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 18 |
+
2026-08-25 21:18:50,635 INFO MainThread:244 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 19 |
+
2026-08-25 21:18:50,635 INFO MainThread:244 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 20 |
+
2026-08-25 21:18:50,643 INFO MainThread:244 [wandb_init.py:init():1094] run started, returning control to user process
|
epfl_final/SparseAtt/wandb/run-20260825_211848-rgmkfpnl/run-rgmkfpnl.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c4ad58bcddb9e1d7768e6959b1375723931d0117c844f2320e58dd3bc81e931b
|
| 3 |
+
size 4227072
|
epfl_final/SparseAtt/wandb/run-20260826_034845-rgmkfpnl/files/output.log
ADDED
|
@@ -0,0 +1,429 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Starting training...
|
| 2 |
+
Validation at step 28250: Loss = 2.5511, Perplexity = 12.82
|
| 3 |
+
wandb: WARNING Tried to log to step 28250 that is less than the current step 28321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 4 |
+
Switching attention mode for step 28251: dense (dense step interval)
|
| 5 |
+
Step 28260/50462 | Loss: 2.5321 | LR: 2.08e-04 | 76912.1ms/step | 0.8 samples/s
|
| 6 |
+
wandb: WARNING Tried to log to step 28260 that is less than the current step 28321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 7 |
+
Step 28270/50462 | Loss: 2.5436 | LR: 2.08e-04 | 5139.4ms/step | 11.7 samples/s
|
| 8 |
+
wandb: WARNING Tried to log to step 28270 that is less than the current step 28321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 9 |
+
Step 28280/50462 | Loss: 2.5219 | LR: 2.08e-04 | 5143.6ms/step | 11.7 samples/s
|
| 10 |
+
wandb: WARNING Tried to log to step 28280 that is less than the current step 28321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 11 |
+
Step 28290/50462 | Loss: 2.5452 | LR: 2.08e-04 | 5144.2ms/step | 11.7 samples/s
|
| 12 |
+
wandb: WARNING Tried to log to step 28290 that is less than the current step 28321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 13 |
+
Step 28300/50462 | Loss: 2.5472 | LR: 2.08e-04 | 5148.4ms/step | 11.7 samples/s
|
| 14 |
+
wandb: WARNING Tried to log to step 28300 that is less than the current step 28321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 15 |
+
Step 28310/50462 | Loss: 2.5197 | LR: 2.07e-04 | 5142.9ms/step | 11.7 samples/s
|
| 16 |
+
wandb: WARNING Tried to log to step 28310 that is less than the current step 28321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 17 |
+
Step 28320/50462 | Loss: 2.5512 | LR: 2.07e-04 | 5144.7ms/step | 11.7 samples/s
|
| 18 |
+
wandb: WARNING Tried to log to step 28320 that is less than the current step 28321. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 19 |
+
Step 28330/50462 | Loss: 2.5459 | LR: 2.07e-04 | 5144.8ms/step | 11.7 samples/s
|
| 20 |
+
Step 28340/50462 | Loss: 2.5545 | LR: 2.07e-04 | 5145.0ms/step | 11.7 samples/s
|
| 21 |
+
Step 28350/50462 | Loss: 2.5360 | LR: 2.07e-04 | 5146.2ms/step | 11.7 samples/s
|
| 22 |
+
Step 28360/50462 | Loss: 2.5467 | LR: 2.07e-04 | 5141.6ms/step | 11.7 samples/s
|
| 23 |
+
Step 28370/50462 | Loss: 2.5317 | LR: 2.07e-04 | 5144.0ms/step | 11.7 samples/s
|
| 24 |
+
Step 28380/50462 | Loss: 2.5360 | LR: 2.06e-04 | 5136.2ms/step | 11.7 samples/s
|
| 25 |
+
Step 28390/50462 | Loss: 2.5145 | LR: 2.06e-04 | 5140.8ms/step | 11.7 samples/s
|
| 26 |
+
Step 28400/50462 | Loss: 2.5183 | LR: 2.06e-04 | 5140.7ms/step | 11.7 samples/s
|
| 27 |
+
Step 28410/50462 | Loss: 2.5360 | LR: 2.06e-04 | 5142.0ms/step | 11.7 samples/s
|
| 28 |
+
Step 28420/50462 | Loss: 2.5411 | LR: 2.06e-04 | 5139.5ms/step | 11.7 samples/s
|
| 29 |
+
Step 28430/50462 | Loss: 2.5263 | LR: 2.06e-04 | 5136.6ms/step | 11.7 samples/s
|
| 30 |
+
Step 28440/50462 | Loss: 2.5300 | LR: 2.05e-04 | 5140.9ms/step | 11.7 samples/s
|
| 31 |
+
Step 28450/50462 | Loss: 2.5634 | LR: 2.05e-04 | 5138.3ms/step | 11.7 samples/s
|
| 32 |
+
Step 28460/50462 | Loss: 2.5436 | LR: 2.05e-04 | 5139.3ms/step | 11.7 samples/s
|
| 33 |
+
Step 28470/50462 | Loss: 2.5500 | LR: 2.05e-04 | 5144.1ms/step | 11.7 samples/s
|
| 34 |
+
Step 28480/50462 | Loss: 2.5425 | LR: 2.05e-04 | 5140.5ms/step | 11.7 samples/s
|
| 35 |
+
Step 28490/50462 | Loss: 2.5057 | LR: 2.05e-04 | 5132.6ms/step | 11.7 samples/s
|
| 36 |
+
Step 28500/50462 | Loss: 2.5371 | LR: 2.05e-04 | 5137.4ms/step | 11.7 samples/s
|
| 37 |
+
Validation at step 28500: Loss = 2.5497, Perplexity = 12.80
|
| 38 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-28500
|
| 39 |
+
Step 28510/50462 | Loss: 2.5348 | LR: 2.04e-04 | 9964.7ms/step | 6.0 samples/s
|
| 40 |
+
Step 28520/50462 | Loss: 2.5404 | LR: 2.04e-04 | 5147.3ms/step | 11.7 samples/s
|
| 41 |
+
Step 28530/50462 | Loss: 2.5389 | LR: 2.04e-04 | 5149.2ms/step | 11.7 samples/s
|
| 42 |
+
Step 28540/50462 | Loss: 2.5463 | LR: 2.04e-04 | 5149.7ms/step | 11.7 samples/s
|
| 43 |
+
Step 28550/50462 | Loss: 2.5395 | LR: 2.04e-04 | 5144.5ms/step | 11.7 samples/s
|
| 44 |
+
Step 28560/50462 | Loss: 2.5355 | LR: 2.04e-04 | 5150.0ms/step | 11.7 samples/s
|
| 45 |
+
Step 28570/50462 | Loss: 2.5366 | LR: 2.03e-04 | 5145.3ms/step | 11.7 samples/s
|
| 46 |
+
Step 28580/50462 | Loss: 2.5209 | LR: 2.03e-04 | 5147.2ms/step | 11.7 samples/s
|
| 47 |
+
Step 28590/50462 | Loss: 2.5474 | LR: 2.03e-04 | 5150.7ms/step | 11.6 samples/s
|
| 48 |
+
Step 28600/50462 | Loss: 2.5342 | LR: 2.03e-04 | 5147.2ms/step | 11.7 samples/s
|
| 49 |
+
Step 28610/50462 | Loss: 2.5474 | LR: 2.03e-04 | 5150.3ms/step | 11.6 samples/s
|
| 50 |
+
Step 28620/50462 | Loss: 2.5448 | LR: 2.03e-04 | 5148.3ms/step | 11.7 samples/s
|
| 51 |
+
Step 28630/50462 | Loss: 2.5446 | LR: 2.03e-04 | 5151.9ms/step | 11.6 samples/s
|
| 52 |
+
Step 28640/50462 | Loss: 2.5584 | LR: 2.02e-04 | 5150.2ms/step | 11.7 samples/s
|
| 53 |
+
Step 28650/50462 | Loss: 2.5331 | LR: 2.02e-04 | 5152.5ms/step | 11.6 samples/s
|
| 54 |
+
Step 28660/50462 | Loss: 2.5328 | LR: 2.02e-04 | 5145.3ms/step | 11.7 samples/s
|
| 55 |
+
Step 28670/50462 | Loss: 2.5057 | LR: 2.02e-04 | 5145.2ms/step | 11.7 samples/s
|
| 56 |
+
Step 28680/50462 | Loss: 2.5223 | LR: 2.02e-04 | 5147.1ms/step | 11.7 samples/s
|
| 57 |
+
Step 28690/50462 | Loss: 2.5598 | LR: 2.02e-04 | 5144.0ms/step | 11.7 samples/s
|
| 58 |
+
Step 28700/50462 | Loss: 2.5483 | LR: 2.01e-04 | 5142.3ms/step | 11.7 samples/s
|
| 59 |
+
Step 28710/50462 | Loss: 2.5629 | LR: 2.01e-04 | 5151.0ms/step | 11.6 samples/s
|
| 60 |
+
Step 28720/50462 | Loss: 2.5199 | LR: 2.01e-04 | 5146.9ms/step | 11.7 samples/s
|
| 61 |
+
Step 28730/50462 | Loss: 2.5480 | LR: 2.01e-04 | 5152.2ms/step | 11.6 samples/s
|
| 62 |
+
Step 28740/50462 | Loss: 2.5272 | LR: 2.01e-04 | 5154.1ms/step | 11.6 samples/s
|
| 63 |
+
Step 28750/50462 | Loss: 2.5215 | LR: 2.01e-04 | 5145.8ms/step | 11.7 samples/s
|
| 64 |
+
Validation at step 28750: Loss = 2.5477, Perplexity = 12.78
|
| 65 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-28750
|
| 66 |
+
Step 28760/50462 | Loss: 2.5648 | LR: 2.00e-04 | 9987.8ms/step | 6.0 samples/s
|
| 67 |
+
Step 28770/50462 | Loss: 2.5296 | LR: 2.00e-04 | 5150.4ms/step | 11.6 samples/s
|
| 68 |
+
Step 28780/50462 | Loss: 2.5427 | LR: 2.00e-04 | 5154.3ms/step | 11.6 samples/s
|
| 69 |
+
Step 28790/50462 | Loss: 2.5264 | LR: 2.00e-04 | 5157.3ms/step | 11.6 samples/s
|
| 70 |
+
Step 28800/50462 | Loss: 2.5500 | LR: 2.00e-04 | 5147.7ms/step | 11.7 samples/s
|
| 71 |
+
Step 28810/50462 | Loss: 2.5199 | LR: 2.00e-04 | 5146.6ms/step | 11.7 samples/s
|
| 72 |
+
Step 28820/50462 | Loss: 2.5492 | LR: 2.00e-04 | 5155.7ms/step | 11.6 samples/s
|
| 73 |
+
Step 28830/50462 | Loss: 2.5150 | LR: 1.99e-04 | 5146.3ms/step | 11.7 samples/s
|
| 74 |
+
Step 28840/50462 | Loss: 2.5315 | LR: 1.99e-04 | 5147.9ms/step | 11.7 samples/s
|
| 75 |
+
Step 28850/50462 | Loss: 2.5170 | LR: 1.99e-04 | 5151.2ms/step | 11.6 samples/s
|
| 76 |
+
Step 28860/50462 | Loss: 2.5327 | LR: 1.99e-04 | 5148.6ms/step | 11.7 samples/s
|
| 77 |
+
Step 28870/50462 | Loss: 2.5415 | LR: 1.99e-04 | 5151.5ms/step | 11.6 samples/s
|
| 78 |
+
Step 28880/50462 | Loss: 2.5410 | LR: 1.99e-04 | 5140.1ms/step | 11.7 samples/s
|
| 79 |
+
Step 28890/50462 | Loss: 2.5383 | LR: 1.98e-04 | 5142.3ms/step | 11.7 samples/s
|
| 80 |
+
Step 28900/50462 | Loss: 2.5227 | LR: 1.98e-04 | 5139.4ms/step | 11.7 samples/s
|
| 81 |
+
Step 28910/50462 | Loss: 2.5265 | LR: 1.98e-04 | 5146.5ms/step | 11.7 samples/s
|
| 82 |
+
Step 28920/50462 | Loss: 2.5359 | LR: 1.98e-04 | 5150.8ms/step | 11.6 samples/s
|
| 83 |
+
Step 28930/50462 | Loss: 2.5423 | LR: 1.98e-04 | 5150.3ms/step | 11.6 samples/s
|
| 84 |
+
Step 28940/50462 | Loss: 2.5474 | LR: 1.98e-04 | 5143.3ms/step | 11.7 samples/s
|
| 85 |
+
Step 28950/50462 | Loss: 2.5194 | LR: 1.98e-04 | 5149.7ms/step | 11.7 samples/s
|
| 86 |
+
Step 28960/50462 | Loss: 2.5347 | LR: 1.97e-04 | 5148.5ms/step | 11.7 samples/s
|
| 87 |
+
Step 28970/50462 | Loss: 2.5122 | LR: 1.97e-04 | 5149.6ms/step | 11.7 samples/s
|
| 88 |
+
Step 28980/50462 | Loss: 2.5339 | LR: 1.97e-04 | 5145.8ms/step | 11.7 samples/s
|
| 89 |
+
Step 28990/50462 | Loss: 2.5345 | LR: 1.97e-04 | 5147.5ms/step | 11.7 samples/s
|
| 90 |
+
Step 29000/50462 | Loss: 2.5544 | LR: 1.97e-04 | 5147.8ms/step | 11.7 samples/s
|
| 91 |
+
Validation at step 29000: Loss = 2.5463, Perplexity = 12.76
|
| 92 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-29000
|
| 93 |
+
Step 29010/50462 | Loss: 2.5448 | LR: 1.97e-04 | 9953.1ms/step | 6.0 samples/s
|
| 94 |
+
Step 29020/50462 | Loss: 2.5409 | LR: 1.96e-04 | 5135.4ms/step | 11.7 samples/s
|
| 95 |
+
Step 29030/50462 | Loss: 2.5424 | LR: 1.96e-04 | 5138.4ms/step | 11.7 samples/s
|
| 96 |
+
Step 29040/50462 | Loss: 2.5426 | LR: 1.96e-04 | 5137.3ms/step | 11.7 samples/s
|
| 97 |
+
Step 29050/50462 | Loss: 2.5210 | LR: 1.96e-04 | 5138.9ms/step | 11.7 samples/s
|
| 98 |
+
Step 29060/50462 | Loss: 2.5210 | LR: 1.96e-04 | 5136.4ms/step | 11.7 samples/s
|
| 99 |
+
Step 29070/50462 | Loss: 2.5534 | LR: 1.96e-04 | 5138.7ms/step | 11.7 samples/s
|
| 100 |
+
Step 29080/50462 | Loss: 2.5518 | LR: 1.96e-04 | 5139.3ms/step | 11.7 samples/s
|
| 101 |
+
Step 29090/50462 | Loss: 2.5484 | LR: 1.95e-04 | 5144.5ms/step | 11.7 samples/s
|
| 102 |
+
Step 29100/50462 | Loss: 2.5496 | LR: 1.95e-04 | 5139.3ms/step | 11.7 samples/s
|
| 103 |
+
Step 29110/50462 | Loss: 2.5552 | LR: 1.95e-04 | 5132.6ms/step | 11.7 samples/s
|
| 104 |
+
Step 29120/50462 | Loss: 2.5222 | LR: 1.95e-04 | 5136.4ms/step | 11.7 samples/s
|
| 105 |
+
Step 29130/50462 | Loss: 2.5404 | LR: 1.95e-04 | 5139.6ms/step | 11.7 samples/s
|
| 106 |
+
Step 29140/50462 | Loss: 2.5239 | LR: 1.95e-04 | 5142.1ms/step | 11.7 samples/s
|
| 107 |
+
Step 29150/50462 | Loss: 2.5382 | LR: 1.94e-04 | 5137.7ms/step | 11.7 samples/s
|
| 108 |
+
Step 29160/50462 | Loss: 2.5373 | LR: 1.94e-04 | 5138.3ms/step | 11.7 samples/s
|
| 109 |
+
Step 29170/50462 | Loss: 2.4782 | LR: 1.94e-04 | 5139.4ms/step | 11.7 samples/s
|
| 110 |
+
Step 29180/50462 | Loss: 2.5339 | LR: 1.94e-04 | 5136.9ms/step | 11.7 samples/s
|
| 111 |
+
Step 29190/50462 | Loss: 2.5229 | LR: 1.94e-04 | 5137.2ms/step | 11.7 samples/s
|
| 112 |
+
Step 29200/50462 | Loss: 2.5423 | LR: 1.94e-04 | 5136.0ms/step | 11.7 samples/s
|
| 113 |
+
Step 29210/50462 | Loss: 2.5133 | LR: 1.94e-04 | 5142.1ms/step | 11.7 samples/s
|
| 114 |
+
Step 29220/50462 | Loss: 2.5252 | LR: 1.93e-04 | 5135.9ms/step | 11.7 samples/s
|
| 115 |
+
Step 29230/50462 | Loss: 2.5181 | LR: 1.93e-04 | 5135.7ms/step | 11.7 samples/s
|
| 116 |
+
Step 29240/50462 | Loss: 2.5331 | LR: 1.93e-04 | 5135.0ms/step | 11.7 samples/s
|
| 117 |
+
Step 29250/50462 | Loss: 2.5178 | LR: 1.93e-04 | 5136.2ms/step | 11.7 samples/s
|
| 118 |
+
Validation at step 29250: Loss = 2.5451, Perplexity = 12.74
|
| 119 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-29250
|
| 120 |
+
Step 29260/50462 | Loss: 2.5439 | LR: 1.93e-04 | 9952.6ms/step | 6.0 samples/s
|
| 121 |
+
Step 29270/50462 | Loss: 2.5307 | LR: 1.93e-04 | 5138.0ms/step | 11.7 samples/s
|
| 122 |
+
Step 29280/50462 | Loss: 2.5416 | LR: 1.92e-04 | 5138.1ms/step | 11.7 samples/s
|
| 123 |
+
Step 29290/50462 | Loss: 2.5319 | LR: 1.92e-04 | 5138.4ms/step | 11.7 samples/s
|
| 124 |
+
Step 29300/50462 | Loss: 2.5354 | LR: 1.92e-04 | 5135.3ms/step | 11.7 samples/s
|
| 125 |
+
Step 29310/50462 | Loss: 2.5549 | LR: 1.92e-04 | 5131.9ms/step | 11.7 samples/s
|
| 126 |
+
Step 29320/50462 | Loss: 2.5470 | LR: 1.92e-04 | 5141.0ms/step | 11.7 samples/s
|
| 127 |
+
Step 29330/50462 | Loss: 2.5519 | LR: 1.92e-04 | 5135.1ms/step | 11.7 samples/s
|
| 128 |
+
Step 29340/50462 | Loss: 2.5504 | LR: 1.92e-04 | 5132.8ms/step | 11.7 samples/s
|
| 129 |
+
Step 29350/50462 | Loss: 2.5222 | LR: 1.91e-04 | 5135.4ms/step | 11.7 samples/s
|
| 130 |
+
Step 29360/50462 | Loss: 2.5275 | LR: 1.91e-04 | 5132.7ms/step | 11.7 samples/s
|
| 131 |
+
Step 29370/50462 | Loss: 2.5207 | LR: 1.91e-04 | 5136.7ms/step | 11.7 samples/s
|
| 132 |
+
Step 29380/50462 | Loss: 2.5210 | LR: 1.91e-04 | 5130.8ms/step | 11.7 samples/s
|
| 133 |
+
Step 29390/50462 | Loss: 2.5474 | LR: 1.91e-04 | 5133.8ms/step | 11.7 samples/s
|
| 134 |
+
Step 29400/50462 | Loss: 2.5374 | LR: 1.91e-04 | 5138.5ms/step | 11.7 samples/s
|
| 135 |
+
Step 29410/50462 | Loss: 2.5266 | LR: 1.90e-04 | 5139.6ms/step | 11.7 samples/s
|
| 136 |
+
Step 29420/50462 | Loss: 2.5165 | LR: 1.90e-04 | 5145.1ms/step | 11.7 samples/s
|
| 137 |
+
Step 29430/50462 | Loss: 2.5440 | LR: 1.90e-04 | 5144.7ms/step | 11.7 samples/s
|
| 138 |
+
Step 29440/50462 | Loss: 2.5247 | LR: 1.90e-04 | 5141.3ms/step | 11.7 samples/s
|
| 139 |
+
Step 29450/50462 | Loss: 2.5191 | LR: 1.90e-04 | 5138.6ms/step | 11.7 samples/s
|
| 140 |
+
Step 29460/50462 | Loss: 2.5407 | LR: 1.90e-04 | 5137.7ms/step | 11.7 samples/s
|
| 141 |
+
Step 29470/50462 | Loss: 2.5446 | LR: 1.90e-04 | 5132.6ms/step | 11.7 samples/s
|
| 142 |
+
Step 29480/50462 | Loss: 2.5514 | LR: 1.89e-04 | 5135.7ms/step | 11.7 samples/s
|
| 143 |
+
Step 29490/50462 | Loss: 2.5377 | LR: 1.89e-04 | 5132.9ms/step | 11.7 samples/s
|
| 144 |
+
Step 29500/50462 | Loss: 2.5328 | LR: 1.89e-04 | 5139.2ms/step | 11.7 samples/s
|
| 145 |
+
Validation at step 29500: Loss = 2.5437, Perplexity = 12.73
|
| 146 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-29500
|
| 147 |
+
Step 29510/50462 | Loss: 2.5185 | LR: 1.89e-04 | 9944.8ms/step | 6.0 samples/s
|
| 148 |
+
Step 29520/50462 | Loss: 2.5428 | LR: 1.89e-04 | 5139.4ms/step | 11.7 samples/s
|
| 149 |
+
Step 29530/50462 | Loss: 2.5175 | LR: 1.89e-04 | 5145.1ms/step | 11.7 samples/s
|
| 150 |
+
Step 29540/50462 | Loss: 2.5268 | LR: 1.88e-04 | 5147.5ms/step | 11.7 samples/s
|
| 151 |
+
Step 29550/50462 | Loss: 2.5328 | LR: 1.88e-04 | 5143.5ms/step | 11.7 samples/s
|
| 152 |
+
Step 29560/50462 | Loss: 2.5377 | LR: 1.88e-04 | 5142.1ms/step | 11.7 samples/s
|
| 153 |
+
Step 29570/50462 | Loss: 2.5374 | LR: 1.88e-04 | 5142.1ms/step | 11.7 samples/s
|
| 154 |
+
Step 29580/50462 | Loss: 2.5672 | LR: 1.88e-04 | 5146.8ms/step | 11.7 samples/s
|
| 155 |
+
Step 29590/50462 | Loss: 2.5259 | LR: 1.88e-04 | 5145.8ms/step | 11.7 samples/s
|
| 156 |
+
Step 29600/50462 | Loss: 2.5331 | LR: 1.88e-04 | 5149.2ms/step | 11.7 samples/s
|
| 157 |
+
Step 29610/50462 | Loss: 2.5265 | LR: 1.87e-04 | 5150.7ms/step | 11.6 samples/s
|
| 158 |
+
Step 29620/50462 | Loss: 2.5408 | LR: 1.87e-04 | 5148.1ms/step | 11.7 samples/s
|
| 159 |
+
Step 29630/50462 | Loss: 2.5457 | LR: 1.87e-04 | 5140.7ms/step | 11.7 samples/s
|
| 160 |
+
Step 29640/50462 | Loss: 2.5242 | LR: 1.87e-04 | 5145.3ms/step | 11.7 samples/s
|
| 161 |
+
Step 29650/50462 | Loss: 2.5293 | LR: 1.87e-04 | 5149.8ms/step | 11.7 samples/s
|
| 162 |
+
Step 29660/50462 | Loss: 2.5272 | LR: 1.87e-04 | 5140.6ms/step | 11.7 samples/s
|
| 163 |
+
Step 29670/50462 | Loss: 2.5336 | LR: 1.86e-04 | 5140.2ms/step | 11.7 samples/s
|
| 164 |
+
Step 29680/50462 | Loss: 2.5255 | LR: 1.86e-04 | 5144.6ms/step | 11.7 samples/s
|
| 165 |
+
Step 29690/50462 | Loss: 2.5065 | LR: 1.86e-04 | 5145.0ms/step | 11.7 samples/s
|
| 166 |
+
Step 29700/50462 | Loss: 2.5423 | LR: 1.86e-04 | 5144.1ms/step | 11.7 samples/s
|
| 167 |
+
Step 29710/50462 | Loss: 2.5312 | LR: 1.86e-04 | 5147.7ms/step | 11.7 samples/s
|
| 168 |
+
Step 29720/50462 | Loss: 2.5327 | LR: 1.86e-04 | 5139.8ms/step | 11.7 samples/s
|
| 169 |
+
Step 29730/50462 | Loss: 2.5201 | LR: 1.86e-04 | 5144.1ms/step | 11.7 samples/s
|
| 170 |
+
Step 29740/50462 | Loss: 2.5250 | LR: 1.85e-04 | 5148.1ms/step | 11.7 samples/s
|
| 171 |
+
Step 29750/50462 | Loss: 2.5408 | LR: 1.85e-04 | 5148.2ms/step | 11.7 samples/s
|
| 172 |
+
Validation at step 29750: Loss = 2.5424, Perplexity = 12.71
|
| 173 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-29750
|
| 174 |
+
Step 29760/50462 | Loss: 2.5271 | LR: 1.85e-04 | 10007.9ms/step | 6.0 samples/s
|
| 175 |
+
Step 29770/50462 | Loss: 2.5362 | LR: 1.85e-04 | 5138.8ms/step | 11.7 samples/s
|
| 176 |
+
Step 29780/50462 | Loss: 2.5315 | LR: 1.85e-04 | 5141.3ms/step | 11.7 samples/s
|
| 177 |
+
Step 29790/50462 | Loss: 2.5265 | LR: 1.85e-04 | 5138.5ms/step | 11.7 samples/s
|
| 178 |
+
Step 29800/50462 | Loss: 2.5272 | LR: 1.85e-04 | 5143.6ms/step | 11.7 samples/s
|
| 179 |
+
Step 29810/50462 | Loss: 2.5472 | LR: 1.84e-04 | 5138.1ms/step | 11.7 samples/s
|
| 180 |
+
Step 29820/50462 | Loss: 2.5302 | LR: 1.84e-04 | 5147.5ms/step | 11.7 samples/s
|
| 181 |
+
Step 29830/50462 | Loss: 2.5165 | LR: 1.84e-04 | 5153.2ms/step | 11.6 samples/s
|
| 182 |
+
Step 29840/50462 | Loss: 2.5171 | LR: 1.84e-04 | 5155.3ms/step | 11.6 samples/s
|
| 183 |
+
Step 29850/50462 | Loss: 2.5390 | LR: 1.84e-04 | 5138.9ms/step | 11.7 samples/s
|
| 184 |
+
Step 29860/50462 | Loss: 2.5055 | LR: 1.84e-04 | 5141.5ms/step | 11.7 samples/s
|
| 185 |
+
Step 29870/50462 | Loss: 2.5327 | LR: 1.83e-04 | 5149.5ms/step | 11.7 samples/s
|
| 186 |
+
Step 29880/50462 | Loss: 2.5393 | LR: 1.83e-04 | 5145.8ms/step | 11.7 samples/s
|
| 187 |
+
Step 29890/50462 | Loss: 2.5329 | LR: 1.83e-04 | 5142.9ms/step | 11.7 samples/s
|
| 188 |
+
Step 29900/50462 | Loss: 2.5094 | LR: 1.83e-04 | 5144.0ms/step | 11.7 samples/s
|
| 189 |
+
Step 29910/50462 | Loss: 2.5271 | LR: 1.83e-04 | 5146.8ms/step | 11.7 samples/s
|
| 190 |
+
Step 29920/50462 | Loss: 2.5292 | LR: 1.83e-04 | 5139.3ms/step | 11.7 samples/s
|
| 191 |
+
Step 29930/50462 | Loss: 2.5389 | LR: 1.83e-04 | 5143.6ms/step | 11.7 samples/s
|
| 192 |
+
Step 29940/50462 | Loss: 2.5375 | LR: 1.82e-04 | 5139.8ms/step | 11.7 samples/s
|
| 193 |
+
Step 29950/50462 | Loss: 2.5272 | LR: 1.82e-04 | 5145.7ms/step | 11.7 samples/s
|
| 194 |
+
Step 29960/50462 | Loss: 2.5328 | LR: 1.82e-04 | 5146.1ms/step | 11.7 samples/s
|
| 195 |
+
Step 29970/50462 | Loss: 2.5421 | LR: 1.82e-04 | 5152.0ms/step | 11.6 samples/s
|
| 196 |
+
Step 29980/50462 | Loss: 2.5263 | LR: 1.82e-04 | 5147.3ms/step | 11.7 samples/s
|
| 197 |
+
Step 29990/50462 | Loss: 2.5593 | LR: 1.82e-04 | 5146.8ms/step | 11.7 samples/s
|
| 198 |
+
Step 30000/50462 | Loss: 2.5241 | LR: 1.81e-04 | 5147.1ms/step | 11.7 samples/s
|
| 199 |
+
Validation at step 30000: Loss = 2.5412, Perplexity = 12.69
|
| 200 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-30000
|
| 201 |
+
Step 30010/50462 | Loss: 2.5122 | LR: 1.81e-04 | 9964.1ms/step | 6.0 samples/s
|
| 202 |
+
Step 30020/50462 | Loss: 2.5166 | LR: 1.81e-04 | 5129.9ms/step | 11.7 samples/s
|
| 203 |
+
Step 30030/50462 | Loss: 2.5387 | LR: 1.81e-04 | 5133.3ms/step | 11.7 samples/s
|
| 204 |
+
Step 30040/50462 | Loss: 2.5279 | LR: 1.81e-04 | 5127.8ms/step | 11.7 samples/s
|
| 205 |
+
Step 30050/50462 | Loss: 2.5394 | LR: 1.81e-04 | 5129.6ms/step | 11.7 samples/s
|
| 206 |
+
Step 30060/50462 | Loss: 2.5046 | LR: 1.81e-04 | 5135.1ms/step | 11.7 samples/s
|
| 207 |
+
Step 30070/50462 | Loss: 2.5198 | LR: 1.80e-04 | 5133.2ms/step | 11.7 samples/s
|
| 208 |
+
Step 30080/50462 | Loss: 2.5059 | LR: 1.80e-04 | 5129.2ms/step | 11.7 samples/s
|
| 209 |
+
Step 30090/50462 | Loss: 2.5388 | LR: 1.80e-04 | 5129.8ms/step | 11.7 samples/s
|
| 210 |
+
Step 30100/50462 | Loss: 2.5356 | LR: 1.80e-04 | 5129.5ms/step | 11.7 samples/s
|
| 211 |
+
Step 30110/50462 | Loss: 2.5330 | LR: 1.80e-04 | 5139.1ms/step | 11.7 samples/s
|
| 212 |
+
Step 30120/50462 | Loss: 2.5094 | LR: 1.80e-04 | 5138.9ms/step | 11.7 samples/s
|
| 213 |
+
Step 30130/50462 | Loss: 2.5400 | LR: 1.79e-04 | 5140.5ms/step | 11.7 samples/s
|
| 214 |
+
Step 30140/50462 | Loss: 2.5472 | LR: 1.79e-04 | 5133.4ms/step | 11.7 samples/s
|
| 215 |
+
Step 30150/50462 | Loss: 2.5217 | LR: 1.79e-04 | 5130.2ms/step | 11.7 samples/s
|
| 216 |
+
Step 30160/50462 | Loss: 2.5274 | LR: 1.79e-04 | 5141.0ms/step | 11.7 samples/s
|
| 217 |
+
Step 30170/50462 | Loss: 2.5281 | LR: 1.79e-04 | 5136.7ms/step | 11.7 samples/s
|
| 218 |
+
Step 30180/50462 | Loss: 2.5324 | LR: 1.79e-04 | 5142.3ms/step | 11.7 samples/s
|
| 219 |
+
Step 30190/50462 | Loss: 2.5035 | LR: 1.79e-04 | 5137.5ms/step | 11.7 samples/s
|
| 220 |
+
Step 30200/50462 | Loss: 2.5499 | LR: 1.78e-04 | 5139.9ms/step | 11.7 samples/s
|
| 221 |
+
Step 30210/50462 | Loss: 2.5274 | LR: 1.78e-04 | 5138.4ms/step | 11.7 samples/s
|
| 222 |
+
Step 30220/50462 | Loss: 2.5143 | LR: 1.78e-04 | 5134.4ms/step | 11.7 samples/s
|
| 223 |
+
Step 30230/50462 | Loss: 2.5193 | LR: 1.78e-04 | 5133.9ms/step | 11.7 samples/s
|
| 224 |
+
Step 30240/50462 | Loss: 2.5251 | LR: 1.78e-04 | 5140.1ms/step | 11.7 samples/s
|
| 225 |
+
Step 30250/50462 | Loss: 2.5539 | LR: 1.78e-04 | 5136.9ms/step | 11.7 samples/s
|
| 226 |
+
Validation at step 30250: Loss = 2.5400, Perplexity = 12.68
|
| 227 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-30250
|
| 228 |
+
Step 30260/50462 | Loss: 2.5122 | LR: 1.78e-04 | 9945.5ms/step | 6.0 samples/s
|
| 229 |
+
Step 30270/50462 | Loss: 2.5175 | LR: 1.77e-04 | 5144.7ms/step | 11.7 samples/s
|
| 230 |
+
Step 30280/50462 | Loss: 2.5336 | LR: 1.77e-04 | 5140.7ms/step | 11.7 samples/s
|
| 231 |
+
Step 30290/50462 | Loss: 2.5398 | LR: 1.77e-04 | 5145.6ms/step | 11.7 samples/s
|
| 232 |
+
Step 30300/50462 | Loss: 2.5386 | LR: 1.77e-04 | 5146.3ms/step | 11.7 samples/s
|
| 233 |
+
Step 30310/50462 | Loss: 2.5192 | LR: 1.77e-04 | 5147.3ms/step | 11.7 samples/s
|
| 234 |
+
Step 30320/50462 | Loss: 2.5271 | LR: 1.77e-04 | 5143.8ms/step | 11.7 samples/s
|
| 235 |
+
Step 30330/50462 | Loss: 2.5433 | LR: 1.76e-04 | 5146.0ms/step | 11.7 samples/s
|
| 236 |
+
Step 30340/50462 | Loss: 2.5222 | LR: 1.76e-04 | 5144.2ms/step | 11.7 samples/s
|
| 237 |
+
Step 30350/50462 | Loss: 2.5191 | LR: 1.76e-04 | 5152.5ms/step | 11.6 samples/s
|
| 238 |
+
Step 30360/50462 | Loss: 2.5035 | LR: 1.76e-04 | 5147.0ms/step | 11.7 samples/s
|
| 239 |
+
Step 30370/50462 | Loss: 2.5282 | LR: 1.76e-04 | 5150.2ms/step | 11.7 samples/s
|
| 240 |
+
Step 30380/50462 | Loss: 2.5425 | LR: 1.76e-04 | 5149.3ms/step | 11.7 samples/s
|
| 241 |
+
Step 30390/50462 | Loss: 2.5307 | LR: 1.76e-04 | 5144.4ms/step | 11.7 samples/s
|
| 242 |
+
Step 30400/50462 | Loss: 2.5262 | LR: 1.75e-04 | 5140.9ms/step | 11.7 samples/s
|
| 243 |
+
Step 30410/50462 | Loss: 2.5053 | LR: 1.75e-04 | 5146.7ms/step | 11.7 samples/s
|
| 244 |
+
Step 30420/50462 | Loss: 2.5265 | LR: 1.75e-04 | 5145.6ms/step | 11.7 samples/s
|
| 245 |
+
Step 30430/50462 | Loss: 2.5347 | LR: 1.75e-04 | 5145.1ms/step | 11.7 samples/s
|
| 246 |
+
Step 30440/50462 | Loss: 2.5251 | LR: 1.75e-04 | 5147.0ms/step | 11.7 samples/s
|
| 247 |
+
Step 30450/50462 | Loss: 2.5256 | LR: 1.75e-04 | 5148.4ms/step | 11.7 samples/s
|
| 248 |
+
Step 30460/50462 | Loss: 2.5226 | LR: 1.75e-04 | 5143.1ms/step | 11.7 samples/s
|
| 249 |
+
Step 30470/50462 | Loss: 2.5142 | LR: 1.74e-04 | 5143.3ms/step | 11.7 samples/s
|
| 250 |
+
Step 30480/50462 | Loss: 2.5086 | LR: 1.74e-04 | 5149.7ms/step | 11.7 samples/s
|
| 251 |
+
Step 30490/50462 | Loss: 2.5353 | LR: 1.74e-04 | 5147.8ms/step | 11.7 samples/s
|
| 252 |
+
Step 30500/50462 | Loss: 2.5111 | LR: 1.74e-04 | 5145.0ms/step | 11.7 samples/s
|
| 253 |
+
Validation at step 30500: Loss = 2.5385, Perplexity = 12.66
|
| 254 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-30500
|
| 255 |
+
Step 30510/50462 | Loss: 2.5266 | LR: 1.74e-04 | 9961.6ms/step | 6.0 samples/s
|
| 256 |
+
Step 30520/50462 | Loss: 2.5220 | LR: 1.74e-04 | 5144.3ms/step | 11.7 samples/s
|
| 257 |
+
Step 30530/50462 | Loss: 2.5127 | LR: 1.73e-04 | 5139.3ms/step | 11.7 samples/s
|
| 258 |
+
Step 30540/50462 | Loss: 2.5188 | LR: 1.73e-04 | 5144.5ms/step | 11.7 samples/s
|
| 259 |
+
Step 30550/50462 | Loss: 2.5182 | LR: 1.73e-04 | 5144.2ms/step | 11.7 samples/s
|
| 260 |
+
Step 30560/50462 | Loss: 2.4971 | LR: 1.73e-04 | 5151.7ms/step | 11.6 samples/s
|
| 261 |
+
Step 30570/50462 | Loss: 2.5434 | LR: 1.73e-04 | 5152.2ms/step | 11.6 samples/s
|
| 262 |
+
Step 30580/50462 | Loss: 2.5222 | LR: 1.73e-04 | 5156.5ms/step | 11.6 samples/s
|
| 263 |
+
Step 30590/50462 | Loss: 2.5259 | LR: 1.73e-04 | 5147.5ms/step | 11.7 samples/s
|
| 264 |
+
Step 30600/50462 | Loss: 2.4988 | LR: 1.72e-04 | 5151.6ms/step | 11.6 samples/s
|
| 265 |
+
Step 30610/50462 | Loss: 2.5448 | LR: 1.72e-04 | 5150.4ms/step | 11.6 samples/s
|
| 266 |
+
Step 30620/50462 | Loss: 2.5127 | LR: 1.72e-04 | 5144.1ms/step | 11.7 samples/s
|
| 267 |
+
Step 30630/50462 | Loss: 2.5224 | LR: 1.72e-04 | 5145.4ms/step | 11.7 samples/s
|
| 268 |
+
Step 30640/50462 | Loss: 2.5325 | LR: 1.72e-04 | 5148.0ms/step | 11.7 samples/s
|
| 269 |
+
Step 30650/50462 | Loss: 2.5178 | LR: 1.72e-04 | 5141.0ms/step | 11.7 samples/s
|
| 270 |
+
Step 30660/50462 | Loss: 2.5372 | LR: 1.72e-04 | 5150.9ms/step | 11.6 samples/s
|
| 271 |
+
Step 30670/50462 | Loss: 2.5402 | LR: 1.71e-04 | 5144.8ms/step | 11.7 samples/s
|
| 272 |
+
Step 30680/50462 | Loss: 2.5405 | LR: 1.71e-04 | 5143.8ms/step | 11.7 samples/s
|
| 273 |
+
Step 30690/50462 | Loss: 2.4944 | LR: 1.71e-04 | 5147.4ms/step | 11.7 samples/s
|
| 274 |
+
Step 30700/50462 | Loss: 2.5313 | LR: 1.71e-04 | 5152.3ms/step | 11.6 samples/s
|
| 275 |
+
Step 30710/50462 | Loss: 2.5200 | LR: 1.71e-04 | 5142.8ms/step | 11.7 samples/s
|
| 276 |
+
Step 30720/50462 | Loss: 2.5346 | LR: 1.71e-04 | 5149.0ms/step | 11.7 samples/s
|
| 277 |
+
Step 30730/50462 | Loss: 2.5389 | LR: 1.70e-04 | 5151.2ms/step | 11.6 samples/s
|
| 278 |
+
Step 30740/50462 | Loss: 2.5250 | LR: 1.70e-04 | 5144.6ms/step | 11.7 samples/s
|
| 279 |
+
Step 30750/50462 | Loss: 2.5406 | LR: 1.70e-04 | 5150.7ms/step | 11.6 samples/s
|
| 280 |
+
Validation at step 30750: Loss = 2.5372, Perplexity = 12.64
|
| 281 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-30750
|
| 282 |
+
Step 30760/50462 | Loss: 2.5100 | LR: 1.70e-04 | 9955.9ms/step | 6.0 samples/s
|
| 283 |
+
Step 30770/50462 | Loss: 2.5020 | LR: 1.70e-04 | 5141.9ms/step | 11.7 samples/s
|
| 284 |
+
Step 30780/50462 | Loss: 2.5337 | LR: 1.70e-04 | 5145.8ms/step | 11.7 samples/s
|
| 285 |
+
Step 30790/50462 | Loss: 2.4997 | LR: 1.70e-04 | 5158.2ms/step | 11.6 samples/s
|
| 286 |
+
Step 30800/50462 | Loss: 2.5205 | LR: 1.69e-04 | 5156.8ms/step | 11.6 samples/s
|
| 287 |
+
Step 30810/50462 | Loss: 2.5365 | LR: 1.69e-04 | 5151.3ms/step | 11.6 samples/s
|
| 288 |
+
Step 30820/50462 | Loss: 2.5294 | LR: 1.69e-04 | 5143.4ms/step | 11.7 samples/s
|
| 289 |
+
Step 30830/50462 | Loss: 2.4878 | LR: 1.69e-04 | 5148.8ms/step | 11.7 samples/s
|
| 290 |
+
Step 30840/50462 | Loss: 2.5177 | LR: 1.69e-04 | 5143.4ms/step | 11.7 samples/s
|
| 291 |
+
Step 30850/50462 | Loss: 2.5314 | LR: 1.69e-04 | 5144.5ms/step | 11.7 samples/s
|
| 292 |
+
Step 30860/50462 | Loss: 2.5213 | LR: 1.69e-04 | 5147.4ms/step | 11.7 samples/s
|
| 293 |
+
Step 30870/50462 | Loss: 2.4878 | LR: 1.68e-04 | 5149.3ms/step | 11.7 samples/s
|
| 294 |
+
Step 30880/50462 | Loss: 2.5344 | LR: 1.68e-04 | 5149.4ms/step | 11.7 samples/s
|
| 295 |
+
Step 30890/50462 | Loss: 2.5251 | LR: 1.68e-04 | 5148.8ms/step | 11.7 samples/s
|
| 296 |
+
Step 30900/50462 | Loss: 2.5364 | LR: 1.68e-04 | 5147.1ms/step | 11.7 samples/s
|
| 297 |
+
Step 30910/50462 | Loss: 2.5404 | LR: 1.68e-04 | 5149.5ms/step | 11.7 samples/s
|
| 298 |
+
Step 30920/50462 | Loss: 2.5313 | LR: 1.68e-04 | 5150.5ms/step | 11.6 samples/s
|
| 299 |
+
Step 30930/50462 | Loss: 2.5581 | LR: 1.67e-04 | 5147.8ms/step | 11.7 samples/s
|
| 300 |
+
Step 30940/50462 | Loss: 2.5334 | LR: 1.67e-04 | 5147.3ms/step | 11.7 samples/s
|
| 301 |
+
Step 30950/50462 | Loss: 2.5254 | LR: 1.67e-04 | 5148.5ms/step | 11.7 samples/s
|
| 302 |
+
Step 30960/50462 | Loss: 2.5400 | LR: 1.67e-04 | 5154.7ms/step | 11.6 samples/s
|
| 303 |
+
Step 30970/50462 | Loss: 2.5188 | LR: 1.67e-04 | 5156.5ms/step | 11.6 samples/s
|
| 304 |
+
Step 30980/50462 | Loss: 2.5286 | LR: 1.67e-04 | 5152.7ms/step | 11.6 samples/s
|
| 305 |
+
Step 30990/50462 | Loss: 2.5265 | LR: 1.67e-04 | 5150.1ms/step | 11.7 samples/s
|
| 306 |
+
Step 31000/50462 | Loss: 2.5254 | LR: 1.66e-04 | 5151.1ms/step | 11.6 samples/s
|
| 307 |
+
Validation at step 31000: Loss = 2.5363, Perplexity = 12.63
|
| 308 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-31000
|
| 309 |
+
Step 31010/50462 | Loss: 2.5515 | LR: 1.66e-04 | 9975.4ms/step | 6.0 samples/s
|
| 310 |
+
Step 31020/50462 | Loss: 2.5364 | LR: 1.66e-04 | 5142.3ms/step | 11.7 samples/s
|
| 311 |
+
Step 31030/50462 | Loss: 2.5313 | LR: 1.66e-04 | 5136.7ms/step | 11.7 samples/s
|
| 312 |
+
Step 31040/50462 | Loss: 2.5236 | LR: 1.66e-04 | 5142.6ms/step | 11.7 samples/s
|
| 313 |
+
Step 31050/50462 | Loss: 2.5110 | LR: 1.66e-04 | 5143.2ms/step | 11.7 samples/s
|
| 314 |
+
Step 31060/50462 | Loss: 2.5398 | LR: 1.66e-04 | 5141.8ms/step | 11.7 samples/s
|
| 315 |
+
Step 31070/50462 | Loss: 2.5341 | LR: 1.65e-04 | 5140.5ms/step | 11.7 samples/s
|
| 316 |
+
Step 31080/50462 | Loss: 2.5368 | LR: 1.65e-04 | 5141.8ms/step | 11.7 samples/s
|
| 317 |
+
Step 31090/50462 | Loss: 2.5110 | LR: 1.65e-04 | 5140.6ms/step | 11.7 samples/s
|
| 318 |
+
Step 31100/50462 | Loss: 2.5243 | LR: 1.65e-04 | 5140.8ms/step | 11.7 samples/s
|
| 319 |
+
Step 31110/50462 | Loss: 2.5508 | LR: 1.65e-04 | 5138.1ms/step | 11.7 samples/s
|
| 320 |
+
Step 31120/50462 | Loss: 2.5251 | LR: 1.65e-04 | 5134.1ms/step | 11.7 samples/s
|
| 321 |
+
Step 31130/50462 | Loss: 2.5051 | LR: 1.64e-04 | 5140.0ms/step | 11.7 samples/s
|
| 322 |
+
Step 31140/50462 | Loss: 2.5174 | LR: 1.64e-04 | 5146.0ms/step | 11.7 samples/s
|
| 323 |
+
Step 31150/50462 | Loss: 2.5170 | LR: 1.64e-04 | 5141.5ms/step | 11.7 samples/s
|
| 324 |
+
Step 31160/50462 | Loss: 2.5268 | LR: 1.64e-04 | 5140.2ms/step | 11.7 samples/s
|
| 325 |
+
Step 31170/50462 | Loss: 2.5207 | LR: 1.64e-04 | 5140.6ms/step | 11.7 samples/s
|
| 326 |
+
Step 31180/50462 | Loss: 2.5312 | LR: 1.64e-04 | 5138.6ms/step | 11.7 samples/s
|
| 327 |
+
Step 31190/50462 | Loss: 2.5342 | LR: 1.64e-04 | 5140.0ms/step | 11.7 samples/s
|
| 328 |
+
Step 31200/50462 | Loss: 2.5385 | LR: 1.63e-04 | 5142.1ms/step | 11.7 samples/s
|
| 329 |
+
Step 31210/50462 | Loss: 2.5112 | LR: 1.63e-04 | 5144.2ms/step | 11.7 samples/s
|
| 330 |
+
Step 31220/50462 | Loss: 2.5466 | LR: 1.63e-04 | 5142.3ms/step | 11.7 samples/s
|
| 331 |
+
Step 31230/50462 | Loss: 2.5345 | LR: 1.63e-04 | 5140.5ms/step | 11.7 samples/s
|
| 332 |
+
Step 31240/50462 | Loss: 2.5363 | LR: 1.63e-04 | 5138.7ms/step | 11.7 samples/s
|
| 333 |
+
Step 31250/50462 | Loss: 2.5174 | LR: 1.63e-04 | 5142.0ms/step | 11.7 samples/s
|
| 334 |
+
Validation at step 31250: Loss = 2.5350, Perplexity = 12.62
|
| 335 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-31250
|
| 336 |
+
Step 31260/50462 | Loss: 2.5199 | LR: 1.63e-04 | 9946.1ms/step | 6.0 samples/s
|
| 337 |
+
Step 31270/50462 | Loss: 2.5336 | LR: 1.62e-04 | 5154.0ms/step | 11.6 samples/s
|
| 338 |
+
Step 31280/50462 | Loss: 2.5056 | LR: 1.62e-04 | 5152.6ms/step | 11.6 samples/s
|
| 339 |
+
Step 31290/50462 | Loss: 2.5165 | LR: 1.62e-04 | 5146.2ms/step | 11.7 samples/s
|
| 340 |
+
Step 31300/50462 | Loss: 2.5397 | LR: 1.62e-04 | 5151.8ms/step | 11.6 samples/s
|
| 341 |
+
Step 31310/50462 | Loss: 2.5255 | LR: 1.62e-04 | 5144.0ms/step | 11.7 samples/s
|
| 342 |
+
Step 31320/50462 | Loss: 2.5186 | LR: 1.62e-04 | 5149.4ms/step | 11.7 samples/s
|
| 343 |
+
Step 31330/50462 | Loss: 2.5165 | LR: 1.62e-04 | 5142.6ms/step | 11.7 samples/s
|
| 344 |
+
Step 31340/50462 | Loss: 2.5136 | LR: 1.61e-04 | 5150.0ms/step | 11.7 samples/s
|
| 345 |
+
Step 31350/50462 | Loss: 2.5411 | LR: 1.61e-04 | 5145.7ms/step | 11.7 samples/s
|
| 346 |
+
Step 31360/50462 | Loss: 2.5102 | LR: 1.61e-04 | 5143.5ms/step | 11.7 samples/s
|
| 347 |
+
Step 31370/50462 | Loss: 2.5493 | LR: 1.61e-04 | 5148.8ms/step | 11.7 samples/s
|
| 348 |
+
Step 31380/50462 | Loss: 2.5335 | LR: 1.61e-04 | 5146.7ms/step | 11.7 samples/s
|
| 349 |
+
Step 31390/50462 | Loss: 2.5218 | LR: 1.61e-04 | 5151.1ms/step | 11.6 samples/s
|
| 350 |
+
Step 31400/50462 | Loss: 2.5431 | LR: 1.60e-04 | 5144.5ms/step | 11.7 samples/s
|
| 351 |
+
Step 31410/50462 | Loss: 2.5111 | LR: 1.60e-04 | 5140.9ms/step | 11.7 samples/s
|
| 352 |
+
Step 31420/50462 | Loss: 2.5354 | LR: 1.60e-04 | 5142.9ms/step | 11.7 samples/s
|
| 353 |
+
Step 31430/50462 | Loss: 2.5124 | LR: 1.60e-04 | 5144.6ms/step | 11.7 samples/s
|
| 354 |
+
Step 31440/50462 | Loss: 2.5088 | LR: 1.60e-04 | 5146.5ms/step | 11.7 samples/s
|
| 355 |
+
Step 31450/50462 | Loss: 2.5154 | LR: 1.60e-04 | 5140.7ms/step | 11.7 samples/s
|
| 356 |
+
Step 31460/50462 | Loss: 2.5273 | LR: 1.60e-04 | 5144.6ms/step | 11.7 samples/s
|
| 357 |
+
Step 31470/50462 | Loss: 2.4992 | LR: 1.59e-04 | 5144.1ms/step | 11.7 samples/s
|
| 358 |
+
Step 31480/50462 | Loss: 2.5265 | LR: 1.59e-04 | 5151.3ms/step | 11.6 samples/s
|
| 359 |
+
Step 31490/50462 | Loss: 2.5165 | LR: 1.59e-04 | 5153.1ms/step | 11.6 samples/s
|
| 360 |
+
Step 31500/50462 | Loss: 2.5075 | LR: 1.59e-04 | 5142.7ms/step | 11.7 samples/s
|
| 361 |
+
Validation at step 31500: Loss = 2.5343, Perplexity = 12.61
|
| 362 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-31500
|
| 363 |
+
Step 31510/50462 | Loss: 2.5341 | LR: 1.59e-04 | 9943.6ms/step | 6.0 samples/s
|
| 364 |
+
Step 31520/50462 | Loss: 2.5090 | LR: 1.59e-04 | 5138.3ms/step | 11.7 samples/s
|
| 365 |
+
Step 31530/50462 | Loss: 2.5263 | LR: 1.59e-04 | 5144.1ms/step | 11.7 samples/s
|
| 366 |
+
Step 31540/50462 | Loss: 2.5157 | LR: 1.58e-04 | 5142.4ms/step | 11.7 samples/s
|
| 367 |
+
Step 31550/50462 | Loss: 2.5256 | LR: 1.58e-04 | 5130.8ms/step | 11.7 samples/s
|
| 368 |
+
Step 31560/50462 | Loss: 2.5026 | LR: 1.58e-04 | 5141.4ms/step | 11.7 samples/s
|
| 369 |
+
Step 31570/50462 | Loss: 2.5285 | LR: 1.58e-04 | 5137.0ms/step | 11.7 samples/s
|
| 370 |
+
Step 31580/50462 | Loss: 2.5375 | LR: 1.58e-04 | 5137.4ms/step | 11.7 samples/s
|
| 371 |
+
Step 31590/50462 | Loss: 2.5116 | LR: 1.58e-04 | 5140.3ms/step | 11.7 samples/s
|
| 372 |
+
Step 31600/50462 | Loss: 2.5131 | LR: 1.58e-04 | 5131.6ms/step | 11.7 samples/s
|
| 373 |
+
Step 31610/50462 | Loss: 2.5202 | LR: 1.57e-04 | 5141.6ms/step | 11.7 samples/s
|
| 374 |
+
Step 31620/50462 | Loss: 2.5300 | LR: 1.57e-04 | 5139.8ms/step | 11.7 samples/s
|
| 375 |
+
Step 31630/50462 | Loss: 2.5382 | LR: 1.57e-04 | 5134.2ms/step | 11.7 samples/s
|
| 376 |
+
Step 31640/50462 | Loss: 2.5199 | LR: 1.57e-04 | 5134.2ms/step | 11.7 samples/s
|
| 377 |
+
Step 31650/50462 | Loss: 2.5267 | LR: 1.57e-04 | 5141.9ms/step | 11.7 samples/s
|
| 378 |
+
Step 31660/50462 | Loss: 2.5279 | LR: 1.57e-04 | 5134.7ms/step | 11.7 samples/s
|
| 379 |
+
Step 31670/50462 | Loss: 2.5177 | LR: 1.57e-04 | 5137.8ms/step | 11.7 samples/s
|
| 380 |
+
Step 31680/50462 | Loss: 2.5320 | LR: 1.56e-04 | 5138.7ms/step | 11.7 samples/s
|
| 381 |
+
Step 31690/50462 | Loss: 2.5197 | LR: 1.56e-04 | 5144.9ms/step | 11.7 samples/s
|
| 382 |
+
Step 31700/50462 | Loss: 2.5384 | LR: 1.56e-04 | 5141.6ms/step | 11.7 samples/s
|
| 383 |
+
Step 31710/50462 | Loss: 2.5334 | LR: 1.56e-04 | 5143.4ms/step | 11.7 samples/s
|
| 384 |
+
Step 31720/50462 | Loss: 2.5329 | LR: 1.56e-04 | 5139.4ms/step | 11.7 samples/s
|
| 385 |
+
Step 31730/50462 | Loss: 2.5299 | LR: 1.56e-04 | 5140.3ms/step | 11.7 samples/s
|
| 386 |
+
Step 31740/50462 | Loss: 2.5305 | LR: 1.55e-04 | 5137.7ms/step | 11.7 samples/s
|
| 387 |
+
Step 31750/50462 | Loss: 2.4836 | LR: 1.55e-04 | 5130.3ms/step | 11.7 samples/s
|
| 388 |
+
Validation at step 31750: Loss = 2.5330, Perplexity = 12.59
|
| 389 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-31750
|
| 390 |
+
Step 31760/50462 | Loss: 2.5261 | LR: 1.55e-04 | 9947.7ms/step | 6.0 samples/s
|
| 391 |
+
Step 31770/50462 | Loss: 2.5266 | LR: 1.55e-04 | 5145.3ms/step | 11.7 samples/s
|
| 392 |
+
Step 31780/50462 | Loss: 2.5081 | LR: 1.55e-04 | 5149.0ms/step | 11.7 samples/s
|
| 393 |
+
Step 31790/50462 | Loss: 2.5100 | LR: 1.55e-04 | 5151.0ms/step | 11.6 samples/s
|
| 394 |
+
Step 31800/50462 | Loss: 2.5277 | LR: 1.55e-04 | 5144.3ms/step | 11.7 samples/s
|
| 395 |
+
Step 31810/50462 | Loss: 2.4963 | LR: 1.54e-04 | 5148.3ms/step | 11.7 samples/s
|
| 396 |
+
Step 31820/50462 | Loss: 2.5122 | LR: 1.54e-04 | 5151.3ms/step | 11.6 samples/s
|
| 397 |
+
Step 31830/50462 | Loss: 2.5154 | LR: 1.54e-04 | 5150.8ms/step | 11.6 samples/s
|
| 398 |
+
Step 31840/50462 | Loss: 2.5283 | LR: 1.54e-04 | 5145.3ms/step | 11.7 samples/s
|
| 399 |
+
Step 31850/50462 | Loss: 2.4963 | LR: 1.54e-04 | 5145.5ms/step | 11.7 samples/s
|
| 400 |
+
Step 31860/50462 | Loss: 2.5295 | LR: 1.54e-04 | 5147.2ms/step | 11.7 samples/s
|
| 401 |
+
Step 31870/50462 | Loss: 2.5388 | LR: 1.54e-04 | 5149.5ms/step | 11.7 samples/s
|
| 402 |
+
Step 31880/50462 | Loss: 2.5248 | LR: 1.53e-04 | 5154.4ms/step | 11.6 samples/s
|
| 403 |
+
Step 31890/50462 | Loss: 2.5228 | LR: 1.53e-04 | 5153.6ms/step | 11.6 samples/s
|
| 404 |
+
Step 31900/50462 | Loss: 2.5279 | LR: 1.53e-04 | 5152.8ms/step | 11.6 samples/s
|
| 405 |
+
Step 31910/50462 | Loss: 2.5341 | LR: 1.53e-04 | 5150.9ms/step | 11.6 samples/s
|
| 406 |
+
Step 31920/50462 | Loss: 2.5112 | LR: 1.53e-04 | 5144.5ms/step | 11.7 samples/s
|
| 407 |
+
Step 31930/50462 | Loss: 2.5328 | LR: 1.53e-04 | 5149.8ms/step | 11.7 samples/s
|
| 408 |
+
Step 31940/50462 | Loss: 2.5061 | LR: 1.53e-04 | 5142.5ms/step | 11.7 samples/s
|
| 409 |
+
Step 31950/50462 | Loss: 2.5239 | LR: 1.52e-04 | 5152.1ms/step | 11.6 samples/s
|
| 410 |
+
Step 31960/50462 | Loss: 2.5142 | LR: 1.52e-04 | 5149.9ms/step | 11.7 samples/s
|
| 411 |
+
Step 31970/50462 | Loss: 2.5299 | LR: 1.52e-04 | 5145.2ms/step | 11.7 samples/s
|
| 412 |
+
Step 31980/50462 | Loss: 2.5221 | LR: 1.52e-04 | 5149.0ms/step | 11.7 samples/s
|
| 413 |
+
Step 31990/50462 | Loss: 2.5190 | LR: 1.52e-04 | 5144.5ms/step | 11.7 samples/s
|
| 414 |
+
Step 32000/50462 | Loss: 2.5284 | LR: 1.52e-04 | 5153.6ms/step | 11.6 samples/s
|
| 415 |
+
Validation at step 32000: Loss = 2.5319, Perplexity = 12.58
|
| 416 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-32000
|
| 417 |
+
Step 32010/50462 | Loss: 2.5123 | LR: 1.52e-04 | 9971.0ms/step | 6.0 samples/s
|
| 418 |
+
Step 32020/50462 | Loss: 2.5099 | LR: 1.51e-04 | 5139.9ms/step | 11.7 samples/s
|
| 419 |
+
Step 32030/50462 | Loss: 2.5202 | LR: 1.51e-04 | 5149.4ms/step | 11.7 samples/s
|
| 420 |
+
Step 32040/50462 | Loss: 2.5107 | LR: 1.51e-04 | 5146.4ms/step | 11.7 samples/s
|
| 421 |
+
Step 32050/50462 | Loss: 2.4930 | LR: 1.51e-04 | 5146.7ms/step | 11.7 samples/s
|
| 422 |
+
Step 32060/50462 | Loss: 2.5326 | LR: 1.51e-04 | 5149.2ms/step | 11.7 samples/s
|
| 423 |
+
Step 32070/50462 | Loss: 2.5345 | LR: 1.51e-04 | 5152.7ms/step | 11.6 samples/s
|
| 424 |
+
Step 32080/50462 | Loss: 2.5027 | LR: 1.51e-04 | 5149.6ms/step | 11.7 samples/s
|
| 425 |
+
Step 32090/50462 | Loss: 2.5296 | LR: 1.50e-04 | 5146.0ms/step | 11.7 samples/s
|
| 426 |
+
Step 32100/50462 | Loss: 2.5338 | LR: 1.50e-04 | 5149.3ms/step | 11.7 samples/s
|
| 427 |
+
Step 32110/50462 | Loss: 2.5115 | LR: 1.50e-04 | 5148.1ms/step | 11.7 samples/s
|
| 428 |
+
Step 32120/50462 | Loss: 2.5324 | LR: 1.50e-04 | 5144.5ms/step | 11.7 samples/s
|
| 429 |
+
Step 32130/50462 | Loss: 2.4905 | LR: 1.50e-04 | 5148.7ms/step | 11.7 samples/s
|
epfl_final/SparseAtt/wandb/run-20260826_034845-rgmkfpnl/files/requirements.txt
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip==26.0.1
|
| 2 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 3 |
+
mpmath==1.3.0
|
| 4 |
+
cuda-toolkit==13.0.2
|
| 5 |
+
zstandard==0.25.0
|
| 6 |
+
xxhash==3.7.0
|
| 7 |
+
urllib3==2.6.3
|
| 8 |
+
typing_extensions==4.15.0
|
| 9 |
+
triton==3.6.0
|
| 10 |
+
tqdm==4.67.3
|
| 11 |
+
torchinfo==1.8.0
|
| 12 |
+
sympy==1.14.0
|
| 13 |
+
smmap==5.0.3
|
| 14 |
+
six==1.17.0
|
| 15 |
+
shellingham==1.5.4
|
| 16 |
+
setuptools==81.0.0
|
| 17 |
+
safetensors==0.7.0
|
| 18 |
+
regex==2026.4.4
|
| 19 |
+
PyYAML==6.0.3
|
| 20 |
+
Pygments==2.20.0
|
| 21 |
+
pyarrow==24.0.0
|
| 22 |
+
psutil==7.2.2
|
| 23 |
+
protobuf==7.34.1
|
| 24 |
+
propcache==0.4.1
|
| 25 |
+
platformdirs==4.9.6
|
| 26 |
+
packaging==26.2
|
| 27 |
+
nvidia-nvtx==13.0.85
|
| 28 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 29 |
+
nvidia-nvjitlink==13.0.88
|
| 30 |
+
nvidia-nccl-cu13==2.28.9
|
| 31 |
+
nvidia-curand==10.4.0.35
|
| 32 |
+
nvidia-cufile==1.15.1.6
|
| 33 |
+
nvidia-cuda-runtime==13.0.96
|
| 34 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 35 |
+
nvidia-cuda-cupti==13.0.85
|
| 36 |
+
nvidia-cublas==13.1.0.3
|
| 37 |
+
numpy==2.4.4
|
| 38 |
+
networkx==3.6.1
|
| 39 |
+
multidict==6.7.1
|
| 40 |
+
mdurl==0.1.2
|
| 41 |
+
MarkupSafe==3.0.3
|
| 42 |
+
idna==3.13
|
| 43 |
+
hf-xet==1.4.3
|
| 44 |
+
h11==0.16.0
|
| 45 |
+
fsspec==2026.2.0
|
| 46 |
+
frozenlist==1.8.0
|
| 47 |
+
filelock==3.29.0
|
| 48 |
+
dill==0.4.1
|
| 49 |
+
cuda-pathfinder==1.5.4
|
| 50 |
+
click==8.3.3
|
| 51 |
+
charset-normalizer==3.4.7
|
| 52 |
+
certifi==2026.4.22
|
| 53 |
+
attrs==26.1.0
|
| 54 |
+
annotated-types==0.7.0
|
| 55 |
+
annotated-doc==0.0.4
|
| 56 |
+
aiohappyeyeballs==2.6.1
|
| 57 |
+
yarl==1.23.0
|
| 58 |
+
typing-inspection==0.4.2
|
| 59 |
+
sentry-sdk==2.58.0
|
| 60 |
+
requests==2.33.1
|
| 61 |
+
python-dateutil==2.9.0.post0
|
| 62 |
+
pydantic_core==2.46.3
|
| 63 |
+
nvidia-cusparse==12.6.3.3
|
| 64 |
+
nvidia-cufft==12.0.0.61
|
| 65 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 66 |
+
multiprocess==0.70.19
|
| 67 |
+
markdown-it-py==4.0.0
|
| 68 |
+
Jinja2==3.1.6
|
| 69 |
+
httpcore==1.0.9
|
| 70 |
+
gitdb==4.0.12
|
| 71 |
+
cuda-bindings==13.2.0
|
| 72 |
+
anyio==4.13.0
|
| 73 |
+
aiosignal==1.4.0
|
| 74 |
+
rich==15.0.0
|
| 75 |
+
pydantic==2.13.3
|
| 76 |
+
pandas==3.0.2
|
| 77 |
+
nvidia-cusolver==12.0.4.66
|
| 78 |
+
httpx==0.28.1
|
| 79 |
+
GitPython==3.1.49
|
| 80 |
+
aiohttp==3.13.5
|
| 81 |
+
wandb==0.26.1
|
| 82 |
+
typer==0.25.1
|
| 83 |
+
torch==2.11.0
|
| 84 |
+
huggingface_hub==1.13.0
|
| 85 |
+
tokenizers==0.22.2
|
| 86 |
+
datasets==4.8.5
|
| 87 |
+
accelerate==1.13.0
|
| 88 |
+
transformers==5.7.0
|
| 89 |
+
word2number==1.1
|
| 90 |
+
sqlitedict==2.1.0
|
| 91 |
+
pytz==2026.2
|
| 92 |
+
threadpoolctl==3.6.0
|
| 93 |
+
tcolorpy==0.1.7
|
| 94 |
+
tabulate==0.10.0
|
| 95 |
+
scipy==1.17.1
|
| 96 |
+
portalocker==3.2.0
|
| 97 |
+
pathvalidate==3.3.1
|
| 98 |
+
narwhals==2.22.1
|
| 99 |
+
more-itertools==11.1.0
|
| 100 |
+
lxml==6.1.1
|
| 101 |
+
joblib==1.5.3
|
| 102 |
+
colorama==0.4.6
|
| 103 |
+
chardet==6.0.0.post1
|
| 104 |
+
absl-py==2.4.0
|
| 105 |
+
scikit-learn==1.9.0
|
| 106 |
+
sacrebleu==2.6.0
|
| 107 |
+
nltk==3.9.4
|
| 108 |
+
mbstrdecoder==1.1.5
|
| 109 |
+
typepy==1.3.5
|
| 110 |
+
rouge_score==0.1.2
|
| 111 |
+
DataProperty==1.1.1
|
| 112 |
+
tabledata==1.3.5
|
| 113 |
+
evaluate==0.4.6
|
| 114 |
+
pytablewriter==1.2.1
|
| 115 |
+
lm_eval==0.4.12
|
| 116 |
+
pluggy==1.6.0
|
| 117 |
+
iniconfig==2.3.0
|
| 118 |
+
pytest==9.1.0
|
| 119 |
+
py-spy==0.4.2
|
| 120 |
+
pillow==12.2.0
|
| 121 |
+
tifffile==2026.6.1
|
| 122 |
+
imagecodecs==2026.6.26
|
| 123 |
+
zipp==4.1.0
|
| 124 |
+
importlib_metadata==9.0.0
|
| 125 |
+
derivative==0.6.3
|
| 126 |
+
pysindy==2.1.0
|
| 127 |
+
pyparsing==3.3.2
|
| 128 |
+
kiwisolver==1.5.0
|
| 129 |
+
fonttools==4.63.0
|
| 130 |
+
cycler==0.12.1
|
| 131 |
+
contourpy==1.3.3
|
| 132 |
+
matplotlib==3.11.0
|
| 133 |
+
sortedcontainers==2.4.0
|
| 134 |
+
wcwidth==0.8.2
|
| 135 |
+
sigfig==1.3.19
|
| 136 |
+
importlib_resources==7.1.0
|
| 137 |
+
enum-tools==0.13.0
|
| 138 |
+
cramjam==2.11.0
|
| 139 |
+
python-snappy==0.7.3
|
| 140 |
+
compact-json==1.8.2
|
| 141 |
+
numbers-parser==4.18.5
|
| 142 |
+
entmax==1.3
|
epfl_final/SparseAtt/wandb/run-20260826_034845-rgmkfpnl/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.13.13",
|
| 4 |
+
"startedAt": "2026-08-26T03:48:45.664626Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config",
|
| 7 |
+
"configs/p2_1p2b_dense_seed70_2gpu.json"
|
| 8 |
+
],
|
| 9 |
+
"program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
|
| 10 |
+
"codePath": "train.py",
|
| 11 |
+
"codePathLocal": "train.py",
|
| 12 |
+
"git": {
|
| 13 |
+
"remote": "https://github.com/rubentium/SparseAtt.git",
|
| 14 |
+
"commit": "8f862f749eb4d5c17f688eb1562235e3fe5b2550"
|
| 15 |
+
},
|
| 16 |
+
"email": "ruben.navasardyan@mail.utoronto.ca",
|
| 17 |
+
"root": "/mloscratch/homes/navasard/sparse/SparseAtt",
|
| 18 |
+
"host": "h100-workspace-2-gpu-0-19",
|
| 19 |
+
"executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
|
| 20 |
+
"cpu_count": 96,
|
| 21 |
+
"cpu_count_logical": 192,
|
| 22 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 23 |
+
"gpu_count": 2,
|
| 24 |
+
"disk": {
|
| 25 |
+
"/": {
|
| 26 |
+
"total": "9374931091456",
|
| 27 |
+
"used": "6504904605696"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"memory": {
|
| 31 |
+
"total": "1623005732864"
|
| 32 |
+
},
|
| 33 |
+
"gpu_nvidia": [
|
| 34 |
+
{
|
| 35 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 36 |
+
"memoryTotal": "85520809984",
|
| 37 |
+
"cudaCores": 16896,
|
| 38 |
+
"architecture": "Hopper",
|
| 39 |
+
"uuid": "GPU-fd3a94c5-cb60-deec-b7e0-5ab205a565ca"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 43 |
+
"memoryTotal": "85520809984",
|
| 44 |
+
"cudaCores": 16896,
|
| 45 |
+
"architecture": "Hopper",
|
| 46 |
+
"uuid": "GPU-af79466d-4300-4171-b5bd-7697eca27264"
|
| 47 |
+
}
|
| 48 |
+
],
|
| 49 |
+
"cudaVersion": "13.0",
|
| 50 |
+
"writerId": "2jvz4bo069w92gp14il4hh1rqmghk647"
|
| 51 |
+
}
|
epfl_final/SparseAtt/wandb/run-20260826_034845-rgmkfpnl/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|