Hugging Face's logo

rubentium
/
sparse-checkpoints

Model card Files
xet
 Community
rubentium commited on
Commit
7d16a7c
·
verified ·
1 Parent(s): 43c1359

EPFL final checkpoints and W&B history 701-750 of 1463

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +9 -0
  2. epfl_final/SparseAtt/wandb/run-20260820_011125-jqstnvej/run-jqstnvej.wandb +3 -0
  3. epfl_final/SparseAtt/wandb/run-20260820_011125-xfnxdn24/files/output.log +733 -0
  4. epfl_final/SparseAtt/wandb/run-20260820_011125-xfnxdn24/files/requirements.txt +142 -0
  5. epfl_final/SparseAtt/wandb/run-20260820_011125-xfnxdn24/files/wandb-metadata.json +72 -0
  6. epfl_final/SparseAtt/wandb/run-20260820_011125-xfnxdn24/logs/debug-internal.log +0 -0
  7. epfl_final/SparseAtt/wandb/run-20260820_011125-xfnxdn24/logs/debug.log +19 -0
  8. epfl_final/SparseAtt/wandb/run-20260820_011125-xfnxdn24/run-xfnxdn24.wandb +3 -0
  9. epfl_final/SparseAtt/wandb/run-20260820_011128-ysa9iyov/files/output.log +717 -0
  10. epfl_final/SparseAtt/wandb/run-20260820_011128-ysa9iyov/files/requirements.txt +142 -0
  11. epfl_final/SparseAtt/wandb/run-20260820_011128-ysa9iyov/files/wandb-metadata.json +72 -0
  12. epfl_final/SparseAtt/wandb/run-20260820_011128-ysa9iyov/logs/debug-internal.log +0 -0
  13. epfl_final/SparseAtt/wandb/run-20260820_011128-ysa9iyov/logs/debug.log +19 -0
  14. epfl_final/SparseAtt/wandb/run-20260820_011128-ysa9iyov/run-ysa9iyov.wandb +3 -0
  15. epfl_final/SparseAtt/wandb/run-20260820_031315-89kzpaot/files/output.log +0 -0
  16. epfl_final/SparseAtt/wandb/run-20260820_031315-89kzpaot/files/requirements.txt +142 -0
  17. epfl_final/SparseAtt/wandb/run-20260820_031315-89kzpaot/files/wandb-metadata.json +72 -0
  18. epfl_final/SparseAtt/wandb/run-20260820_031315-89kzpaot/logs/debug-internal.log +0 -0
  19. epfl_final/SparseAtt/wandb/run-20260820_031315-89kzpaot/logs/debug.log +19 -0
  20. epfl_final/SparseAtt/wandb/run-20260820_031315-89kzpaot/run-89kzpaot.wandb +3 -0
  21. epfl_final/SparseAtt/wandb/run-20260820_032150-93ukqj9s/files/output.log +0 -0
  22. epfl_final/SparseAtt/wandb/run-20260820_032150-93ukqj9s/files/requirements.txt +142 -0
  23. epfl_final/SparseAtt/wandb/run-20260820_032150-93ukqj9s/files/wandb-metadata.json +72 -0
  24. epfl_final/SparseAtt/wandb/run-20260820_032150-93ukqj9s/logs/debug-internal.log +0 -0
  25. epfl_final/SparseAtt/wandb/run-20260820_032150-93ukqj9s/logs/debug.log +19 -0
  26. epfl_final/SparseAtt/wandb/run-20260820_032150-93ukqj9s/run-93ukqj9s.wandb +3 -0
  27. epfl_final/SparseAtt/wandb/run-20260820_041558-bajm6b15/files/output.log +482 -0
  28. epfl_final/SparseAtt/wandb/run-20260820_041558-bajm6b15/files/requirements.txt +142 -0
  29. epfl_final/SparseAtt/wandb/run-20260820_041558-bajm6b15/files/wandb-metadata.json +72 -0
  30. epfl_final/SparseAtt/wandb/run-20260820_041558-bajm6b15/logs/debug-internal.log +0 -0
  31. epfl_final/SparseAtt/wandb/run-20260820_041558-bajm6b15/logs/debug.log +20 -0
  32. epfl_final/SparseAtt/wandb/run-20260820_041558-bajm6b15/run-bajm6b15.wandb +3 -0
  33. epfl_final/SparseAtt/wandb/run-20260820_041559-jqstnvej/files/output.log +463 -0
  34. epfl_final/SparseAtt/wandb/run-20260820_041559-jqstnvej/files/requirements.txt +142 -0
  35. epfl_final/SparseAtt/wandb/run-20260820_041559-jqstnvej/files/wandb-metadata.json +72 -0
  36. epfl_final/SparseAtt/wandb/run-20260820_041559-jqstnvej/logs/debug-internal.log +0 -0
  37. epfl_final/SparseAtt/wandb/run-20260820_041559-jqstnvej/logs/debug.log +20 -0
  38. epfl_final/SparseAtt/wandb/run-20260820_041559-jqstnvej/run-jqstnvej.wandb +3 -0
  39. epfl_final/SparseAtt/wandb/run-20260820_041559-xfnxdn24/files/output.log +472 -0
  40. epfl_final/SparseAtt/wandb/run-20260820_041559-xfnxdn24/files/requirements.txt +142 -0
  41. epfl_final/SparseAtt/wandb/run-20260820_041559-xfnxdn24/files/wandb-metadata.json +72 -0
  42. epfl_final/SparseAtt/wandb/run-20260820_041559-xfnxdn24/logs/debug-internal.log +0 -0
  43. epfl_final/SparseAtt/wandb/run-20260820_041559-xfnxdn24/logs/debug.log +20 -0
  44. epfl_final/SparseAtt/wandb/run-20260820_041559-xfnxdn24/run-xfnxdn24.wandb +3 -0
  45. epfl_final/SparseAtt/wandb/run-20260820_041559-ysa9iyov/logs/debug.log +20 -0
  46. epfl_final/SparseAtt/wandb/run-20260820_041559-zovy63wq/files/output.log +469 -0
  47. epfl_final/SparseAtt/wandb/run-20260820_041559-zovy63wq/files/requirements.txt +142 -0
  48. epfl_final/SparseAtt/wandb/run-20260820_041559-zovy63wq/files/wandb-metadata.json +72 -0
  49. epfl_final/SparseAtt/wandb/run-20260820_041559-zovy63wq/logs/debug-internal.log +0 -0
  50. epfl_final/SparseAtt/wandb/run-20260820_041559-zovy63wq/logs/debug.log +20 -0
.gitattributes CHANGED
@@ -88,3 +88,12 @@ epfl_final/SparseAtt/wandb/run-20260802_195007-r5no0m0m/run-r5no0m0m.wandb filte
88
  epfl_final/SparseAtt/wandb/run-20260802_195008-c1ekdpek/run-c1ekdpek.wandb filter=lfs diff=lfs merge=lfs -text
89
  epfl_final/SparseAtt/wandb/run-20260820_011124-zovy63wq/run-zovy63wq.wandb filter=lfs diff=lfs merge=lfs -text
90
  epfl_final/SparseAtt/wandb/run-20260820_011125-bajm6b15/run-bajm6b15.wandb filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
88
  epfl_final/SparseAtt/wandb/run-20260802_195008-c1ekdpek/run-c1ekdpek.wandb filter=lfs diff=lfs merge=lfs -text
89
  epfl_final/SparseAtt/wandb/run-20260820_011124-zovy63wq/run-zovy63wq.wandb filter=lfs diff=lfs merge=lfs -text
90
  epfl_final/SparseAtt/wandb/run-20260820_011125-bajm6b15/run-bajm6b15.wandb filter=lfs diff=lfs merge=lfs -text
91
+ epfl_final/SparseAtt/wandb/run-20260820_011125-jqstnvej/run-jqstnvej.wandb filter=lfs diff=lfs merge=lfs -text
92
+ epfl_final/SparseAtt/wandb/run-20260820_011125-xfnxdn24/run-xfnxdn24.wandb filter=lfs diff=lfs merge=lfs -text
93
+ epfl_final/SparseAtt/wandb/run-20260820_011128-ysa9iyov/run-ysa9iyov.wandb filter=lfs diff=lfs merge=lfs -text
94
+ epfl_final/SparseAtt/wandb/run-20260820_031315-89kzpaot/run-89kzpaot.wandb filter=lfs diff=lfs merge=lfs -text
95
+ epfl_final/SparseAtt/wandb/run-20260820_032150-93ukqj9s/run-93ukqj9s.wandb filter=lfs diff=lfs merge=lfs -text
96
+ epfl_final/SparseAtt/wandb/run-20260820_041558-bajm6b15/run-bajm6b15.wandb filter=lfs diff=lfs merge=lfs -text
97
+ epfl_final/SparseAtt/wandb/run-20260820_041559-xfnxdn24/run-xfnxdn24.wandb filter=lfs diff=lfs merge=lfs -text
98
+ epfl_final/SparseAtt/wandb/run-20260820_041559-zovy63wq/run-zovy63wq.wandb filter=lfs diff=lfs merge=lfs -text
99
+ epfl_final/SparseAtt/wandb/run-20260820_041559-jqstnvej/run-jqstnvej.wandb filter=lfs diff=lfs merge=lfs -text
epfl_final/SparseAtt/wandb/run-20260820_011125-jqstnvej/run-jqstnvej.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d6e32da2938eabc8c68b623e3e42e27d85a6ecc51ae846247593d08ccaa20d88
3
+ size 4227072
epfl_final/SparseAtt/wandb/run-20260820_011125-xfnxdn24/files/output.log ADDED
@@ -0,0 +1,733 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Starting training...
2
+ Validation at step 6000: Loss = 3.2430, Perplexity = 25.61
3
+ Switching attention mode for step 6001: dense (dense step interval)
4
+ Step 6010/44000 | Loss: 3.2445 | LR: 4.82e-04 | 15921.6ms/step | 1.3 samples/s
5
+ Step 6020/44000 | Loss: 3.2320 | LR: 4.82e-04 | 1363.3ms/step | 14.7 samples/s
6
+ Step 6030/44000 | Loss: 3.2292 | LR: 4.82e-04 | 1365.0ms/step | 14.7 samples/s
7
+ Step 6040/44000 | Loss: 3.2477 | LR: 4.82e-04 | 1366.2ms/step | 14.6 samples/s
8
+ Step 6050/44000 | Loss: 3.2273 | LR: 4.82e-04 | 1368.1ms/step | 14.6 samples/s
9
+ Step 6060/44000 | Loss: 3.2241 | LR: 4.82e-04 | 1370.5ms/step | 14.6 samples/s
10
+ Step 6070/44000 | Loss: 3.2377 | LR: 4.82e-04 | 1373.3ms/step | 14.6 samples/s
11
+ Step 6080/44000 | Loss: 3.2289 | LR: 4.81e-04 | 1373.2ms/step | 14.6 samples/s
12
+ Step 6090/44000 | Loss: 3.1882 | LR: 4.81e-04 | 1371.1ms/step | 14.6 samples/s
13
+ Step 6100/44000 | Loss: 3.2602 | LR: 4.81e-04 | 1370.5ms/step | 14.6 samples/s
14
+ Step 6110/44000 | Loss: 3.2077 | LR: 4.81e-04 | 1369.7ms/step | 14.6 samples/s
15
+ Step 6120/44000 | Loss: 3.2222 | LR: 4.81e-04 | 1371.2ms/step | 14.6 samples/s
16
+ Step 6130/44000 | Loss: 3.2656 | LR: 4.81e-04 | 1370.4ms/step | 14.6 samples/s
17
+ Step 6140/44000 | Loss: 3.2513 | LR: 4.81e-04 | 1373.1ms/step | 14.6 samples/s
18
+ Step 6150/44000 | Loss: 3.2022 | LR: 4.81e-04 | 1373.7ms/step | 14.6 samples/s
19
+ Step 6160/44000 | Loss: 3.2491 | LR: 4.81e-04 | 1373.4ms/step | 14.6 samples/s
20
+ Step 6170/44000 | Loss: 3.2120 | LR: 4.81e-04 | 1373.2ms/step | 14.6 samples/s
21
+ Step 6180/44000 | Loss: 3.2312 | LR: 4.81e-04 | 1373.3ms/step | 14.6 samples/s
22
+ Step 6190/44000 | Loss: 3.2278 | LR: 4.81e-04 | 1372.8ms/step | 14.6 samples/s
23
+ Step 6200/44000 | Loss: 3.2231 | LR: 4.81e-04 | 1375.2ms/step | 14.5 samples/s
24
+ Step 6210/44000 | Loss: 3.2128 | LR: 4.81e-04 | 1372.8ms/step | 14.6 samples/s
25
+ Step 6220/44000 | Loss: 3.1997 | LR: 4.81e-04 | 1373.8ms/step | 14.6 samples/s
26
+ Step 6230/44000 | Loss: 3.2366 | LR: 4.80e-04 | 1372.1ms/step | 14.6 samples/s
27
+ Step 6240/44000 | Loss: 3.2222 | LR: 4.80e-04 | 1372.8ms/step | 14.6 samples/s
28
+ Step 6250/44000 | Loss: 3.2532 | LR: 4.80e-04 | 1373.5ms/step | 14.6 samples/s
29
+ Step 6260/44000 | Loss: 3.1944 | LR: 4.80e-04 | 1373.7ms/step | 14.6 samples/s
30
+ Step 6270/44000 | Loss: 3.2453 | LR: 4.80e-04 | 1374.5ms/step | 14.6 samples/s
31
+ Step 6280/44000 | Loss: 3.2447 | LR: 4.80e-04 | 1373.9ms/step | 14.6 samples/s
32
+ Step 6290/44000 | Loss: 3.2012 | LR: 4.80e-04 | 1371.8ms/step | 14.6 samples/s
33
+ Step 6300/44000 | Loss: 3.1958 | LR: 4.80e-04 | 1373.1ms/step | 14.6 samples/s
34
+ Step 6310/44000 | Loss: 3.2182 | LR: 4.80e-04 | 1372.9ms/step | 14.6 samples/s
35
+ Step 6320/44000 | Loss: 3.2276 | LR: 4.80e-04 | 1371.1ms/step | 14.6 samples/s
36
+ Step 6330/44000 | Loss: 3.2425 | LR: 4.80e-04 | 1370.4ms/step | 14.6 samples/s
37
+ Step 6340/44000 | Loss: 3.2281 | LR: 4.80e-04 | 1370.4ms/step | 14.6 samples/s
38
+ Step 6350/44000 | Loss: 3.2387 | LR: 4.80e-04 | 1374.1ms/step | 14.6 samples/s
39
+ Step 6360/44000 | Loss: 3.1916 | LR: 4.80e-04 | 1373.8ms/step | 14.6 samples/s
40
+ Step 6370/44000 | Loss: 3.2288 | LR: 4.79e-04 | 1372.6ms/step | 14.6 samples/s
41
+ Step 6380/44000 | Loss: 3.1844 | LR: 4.79e-04 | 1373.2ms/step | 14.6 samples/s
42
+ Step 6390/44000 | Loss: 3.2133 | LR: 4.79e-04 | 1374.2ms/step | 14.6 samples/s
43
+ Step 6400/44000 | Loss: 3.1995 | LR: 4.79e-04 | 1374.8ms/step | 14.5 samples/s
44
+ Step 6410/44000 | Loss: 3.2246 | LR: 4.79e-04 | 1374.4ms/step | 14.6 samples/s
45
+ Step 6420/44000 | Loss: 3.2090 | LR: 4.79e-04 | 1372.9ms/step | 14.6 samples/s
46
+ Step 6430/44000 | Loss: 3.2159 | LR: 4.79e-04 | 1374.4ms/step | 14.6 samples/s
47
+ Step 6440/44000 | Loss: 3.2074 | LR: 4.79e-04 | 1374.1ms/step | 14.6 samples/s
48
+ Step 6450/44000 | Loss: 3.1822 | LR: 4.79e-04 | 1376.1ms/step | 14.5 samples/s
49
+ Step 6460/44000 | Loss: 3.2137 | LR: 4.79e-04 | 1373.7ms/step | 14.6 samples/s
50
+ Step 6470/44000 | Loss: 3.2443 | LR: 4.79e-04 | 1372.7ms/step | 14.6 samples/s
51
+ Step 6480/44000 | Loss: 3.2253 | LR: 4.79e-04 | 1374.7ms/step | 14.5 samples/s
52
+ Step 6490/44000 | Loss: 3.2224 | LR: 4.79e-04 | 1373.9ms/step | 14.6 samples/s
53
+ Step 6500/44000 | Loss: 3.2266 | LR: 4.79e-04 | 1374.1ms/step | 14.6 samples/s
54
+ Step 6510/44000 | Loss: 3.2032 | LR: 4.78e-04 | 1373.3ms/step | 14.6 samples/s
55
+ Step 6520/44000 | Loss: 3.2297 | LR: 4.78e-04 | 1375.0ms/step | 14.5 samples/s
56
+ Step 6530/44000 | Loss: 3.1851 | LR: 4.78e-04 | 1371.5ms/step | 14.6 samples/s
57
+ Step 6540/44000 | Loss: 3.2026 | LR: 4.78e-04 | 1374.1ms/step | 14.6 samples/s
58
+ Step 6550/44000 | Loss: 3.2072 | LR: 4.78e-04 | 1371.2ms/step | 14.6 samples/s
59
+ Step 6560/44000 | Loss: 3.2505 | LR: 4.78e-04 | 1372.1ms/step | 14.6 samples/s
60
+ Step 6570/44000 | Loss: 3.2384 | LR: 4.78e-04 | 1372.8ms/step | 14.6 samples/s
61
+ Step 6580/44000 | Loss: 3.2137 | LR: 4.78e-04 | 1375.6ms/step | 14.5 samples/s
62
+ Step 6590/44000 | Loss: 3.1758 | LR: 4.78e-04 | 1374.9ms/step | 14.5 samples/s
63
+ Step 6600/44000 | Loss: 3.2200 | LR: 4.78e-04 | 1375.7ms/step | 14.5 samples/s
64
+ Step 6610/44000 | Loss: 3.2341 | LR: 4.78e-04 | 1375.1ms/step | 14.5 samples/s
65
+ Step 6620/44000 | Loss: 3.2203 | LR: 4.78e-04 | 1375.3ms/step | 14.5 samples/s
66
+ Step 6630/44000 | Loss: 3.1918 | LR: 4.78e-04 | 1374.5ms/step | 14.6 samples/s
67
+ Step 6640/44000 | Loss: 3.2142 | LR: 4.77e-04 | 1374.7ms/step | 14.5 samples/s
68
+ Step 6650/44000 | Loss: 3.2391 | LR: 4.77e-04 | 1375.3ms/step | 14.5 samples/s
69
+ Step 6660/44000 | Loss: 3.2148 | LR: 4.77e-04 | 1373.0ms/step | 14.6 samples/s
70
+ Step 6670/44000 | Loss: 3.1899 | LR: 4.77e-04 | 1375.0ms/step | 14.5 samples/s
71
+ Step 6680/44000 | Loss: 3.1889 | LR: 4.77e-04 | 1373.6ms/step | 14.6 samples/s
72
+ Step 6690/44000 | Loss: 3.2476 | LR: 4.77e-04 | 1373.0ms/step | 14.6 samples/s
73
+ Step 6700/44000 | Loss: 3.1795 | LR: 4.77e-04 | 1374.7ms/step | 14.5 samples/s
74
+ Step 6710/44000 | Loss: 3.2118 | LR: 4.77e-04 | 1374.2ms/step | 14.6 samples/s
75
+ Step 6720/44000 | Loss: 3.1901 | LR: 4.77e-04 | 1374.0ms/step | 14.6 samples/s
76
+ Step 6730/44000 | Loss: 3.1923 | LR: 4.77e-04 | 1375.2ms/step | 14.5 samples/s
77
+ Step 6740/44000 | Loss: 3.2191 | LR: 4.77e-04 | 1373.8ms/step | 14.6 samples/s
78
+ Step 6750/44000 | Loss: 3.1745 | LR: 4.77e-04 | 1372.6ms/step | 14.6 samples/s
79
+ Step 6760/44000 | Loss: 3.2194 | LR: 4.77e-04 | 1373.5ms/step | 14.6 samples/s
80
+ Step 6770/44000 | Loss: 3.2216 | LR: 4.76e-04 | 1373.3ms/step | 14.6 samples/s
81
+ Step 6780/44000 | Loss: 3.2298 | LR: 4.76e-04 | 1373.5ms/step | 14.6 samples/s
82
+ Step 6790/44000 | Loss: 3.2327 | LR: 4.76e-04 | 1373.2ms/step | 14.6 samples/s
83
+ Step 6800/44000 | Loss: 3.2005 | LR: 4.76e-04 | 1374.5ms/step | 14.6 samples/s
84
+ Step 6810/44000 | Loss: 3.2230 | LR: 4.76e-04 | 1372.1ms/step | 14.6 samples/s
85
+ Step 6820/44000 | Loss: 3.2070 | LR: 4.76e-04 | 1371.9ms/step | 14.6 samples/s
86
+ Step 6830/44000 | Loss: 3.2384 | LR: 4.76e-04 | 1374.1ms/step | 14.6 samples/s
87
+ Step 6840/44000 | Loss: 3.1965 | LR: 4.76e-04 | 1374.2ms/step | 14.6 samples/s
88
+ Step 6850/44000 | Loss: 3.2126 | LR: 4.76e-04 | 1374.4ms/step | 14.6 samples/s
89
+ Step 6860/44000 | Loss: 3.2079 | LR: 4.76e-04 | 1373.0ms/step | 14.6 samples/s
90
+ Step 6870/44000 | Loss: 3.2050 | LR: 4.76e-04 | 1374.5ms/step | 14.6 samples/s
91
+ Step 6880/44000 | Loss: 3.2106 | LR: 4.76e-04 | 1373.8ms/step | 14.6 samples/s
92
+ Step 6890/44000 | Loss: 3.2017 | LR: 4.76e-04 | 1370.7ms/step | 14.6 samples/s
93
+ Step 6900/44000 | Loss: 3.2290 | LR: 4.75e-04 | 1371.5ms/step | 14.6 samples/s
94
+ Step 6910/44000 | Loss: 3.1918 | LR: 4.75e-04 | 1372.2ms/step | 14.6 samples/s
95
+ Step 6920/44000 | Loss: 3.2006 | LR: 4.75e-04 | 1372.4ms/step | 14.6 samples/s
96
+ Step 6930/44000 | Loss: 3.1870 | LR: 4.75e-04 | 1373.3ms/step | 14.6 samples/s
97
+ Step 6940/44000 | Loss: 3.1844 | LR: 4.75e-04 | 1371.0ms/step | 14.6 samples/s
98
+ Step 6950/44000 | Loss: 3.2201 | LR: 4.75e-04 | 1371.1ms/step | 14.6 samples/s
99
+ Step 6960/44000 | Loss: 3.1994 | LR: 4.75e-04 | 1372.0ms/step | 14.6 samples/s
100
+ Step 6970/44000 | Loss: 3.2199 | LR: 4.75e-04 | 1373.5ms/step | 14.6 samples/s
101
+ Step 6980/44000 | Loss: 3.1966 | LR: 4.75e-04 | 1373.0ms/step | 14.6 samples/s
102
+ Step 6990/44000 | Loss: 3.2020 | LR: 4.75e-04 | 1373.2ms/step | 14.6 samples/s
103
+ Step 7000/44000 | Loss: 3.1869 | LR: 4.75e-04 | 1372.6ms/step | 14.6 samples/s
104
+ Validation at step 7000: Loss = 3.2020, Perplexity = 24.58
105
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-7000
106
+ Step 7010/44000 | Loss: 3.2182 | LR: 4.75e-04 | 3855.1ms/step | 5.2 samples/s
107
+ Step 7020/44000 | Loss: 3.1986 | LR: 4.75e-04 | 1374.7ms/step | 14.5 samples/s
108
+ Step 7030/44000 | Loss: 3.1884 | LR: 4.74e-04 | 1374.2ms/step | 14.6 samples/s
109
+ Step 7040/44000 | Loss: 3.2143 | LR: 4.74e-04 | 1375.4ms/step | 14.5 samples/s
110
+ Step 7050/44000 | Loss: 3.2066 | LR: 4.74e-04 | 1374.1ms/step | 14.6 samples/s
111
+ Step 7060/44000 | Loss: 3.1997 | LR: 4.74e-04 | 1375.0ms/step | 14.5 samples/s
112
+ Step 7070/44000 | Loss: 3.2099 | LR: 4.74e-04 | 1376.1ms/step | 14.5 samples/s
113
+ Step 7080/44000 | Loss: 3.1954 | LR: 4.74e-04 | 1376.0ms/step | 14.5 samples/s
114
+ Step 7090/44000 | Loss: 3.2054 | LR: 4.74e-04 | 1375.5ms/step | 14.5 samples/s
115
+ Step 7100/44000 | Loss: 3.1935 | LR: 4.74e-04 | 1375.8ms/step | 14.5 samples/s
116
+ Step 7110/44000 | Loss: 3.1885 | LR: 4.74e-04 | 1374.7ms/step | 14.5 samples/s
117
+ Step 7120/44000 | Loss: 3.1837 | LR: 4.74e-04 | 1376.7ms/step | 14.5 samples/s
118
+ Step 7130/44000 | Loss: 3.1964 | LR: 4.74e-04 | 1376.5ms/step | 14.5 samples/s
119
+ Step 7140/44000 | Loss: 3.1631 | LR: 4.74e-04 | 1379.7ms/step | 14.5 samples/s
120
+ Step 7150/44000 | Loss: 3.2022 | LR: 4.73e-04 | 1377.4ms/step | 14.5 samples/s
121
+ Step 7160/44000 | Loss: 3.1835 | LR: 4.73e-04 | 1376.6ms/step | 14.5 samples/s
122
+ Step 7170/44000 | Loss: 3.2052 | LR: 4.73e-04 | 1375.8ms/step | 14.5 samples/s
123
+ Step 7180/44000 | Loss: 3.2003 | LR: 4.73e-04 | 1374.5ms/step | 14.6 samples/s
124
+ Step 7190/44000 | Loss: 3.1781 | LR: 4.73e-04 | 1376.2ms/step | 14.5 samples/s
125
+ Step 7200/44000 | Loss: 3.1821 | LR: 4.73e-04 | 1376.2ms/step | 14.5 samples/s
126
+ Step 7210/44000 | Loss: 3.1756 | LR: 4.73e-04 | 1375.9ms/step | 14.5 samples/s
127
+ Step 7220/44000 | Loss: 3.2084 | LR: 4.73e-04 | 1375.5ms/step | 14.5 samples/s
128
+ Step 7230/44000 | Loss: 3.1840 | LR: 4.73e-04 | 1374.1ms/step | 14.6 samples/s
129
+ Step 7240/44000 | Loss: 3.1505 | LR: 4.73e-04 | 1376.3ms/step | 14.5 samples/s
130
+ Step 7250/44000 | Loss: 3.1570 | LR: 4.73e-04 | 1377.6ms/step | 14.5 samples/s
131
+ Step 7260/44000 | Loss: 3.1761 | LR: 4.73e-04 | 1376.3ms/step | 14.5 samples/s
132
+ Step 7270/44000 | Loss: 3.1758 | LR: 4.72e-04 | 1376.2ms/step | 14.5 samples/s
133
+ Step 7280/44000 | Loss: 3.2155 | LR: 4.72e-04 | 1372.6ms/step | 14.6 samples/s
134
+ Step 7290/44000 | Loss: 3.2225 | LR: 4.72e-04 | 1367.7ms/step | 14.6 samples/s
135
+ Step 7300/44000 | Loss: 3.2045 | LR: 4.72e-04 | 1369.7ms/step | 14.6 samples/s
136
+ Step 7310/44000 | Loss: 3.1886 | LR: 4.72e-04 | 1376.8ms/step | 14.5 samples/s
137
+ Step 7320/44000 | Loss: 3.1948 | LR: 4.72e-04 | 1376.1ms/step | 14.5 samples/s
138
+ Step 7330/44000 | Loss: 3.2054 | LR: 4.72e-04 | 1372.8ms/step | 14.6 samples/s
139
+ Step 7340/44000 | Loss: 3.1975 | LR: 4.72e-04 | 1368.6ms/step | 14.6 samples/s
140
+ Step 7350/44000 | Loss: 3.2064 | LR: 4.72e-04 | 1368.0ms/step | 14.6 samples/s
141
+ Step 7360/44000 | Loss: 3.1890 | LR: 4.72e-04 | 1371.0ms/step | 14.6 samples/s
142
+ Step 7370/44000 | Loss: 3.2000 | LR: 4.72e-04 | 1374.4ms/step | 14.6 samples/s
143
+ Step 7380/44000 | Loss: 3.1468 | LR: 4.72e-04 | 1376.1ms/step | 14.5 samples/s
144
+ Step 7390/44000 | Loss: 3.2038 | LR: 4.71e-04 | 1376.4ms/step | 14.5 samples/s
145
+ Step 7400/44000 | Loss: 3.1729 | LR: 4.71e-04 | 1375.2ms/step | 14.5 samples/s
146
+ Step 7410/44000 | Loss: 3.1946 | LR: 4.71e-04 | 1369.3ms/step | 14.6 samples/s
147
+ Step 7420/44000 | Loss: 3.1607 | LR: 4.71e-04 | 1368.3ms/step | 14.6 samples/s
148
+ Step 7430/44000 | Loss: 3.1734 | LR: 4.71e-04 | 1368.5ms/step | 14.6 samples/s
149
+ Step 7440/44000 | Loss: 3.1780 | LR: 4.71e-04 | 1368.3ms/step | 14.6 samples/s
150
+ Step 7450/44000 | Loss: 3.1762 | LR: 4.71e-04 | 1367.9ms/step | 14.6 samples/s
151
+ Step 7460/44000 | Loss: 3.1928 | LR: 4.71e-04 | 1367.6ms/step | 14.6 samples/s
152
+ Step 7470/44000 | Loss: 3.1691 | LR: 4.71e-04 | 1376.8ms/step | 14.5 samples/s
153
+ Step 7480/44000 | Loss: 3.1610 | LR: 4.71e-04 | 1380.8ms/step | 14.5 samples/s
154
+ Step 7490/44000 | Loss: 3.1791 | LR: 4.71e-04 | 1378.5ms/step | 14.5 samples/s
155
+ Step 7500/44000 | Loss: 3.1840 | LR: 4.71e-04 | 1369.5ms/step | 14.6 samples/s
156
+ Step 7510/44000 | Loss: 3.1856 | LR: 4.70e-04 | 1369.8ms/step | 14.6 samples/s
157
+ Step 7520/44000 | Loss: 3.1711 | LR: 4.70e-04 | 1368.2ms/step | 14.6 samples/s
158
+ Step 7530/44000 | Loss: 3.2171 | LR: 4.70e-04 | 1368.4ms/step | 14.6 samples/s
159
+ Step 7540/44000 | Loss: 3.1577 | LR: 4.70e-04 | 1370.6ms/step | 14.6 samples/s
160
+ Step 7550/44000 | Loss: 3.1843 | LR: 4.70e-04 | 1369.6ms/step | 14.6 samples/s
161
+ Step 7560/44000 | Loss: 3.1689 | LR: 4.70e-04 | 1369.0ms/step | 14.6 samples/s
162
+ Step 7570/44000 | Loss: 3.1876 | LR: 4.70e-04 | 1368.8ms/step | 14.6 samples/s
163
+ Step 7580/44000 | Loss: 3.1437 | LR: 4.70e-04 | 1368.0ms/step | 14.6 samples/s
164
+ Step 7590/44000 | Loss: 3.1687 | LR: 4.70e-04 | 1368.9ms/step | 14.6 samples/s
165
+ Step 7600/44000 | Loss: 3.1725 | LR: 4.70e-04 | 1375.2ms/step | 14.5 samples/s
166
+ Step 7610/44000 | Loss: 3.1939 | LR: 4.70e-04 | 1374.8ms/step | 14.5 samples/s
167
+ Step 7620/44000 | Loss: 3.1872 | LR: 4.70e-04 | 1375.7ms/step | 14.5 samples/s
168
+ Step 7630/44000 | Loss: 3.1633 | LR: 4.69e-04 | 1369.8ms/step | 14.6 samples/s
169
+ Step 7640/44000 | Loss: 3.1707 | LR: 4.69e-04 | 1367.9ms/step | 14.6 samples/s
170
+ Step 7650/44000 | Loss: 3.1745 | LR: 4.69e-04 | 1373.1ms/step | 14.6 samples/s
171
+ Step 7660/44000 | Loss: 3.1699 | LR: 4.69e-04 | 1374.3ms/step | 14.6 samples/s
172
+ Step 7670/44000 | Loss: 3.1805 | LR: 4.69e-04 | 1376.9ms/step | 14.5 samples/s
173
+ Step 7680/44000 | Loss: 3.1818 | LR: 4.69e-04 | 1373.2ms/step | 14.6 samples/s
174
+ Step 7690/44000 | Loss: 3.1588 | LR: 4.69e-04 | 1373.1ms/step | 14.6 samples/s
175
+ Step 7700/44000 | Loss: 3.2160 | LR: 4.69e-04 | 1376.1ms/step | 14.5 samples/s
176
+ Step 7710/44000 | Loss: 3.1482 | LR: 4.69e-04 | 1374.5ms/step | 14.6 samples/s
177
+ Step 7720/44000 | Loss: 3.1540 | LR: 4.69e-04 | 1374.3ms/step | 14.6 samples/s
178
+ Step 7730/44000 | Loss: 3.1792 | LR: 4.69e-04 | 1376.6ms/step | 14.5 samples/s
179
+ Step 7740/44000 | Loss: 3.1868 | LR: 4.68e-04 | 1373.0ms/step | 14.6 samples/s
180
+ Step 7750/44000 | Loss: 3.2058 | LR: 4.68e-04 | 1374.0ms/step | 14.6 samples/s
181
+ Step 7760/44000 | Loss: 3.1753 | LR: 4.68e-04 | 1375.5ms/step | 14.5 samples/s
182
+ Step 7770/44000 | Loss: 3.1804 | LR: 4.68e-04 | 1373.9ms/step | 14.6 samples/s
183
+ Step 7780/44000 | Loss: 3.1564 | LR: 4.68e-04 | 1374.1ms/step | 14.6 samples/s
184
+ Step 7790/44000 | Loss: 3.1938 | LR: 4.68e-04 | 1372.9ms/step | 14.6 samples/s
185
+ Step 7800/44000 | Loss: 3.1710 | LR: 4.68e-04 | 1374.8ms/step | 14.5 samples/s
186
+ Step 7810/44000 | Loss: 3.1718 | LR: 4.68e-04 | 1376.5ms/step | 14.5 samples/s
187
+ Step 7820/44000 | Loss: 3.1649 | LR: 4.68e-04 | 1373.9ms/step | 14.6 samples/s
188
+ Step 7830/44000 | Loss: 3.1690 | LR: 4.68e-04 | 1373.8ms/step | 14.6 samples/s
189
+ Step 7840/44000 | Loss: 3.1664 | LR: 4.68e-04 | 1375.9ms/step | 14.5 samples/s
190
+ Step 7850/44000 | Loss: 3.1873 | LR: 4.67e-04 | 1373.3ms/step | 14.6 samples/s
191
+ Step 7860/44000 | Loss: 3.1695 | LR: 4.67e-04 | 1368.7ms/step | 14.6 samples/s
192
+ Step 7870/44000 | Loss: 3.1495 | LR: 4.67e-04 | 1367.3ms/step | 14.6 samples/s
193
+ Step 7880/44000 | Loss: 3.1589 | LR: 4.67e-04 | 1367.5ms/step | 14.6 samples/s
194
+ Step 7890/44000 | Loss: 3.1899 | LR: 4.67e-04 | 1367.2ms/step | 14.6 samples/s
195
+ Step 7900/44000 | Loss: 3.1443 | LR: 4.67e-04 | 1367.8ms/step | 14.6 samples/s
196
+ Step 7910/44000 | Loss: 3.1807 | LR: 4.67e-04 | 1367.8ms/step | 14.6 samples/s
197
+ Step 7920/44000 | Loss: 3.1729 | LR: 4.67e-04 | 1367.7ms/step | 14.6 samples/s
198
+ Step 7930/44000 | Loss: 3.1886 | LR: 4.67e-04 | 1368.0ms/step | 14.6 samples/s
199
+ Step 7940/44000 | Loss: 3.1851 | LR: 4.67e-04 | 1372.6ms/step | 14.6 samples/s
200
+ Step 7950/44000 | Loss: 3.1525 | LR: 4.67e-04 | 1374.8ms/step | 14.5 samples/s
201
+ Step 7960/44000 | Loss: 3.1523 | LR: 4.66e-04 | 1373.7ms/step | 14.6 samples/s
202
+ Step 7970/44000 | Loss: 3.1691 | LR: 4.66e-04 | 1374.9ms/step | 14.5 samples/s
203
+ Step 7980/44000 | Loss: 3.1590 | LR: 4.66e-04 | 1374.1ms/step | 14.6 samples/s
204
+ Step 7990/44000 | Loss: 3.1811 | LR: 4.66e-04 | 1368.8ms/step | 14.6 samples/s
205
+ Step 8000/44000 | Loss: 3.1724 | LR: 4.66e-04 | 1366.4ms/step | 14.6 samples/s
206
+ Validation at step 8000: Loss = 3.1700, Perplexity = 23.81
207
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-8000
208
+ Step 8010/44000 | Loss: 3.1763 | LR: 4.66e-04 | 3825.9ms/step | 5.2 samples/s
209
+ Step 8020/44000 | Loss: 3.1703 | LR: 4.66e-04 | 1371.8ms/step | 14.6 samples/s
210
+ Step 8030/44000 | Loss: 3.1998 | LR: 4.66e-04 | 1376.1ms/step | 14.5 samples/s
211
+ Step 8040/44000 | Loss: 3.1558 | LR: 4.66e-04 | 1378.0ms/step | 14.5 samples/s
212
+ Step 8050/44000 | Loss: 3.2043 | LR: 4.66e-04 | 1379.9ms/step | 14.5 samples/s
213
+ Step 8060/44000 | Loss: 3.1568 | LR: 4.66e-04 | 1376.6ms/step | 14.5 samples/s
214
+ Step 8070/44000 | Loss: 3.1673 | LR: 4.65e-04 | 1379.6ms/step | 14.5 samples/s
215
+ Step 8080/44000 | Loss: 3.1835 | LR: 4.65e-04 | 1375.9ms/step | 14.5 samples/s
216
+ Step 8090/44000 | Loss: 3.1719 | LR: 4.65e-04 | 1379.2ms/step | 14.5 samples/s
217
+ Step 8100/44000 | Loss: 3.1674 | LR: 4.65e-04 | 1373.8ms/step | 14.6 samples/s
218
+ Step 8110/44000 | Loss: 3.1428 | LR: 4.65e-04 | 1377.5ms/step | 14.5 samples/s
219
+ Step 8120/44000 | Loss: 3.1635 | LR: 4.65e-04 | 1378.1ms/step | 14.5 samples/s
220
+ Step 8130/44000 | Loss: 3.1580 | LR: 4.65e-04 | 1376.4ms/step | 14.5 samples/s
221
+ Step 8140/44000 | Loss: 3.1821 | LR: 4.65e-04 | 1376.6ms/step | 14.5 samples/s
222
+ Step 8150/44000 | Loss: 3.1520 | LR: 4.65e-04 | 1374.4ms/step | 14.6 samples/s
223
+ Step 8160/44000 | Loss: 3.1861 | LR: 4.65e-04 | 1375.4ms/step | 14.5 samples/s
224
+ Step 8170/44000 | Loss: 3.1925 | LR: 4.65e-04 | 1377.8ms/step | 14.5 samples/s
225
+ Step 8180/44000 | Loss: 3.1330 | LR: 4.64e-04 | 1377.2ms/step | 14.5 samples/s
226
+ Step 8190/44000 | Loss: 3.1635 | LR: 4.64e-04 | 1379.0ms/step | 14.5 samples/s
227
+ Step 8200/44000 | Loss: 3.1542 | LR: 4.64e-04 | 1378.0ms/step | 14.5 samples/s
228
+ Step 8210/44000 | Loss: 3.1401 | LR: 4.64e-04 | 1380.1ms/step | 14.5 samples/s
229
+ Step 8220/44000 | Loss: 3.1462 | LR: 4.64e-04 | 1378.1ms/step | 14.5 samples/s
230
+ Step 8230/44000 | Loss: 3.1451 | LR: 4.64e-04 | 1376.3ms/step | 14.5 samples/s
231
+ Step 8240/44000 | Loss: 3.1620 | LR: 4.64e-04 | 1382.4ms/step | 14.5 samples/s
232
+ Step 8250/44000 | Loss: 3.1848 | LR: 4.64e-04 | 1379.8ms/step | 14.5 samples/s
233
+ Step 8260/44000 | Loss: 3.1417 | LR: 4.64e-04 | 1379.6ms/step | 14.5 samples/s
234
+ Step 8270/44000 | Loss: 3.1524 | LR: 4.64e-04 | 1380.5ms/step | 14.5 samples/s
235
+ Step 8280/44000 | Loss: 3.1416 | LR: 4.64e-04 | 1379.1ms/step | 14.5 samples/s
236
+ Step 8290/44000 | Loss: 3.1631 | LR: 4.63e-04 | 1379.0ms/step | 14.5 samples/s
237
+ Step 8300/44000 | Loss: 3.1837 | LR: 4.63e-04 | 1380.0ms/step | 14.5 samples/s
238
+ Step 8310/44000 | Loss: 3.1415 | LR: 4.63e-04 | 1378.4ms/step | 14.5 samples/s
239
+ Step 8320/44000 | Loss: 3.1485 | LR: 4.63e-04 | 1381.4ms/step | 14.5 samples/s
240
+ Step 8330/44000 | Loss: 3.1503 | LR: 4.63e-04 | 1380.7ms/step | 14.5 samples/s
241
+ Step 8340/44000 | Loss: 3.1598 | LR: 4.63e-04 | 1379.3ms/step | 14.5 samples/s
242
+ Step 8350/44000 | Loss: 3.1651 | LR: 4.63e-04 | 1381.6ms/step | 14.5 samples/s
243
+ Step 8360/44000 | Loss: 3.1267 | LR: 4.63e-04 | 1380.2ms/step | 14.5 samples/s
244
+ Step 8370/44000 | Loss: 3.1714 | LR: 4.63e-04 | 1378.6ms/step | 14.5 samples/s
245
+ Step 8380/44000 | Loss: 3.1538 | LR: 4.63e-04 | 1377.9ms/step | 14.5 samples/s
246
+ Step 8390/44000 | Loss: 3.1354 | LR: 4.62e-04 | 1379.9ms/step | 14.5 samples/s
247
+ Step 8400/44000 | Loss: 3.1465 | LR: 4.62e-04 | 1381.3ms/step | 14.5 samples/s
248
+ Step 8410/44000 | Loss: 3.1233 | LR: 4.62e-04 | 1378.9ms/step | 14.5 samples/s
249
+ Step 8420/44000 | Loss: 3.1918 | LR: 4.62e-04 | 1378.9ms/step | 14.5 samples/s
250
+ Step 8430/44000 | Loss: 3.1699 | LR: 4.62e-04 | 1379.2ms/step | 14.5 samples/s
251
+ Step 8440/44000 | Loss: 3.1743 | LR: 4.62e-04 | 1377.5ms/step | 14.5 samples/s
252
+ Step 8450/44000 | Loss: 3.1423 | LR: 4.62e-04 | 1376.7ms/step | 14.5 samples/s
253
+ Step 8460/44000 | Loss: 3.1477 | LR: 4.62e-04 | 1377.6ms/step | 14.5 samples/s
254
+ Step 8470/44000 | Loss: 3.1451 | LR: 4.62e-04 | 1377.9ms/step | 14.5 samples/s
255
+ Step 8480/44000 | Loss: 3.1738 | LR: 4.62e-04 | 1380.1ms/step | 14.5 samples/s
256
+ Step 8490/44000 | Loss: 3.1549 | LR: 4.62e-04 | 1380.6ms/step | 14.5 samples/s
257
+ Step 8500/44000 | Loss: 3.1455 | LR: 4.61e-04 | 1378.6ms/step | 14.5 samples/s
258
+ Step 8510/44000 | Loss: 3.1410 | LR: 4.61e-04 | 1379.4ms/step | 14.5 samples/s
259
+ Step 8520/44000 | Loss: 3.1421 | LR: 4.61e-04 | 1379.4ms/step | 14.5 samples/s
260
+ Step 8530/44000 | Loss: 3.1647 | LR: 4.61e-04 | 1378.2ms/step | 14.5 samples/s
261
+ Step 8540/44000 | Loss: 3.1325 | LR: 4.61e-04 | 1378.8ms/step | 14.5 samples/s
262
+ Step 8550/44000 | Loss: 3.1571 | LR: 4.61e-04 | 1378.7ms/step | 14.5 samples/s
263
+ Step 8560/44000 | Loss: 3.1742 | LR: 4.61e-04 | 1380.5ms/step | 14.5 samples/s
264
+ Step 8570/44000 | Loss: 3.1410 | LR: 4.61e-04 | 1379.6ms/step | 14.5 samples/s
265
+ Step 8580/44000 | Loss: 3.1554 | LR: 4.61e-04 | 1379.5ms/step | 14.5 samples/s
266
+ Step 8590/44000 | Loss: 3.1431 | LR: 4.61e-04 | 1379.8ms/step | 14.5 samples/s
267
+ Step 8600/44000 | Loss: 3.1456 | LR: 4.60e-04 | 1378.9ms/step | 14.5 samples/s
268
+ Step 8610/44000 | Loss: 3.1638 | LR: 4.60e-04 | 1376.4ms/step | 14.5 samples/s
269
+ Step 8620/44000 | Loss: 3.1544 | LR: 4.60e-04 | 1376.5ms/step | 14.5 samples/s
270
+ Step 8630/44000 | Loss: 3.1507 | LR: 4.60e-04 | 1375.8ms/step | 14.5 samples/s
271
+ Step 8640/44000 | Loss: 3.1333 | LR: 4.60e-04 | 1377.1ms/step | 14.5 samples/s
272
+ Step 8650/44000 | Loss: 3.1695 | LR: 4.60e-04 | 1376.8ms/step | 14.5 samples/s
273
+ Step 8660/44000 | Loss: 3.1677 | LR: 4.60e-04 | 1376.8ms/step | 14.5 samples/s
274
+ Step 8670/44000 | Loss: 3.1711 | LR: 4.60e-04 | 1376.4ms/step | 14.5 samples/s
275
+ Step 8680/44000 | Loss: 3.1458 | LR: 4.60e-04 | 1376.4ms/step | 14.5 samples/s
276
+ Step 8690/44000 | Loss: 3.1586 | LR: 4.60e-04 | 1376.0ms/step | 14.5 samples/s
277
+ Step 8700/44000 | Loss: 3.1438 | LR: 4.59e-04 | 1378.7ms/step | 14.5 samples/s
278
+ Step 8710/44000 | Loss: 3.1476 | LR: 4.59e-04 | 1378.5ms/step | 14.5 samples/s
279
+ Step 8720/44000 | Loss: 3.1529 | LR: 4.59e-04 | 1377.6ms/step | 14.5 samples/s
280
+ Step 8730/44000 | Loss: 3.1643 | LR: 4.59e-04 | 1374.8ms/step | 14.5 samples/s
281
+ Step 8740/44000 | Loss: 3.1561 | LR: 4.59e-04 | 1378.4ms/step | 14.5 samples/s
282
+ Step 8750/44000 | Loss: 3.1321 | LR: 4.59e-04 | 1378.2ms/step | 14.5 samples/s
283
+ Step 8760/44000 | Loss: 3.1329 | LR: 4.59e-04 | 1376.8ms/step | 14.5 samples/s
284
+ Step 8770/44000 | Loss: 3.1366 | LR: 4.59e-04 | 1377.1ms/step | 14.5 samples/s
285
+ Step 8780/44000 | Loss: 3.1257 | LR: 4.59e-04 | 1377.8ms/step | 14.5 samples/s
286
+ Step 8790/44000 | Loss: 3.1727 | LR: 4.59e-04 | 1378.3ms/step | 14.5 samples/s
287
+ Step 8800/44000 | Loss: 3.1205 | LR: 4.58e-04 | 1377.0ms/step | 14.5 samples/s
288
+ Step 8810/44000 | Loss: 3.1371 | LR: 4.58e-04 | 1376.3ms/step | 14.5 samples/s
289
+ Step 8820/44000 | Loss: 3.1782 | LR: 4.58e-04 | 1378.2ms/step | 14.5 samples/s
290
+ Step 8830/44000 | Loss: 3.1524 | LR: 4.58e-04 | 1378.4ms/step | 14.5 samples/s
291
+ Step 8840/44000 | Loss: 3.1315 | LR: 4.58e-04 | 1376.1ms/step | 14.5 samples/s
292
+ Step 8850/44000 | Loss: 3.1351 | LR: 4.58e-04 | 1376.1ms/step | 14.5 samples/s
293
+ Step 8860/44000 | Loss: 3.1553 | LR: 4.58e-04 | 1378.3ms/step | 14.5 samples/s
294
+ Step 8870/44000 | Loss: 3.1625 | LR: 4.58e-04 | 1376.8ms/step | 14.5 samples/s
295
+ Step 8880/44000 | Loss: 3.1433 | LR: 4.58e-04 | 1376.7ms/step | 14.5 samples/s
296
+ Step 8890/44000 | Loss: 3.1242 | LR: 4.58e-04 | 1376.2ms/step | 14.5 samples/s
297
+ Step 8900/44000 | Loss: 3.1515 | LR: 4.57e-04 | 1376.0ms/step | 14.5 samples/s
298
+ Step 8910/44000 | Loss: 3.1420 | LR: 4.57e-04 | 1377.8ms/step | 14.5 samples/s
299
+ Step 8920/44000 | Loss: 3.1557 | LR: 4.57e-04 | 1377.6ms/step | 14.5 samples/s
300
+ Step 8930/44000 | Loss: 3.0762 | LR: 4.57e-04 | 1378.3ms/step | 14.5 samples/s
301
+ Step 8940/44000 | Loss: 3.1286 | LR: 4.57e-04 | 1377.3ms/step | 14.5 samples/s
302
+ Step 8950/44000 | Loss: 3.1349 | LR: 4.57e-04 | 1378.2ms/step | 14.5 samples/s
303
+ Step 8960/44000 | Loss: 3.1219 | LR: 4.57e-04 | 1376.6ms/step | 14.5 samples/s
304
+ Step 8970/44000 | Loss: 3.1607 | LR: 4.57e-04 | 1376.6ms/step | 14.5 samples/s
305
+ Step 8980/44000 | Loss: 3.1704 | LR: 4.57e-04 | 1378.6ms/step | 14.5 samples/s
306
+ Step 8990/44000 | Loss: 3.1504 | LR: 4.57e-04 | 1377.9ms/step | 14.5 samples/s
307
+ Step 9000/44000 | Loss: 3.1532 | LR: 4.56e-04 | 1376.9ms/step | 14.5 samples/s
308
+ Validation at step 9000: Loss = 3.1415, Perplexity = 23.14
309
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-9000
310
+ Step 9010/44000 | Loss: 3.1069 | LR: 4.56e-04 | 3835.4ms/step | 5.2 samples/s
311
+ Step 9020/44000 | Loss: 3.1067 | LR: 4.56e-04 | 1373.9ms/step | 14.6 samples/s
312
+ Step 9030/44000 | Loss: 3.1065 | LR: 4.56e-04 | 1374.5ms/step | 14.6 samples/s
313
+ Step 9040/44000 | Loss: 3.1691 | LR: 4.56e-04 | 1372.8ms/step | 14.6 samples/s
314
+ Step 9050/44000 | Loss: 3.1311 | LR: 4.56e-04 | 1376.6ms/step | 14.5 samples/s
315
+ Step 9060/44000 | Loss: 3.1256 | LR: 4.56e-04 | 1377.3ms/step | 14.5 samples/s
316
+ Step 9070/44000 | Loss: 3.1164 | LR: 4.56e-04 | 1375.3ms/step | 14.5 samples/s
317
+ Step 9080/44000 | Loss: 3.1446 | LR: 4.56e-04 | 1372.6ms/step | 14.6 samples/s
318
+ Step 9090/44000 | Loss: 3.1264 | LR: 4.56e-04 | 1375.7ms/step | 14.5 samples/s
319
+ Step 9100/44000 | Loss: 3.1010 | LR: 4.55e-04 | 1376.1ms/step | 14.5 samples/s
320
+ Step 9110/44000 | Loss: 3.1456 | LR: 4.55e-04 | 1373.8ms/step | 14.6 samples/s
321
+ Step 9120/44000 | Loss: 3.1560 | LR: 4.55e-04 | 1375.7ms/step | 14.5 samples/s
322
+ Step 9130/44000 | Loss: 3.1311 | LR: 4.55e-04 | 1376.2ms/step | 14.5 samples/s
323
+ Step 9140/44000 | Loss: 3.1120 | LR: 4.55e-04 | 1374.2ms/step | 14.6 samples/s
324
+ Step 9150/44000 | Loss: 3.1302 | LR: 4.55e-04 | 1376.9ms/step | 14.5 samples/s
325
+ Step 9160/44000 | Loss: 3.1368 | LR: 4.55e-04 | 1375.2ms/step | 14.5 samples/s
326
+ Step 9170/44000 | Loss: 3.1315 | LR: 4.55e-04 | 1377.5ms/step | 14.5 samples/s
327
+ Step 9180/44000 | Loss: 3.1249 | LR: 4.55e-04 | 1375.9ms/step | 14.5 samples/s
328
+ Step 9190/44000 | Loss: 3.1148 | LR: 4.54e-04 | 1378.4ms/step | 14.5 samples/s
329
+ Step 9200/44000 | Loss: 3.1521 | LR: 4.54e-04 | 1375.1ms/step | 14.5 samples/s
330
+ Step 9210/44000 | Loss: 3.1235 | LR: 4.54e-04 | 1377.0ms/step | 14.5 samples/s
331
+ Step 9220/44000 | Loss: 3.1273 | LR: 4.54e-04 | 1373.7ms/step | 14.6 samples/s
332
+ Step 9230/44000 | Loss: 3.1205 | LR: 4.54e-04 | 1368.1ms/step | 14.6 samples/s
333
+ Step 9240/44000 | Loss: 3.1336 | LR: 4.54e-04 | 1368.8ms/step | 14.6 samples/s
334
+ Step 9250/44000 | Loss: 3.1386 | LR: 4.54e-04 | 1371.4ms/step | 14.6 samples/s
335
+ Step 9260/44000 | Loss: 3.1111 | LR: 4.54e-04 | 1368.8ms/step | 14.6 samples/s
336
+ Step 9270/44000 | Loss: 3.1040 | LR: 4.54e-04 | 1369.3ms/step | 14.6 samples/s
337
+ Step 9280/44000 | Loss: 3.1550 | LR: 4.54e-04 | 1369.5ms/step | 14.6 samples/s
338
+ Step 9290/44000 | Loss: 3.1560 | LR: 4.53e-04 | 1370.4ms/step | 14.6 samples/s
339
+ Step 9300/44000 | Loss: 3.1248 | LR: 4.53e-04 | 1367.7ms/step | 14.6 samples/s
340
+ Step 9310/44000 | Loss: 3.1531 | LR: 4.53e-04 | 1370.8ms/step | 14.6 samples/s
341
+ Step 9320/44000 | Loss: 3.1440 | LR: 4.53e-04 | 1369.8ms/step | 14.6 samples/s
342
+ Step 9330/44000 | Loss: 3.0876 | LR: 4.53e-04 | 1371.1ms/step | 14.6 samples/s
343
+ Step 9340/44000 | Loss: 3.1292 | LR: 4.53e-04 | 1370.0ms/step | 14.6 samples/s
344
+ Step 9350/44000 | Loss: 3.1324 | LR: 4.53e-04 | 1369.7ms/step | 14.6 samples/s
345
+ Step 9360/44000 | Loss: 3.1445 | LR: 4.53e-04 | 1370.8ms/step | 14.6 samples/s
346
+ Step 9370/44000 | Loss: 3.1263 | LR: 4.53e-04 | 1368.1ms/step | 14.6 samples/s
347
+ Step 9380/44000 | Loss: 3.1299 | LR: 4.52e-04 | 1368.3ms/step | 14.6 samples/s
348
+ Step 9390/44000 | Loss: 3.1283 | LR: 4.52e-04 | 1368.7ms/step | 14.6 samples/s
349
+ Step 9400/44000 | Loss: 3.1297 | LR: 4.52e-04 | 1369.9ms/step | 14.6 samples/s
350
+ Step 9410/44000 | Loss: 3.1502 | LR: 4.52e-04 | 1369.2ms/step | 14.6 samples/s
351
+ Step 9420/44000 | Loss: 3.1017 | LR: 4.52e-04 | 1368.8ms/step | 14.6 samples/s
352
+ Step 9430/44000 | Loss: 3.1365 | LR: 4.52e-04 | 1370.0ms/step | 14.6 samples/s
353
+ Step 9440/44000 | Loss: 3.1403 | LR: 4.52e-04 | 1369.7ms/step | 14.6 samples/s
354
+ Step 9450/44000 | Loss: 3.0910 | LR: 4.52e-04 | 1369.4ms/step | 14.6 samples/s
355
+ Step 9460/44000 | Loss: 3.1217 | LR: 4.52e-04 | 1368.1ms/step | 14.6 samples/s
356
+ Step 9470/44000 | Loss: 3.0866 | LR: 4.52e-04 | 1369.1ms/step | 14.6 samples/s
357
+ Step 9480/44000 | Loss: 3.1197 | LR: 4.51e-04 | 1368.5ms/step | 14.6 samples/s
358
+ Step 9490/44000 | Loss: 3.1031 | LR: 4.51e-04 | 1368.7ms/step | 14.6 samples/s
359
+ Step 9500/44000 | Loss: 3.0980 | LR: 4.51e-04 | 1367.9ms/step | 14.6 samples/s
360
+ Step 9510/44000 | Loss: 3.1006 | LR: 4.51e-04 | 1368.3ms/step | 14.6 samples/s
361
+ Step 9520/44000 | Loss: 3.1351 | LR: 4.51e-04 | 1369.2ms/step | 14.6 samples/s
362
+ Step 9530/44000 | Loss: 3.0958 | LR: 4.51e-04 | 1367.3ms/step | 14.6 samples/s
363
+ Step 9540/44000 | Loss: 3.1327 | LR: 4.51e-04 | 1368.9ms/step | 14.6 samples/s
364
+ Step 9550/44000 | Loss: 3.1260 | LR: 4.51e-04 | 1372.4ms/step | 14.6 samples/s
365
+ Step 9560/44000 | Loss: 3.1151 | LR: 4.51e-04 | 1375.7ms/step | 14.5 samples/s
366
+ Step 9570/44000 | Loss: 3.1477 | LR: 4.50e-04 | 1374.6ms/step | 14.5 samples/s
367
+ Step 9580/44000 | Loss: 3.1397 | LR: 4.50e-04 | 1374.7ms/step | 14.5 samples/s
368
+ Step 9590/44000 | Loss: 3.1256 | LR: 4.50e-04 | 1374.4ms/step | 14.6 samples/s
369
+ Step 9600/44000 | Loss: 3.1267 | LR: 4.50e-04 | 1375.2ms/step | 14.5 samples/s
370
+ Step 9610/44000 | Loss: 3.1327 | LR: 4.50e-04 | 1368.1ms/step | 14.6 samples/s
371
+ Step 9620/44000 | Loss: 3.0977 | LR: 4.50e-04 | 1369.6ms/step | 14.6 samples/s
372
+ Step 9630/44000 | Loss: 3.1360 | LR: 4.50e-04 | 1370.1ms/step | 14.6 samples/s
373
+ Step 9640/44000 | Loss: 3.1195 | LR: 4.50e-04 | 1369.0ms/step | 14.6 samples/s
374
+ Step 9650/44000 | Loss: 3.1382 | LR: 4.50e-04 | 1373.3ms/step | 14.6 samples/s
375
+ Step 9660/44000 | Loss: 3.1296 | LR: 4.49e-04 | 1375.8ms/step | 14.5 samples/s
376
+ Step 9670/44000 | Loss: 3.1300 | LR: 4.49e-04 | 1377.9ms/step | 14.5 samples/s
377
+ Step 9680/44000 | Loss: 3.0823 | LR: 4.49e-04 | 1374.5ms/step | 14.6 samples/s
378
+ Step 9690/44000 | Loss: 3.0979 | LR: 4.49e-04 | 1373.5ms/step | 14.6 samples/s
379
+ Step 9700/44000 | Loss: 3.1471 | LR: 4.49e-04 | 1375.7ms/step | 14.5 samples/s
380
+ Step 9710/44000 | Loss: 3.1030 | LR: 4.49e-04 | 1376.1ms/step | 14.5 samples/s
381
+ Step 9720/44000 | Loss: 3.1351 | LR: 4.49e-04 | 1375.4ms/step | 14.5 samples/s
382
+ Step 9730/44000 | Loss: 3.1620 | LR: 4.49e-04 | 1377.0ms/step | 14.5 samples/s
383
+ Step 9740/44000 | Loss: 3.1316 | LR: 4.49e-04 | 1375.2ms/step | 14.5 samples/s
384
+ Step 9750/44000 | Loss: 3.1529 | LR: 4.48e-04 | 1377.9ms/step | 14.5 samples/s
385
+ Step 9760/44000 | Loss: 3.0992 | LR: 4.48e-04 | 1374.1ms/step | 14.6 samples/s
386
+ Step 9770/44000 | Loss: 3.1378 | LR: 4.48e-04 | 1368.1ms/step | 14.6 samples/s
387
+ Step 9780/44000 | Loss: 3.1098 | LR: 4.48e-04 | 1369.1ms/step | 14.6 samples/s
388
+ Step 9790/44000 | Loss: 3.1250 | LR: 4.48e-04 | 1368.2ms/step | 14.6 samples/s
389
+ Step 9800/44000 | Loss: 3.1183 | LR: 4.48e-04 | 1369.1ms/step | 14.6 samples/s
390
+ Step 9810/44000 | Loss: 3.1355 | LR: 4.48e-04 | 1367.0ms/step | 14.6 samples/s
391
+ Step 9820/44000 | Loss: 3.1089 | LR: 4.48e-04 | 1368.0ms/step | 14.6 samples/s
392
+ Step 9830/44000 | Loss: 3.1147 | LR: 4.48e-04 | 1368.0ms/step | 14.6 samples/s
393
+ Step 9840/44000 | Loss: 3.1440 | LR: 4.47e-04 | 1367.0ms/step | 14.6 samples/s
394
+ Step 9850/44000 | Loss: 3.0835 | LR: 4.47e-04 | 1374.2ms/step | 14.6 samples/s
395
+ Step 9860/44000 | Loss: 3.1299 | LR: 4.47e-04 | 1376.5ms/step | 14.5 samples/s
396
+ Step 9870/44000 | Loss: 3.0873 | LR: 4.47e-04 | 1370.5ms/step | 14.6 samples/s
397
+ Step 9880/44000 | Loss: 3.1159 | LR: 4.47e-04 | 1369.6ms/step | 14.6 samples/s
398
+ Step 9890/44000 | Loss: 3.1313 | LR: 4.47e-04 | 1367.1ms/step | 14.6 samples/s
399
+ Step 9900/44000 | Loss: 3.0940 | LR: 4.47e-04 | 1368.1ms/step | 14.6 samples/s
400
+ Step 9910/44000 | Loss: 3.1102 | LR: 4.47e-04 | 1368.1ms/step | 14.6 samples/s
401
+ Step 9920/44000 | Loss: 3.1081 | LR: 4.47e-04 | 1367.9ms/step | 14.6 samples/s
402
+ Step 9930/44000 | Loss: 3.1068 | LR: 4.46e-04 | 1367.8ms/step | 14.6 samples/s
403
+ Step 9940/44000 | Loss: 3.1374 | LR: 4.46e-04 | 1376.4ms/step | 14.5 samples/s
404
+ Step 9950/44000 | Loss: 3.1058 | LR: 4.46e-04 | 1374.8ms/step | 14.5 samples/s
405
+ Step 9960/44000 | Loss: 3.1421 | LR: 4.46e-04 | 1376.6ms/step | 14.5 samples/s
406
+ Step 9970/44000 | Loss: 3.1365 | LR: 4.46e-04 | 1375.3ms/step | 14.5 samples/s
407
+ Step 9980/44000 | Loss: 3.1054 | LR: 4.46e-04 | 1375.0ms/step | 14.5 samples/s
408
+ Step 9990/44000 | Loss: 3.1092 | LR: 4.46e-04 | 1375.7ms/step | 14.5 samples/s
409
+ Step 10000/44000 | Loss: 3.0955 | LR: 4.46e-04 | 1376.0ms/step | 14.5 samples/s
410
+ Validation at step 10000: Loss = 3.1172, Perplexity = 22.58
411
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-10000
412
+ Step 10010/44000 | Loss: 3.1577 | LR: 4.46e-04 | 3850.3ms/step | 5.2 samples/s
413
+ Step 10020/44000 | Loss: 3.1321 | LR: 4.45e-04 | 1374.9ms/step | 14.5 samples/s
414
+ Step 10030/44000 | Loss: 3.1147 | LR: 4.45e-04 | 1378.1ms/step | 14.5 samples/s
415
+ Step 10040/44000 | Loss: 3.1116 | LR: 4.45e-04 | 1377.0ms/step | 14.5 samples/s
416
+ Step 10050/44000 | Loss: 3.1121 | LR: 4.45e-04 | 1376.2ms/step | 14.5 samples/s
417
+ Step 10060/44000 | Loss: 3.1021 | LR: 4.45e-04 | 1375.9ms/step | 14.5 samples/s
418
+ Step 10070/44000 | Loss: 3.1353 | LR: 4.45e-04 | 1373.1ms/step | 14.6 samples/s
419
+ Step 10080/44000 | Loss: 3.0877 | LR: 4.45e-04 | 1374.4ms/step | 14.6 samples/s
420
+ Step 10090/44000 | Loss: 3.1343 | LR: 4.45e-04 | 1374.7ms/step | 14.5 samples/s
421
+ Step 10100/44000 | Loss: 3.1085 | LR: 4.45e-04 | 1375.5ms/step | 14.5 samples/s
422
+ Step 10110/44000 | Loss: 3.1264 | LR: 4.44e-04 | 1374.8ms/step | 14.5 samples/s
423
+ Step 10120/44000 | Loss: 3.1049 | LR: 4.44e-04 | 1375.3ms/step | 14.5 samples/s
424
+ Step 10130/44000 | Loss: 3.1305 | LR: 4.44e-04 | 1374.9ms/step | 14.5 samples/s
425
+ Step 10140/44000 | Loss: 3.1432 | LR: 4.44e-04 | 1377.1ms/step | 14.5 samples/s
426
+ Step 10150/44000 | Loss: 3.1200 | LR: 4.44e-04 | 1374.5ms/step | 14.6 samples/s
427
+ Step 10160/44000 | Loss: 3.1051 | LR: 4.44e-04 | 1374.4ms/step | 14.6 samples/s
428
+ Step 10170/44000 | Loss: 3.0985 | LR: 4.44e-04 | 1376.4ms/step | 14.5 samples/s
429
+ Step 10180/44000 | Loss: 3.1284 | LR: 4.44e-04 | 1377.2ms/step | 14.5 samples/s
430
+ Step 10190/44000 | Loss: 3.0833 | LR: 4.43e-04 | 1377.8ms/step | 14.5 samples/s
431
+ Step 10200/44000 | Loss: 3.1028 | LR: 4.43e-04 | 1377.6ms/step | 14.5 samples/s
432
+ Step 10210/44000 | Loss: 3.1018 | LR: 4.43e-04 | 1377.3ms/step | 14.5 samples/s
433
+ Step 10220/44000 | Loss: 3.1118 | LR: 4.43e-04 | 1378.3ms/step | 14.5 samples/s
434
+ Step 10230/44000 | Loss: 3.1247 | LR: 4.43e-04 | 1376.4ms/step | 14.5 samples/s
435
+ Step 10240/44000 | Loss: 3.1250 | LR: 4.43e-04 | 1376.5ms/step | 14.5 samples/s
436
+ Step 10250/44000 | Loss: 3.1296 | LR: 4.43e-04 | 1377.3ms/step | 14.5 samples/s
437
+ Step 10260/44000 | Loss: 3.1431 | LR: 4.43e-04 | 1376.7ms/step | 14.5 samples/s
438
+ Step 10270/44000 | Loss: 3.0910 | LR: 4.43e-04 | 1377.4ms/step | 14.5 samples/s
439
+ Step 10280/44000 | Loss: 3.0994 | LR: 4.42e-04 | 1376.3ms/step | 14.5 samples/s
440
+ Step 10290/44000 | Loss: 3.1131 | LR: 4.42e-04 | 1375.8ms/step | 14.5 samples/s
441
+ Step 10300/44000 | Loss: 3.0805 | LR: 4.42e-04 | 1378.3ms/step | 14.5 samples/s
442
+ Step 10310/44000 | Loss: 3.1373 | LR: 4.42e-04 | 1376.0ms/step | 14.5 samples/s
443
+ Step 10320/44000 | Loss: 3.1321 | LR: 4.42e-04 | 1376.0ms/step | 14.5 samples/s
444
+ Step 10330/44000 | Loss: 3.1254 | LR: 4.42e-04 | 1375.7ms/step | 14.5 samples/s
445
+ Step 10340/44000 | Loss: 3.1153 | LR: 4.42e-04 | 1376.1ms/step | 14.5 samples/s
446
+ Step 10350/44000 | Loss: 3.1168 | LR: 4.42e-04 | 1377.3ms/step | 14.5 samples/s
447
+ Step 10360/44000 | Loss: 3.0953 | LR: 4.42e-04 | 1374.8ms/step | 14.5 samples/s
448
+ Step 10370/44000 | Loss: 3.1419 | LR: 4.41e-04 | 1374.5ms/step | 14.6 samples/s
449
+ Step 10380/44000 | Loss: 3.1200 | LR: 4.41e-04 | 1374.4ms/step | 14.6 samples/s
450
+ Step 10390/44000 | Loss: 3.1170 | LR: 4.41e-04 | 1373.9ms/step | 14.6 samples/s
451
+ Step 10400/44000 | Loss: 3.1196 | LR: 4.41e-04 | 1376.6ms/step | 14.5 samples/s
452
+ Step 10410/44000 | Loss: 3.1096 | LR: 4.41e-04 | 1376.4ms/step | 14.5 samples/s
453
+ Step 10420/44000 | Loss: 3.1237 | LR: 4.41e-04 | 1375.9ms/step | 14.5 samples/s
454
+ Step 10430/44000 | Loss: 3.1297 | LR: 4.41e-04 | 1376.1ms/step | 14.5 samples/s
455
+ Step 10440/44000 | Loss: 3.1274 | LR: 4.41e-04 | 1377.7ms/step | 14.5 samples/s
456
+ Step 10450/44000 | Loss: 3.1136 | LR: 4.40e-04 | 1376.5ms/step | 14.5 samples/s
457
+ Step 10460/44000 | Loss: 3.1078 | LR: 4.40e-04 | 1378.2ms/step | 14.5 samples/s
458
+ Step 10470/44000 | Loss: 3.1171 | LR: 4.40e-04 | 1376.6ms/step | 14.5 samples/s
459
+ Step 10480/44000 | Loss: 3.0902 | LR: 4.40e-04 | 1374.9ms/step | 14.5 samples/s
460
+ Step 10490/44000 | Loss: 3.0957 | LR: 4.40e-04 | 1375.9ms/step | 14.5 samples/s
461
+ Step 10500/44000 | Loss: 3.1322 | LR: 4.40e-04 | 1376.0ms/step | 14.5 samples/s
462
+ Step 10510/44000 | Loss: 3.1123 | LR: 4.40e-04 | 1378.0ms/step | 14.5 samples/s
463
+ Step 10520/44000 | Loss: 3.0964 | LR: 4.40e-04 | 1376.6ms/step | 14.5 samples/s
464
+ Step 10530/44000 | Loss: 3.1096 | LR: 4.40e-04 | 1374.9ms/step | 14.5 samples/s
465
+ Step 10540/44000 | Loss: 3.1352 | LR: 4.39e-04 | 1374.9ms/step | 14.5 samples/s
466
+ Step 10550/44000 | Loss: 3.0738 | LR: 4.39e-04 | 1374.7ms/step | 14.5 samples/s
467
+ Step 10560/44000 | Loss: 3.1273 | LR: 4.39e-04 | 1376.7ms/step | 14.5 samples/s
468
+ Step 10570/44000 | Loss: 3.1409 | LR: 4.39e-04 | 1376.0ms/step | 14.5 samples/s
469
+ Step 10580/44000 | Loss: 3.1175 | LR: 4.39e-04 | 1376.0ms/step | 14.5 samples/s
470
+ Step 10590/44000 | Loss: 3.0960 | LR: 4.39e-04 | 1376.3ms/step | 14.5 samples/s
471
+ Step 10600/44000 | Loss: 3.1233 | LR: 4.39e-04 | 1376.0ms/step | 14.5 samples/s
472
+ Step 10610/44000 | Loss: 3.1147 | LR: 4.39e-04 | 1375.2ms/step | 14.5 samples/s
473
+ Step 10620/44000 | Loss: 3.0829 | LR: 4.38e-04 | 1377.1ms/step | 14.5 samples/s
474
+ Step 10630/44000 | Loss: 3.0911 | LR: 4.38e-04 | 1376.0ms/step | 14.5 samples/s
475
+ Step 10640/44000 | Loss: 3.1074 | LR: 4.38e-04 | 1375.3ms/step | 14.5 samples/s
476
+ Step 10650/44000 | Loss: 3.1211 | LR: 4.38e-04 | 1375.5ms/step | 14.5 samples/s
477
+ Step 10660/44000 | Loss: 3.0993 | LR: 4.38e-04 | 1374.4ms/step | 14.6 samples/s
478
+ Step 10670/44000 | Loss: 3.1174 | LR: 4.38e-04 | 1374.4ms/step | 14.6 samples/s
479
+ Step 10680/44000 | Loss: 3.1108 | LR: 4.38e-04 | 1375.5ms/step | 14.5 samples/s
480
+ Step 10690/44000 | Loss: 3.0950 | LR: 4.38e-04 | 1376.0ms/step | 14.5 samples/s
481
+ Step 10700/44000 | Loss: 3.0975 | LR: 4.37e-04 | 1376.6ms/step | 14.5 samples/s
482
+ Step 10710/44000 | Loss: 3.0881 | LR: 4.37e-04 | 1375.9ms/step | 14.5 samples/s
483
+ Step 10720/44000 | Loss: 3.0976 | LR: 4.37e-04 | 1376.4ms/step | 14.5 samples/s
484
+ Step 10730/44000 | Loss: 3.1341 | LR: 4.37e-04 | 1375.7ms/step | 14.5 samples/s
485
+ Step 10740/44000 | Loss: 3.1056 | LR: 4.37e-04 | 1373.7ms/step | 14.6 samples/s
486
+ Step 10750/44000 | Loss: 3.1045 | LR: 4.37e-04 | 1374.4ms/step | 14.6 samples/s
487
+ Step 10760/44000 | Loss: 3.0856 | LR: 4.37e-04 | 1376.2ms/step | 14.5 samples/s
488
+ Step 10770/44000 | Loss: 3.0984 | LR: 4.37e-04 | 1374.9ms/step | 14.5 samples/s
489
+ Step 10780/44000 | Loss: 3.1071 | LR: 4.37e-04 | 1375.7ms/step | 14.5 samples/s
490
+ Step 10790/44000 | Loss: 3.0717 | LR: 4.36e-04 | 1375.2ms/step | 14.5 samples/s
491
+ Step 10800/44000 | Loss: 3.1103 | LR: 4.36e-04 | 1372.6ms/step | 14.6 samples/s
492
+ Step 10810/44000 | Loss: 3.0996 | LR: 4.36e-04 | 1372.8ms/step | 14.6 samples/s
493
+ Step 10820/44000 | Loss: 3.1098 | LR: 4.36e-04 | 1374.4ms/step | 14.6 samples/s
494
+ Step 10830/44000 | Loss: 3.1121 | LR: 4.36e-04 | 1376.3ms/step | 14.5 samples/s
495
+ Step 10840/44000 | Loss: 3.1197 | LR: 4.36e-04 | 1373.8ms/step | 14.6 samples/s
496
+ Step 10850/44000 | Loss: 3.1029 | LR: 4.36e-04 | 1374.6ms/step | 14.6 samples/s
497
+ Step 10860/44000 | Loss: 3.1019 | LR: 4.36e-04 | 1375.8ms/step | 14.5 samples/s
498
+ Step 10870/44000 | Loss: 3.0969 | LR: 4.35e-04 | 1376.1ms/step | 14.5 samples/s
499
+ Step 10880/44000 | Loss: 3.1032 | LR: 4.35e-04 | 1373.3ms/step | 14.6 samples/s
500
+ Step 10890/44000 | Loss: 3.0805 | LR: 4.35e-04 | 1376.9ms/step | 14.5 samples/s
501
+ Step 10900/44000 | Loss: 3.0826 | LR: 4.35e-04 | 1373.6ms/step | 14.6 samples/s
502
+ Step 10910/44000 | Loss: 3.0908 | LR: 4.35e-04 | 1374.5ms/step | 14.6 samples/s
503
+ Step 10920/44000 | Loss: 3.0796 | LR: 4.35e-04 | 1377.3ms/step | 14.5 samples/s
504
+ Step 10930/44000 | Loss: 3.0787 | LR: 4.35e-04 | 1374.5ms/step | 14.6 samples/s
505
+ Step 10940/44000 | Loss: 3.1248 | LR: 4.35e-04 | 1374.7ms/step | 14.5 samples/s
506
+ Step 10950/44000 | Loss: 3.0740 | LR: 4.34e-04 | 1374.6ms/step | 14.5 samples/s
507
+ Step 10960/44000 | Loss: 3.0370 | LR: 4.34e-04 | 1375.5ms/step | 14.5 samples/s
508
+ Step 10970/44000 | Loss: 3.0935 | LR: 4.34e-04 | 1376.2ms/step | 14.5 samples/s
509
+ Step 10980/44000 | Loss: 3.1175 | LR: 4.34e-04 | 1375.0ms/step | 14.5 samples/s
510
+ Step 10990/44000 | Loss: 3.1092 | LR: 4.34e-04 | 1376.0ms/step | 14.5 samples/s
511
+ Step 11000/44000 | Loss: 3.0943 | LR: 4.34e-04 | 1376.3ms/step | 14.5 samples/s
512
+ Validation at step 11000: Loss = 3.0969, Perplexity = 22.13
513
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-11000
514
+ Step 11010/44000 | Loss: 3.1211 | LR: 4.34e-04 | 3832.1ms/step | 5.2 samples/s
515
+ Step 11020/44000 | Loss: 3.0622 | LR: 4.34e-04 | 1377.8ms/step | 14.5 samples/s
516
+ Step 11030/44000 | Loss: 3.0867 | LR: 4.33e-04 | 1375.7ms/step | 14.5 samples/s
517
+ Step 11040/44000 | Loss: 3.0748 | LR: 4.33e-04 | 1376.4ms/step | 14.5 samples/s
518
+ Step 11050/44000 | Loss: 3.1056 | LR: 4.33e-04 | 1376.0ms/step | 14.5 samples/s
519
+ Step 11060/44000 | Loss: 3.1100 | LR: 4.33e-04 | 1376.3ms/step | 14.5 samples/s
520
+ Step 11070/44000 | Loss: 3.0880 | LR: 4.33e-04 | 1376.7ms/step | 14.5 samples/s
521
+ Step 11080/44000 | Loss: 3.0939 | LR: 4.33e-04 | 1375.4ms/step | 14.5 samples/s
522
+ Step 11090/44000 | Loss: 3.0772 | LR: 4.33e-04 | 1376.6ms/step | 14.5 samples/s
523
+ Step 11100/44000 | Loss: 3.1127 | LR: 4.33e-04 | 1375.7ms/step | 14.5 samples/s
524
+ Step 11110/44000 | Loss: 3.0798 | LR: 4.32e-04 | 1375.4ms/step | 14.5 samples/s
525
+ Step 11120/44000 | Loss: 3.1024 | LR: 4.32e-04 | 1375.3ms/step | 14.5 samples/s
526
+ Step 11130/44000 | Loss: 3.0381 | LR: 4.32e-04 | 1374.9ms/step | 14.5 samples/s
527
+ Step 11140/44000 | Loss: 3.1133 | LR: 4.32e-04 | 1374.8ms/step | 14.5 samples/s
528
+ Step 11150/44000 | Loss: 3.0788 | LR: 4.32e-04 | 1376.0ms/step | 14.5 samples/s
529
+ Step 11160/44000 | Loss: 3.0934 | LR: 4.32e-04 | 1376.2ms/step | 14.5 samples/s
530
+ Step 11170/44000 | Loss: 3.1054 | LR: 4.32e-04 | 1376.3ms/step | 14.5 samples/s
531
+ Step 11180/44000 | Loss: 3.1096 | LR: 4.32e-04 | 1377.8ms/step | 14.5 samples/s
532
+ Step 11190/44000 | Loss: 3.0749 | LR: 4.31e-04 | 1377.4ms/step | 14.5 samples/s
533
+ Step 11200/44000 | Loss: 3.0931 | LR: 4.31e-04 | 1376.8ms/step | 14.5 samples/s
534
+ Step 11210/44000 | Loss: 3.0849 | LR: 4.31e-04 | 1377.7ms/step | 14.5 samples/s
535
+ Step 11220/44000 | Loss: 3.0990 | LR: 4.31e-04 | 1377.3ms/step | 14.5 samples/s
536
+ Step 11230/44000 | Loss: 3.1051 | LR: 4.31e-04 | 1368.3ms/step | 14.6 samples/s
537
+ Step 11240/44000 | Loss: 3.0623 | LR: 4.31e-04 | 1368.3ms/step | 14.6 samples/s
538
+ Step 11250/44000 | Loss: 3.0780 | LR: 4.31e-04 | 1366.1ms/step | 14.6 samples/s
539
+ Step 11260/44000 | Loss: 3.0475 | LR: 4.31e-04 | 1369.3ms/step | 14.6 samples/s
540
+ Step 11270/44000 | Loss: 3.0910 | LR: 4.30e-04 | 1368.2ms/step | 14.6 samples/s
541
+ Step 11280/44000 | Loss: 3.1075 | LR: 4.30e-04 | 1369.7ms/step | 14.6 samples/s
542
+ Step 11290/44000 | Loss: 3.0896 | LR: 4.30e-04 | 1369.1ms/step | 14.6 samples/s
543
+ Step 11300/44000 | Loss: 3.0585 | LR: 4.30e-04 | 1368.4ms/step | 14.6 samples/s
544
+ Step 11310/44000 | Loss: 3.0636 | LR: 4.30e-04 | 1367.8ms/step | 14.6 samples/s
545
+ Step 11320/44000 | Loss: 3.0840 | LR: 4.30e-04 | 1371.1ms/step | 14.6 samples/s
546
+ Step 11330/44000 | Loss: 3.0912 | LR: 4.30e-04 | 1369.0ms/step | 14.6 samples/s
547
+ Step 11340/44000 | Loss: 3.1084 | LR: 4.30e-04 | 1368.5ms/step | 14.6 samples/s
548
+ Step 11350/44000 | Loss: 3.0879 | LR: 4.29e-04 | 1368.4ms/step | 14.6 samples/s
549
+ Step 11360/44000 | Loss: 3.1062 | LR: 4.29e-04 | 1367.0ms/step | 14.6 samples/s
550
+ Step 11370/44000 | Loss: 3.0955 | LR: 4.29e-04 | 1368.4ms/step | 14.6 samples/s
551
+ Step 11380/44000 | Loss: 3.0599 | LR: 4.29e-04 | 1369.2ms/step | 14.6 samples/s
552
+ Step 11390/44000 | Loss: 3.0934 | LR: 4.29e-04 | 1375.0ms/step | 14.5 samples/s
553
+ Step 11400/44000 | Loss: 3.0717 | LR: 4.29e-04 | 1374.7ms/step | 14.5 samples/s
554
+ Step 11410/44000 | Loss: 3.0621 | LR: 4.29e-04 | 1369.5ms/step | 14.6 samples/s
555
+ Step 11420/44000 | Loss: 3.0803 | LR: 4.29e-04 | 1370.0ms/step | 14.6 samples/s
556
+ Step 11430/44000 | Loss: 3.0850 | LR: 4.28e-04 | 1374.5ms/step | 14.6 samples/s
557
+ Step 11440/44000 | Loss: 3.0717 | LR: 4.28e-04 | 1377.2ms/step | 14.5 samples/s
558
+ Step 11450/44000 | Loss: 3.0824 | LR: 4.28e-04 | 1376.2ms/step | 14.5 samples/s
559
+ Step 11460/44000 | Loss: 3.0860 | LR: 4.28e-04 | 1375.9ms/step | 14.5 samples/s
560
+ Step 11470/44000 | Loss: 3.0964 | LR: 4.28e-04 | 1375.0ms/step | 14.5 samples/s
561
+ Step 11480/44000 | Loss: 3.0860 | LR: 4.28e-04 | 1377.0ms/step | 14.5 samples/s
562
+ Step 11490/44000 | Loss: 3.0804 | LR: 4.28e-04 | 1377.5ms/step | 14.5 samples/s
563
+ Step 11500/44000 | Loss: 3.0837 | LR: 4.28e-04 | 1377.6ms/step | 14.5 samples/s
564
+ Step 11510/44000 | Loss: 3.0899 | LR: 4.27e-04 | 1375.6ms/step | 14.5 samples/s
565
+ Step 11520/44000 | Loss: 3.1188 | LR: 4.27e-04 | 1376.7ms/step | 14.5 samples/s
566
+ Step 11530/44000 | Loss: 3.0671 | LR: 4.27e-04 | 1378.3ms/step | 14.5 samples/s
567
+ Step 11540/44000 | Loss: 3.1175 | LR: 4.27e-04 | 1375.0ms/step | 14.5 samples/s
568
+ Step 11550/44000 | Loss: 3.1095 | LR: 4.27e-04 | 1381.8ms/step | 14.5 samples/s
569
+ Step 11560/44000 | Loss: 3.0602 | LR: 4.27e-04 | 1377.3ms/step | 14.5 samples/s
570
+ Step 11570/44000 | Loss: 3.0885 | LR: 4.27e-04 | 1373.3ms/step | 14.6 samples/s
571
+ Step 11580/44000 | Loss: 3.0708 | LR: 4.27e-04 | 1376.1ms/step | 14.5 samples/s
572
+ Step 11590/44000 | Loss: 3.1099 | LR: 4.26e-04 | 1374.6ms/step | 14.5 samples/s
573
+ Step 11600/44000 | Loss: 3.0879 | LR: 4.26e-04 | 1375.2ms/step | 14.5 samples/s
574
+ Step 11610/44000 | Loss: 3.0879 | LR: 4.26e-04 | 1373.0ms/step | 14.6 samples/s
575
+ Step 11620/44000 | Loss: 3.0705 | LR: 4.26e-04 | 1375.4ms/step | 14.5 samples/s
576
+ Step 11630/44000 | Loss: 3.0633 | LR: 4.26e-04 | 1375.8ms/step | 14.5 samples/s
577
+ Step 11640/44000 | Loss: 3.0600 | LR: 4.26e-04 | 1374.4ms/step | 14.6 samples/s
578
+ Step 11650/44000 | Loss: 3.0727 | LR: 4.26e-04 | 1373.5ms/step | 14.6 samples/s
579
+ Step 11660/44000 | Loss: 3.0803 | LR: 4.26e-04 | 1374.9ms/step | 14.5 samples/s
580
+ Step 11670/44000 | Loss: 3.0840 | LR: 4.25e-04 | 1377.1ms/step | 14.5 samples/s
581
+ Step 11680/44000 | Loss: 3.0877 | LR: 4.25e-04 | 1376.5ms/step | 14.5 samples/s
582
+ Step 11690/44000 | Loss: 3.0806 | LR: 4.25e-04 | 1375.2ms/step | 14.5 samples/s
583
+ Step 11700/44000 | Loss: 3.0938 | LR: 4.25e-04 | 1373.2ms/step | 14.6 samples/s
584
+ Step 11710/44000 | Loss: 3.0897 | LR: 4.25e-04 | 1374.5ms/step | 14.6 samples/s
585
+ Step 11720/44000 | Loss: 3.0731 | LR: 4.25e-04 | 1372.8ms/step | 14.6 samples/s
586
+ Step 11730/44000 | Loss: 3.0878 | LR: 4.25e-04 | 1375.7ms/step | 14.5 samples/s
587
+ Step 11740/44000 | Loss: 3.0579 | LR: 4.24e-04 | 1375.4ms/step | 14.5 samples/s
588
+ Step 11750/44000 | Loss: 3.1060 | LR: 4.24e-04 | 1376.4ms/step | 14.5 samples/s
589
+ Step 11760/44000 | Loss: 3.0475 | LR: 4.24e-04 | 1376.7ms/step | 14.5 samples/s
590
+ Step 11770/44000 | Loss: 3.0894 | LR: 4.24e-04 | 1375.9ms/step | 14.5 samples/s
591
+ Step 11780/44000 | Loss: 3.1142 | LR: 4.24e-04 | 1376.1ms/step | 14.5 samples/s
592
+ Step 11790/44000 | Loss: 3.0960 | LR: 4.24e-04 | 1380.3ms/step | 14.5 samples/s
593
+ Step 11800/44000 | Loss: 3.0810 | LR: 4.24e-04 | 1381.3ms/step | 14.5 samples/s
594
+ Step 11810/44000 | Loss: 3.0988 | LR: 4.24e-04 | 1378.1ms/step | 14.5 samples/s
595
+ Step 11820/44000 | Loss: 3.0816 | LR: 4.23e-04 | 1378.4ms/step | 14.5 samples/s
596
+ Step 11830/44000 | Loss: 3.1038 | LR: 4.23e-04 | 1379.9ms/step | 14.5 samples/s
597
+ Step 11840/44000 | Loss: 3.0876 | LR: 4.23e-04 | 1380.2ms/step | 14.5 samples/s
598
+ Step 11850/44000 | Loss: 3.0735 | LR: 4.23e-04 | 1378.7ms/step | 14.5 samples/s
599
+ Step 11860/44000 | Loss: 3.0879 | LR: 4.23e-04 | 1379.4ms/step | 14.5 samples/s
600
+ Step 11870/44000 | Loss: 3.0623 | LR: 4.23e-04 | 1379.9ms/step | 14.5 samples/s
601
+ Step 11880/44000 | Loss: 3.0797 | LR: 4.23e-04 | 1378.1ms/step | 14.5 samples/s
602
+ Step 11890/44000 | Loss: 3.0711 | LR: 4.23e-04 | 1377.7ms/step | 14.5 samples/s
603
+ Step 11900/44000 | Loss: 3.0991 | LR: 4.22e-04 | 1377.7ms/step | 14.5 samples/s
604
+ Step 11910/44000 | Loss: 3.0638 | LR: 4.22e-04 | 1378.2ms/step | 14.5 samples/s
605
+ Step 11920/44000 | Loss: 3.0691 | LR: 4.22e-04 | 1375.3ms/step | 14.5 samples/s
606
+ Step 11930/44000 | Loss: 3.0873 | LR: 4.22e-04 | 1374.9ms/step | 14.5 samples/s
607
+ Step 11940/44000 | Loss: 3.0698 | LR: 4.22e-04 | 1375.0ms/step | 14.5 samples/s
608
+ Step 11950/44000 | Loss: 3.0666 | LR: 4.22e-04 | 1376.1ms/step | 14.5 samples/s
609
+ Step 11960/44000 | Loss: 3.0588 | LR: 4.22e-04 | 1375.0ms/step | 14.5 samples/s
610
+ Step 11970/44000 | Loss: 3.1018 | LR: 4.21e-04 | 1376.0ms/step | 14.5 samples/s
611
+ Step 11980/44000 | Loss: 3.0509 | LR: 4.21e-04 | 1373.8ms/step | 14.6 samples/s
612
+ Step 11990/44000 | Loss: 3.0913 | LR: 4.21e-04 | 1375.4ms/step | 14.5 samples/s
613
+ Step 12000/44000 | Loss: 3.1078 | LR: 4.21e-04 | 1377.3ms/step | 14.5 samples/s
614
+ Validation at step 12000: Loss = 3.0785, Perplexity = 21.73
615
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-12000
616
+ Step 12010/44000 | Loss: 3.0695 | LR: 4.21e-04 | 3943.1ms/step | 5.1 samples/s
617
+ Step 12020/44000 | Loss: 3.0984 | LR: 4.21e-04 | 1372.6ms/step | 14.6 samples/s
618
+ Step 12030/44000 | Loss: 3.0893 | LR: 4.21e-04 | 1372.3ms/step | 14.6 samples/s
619
+ Step 12040/44000 | Loss: 3.0485 | LR: 4.21e-04 | 1375.4ms/step | 14.5 samples/s
620
+ Step 12050/44000 | Loss: 3.0721 | LR: 4.20e-04 | 1377.5ms/step | 14.5 samples/s
621
+ Step 12060/44000 | Loss: 3.1008 | LR: 4.20e-04 | 1376.6ms/step | 14.5 samples/s
622
+ Step 12070/44000 | Loss: 3.0752 | LR: 4.20e-04 | 1377.8ms/step | 14.5 samples/s
623
+ Step 12080/44000 | Loss: 3.0788 | LR: 4.20e-04 | 1379.0ms/step | 14.5 samples/s
624
+ Step 12090/44000 | Loss: 3.1005 | LR: 4.20e-04 | 1377.9ms/step | 14.5 samples/s
625
+ Step 12100/44000 | Loss: 3.0851 | LR: 4.20e-04 | 1377.5ms/step | 14.5 samples/s
626
+ Step 12110/44000 | Loss: 3.0940 | LR: 4.20e-04 | 1376.9ms/step | 14.5 samples/s
627
+ Step 12120/44000 | Loss: 3.0943 | LR: 4.19e-04 | 1378.5ms/step | 14.5 samples/s
628
+ Step 12130/44000 | Loss: 3.0485 | LR: 4.19e-04 | 1376.3ms/step | 14.5 samples/s
629
+ Step 12140/44000 | Loss: 3.0564 | LR: 4.19e-04 | 1376.4ms/step | 14.5 samples/s
630
+ Step 12150/44000 | Loss: 3.0719 | LR: 4.19e-04 | 1379.6ms/step | 14.5 samples/s
631
+ Step 12160/44000 | Loss: 3.1018 | LR: 4.19e-04 | 1379.6ms/step | 14.5 samples/s
632
+ Step 12170/44000 | Loss: 3.0638 | LR: 4.19e-04 | 1378.5ms/step | 14.5 samples/s
633
+ Step 12180/44000 | Loss: 3.0824 | LR: 4.19e-04 | 1380.0ms/step | 14.5 samples/s
634
+ Step 12190/44000 | Loss: 3.0772 | LR: 4.19e-04 | 1377.9ms/step | 14.5 samples/s
635
+ Step 12200/44000 | Loss: 3.0660 | LR: 4.18e-04 | 1377.0ms/step | 14.5 samples/s
636
+ Step 12210/44000 | Loss: 3.0686 | LR: 4.18e-04 | 1379.0ms/step | 14.5 samples/s
637
+ Step 12220/44000 | Loss: 3.0834 | LR: 4.18e-04 | 1379.9ms/step | 14.5 samples/s
638
+ Step 12230/44000 | Loss: 3.0601 | LR: 4.18e-04 | 1379.9ms/step | 14.5 samples/s
639
+ Step 12240/44000 | Loss: 3.1170 | LR: 4.18e-04 | 1378.3ms/step | 14.5 samples/s
640
+ Step 12250/44000 | Loss: 3.0921 | LR: 4.18e-04 | 1378.6ms/step | 14.5 samples/s
641
+ Step 12260/44000 | Loss: 3.0651 | LR: 4.18e-04 | 1378.3ms/step | 14.5 samples/s
642
+ Step 12270/44000 | Loss: 3.0860 | LR: 4.17e-04 | 1378.2ms/step | 14.5 samples/s
643
+ Step 12280/44000 | Loss: 3.0773 | LR: 4.17e-04 | 1378.9ms/step | 14.5 samples/s
644
+ Step 12290/44000 | Loss: 3.0516 | LR: 4.17e-04 | 1377.2ms/step | 14.5 samples/s
645
+ Step 12300/44000 | Loss: 3.0851 | LR: 4.17e-04 | 1377.8ms/step | 14.5 samples/s
646
+ Step 12310/44000 | Loss: 3.0693 | LR: 4.17e-04 | 1379.4ms/step | 14.5 samples/s
647
+ Step 12320/44000 | Loss: 3.1017 | LR: 4.17e-04 | 1378.0ms/step | 14.5 samples/s
648
+ Step 12330/44000 | Loss: 3.0398 | LR: 4.17e-04 | 1379.1ms/step | 14.5 samples/s
649
+ Step 12340/44000 | Loss: 3.0679 | LR: 4.17e-04 | 1378.3ms/step | 14.5 samples/s
650
+ Step 12350/44000 | Loss: 3.0735 | LR: 4.16e-04 | 1378.3ms/step | 14.5 samples/s
651
+ Step 12360/44000 | Loss: 3.0593 | LR: 4.16e-04 | 1379.8ms/step | 14.5 samples/s
652
+ Step 12370/44000 | Loss: 3.0761 | LR: 4.16e-04 | 1378.0ms/step | 14.5 samples/s
653
+ Step 12380/44000 | Loss: 3.0792 | LR: 4.16e-04 | 1377.4ms/step | 14.5 samples/s
654
+ Step 12390/44000 | Loss: 3.0583 | LR: 4.16e-04 | 1378.3ms/step | 14.5 samples/s
655
+ Step 12400/44000 | Loss: 3.0592 | LR: 4.16e-04 | 1378.4ms/step | 14.5 samples/s
656
+ Step 12410/44000 | Loss: 3.0648 | LR: 4.16e-04 | 1380.0ms/step | 14.5 samples/s
657
+ Step 12420/44000 | Loss: 3.0735 | LR: 4.15e-04 | 1378.0ms/step | 14.5 samples/s
658
+ Step 12430/44000 | Loss: 3.0795 | LR: 4.15e-04 | 1377.4ms/step | 14.5 samples/s
659
+ Step 12440/44000 | Loss: 3.0746 | LR: 4.15e-04 | 1378.8ms/step | 14.5 samples/s
660
+ Step 12450/44000 | Loss: 3.0797 | LR: 4.15e-04 | 1379.5ms/step | 14.5 samples/s
661
+ Step 12460/44000 | Loss: 3.0944 | LR: 4.15e-04 | 1380.4ms/step | 14.5 samples/s
662
+ Step 12470/44000 | Loss: 3.0832 | LR: 4.15e-04 | 1380.1ms/step | 14.5 samples/s
663
+ Step 12480/44000 | Loss: 3.0779 | LR: 4.15e-04 | 1381.2ms/step | 14.5 samples/s
664
+ Step 12490/44000 | Loss: 3.0669 | LR: 4.14e-04 | 1377.8ms/step | 14.5 samples/s
665
+ Step 12500/44000 | Loss: 3.0753 | LR: 4.14e-04 | 1380.9ms/step | 14.5 samples/s
666
+ Step 12510/44000 | Loss: 3.0940 | LR: 4.14e-04 | 1381.9ms/step | 14.5 samples/s
667
+ Step 12520/44000 | Loss: 3.0361 | LR: 4.14e-04 | 1380.1ms/step | 14.5 samples/s
668
+ Step 12530/44000 | Loss: 3.0890 | LR: 4.14e-04 | 1380.2ms/step | 14.5 samples/s
669
+ Step 12540/44000 | Loss: 3.0586 | LR: 4.14e-04 | 1379.7ms/step | 14.5 samples/s
670
+ Step 12550/44000 | Loss: 3.0954 | LR: 4.14e-04 | 1376.1ms/step | 14.5 samples/s
671
+ Step 12560/44000 | Loss: 3.0942 | LR: 4.14e-04 | 1375.5ms/step | 14.5 samples/s
672
+ Step 12570/44000 | Loss: 3.0790 | LR: 4.13e-04 | 1376.9ms/step | 14.5 samples/s
673
+ Step 12580/44000 | Loss: 3.0625 | LR: 4.13e-04 | 1377.7ms/step | 14.5 samples/s
674
+ Step 12590/44000 | Loss: 3.0806 | LR: 4.13e-04 | 1378.0ms/step | 14.5 samples/s
675
+ Step 12600/44000 | Loss: 3.0706 | LR: 4.13e-04 | 1374.8ms/step | 14.5 samples/s
676
+ Step 12610/44000 | Loss: 3.0925 | LR: 4.13e-04 | 1376.4ms/step | 14.5 samples/s
677
+ Step 12620/44000 | Loss: 3.0754 | LR: 4.13e-04 | 1376.4ms/step | 14.5 samples/s
678
+ Step 12630/44000 | Loss: 3.0472 | LR: 4.13e-04 | 1376.2ms/step | 14.5 samples/s
679
+ Step 12640/44000 | Loss: 3.0778 | LR: 4.12e-04 | 1376.5ms/step | 14.5 samples/s
680
+ Step 12650/44000 | Loss: 3.0278 | LR: 4.12e-04 | 1377.2ms/step | 14.5 samples/s
681
+ Step 12660/44000 | Loss: 3.0597 | LR: 4.12e-04 | 1377.8ms/step | 14.5 samples/s
682
+ Step 12670/44000 | Loss: 3.0610 | LR: 4.12e-04 | 1379.1ms/step | 14.5 samples/s
683
+ Step 12680/44000 | Loss: 3.0325 | LR: 4.12e-04 | 1377.5ms/step | 14.5 samples/s
684
+ Step 12690/44000 | Loss: 3.0713 | LR: 4.12e-04 | 1379.0ms/step | 14.5 samples/s
685
+ Step 12700/44000 | Loss: 3.0465 | LR: 4.12e-04 | 1375.3ms/step | 14.5 samples/s
686
+ Step 12710/44000 | Loss: 3.0677 | LR: 4.11e-04 | 1375.4ms/step | 14.5 samples/s
687
+ Step 12720/44000 | Loss: 3.0690 | LR: 4.11e-04 | 1376.7ms/step | 14.5 samples/s
688
+ Step 12730/44000 | Loss: 3.0304 | LR: 4.11e-04 | 1380.4ms/step | 14.5 samples/s
689
+ Step 12740/44000 | Loss: 3.0626 | LR: 4.11e-04 | 1377.2ms/step | 14.5 samples/s
690
+ Step 12750/44000 | Loss: 3.0513 | LR: 4.11e-04 | 1376.6ms/step | 14.5 samples/s
691
+ Step 12760/44000 | Loss: 3.0534 | LR: 4.11e-04 | 1377.9ms/step | 14.5 samples/s
692
+ Step 12770/44000 | Loss: 3.0609 | LR: 4.11e-04 | 1377.8ms/step | 14.5 samples/s
693
+ Step 12780/44000 | Loss: 3.0386 | LR: 4.10e-04 | 1378.0ms/step | 14.5 samples/s
694
+ Step 12790/44000 | Loss: 3.0572 | LR: 4.10e-04 | 1377.8ms/step | 14.5 samples/s
695
+ Step 12800/44000 | Loss: 3.0674 | LR: 4.10e-04 | 1378.1ms/step | 14.5 samples/s
696
+ Step 12810/44000 | Loss: 3.0511 | LR: 4.10e-04 | 1378.0ms/step | 14.5 samples/s
697
+ Step 12820/44000 | Loss: 3.0646 | LR: 4.10e-04 | 1376.2ms/step | 14.5 samples/s
698
+ Step 12830/44000 | Loss: 3.0470 | LR: 4.10e-04 | 1378.0ms/step | 14.5 samples/s
699
+ Step 12840/44000 | Loss: 3.0640 | LR: 4.10e-04 | 1379.9ms/step | 14.5 samples/s
700
+ Step 12850/44000 | Loss: 3.0479 | LR: 4.09e-04 | 1376.8ms/step | 14.5 samples/s
701
+ Step 12860/44000 | Loss: 3.0857 | LR: 4.09e-04 | 1379.3ms/step | 14.5 samples/s
702
+ Step 12870/44000 | Loss: 3.0733 | LR: 4.09e-04 | 1376.9ms/step | 14.5 samples/s
703
+ Step 12880/44000 | Loss: 3.0709 | LR: 4.09e-04 | 1378.5ms/step | 14.5 samples/s
704
+ Step 12890/44000 | Loss: 3.0448 | LR: 4.09e-04 | 1377.7ms/step | 14.5 samples/s
705
+ Step 12900/44000 | Loss: 3.0671 | LR: 4.09e-04 | 1375.8ms/step | 14.5 samples/s
706
+ Step 12910/44000 | Loss: 3.0943 | LR: 4.09e-04 | 1378.7ms/step | 14.5 samples/s
707
+ Step 12920/44000 | Loss: 3.0598 | LR: 4.09e-04 | 1376.1ms/step | 14.5 samples/s
708
+ Step 12930/44000 | Loss: 3.0691 | LR: 4.08e-04 | 1375.7ms/step | 14.5 samples/s
709
+ Step 12940/44000 | Loss: 3.0809 | LR: 4.08e-04 | 1376.7ms/step | 14.5 samples/s
710
+ Step 12950/44000 | Loss: 3.0521 | LR: 4.08e-04 | 1376.0ms/step | 14.5 samples/s
711
+ Step 12960/44000 | Loss: 3.0496 | LR: 4.08e-04 | 1375.9ms/step | 14.5 samples/s
712
+ Step 12970/44000 | Loss: 3.0801 | LR: 4.08e-04 | 1377.8ms/step | 14.5 samples/s
713
+ Step 12980/44000 | Loss: 3.0637 | LR: 4.08e-04 | 1378.4ms/step | 14.5 samples/s
714
+ Step 12990/44000 | Loss: 3.0781 | LR: 4.08e-04 | 1378.1ms/step | 14.5 samples/s
715
+ Step 13000/44000 | Loss: 3.0612 | LR: 4.07e-04 | 1375.0ms/step | 14.5 samples/s
716
+ Validation at step 13000: Loss = 3.0601, Perplexity = 21.33
717
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-13000
718
+ Step 13010/44000 | Loss: 3.0525 | LR: 4.07e-04 | 3820.7ms/step | 5.2 samples/s
719
+ Step 13020/44000 | Loss: 3.0433 | LR: 4.07e-04 | 1372.0ms/step | 14.6 samples/s
720
+ Step 13030/44000 | Loss: 3.0306 | LR: 4.07e-04 | 1369.9ms/step | 14.6 samples/s
721
+ Step 13040/44000 | Loss: 3.0661 | LR: 4.07e-04 | 1375.7ms/step | 14.5 samples/s
722
+ Step 13050/44000 | Loss: 3.0451 | LR: 4.07e-04 | 1374.6ms/step | 14.6 samples/s
723
+ Step 13060/44000 | Loss: 3.0528 | LR: 4.07e-04 | 1377.3ms/step | 14.5 samples/s
724
+ Step 13070/44000 | Loss: 3.0593 | LR: 4.06e-04 | 1378.6ms/step | 14.5 samples/s
725
+ Step 13080/44000 | Loss: 3.0376 | LR: 4.06e-04 | 1377.0ms/step | 14.5 samples/s
726
+ Step 13090/44000 | Loss: 3.0893 | LR: 4.06e-04 | 1376.0ms/step | 14.5 samples/s
727
+ Step 13100/44000 | Loss: 3.0524 | LR: 4.06e-04 | 1375.1ms/step | 14.5 samples/s
728
+ Step 13110/44000 | Loss: 3.0893 | LR: 4.06e-04 | 1374.7ms/step | 14.5 samples/s
729
+ Step 13120/44000 | Loss: 3.0876 | LR: 4.06e-04 | 1375.2ms/step | 14.5 samples/s
730
+ Step 13130/44000 | Loss: 3.0559 | LR: 4.06e-04 | 1378.2ms/step | 14.5 samples/s
731
+ Step 13140/44000 | Loss: 3.0262 | LR: 4.05e-04 | 1378.9ms/step | 14.5 samples/s
732
+ Step 13150/44000 | Loss: 3.0955 | LR: 4.05e-04 | 1376.5ms/step | 14.5 samples/s
733
+ Step 13160/44000 | Loss: 3.0439 | LR: 4.05e-04 | 1376.2ms/step | 14.5 samples/s
epfl_final/SparseAtt/wandb/run-20260820_011125-xfnxdn24/files/requirements.txt ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pip==26.0.1
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.6.3
8
+ typing_extensions==4.15.0
9
+ triton==3.6.0
10
+ tqdm==4.67.3
11
+ torchinfo==1.8.0
12
+ sympy==1.14.0
13
+ smmap==5.0.3
14
+ six==1.17.0
15
+ shellingham==1.5.4
16
+ setuptools==81.0.0
17
+ safetensors==0.7.0
18
+ regex==2026.4.4
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.4.1
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.4.4
38
+ networkx==3.6.1
39
+ multidict==6.7.1
40
+ mdurl==0.1.2
41
+ MarkupSafe==3.0.3
42
+ idna==3.13
43
+ hf-xet==1.4.3
44
+ h11==0.16.0
45
+ fsspec==2026.2.0
46
+ frozenlist==1.8.0
47
+ filelock==3.29.0
48
+ dill==0.4.1
49
+ cuda-pathfinder==1.5.4
50
+ click==8.3.3
51
+ charset-normalizer==3.4.7
52
+ certifi==2026.4.22
53
+ attrs==26.1.0
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ yarl==1.23.0
58
+ typing-inspection==0.4.2
59
+ sentry-sdk==2.58.0
60
+ requests==2.33.1
61
+ python-dateutil==2.9.0.post0
62
+ pydantic_core==2.46.3
63
+ nvidia-cusparse==12.6.3.3
64
+ nvidia-cufft==12.0.0.61
65
+ nvidia-cudnn-cu13==9.19.0.56
66
+ multiprocess==0.70.19
67
+ markdown-it-py==4.0.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ cuda-bindings==13.2.0
72
+ anyio==4.13.0
73
+ aiosignal==1.4.0
74
+ rich==15.0.0
75
+ pydantic==2.13.3
76
+ pandas==3.0.2
77
+ nvidia-cusolver==12.0.4.66
78
+ httpx==0.28.1
79
+ GitPython==3.1.49
80
+ aiohttp==3.13.5
81
+ wandb==0.26.1
82
+ typer==0.25.1
83
+ torch==2.11.0
84
+ huggingface_hub==1.13.0
85
+ tokenizers==0.22.2
86
+ datasets==4.8.5
87
+ accelerate==1.13.0
88
+ transformers==5.7.0
89
+ word2number==1.1
90
+ sqlitedict==2.1.0
91
+ pytz==2026.2
92
+ threadpoolctl==3.6.0
93
+ tcolorpy==0.1.7
94
+ tabulate==0.10.0
95
+ scipy==1.17.1
96
+ portalocker==3.2.0
97
+ pathvalidate==3.3.1
98
+ narwhals==2.22.1
99
+ more-itertools==11.1.0
100
+ lxml==6.1.1
101
+ joblib==1.5.3
102
+ colorama==0.4.6
103
+ chardet==6.0.0.post1
104
+ absl-py==2.4.0
105
+ scikit-learn==1.9.0
106
+ sacrebleu==2.6.0
107
+ nltk==3.9.4
108
+ mbstrdecoder==1.1.5
109
+ typepy==1.3.5
110
+ rouge_score==0.1.2
111
+ DataProperty==1.1.1
112
+ tabledata==1.3.5
113
+ evaluate==0.4.6
114
+ pytablewriter==1.2.1
115
+ lm_eval==0.4.12
116
+ pluggy==1.6.0
117
+ iniconfig==2.3.0
118
+ pytest==9.1.0
119
+ py-spy==0.4.2
120
+ pillow==12.2.0
121
+ tifffile==2026.6.1
122
+ imagecodecs==2026.6.26
123
+ zipp==4.1.0
124
+ importlib_metadata==9.0.0
125
+ derivative==0.6.3
126
+ pysindy==2.1.0
127
+ pyparsing==3.3.2
128
+ kiwisolver==1.5.0
129
+ fonttools==4.63.0
130
+ cycler==0.12.1
131
+ contourpy==1.3.3
132
+ matplotlib==3.11.0
133
+ sortedcontainers==2.4.0
134
+ wcwidth==0.8.2
135
+ sigfig==1.3.19
136
+ importlib_resources==7.1.0
137
+ enum-tools==0.13.0
138
+ cramjam==2.11.0
139
+ python-snappy==0.7.3
140
+ compact-json==1.8.2
141
+ numbers-parser==4.18.5
142
+ entmax==1.3
epfl_final/SparseAtt/wandb/run-20260820_011125-xfnxdn24/files/wandb-metadata.json ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.13.13",
4
+ "startedAt": "2026-08-20T01:11:25.993256Z",
5
+ "args": [
6
+ "--config",
7
+ "/mloscratch/homes/navasard/sparse/configs_ropectl/resume_a1p0_wandb/seed45.json"
8
+ ],
9
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
10
+ "codePath": "train.py",
11
+ "codePathLocal": "train.py",
12
+ "git": {
13
+ "remote": "https://github.com/rubentium/TokenPrune.git",
14
+ "commit": "5637886f0cc760009989b14e68c6ae06d4827418"
15
+ },
16
+ "email": "ruben.navasardyan@mail.utoronto.ca",
17
+ "root": "/mloscratch/homes/navasard/sparse/SparseAtt",
18
+ "host": "h100-workspace-5-gpu-0-1",
19
+ "executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
20
+ "cpu_count": 96,
21
+ "cpu_count_logical": 192,
22
+ "gpu": "NVIDIA H100 80GB HBM3",
23
+ "gpu_count": 5,
24
+ "disk": {
25
+ "/": {
26
+ "total": "9374931091456",
27
+ "used": "6373417115648"
28
+ }
29
+ },
30
+ "memory": {
31
+ "total": "1623005732864"
32
+ },
33
+ "gpu_nvidia": [
34
+ {
35
+ "name": "NVIDIA H100 80GB HBM3",
36
+ "memoryTotal": "85520809984",
37
+ "cudaCores": 16896,
38
+ "architecture": "Hopper",
39
+ "uuid": "GPU-51f13867-a72c-7b65-ff60-661787f0c18a"
40
+ },
41
+ {
42
+ "name": "NVIDIA H100 80GB HBM3",
43
+ "memoryTotal": "85520809984",
44
+ "cudaCores": 16896,
45
+ "architecture": "Hopper",
46
+ "uuid": "GPU-a2122da0-1020-9f78-7e6f-a1d5011f694b"
47
+ },
48
+ {
49
+ "name": "NVIDIA H100 80GB HBM3",
50
+ "memoryTotal": "85520809984",
51
+ "cudaCores": 16896,
52
+ "architecture": "Hopper",
53
+ "uuid": "GPU-f65cda91-ff87-9bff-fb3d-3e1b3f9f8cf0"
54
+ },
55
+ {
56
+ "name": "NVIDIA H100 80GB HBM3",
57
+ "memoryTotal": "85520809984",
58
+ "cudaCores": 16896,
59
+ "architecture": "Hopper",
60
+ "uuid": "GPU-4c1d91e9-bf01-f8fd-1e8b-79aa8e0831df"
61
+ },
62
+ {
63
+ "name": "NVIDIA H100 80GB HBM3",
64
+ "memoryTotal": "85520809984",
65
+ "cudaCores": 16896,
66
+ "architecture": "Hopper",
67
+ "uuid": "GPU-bdb5e23e-0700-20a0-0b21-1ad0901f08b7"
68
+ }
69
+ ],
70
+ "cudaVersion": "13.0",
71
+ "writerId": "ni9elbsjzhqeurotw3zb64wsjaa7hqn9"
72
+ }
epfl_final/SparseAtt/wandb/run-20260820_011125-xfnxdn24/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/SparseAtt/wandb/run-20260820_011125-xfnxdn24/logs/debug.log ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-08-20 01:11:26,057 INFO MainThread:9358 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-08-20 01:11:26,057 INFO MainThread:9358 [wandb_setup.py:_flush():81] Configure stats pid to 9358
3
+ 2026-08-20 01:11:26,058 INFO MainThread:9358 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-08-20 01:11:26,058 INFO MainThread:9358 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_011125-xfnxdn24/logs/debug.log
5
+ 2026-08-20 01:11:26,058 INFO MainThread:9358 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_011125-xfnxdn24/logs/debug-internal.log
6
+ 2026-08-20 01:11:26,058 INFO MainThread:9358 [wandb_init.py:init():850] calling init triggers
7
+ 2026-08-20 01:11:26,058 INFO MainThread:9358 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-6000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 1000, 'eval_batches': 50, 'logging_steps': 10, 'seed': 45, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'l8k365m_dense_ropectl_a1p0_seed45', 'wandb_tags': ['qwen3', '365m', '8k', 'dense', 'rope-control'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': 0.0, 'rope_train_position_scale_set': [1.0, 0.9896339488176222, 0.9692159692159692, 0.9420289855072463, 0.8730158730158729], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
9
+ 2026-08-20 01:11:26,058 INFO MainThread:9358 [wandb_init.py:init():898] starting backend
10
+ 2026-08-20 01:11:26,326 INFO MainThread:9358 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-08-20 01:11:26,662 INFO MainThread:9358 [wandb_init.py:init():918] backend started and connected
12
+ 2026-08-20 01:11:26,669 INFO MainThread:9358 [wandb_init.py:init():988] updated telemetry
13
+ 2026-08-20 01:11:26,683 INFO MainThread:9358 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-08-20 01:11:27,036 INFO MainThread:9358 [wandb_init.py:init():1056] starting run threads in backend
15
+ 2026-08-20 01:11:27,218 INFO MainThread:9358 [wandb_run.py:_console_start():2554] atexit reg
16
+ 2026-08-20 01:11:27,218 INFO MainThread:9358 [wandb_run.py:_redirect():2403] redirect: wrap_raw
17
+ 2026-08-20 01:11:27,218 INFO MainThread:9358 [wandb_run.py:_redirect():2472] Wrapping output streams.
18
+ 2026-08-20 01:11:27,218 INFO MainThread:9358 [wandb_run.py:_redirect():2495] Redirects installed.
19
+ 2026-08-20 01:11:27,222 INFO MainThread:9358 [wandb_init.py:init():1094] run started, returning control to user process
epfl_final/SparseAtt/wandb/run-20260820_011125-xfnxdn24/run-xfnxdn24.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c946b56d1c7fd808ef42f67917e59042b1464abf809ac1e6e5138a2bb22af1bd
3
+ size 4259840
epfl_final/SparseAtt/wandb/run-20260820_011128-ysa9iyov/files/output.log ADDED
@@ -0,0 +1,717 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Starting training...
2
+ Validation at step 7000: Loss = 3.1887, Perplexity = 24.26
3
+ Switching attention mode for step 7001: dense (dense step interval)
4
+ Step 7010/44000 | Loss: 3.2378 | LR: 4.75e-04 | 16610.4ms/step | 1.2 samples/s
5
+ Step 7020/44000 | Loss: 3.2161 | LR: 4.75e-04 | 1400.2ms/step | 14.3 samples/s
6
+ Step 7030/44000 | Loss: 3.1930 | LR: 4.74e-04 | 1400.1ms/step | 14.3 samples/s
7
+ Step 7040/44000 | Loss: 3.2021 | LR: 4.74e-04 | 1400.0ms/step | 14.3 samples/s
8
+ Step 7050/44000 | Loss: 3.1891 | LR: 4.74e-04 | 1400.0ms/step | 14.3 samples/s
9
+ Step 7060/44000 | Loss: 3.1711 | LR: 4.74e-04 | 1400.1ms/step | 14.3 samples/s
10
+ Step 7070/44000 | Loss: 3.2081 | LR: 4.74e-04 | 1400.0ms/step | 14.3 samples/s
11
+ Step 7080/44000 | Loss: 3.1494 | LR: 4.74e-04 | 1399.9ms/step | 14.3 samples/s
12
+ Step 7090/44000 | Loss: 3.2019 | LR: 4.74e-04 | 1400.0ms/step | 14.3 samples/s
13
+ Step 7100/44000 | Loss: 3.1880 | LR: 4.74e-04 | 1400.0ms/step | 14.3 samples/s
14
+ Step 7110/44000 | Loss: 3.2000 | LR: 4.74e-04 | 1400.0ms/step | 14.3 samples/s
15
+ Step 7120/44000 | Loss: 3.2023 | LR: 4.74e-04 | 1399.9ms/step | 14.3 samples/s
16
+ Step 7130/44000 | Loss: 3.2013 | LR: 4.74e-04 | 1400.3ms/step | 14.3 samples/s
17
+ Step 7140/44000 | Loss: 3.1745 | LR: 4.74e-04 | 1400.3ms/step | 14.3 samples/s
18
+ Step 7150/44000 | Loss: 3.2055 | LR: 4.73e-04 | 1399.9ms/step | 14.3 samples/s
19
+ Step 7160/44000 | Loss: 3.1701 | LR: 4.73e-04 | 1399.6ms/step | 14.3 samples/s
20
+ Step 7170/44000 | Loss: 3.1819 | LR: 4.73e-04 | 1400.1ms/step | 14.3 samples/s
21
+ Step 7180/44000 | Loss: 3.2145 | LR: 4.73e-04 | 1400.0ms/step | 14.3 samples/s
22
+ Step 7190/44000 | Loss: 3.2191 | LR: 4.73e-04 | 1399.8ms/step | 14.3 samples/s
23
+ Step 7200/44000 | Loss: 3.2221 | LR: 4.73e-04 | 1399.9ms/step | 14.3 samples/s
24
+ Step 7210/44000 | Loss: 3.1904 | LR: 4.73e-04 | 1400.5ms/step | 14.3 samples/s
25
+ Step 7220/44000 | Loss: 3.2112 | LR: 4.73e-04 | 1401.8ms/step | 14.3 samples/s
26
+ Step 7230/44000 | Loss: 3.1888 | LR: 4.73e-04 | 1401.2ms/step | 14.3 samples/s
27
+ Step 7240/44000 | Loss: 3.2273 | LR: 4.73e-04 | 1404.1ms/step | 14.2 samples/s
28
+ Step 7250/44000 | Loss: 3.1857 | LR: 4.73e-04 | 1404.8ms/step | 14.2 samples/s
29
+ Step 7260/44000 | Loss: 3.1968 | LR: 4.73e-04 | 1406.1ms/step | 14.2 samples/s
30
+ Step 7270/44000 | Loss: 3.1982 | LR: 4.72e-04 | 1402.1ms/step | 14.3 samples/s
31
+ Step 7280/44000 | Loss: 3.1973 | LR: 4.72e-04 | 1399.7ms/step | 14.3 samples/s
32
+ Step 7290/44000 | Loss: 3.1980 | LR: 4.72e-04 | 1400.1ms/step | 14.3 samples/s
33
+ Step 7300/44000 | Loss: 3.1992 | LR: 4.72e-04 | 1399.8ms/step | 14.3 samples/s
34
+ Step 7310/44000 | Loss: 3.2175 | LR: 4.72e-04 | 1399.9ms/step | 14.3 samples/s
35
+ Step 7320/44000 | Loss: 3.1606 | LR: 4.72e-04 | 1400.1ms/step | 14.3 samples/s
36
+ Step 7330/44000 | Loss: 3.1815 | LR: 4.72e-04 | 1401.9ms/step | 14.3 samples/s
37
+ Step 7340/44000 | Loss: 3.1827 | LR: 4.72e-04 | 1400.1ms/step | 14.3 samples/s
38
+ Step 7350/44000 | Loss: 3.1640 | LR: 4.72e-04 | 1404.0ms/step | 14.2 samples/s
39
+ Step 7360/44000 | Loss: 3.2107 | LR: 4.72e-04 | 1400.5ms/step | 14.3 samples/s
40
+ Step 7370/44000 | Loss: 3.1788 | LR: 4.72e-04 | 1408.4ms/step | 14.2 samples/s
41
+ Step 7380/44000 | Loss: 3.2138 | LR: 4.72e-04 | 1414.2ms/step | 14.1 samples/s
42
+ Step 7390/44000 | Loss: 3.1946 | LR: 4.71e-04 | 1413.2ms/step | 14.2 samples/s
43
+ Step 7400/44000 | Loss: 3.1901 | LR: 4.71e-04 | 1408.4ms/step | 14.2 samples/s
44
+ Step 7410/44000 | Loss: 3.2026 | LR: 4.71e-04 | 1409.3ms/step | 14.2 samples/s
45
+ Step 7420/44000 | Loss: 3.1861 | LR: 4.71e-04 | 1414.2ms/step | 14.1 samples/s
46
+ Step 7430/44000 | Loss: 3.2039 | LR: 4.71e-04 | 1416.1ms/step | 14.1 samples/s
47
+ Step 7440/44000 | Loss: 3.2059 | LR: 4.71e-04 | 1415.0ms/step | 14.1 samples/s
48
+ Step 7450/44000 | Loss: 3.1806 | LR: 4.71e-04 | 1414.1ms/step | 14.1 samples/s
49
+ Step 7460/44000 | Loss: 3.1856 | LR: 4.71e-04 | 1412.9ms/step | 14.2 samples/s
50
+ Step 7470/44000 | Loss: 3.1765 | LR: 4.71e-04 | 1410.5ms/step | 14.2 samples/s
51
+ Step 7480/44000 | Loss: 3.1932 | LR: 4.71e-04 | 1407.6ms/step | 14.2 samples/s
52
+ Step 7490/44000 | Loss: 3.1632 | LR: 4.71e-04 | 1404.3ms/step | 14.2 samples/s
53
+ Step 7500/44000 | Loss: 3.1904 | LR: 4.71e-04 | 1406.2ms/step | 14.2 samples/s
54
+ Step 7510/44000 | Loss: 3.1920 | LR: 4.70e-04 | 1406.9ms/step | 14.2 samples/s
55
+ Step 7520/44000 | Loss: 3.1792 | LR: 4.70e-04 | 1406.2ms/step | 14.2 samples/s
56
+ Step 7530/44000 | Loss: 3.2126 | LR: 4.70e-04 | 1399.9ms/step | 14.3 samples/s
57
+ Step 7540/44000 | Loss: 3.1964 | LR: 4.70e-04 | 1400.3ms/step | 14.3 samples/s
58
+ Step 7550/44000 | Loss: 3.1889 | LR: 4.70e-04 | 1401.7ms/step | 14.3 samples/s
59
+ Step 7560/44000 | Loss: 3.1665 | LR: 4.70e-04 | 1402.1ms/step | 14.3 samples/s
60
+ Step 7570/44000 | Loss: 3.1858 | LR: 4.70e-04 | 1404.1ms/step | 14.2 samples/s
61
+ Step 7580/44000 | Loss: 3.1450 | LR: 4.70e-04 | 1405.4ms/step | 14.2 samples/s
62
+ Step 7590/44000 | Loss: 3.1720 | LR: 4.70e-04 | 1405.2ms/step | 14.2 samples/s
63
+ Step 7600/44000 | Loss: 3.1793 | LR: 4.70e-04 | 1407.3ms/step | 14.2 samples/s
64
+ Step 7610/44000 | Loss: 3.1825 | LR: 4.70e-04 | 1402.1ms/step | 14.3 samples/s
65
+ Step 7620/44000 | Loss: 3.1902 | LR: 4.70e-04 | 1402.8ms/step | 14.3 samples/s
66
+ Step 7630/44000 | Loss: 3.1696 | LR: 4.69e-04 | 1400.8ms/step | 14.3 samples/s
67
+ Step 7640/44000 | Loss: 3.1666 | LR: 4.69e-04 | 1400.0ms/step | 14.3 samples/s
68
+ Step 7650/44000 | Loss: 3.1633 | LR: 4.69e-04 | 1402.7ms/step | 14.3 samples/s
69
+ Step 7660/44000 | Loss: 3.1983 | LR: 4.69e-04 | 1399.6ms/step | 14.3 samples/s
70
+ Step 7670/44000 | Loss: 3.1791 | LR: 4.69e-04 | 1400.2ms/step | 14.3 samples/s
71
+ Step 7680/44000 | Loss: 3.1686 | LR: 4.69e-04 | 1401.8ms/step | 14.3 samples/s
72
+ Step 7690/44000 | Loss: 3.1692 | LR: 4.69e-04 | 1405.8ms/step | 14.2 samples/s
73
+ Step 7700/44000 | Loss: 3.2092 | LR: 4.69e-04 | 1402.1ms/step | 14.3 samples/s
74
+ Step 7710/44000 | Loss: 3.1626 | LR: 4.69e-04 | 1402.1ms/step | 14.3 samples/s
75
+ Step 7720/44000 | Loss: 3.1576 | LR: 4.69e-04 | 1402.2ms/step | 14.3 samples/s
76
+ Step 7730/44000 | Loss: 3.1353 | LR: 4.69e-04 | 1402.0ms/step | 14.3 samples/s
77
+ Step 7740/44000 | Loss: 3.1613 | LR: 4.68e-04 | 1400.2ms/step | 14.3 samples/s
78
+ Step 7750/44000 | Loss: 3.2006 | LR: 4.68e-04 | 1404.0ms/step | 14.2 samples/s
79
+ Step 7760/44000 | Loss: 3.1672 | LR: 4.68e-04 | 1400.0ms/step | 14.3 samples/s
80
+ Step 7770/44000 | Loss: 3.1725 | LR: 4.68e-04 | 1402.9ms/step | 14.3 samples/s
81
+ Step 7780/44000 | Loss: 3.1748 | LR: 4.68e-04 | 1401.3ms/step | 14.3 samples/s
82
+ Step 7790/44000 | Loss: 3.1730 | LR: 4.68e-04 | 1401.2ms/step | 14.3 samples/s
83
+ Step 7800/44000 | Loss: 3.1990 | LR: 4.68e-04 | 1400.0ms/step | 14.3 samples/s
84
+ Step 7810/44000 | Loss: 3.1459 | LR: 4.68e-04 | 1400.4ms/step | 14.3 samples/s
85
+ Step 7820/44000 | Loss: 3.1935 | LR: 4.68e-04 | 1401.9ms/step | 14.3 samples/s
86
+ Step 7830/44000 | Loss: 3.1653 | LR: 4.68e-04 | 1399.8ms/step | 14.3 samples/s
87
+ Step 7840/44000 | Loss: 3.1482 | LR: 4.68e-04 | 1400.1ms/step | 14.3 samples/s
88
+ Step 7850/44000 | Loss: 3.1597 | LR: 4.67e-04 | 1400.4ms/step | 14.3 samples/s
89
+ Step 7860/44000 | Loss: 3.1516 | LR: 4.67e-04 | 1403.7ms/step | 14.2 samples/s
90
+ Step 7870/44000 | Loss: 3.1740 | LR: 4.67e-04 | 1400.1ms/step | 14.3 samples/s
91
+ Step 7880/44000 | Loss: 3.1948 | LR: 4.67e-04 | 1399.5ms/step | 14.3 samples/s
92
+ Step 7890/44000 | Loss: 3.1725 | LR: 4.67e-04 | 1400.2ms/step | 14.3 samples/s
93
+ Step 7900/44000 | Loss: 3.1784 | LR: 4.67e-04 | 1400.3ms/step | 14.3 samples/s
94
+ Step 7910/44000 | Loss: 3.1622 | LR: 4.67e-04 | 1401.8ms/step | 14.3 samples/s
95
+ Step 7920/44000 | Loss: 3.1737 | LR: 4.67e-04 | 1399.9ms/step | 14.3 samples/s
96
+ Step 7930/44000 | Loss: 3.1807 | LR: 4.67e-04 | 1400.0ms/step | 14.3 samples/s
97
+ Step 7940/44000 | Loss: 3.1674 | LR: 4.67e-04 | 1399.9ms/step | 14.3 samples/s
98
+ Step 7950/44000 | Loss: 3.1637 | LR: 4.67e-04 | 1400.1ms/step | 14.3 samples/s
99
+ Step 7960/44000 | Loss: 3.1523 | LR: 4.66e-04 | 1400.1ms/step | 14.3 samples/s
100
+ Step 7970/44000 | Loss: 3.1872 | LR: 4.66e-04 | 1399.9ms/step | 14.3 samples/s
101
+ Step 7980/44000 | Loss: 3.1797 | LR: 4.66e-04 | 1400.2ms/step | 14.3 samples/s
102
+ Step 7990/44000 | Loss: 3.1744 | LR: 4.66e-04 | 1399.9ms/step | 14.3 samples/s
103
+ Step 8000/44000 | Loss: 3.1619 | LR: 4.66e-04 | 1400.0ms/step | 14.3 samples/s
104
+ Validation at step 8000: Loss = 3.1563, Perplexity = 23.48
105
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-8000
106
+ Step 8010/44000 | Loss: 3.1459 | LR: 4.66e-04 | 3908.1ms/step | 5.1 samples/s
107
+ Step 8020/44000 | Loss: 3.1432 | LR: 4.66e-04 | 1404.6ms/step | 14.2 samples/s
108
+ Step 8030/44000 | Loss: 3.1785 | LR: 4.66e-04 | 1406.6ms/step | 14.2 samples/s
109
+ Step 8040/44000 | Loss: 3.1637 | LR: 4.66e-04 | 1402.9ms/step | 14.3 samples/s
110
+ Step 8050/44000 | Loss: 3.1979 | LR: 4.66e-04 | 1405.0ms/step | 14.2 samples/s
111
+ Step 8060/44000 | Loss: 3.1732 | LR: 4.66e-04 | 1407.1ms/step | 14.2 samples/s
112
+ Step 8070/44000 | Loss: 3.1650 | LR: 4.65e-04 | 1399.8ms/step | 14.3 samples/s
113
+ Step 8080/44000 | Loss: 3.1598 | LR: 4.65e-04 | 1402.4ms/step | 14.3 samples/s
114
+ Step 8090/44000 | Loss: 3.1856 | LR: 4.65e-04 | 1399.5ms/step | 14.3 samples/s
115
+ Step 8100/44000 | Loss: 3.1875 | LR: 4.65e-04 | 1400.1ms/step | 14.3 samples/s
116
+ Step 8110/44000 | Loss: 3.1602 | LR: 4.65e-04 | 1403.1ms/step | 14.3 samples/s
117
+ Step 8120/44000 | Loss: 3.1507 | LR: 4.65e-04 | 1404.7ms/step | 14.2 samples/s
118
+ Step 8130/44000 | Loss: 3.1630 | LR: 4.65e-04 | 1400.9ms/step | 14.3 samples/s
119
+ Step 8140/44000 | Loss: 3.1921 | LR: 4.65e-04 | 1402.0ms/step | 14.3 samples/s
120
+ Step 8150/44000 | Loss: 3.1427 | LR: 4.65e-04 | 1403.5ms/step | 14.3 samples/s
121
+ Step 8160/44000 | Loss: 3.1760 | LR: 4.65e-04 | 1404.2ms/step | 14.2 samples/s
122
+ Step 8170/44000 | Loss: 3.1352 | LR: 4.65e-04 | 1405.8ms/step | 14.2 samples/s
123
+ Step 8180/44000 | Loss: 3.1670 | LR: 4.64e-04 | 1400.0ms/step | 14.3 samples/s
124
+ Step 8190/44000 | Loss: 3.1871 | LR: 4.64e-04 | 1399.8ms/step | 14.3 samples/s
125
+ Step 8200/44000 | Loss: 3.1693 | LR: 4.64e-04 | 1401.0ms/step | 14.3 samples/s
126
+ Step 8210/44000 | Loss: 3.1524 | LR: 4.64e-04 | 1403.7ms/step | 14.2 samples/s
127
+ Step 8220/44000 | Loss: 3.1969 | LR: 4.64e-04 | 1407.3ms/step | 14.2 samples/s
128
+ Step 8230/44000 | Loss: 3.1496 | LR: 4.64e-04 | 1410.8ms/step | 14.2 samples/s
129
+ Step 8240/44000 | Loss: 3.1233 | LR: 4.64e-04 | 1407.9ms/step | 14.2 samples/s
130
+ Step 8250/44000 | Loss: 3.1729 | LR: 4.64e-04 | 1409.8ms/step | 14.2 samples/s
131
+ Step 8260/44000 | Loss: 3.1513 | LR: 4.64e-04 | 1407.8ms/step | 14.2 samples/s
132
+ Step 8270/44000 | Loss: 3.1564 | LR: 4.64e-04 | 1402.0ms/step | 14.3 samples/s
133
+ Step 8280/44000 | Loss: 3.1445 | LR: 4.64e-04 | 1406.9ms/step | 14.2 samples/s
134
+ Step 8290/44000 | Loss: 3.1641 | LR: 4.63e-04 | 1407.6ms/step | 14.2 samples/s
135
+ Step 8300/44000 | Loss: 3.1809 | LR: 4.63e-04 | 1410.4ms/step | 14.2 samples/s
136
+ Step 8310/44000 | Loss: 3.1408 | LR: 4.63e-04 | 1409.3ms/step | 14.2 samples/s
137
+ Step 8320/44000 | Loss: 3.1668 | LR: 4.63e-04 | 1407.8ms/step | 14.2 samples/s
138
+ Step 8330/44000 | Loss: 3.1570 | LR: 4.63e-04 | 1402.1ms/step | 14.3 samples/s
139
+ Step 8340/44000 | Loss: 3.1659 | LR: 4.63e-04 | 1401.7ms/step | 14.3 samples/s
140
+ Step 8350/44000 | Loss: 3.1385 | LR: 4.63e-04 | 1405.4ms/step | 14.2 samples/s
141
+ Step 8360/44000 | Loss: 3.1767 | LR: 4.63e-04 | 1407.3ms/step | 14.2 samples/s
142
+ Step 8370/44000 | Loss: 3.1355 | LR: 4.63e-04 | 1407.5ms/step | 14.2 samples/s
143
+ Step 8380/44000 | Loss: 3.1747 | LR: 4.63e-04 | 1407.8ms/step | 14.2 samples/s
144
+ Step 8390/44000 | Loss: 3.1449 | LR: 4.62e-04 | 1408.7ms/step | 14.2 samples/s
145
+ Step 8400/44000 | Loss: 3.1249 | LR: 4.62e-04 | 1407.6ms/step | 14.2 samples/s
146
+ Step 8410/44000 | Loss: 3.1510 | LR: 4.62e-04 | 1409.7ms/step | 14.2 samples/s
147
+ Step 8420/44000 | Loss: 3.1495 | LR: 4.62e-04 | 1405.9ms/step | 14.2 samples/s
148
+ Step 8430/44000 | Loss: 3.1598 | LR: 4.62e-04 | 1402.2ms/step | 14.3 samples/s
149
+ Step 8440/44000 | Loss: 3.1814 | LR: 4.62e-04 | 1405.9ms/step | 14.2 samples/s
150
+ Step 8450/44000 | Loss: 3.1653 | LR: 4.62e-04 | 1409.0ms/step | 14.2 samples/s
151
+ Step 8460/44000 | Loss: 3.1412 | LR: 4.62e-04 | 1409.4ms/step | 14.2 samples/s
152
+ Step 8470/44000 | Loss: 3.1490 | LR: 4.62e-04 | 1411.8ms/step | 14.2 samples/s
153
+ Step 8480/44000 | Loss: 3.1478 | LR: 4.62e-04 | 1407.9ms/step | 14.2 samples/s
154
+ Step 8490/44000 | Loss: 3.1341 | LR: 4.62e-04 | 1408.0ms/step | 14.2 samples/s
155
+ Step 8500/44000 | Loss: 3.1498 | LR: 4.61e-04 | 1410.3ms/step | 14.2 samples/s
156
+ Step 8510/44000 | Loss: 3.1437 | LR: 4.61e-04 | 1409.0ms/step | 14.2 samples/s
157
+ Step 8520/44000 | Loss: 3.1271 | LR: 4.61e-04 | 1412.0ms/step | 14.2 samples/s
158
+ Step 8530/44000 | Loss: 3.1414 | LR: 4.61e-04 | 1410.6ms/step | 14.2 samples/s
159
+ Step 8540/44000 | Loss: 3.1360 | LR: 4.61e-04 | 1408.1ms/step | 14.2 samples/s
160
+ Step 8550/44000 | Loss: 3.1629 | LR: 4.61e-04 | 1400.0ms/step | 14.3 samples/s
161
+ Step 8560/44000 | Loss: 3.1589 | LR: 4.61e-04 | 1400.0ms/step | 14.3 samples/s
162
+ Step 8570/44000 | Loss: 3.1710 | LR: 4.61e-04 | 1404.1ms/step | 14.2 samples/s
163
+ Step 8580/44000 | Loss: 3.1549 | LR: 4.61e-04 | 1405.2ms/step | 14.2 samples/s
164
+ Step 8590/44000 | Loss: 3.1574 | LR: 4.61e-04 | 1408.8ms/step | 14.2 samples/s
165
+ Step 8600/44000 | Loss: 3.1681 | LR: 4.60e-04 | 1403.7ms/step | 14.2 samples/s
166
+ Step 8610/44000 | Loss: 3.1451 | LR: 4.60e-04 | 1400.0ms/step | 14.3 samples/s
167
+ Step 8620/44000 | Loss: 3.1428 | LR: 4.60e-04 | 1400.1ms/step | 14.3 samples/s
168
+ Step 8630/44000 | Loss: 3.1948 | LR: 4.60e-04 | 1400.1ms/step | 14.3 samples/s
169
+ Step 8640/44000 | Loss: 3.1635 | LR: 4.60e-04 | 1400.0ms/step | 14.3 samples/s
170
+ Step 8650/44000 | Loss: 3.1668 | LR: 4.60e-04 | 1400.1ms/step | 14.3 samples/s
171
+ Step 8660/44000 | Loss: 3.1750 | LR: 4.60e-04 | 1399.7ms/step | 14.3 samples/s
172
+ Step 8670/44000 | Loss: 3.1501 | LR: 4.60e-04 | 1400.3ms/step | 14.3 samples/s
173
+ Step 8680/44000 | Loss: 3.1544 | LR: 4.60e-04 | 1401.8ms/step | 14.3 samples/s
174
+ Step 8690/44000 | Loss: 3.1439 | LR: 4.60e-04 | 1400.0ms/step | 14.3 samples/s
175
+ Step 8700/44000 | Loss: 3.1635 | LR: 4.59e-04 | 1400.1ms/step | 14.3 samples/s
176
+ Step 8710/44000 | Loss: 3.1489 | LR: 4.59e-04 | 1399.9ms/step | 14.3 samples/s
177
+ Step 8720/44000 | Loss: 3.1516 | LR: 4.59e-04 | 1402.0ms/step | 14.3 samples/s
178
+ Step 8730/44000 | Loss: 3.1324 | LR: 4.59e-04 | 1402.3ms/step | 14.3 samples/s
179
+ Step 8740/44000 | Loss: 3.1523 | LR: 4.59e-04 | 1399.7ms/step | 14.3 samples/s
180
+ Step 8750/44000 | Loss: 3.1518 | LR: 4.59e-04 | 1402.3ms/step | 14.3 samples/s
181
+ Step 8760/44000 | Loss: 3.1588 | LR: 4.59e-04 | 1403.7ms/step | 14.2 samples/s
182
+ Step 8770/44000 | Loss: 3.1553 | LR: 4.59e-04 | 1402.5ms/step | 14.3 samples/s
183
+ Step 8780/44000 | Loss: 3.1433 | LR: 4.59e-04 | 1399.8ms/step | 14.3 samples/s
184
+ Step 8790/44000 | Loss: 3.1506 | LR: 4.59e-04 | 1403.8ms/step | 14.2 samples/s
185
+ Step 8800/44000 | Loss: 3.1292 | LR: 4.58e-04 | 1403.9ms/step | 14.2 samples/s
186
+ Step 8810/44000 | Loss: 3.1508 | LR: 4.58e-04 | 1404.4ms/step | 14.2 samples/s
187
+ Step 8820/44000 | Loss: 3.1126 | LR: 4.58e-04 | 1403.9ms/step | 14.2 samples/s
188
+ Step 8830/44000 | Loss: 3.1109 | LR: 4.58e-04 | 1403.9ms/step | 14.2 samples/s
189
+ Step 8840/44000 | Loss: 3.1422 | LR: 4.58e-04 | 1404.6ms/step | 14.2 samples/s
190
+ Step 8850/44000 | Loss: 3.1418 | LR: 4.58e-04 | 1405.9ms/step | 14.2 samples/s
191
+ Step 8860/44000 | Loss: 3.1213 | LR: 4.58e-04 | 1408.0ms/step | 14.2 samples/s
192
+ Step 8870/44000 | Loss: 3.1615 | LR: 4.58e-04 | 1407.5ms/step | 14.2 samples/s
193
+ Step 8880/44000 | Loss: 3.1442 | LR: 4.58e-04 | 1403.8ms/step | 14.2 samples/s
194
+ Step 8890/44000 | Loss: 3.1357 | LR: 4.58e-04 | 1400.1ms/step | 14.3 samples/s
195
+ Step 8900/44000 | Loss: 3.1370 | LR: 4.57e-04 | 1404.8ms/step | 14.2 samples/s
196
+ Step 8910/44000 | Loss: 3.1190 | LR: 4.57e-04 | 1405.3ms/step | 14.2 samples/s
197
+ Step 8920/44000 | Loss: 3.1601 | LR: 4.57e-04 | 1404.2ms/step | 14.2 samples/s
198
+ Step 8930/44000 | Loss: 3.1652 | LR: 4.57e-04 | 1406.1ms/step | 14.2 samples/s
199
+ Step 8940/44000 | Loss: 3.1143 | LR: 4.57e-04 | 1407.7ms/step | 14.2 samples/s
200
+ Step 8950/44000 | Loss: 3.1463 | LR: 4.57e-04 | 1402.3ms/step | 14.3 samples/s
201
+ Step 8960/44000 | Loss: 3.1255 | LR: 4.57e-04 | 1408.1ms/step | 14.2 samples/s
202
+ Step 8970/44000 | Loss: 3.1358 | LR: 4.57e-04 | 1407.5ms/step | 14.2 samples/s
203
+ Step 8980/44000 | Loss: 3.1236 | LR: 4.57e-04 | 1401.6ms/step | 14.3 samples/s
204
+ Step 8990/44000 | Loss: 3.1252 | LR: 4.57e-04 | 1400.3ms/step | 14.3 samples/s
205
+ Step 9000/44000 | Loss: 3.1499 | LR: 4.56e-04 | 1401.9ms/step | 14.3 samples/s
206
+ Validation at step 9000: Loss = 3.1276, Perplexity = 22.82
207
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-9000
208
+ Step 9010/44000 | Loss: 3.1391 | LR: 4.56e-04 | 3930.4ms/step | 5.1 samples/s
209
+ Step 9020/44000 | Loss: 3.1061 | LR: 4.56e-04 | 1403.9ms/step | 14.2 samples/s
210
+ Step 9030/44000 | Loss: 3.1733 | LR: 4.56e-04 | 1401.8ms/step | 14.3 samples/s
211
+ Step 9040/44000 | Loss: 3.1372 | LR: 4.56e-04 | 1404.7ms/step | 14.2 samples/s
212
+ Step 9050/44000 | Loss: 3.1105 | LR: 4.56e-04 | 1401.3ms/step | 14.3 samples/s
213
+ Step 9060/44000 | Loss: 3.1341 | LR: 4.56e-04 | 1401.2ms/step | 14.3 samples/s
214
+ Step 9070/44000 | Loss: 3.1049 | LR: 4.56e-04 | 1400.7ms/step | 14.3 samples/s
215
+ Step 9080/44000 | Loss: 3.1517 | LR: 4.56e-04 | 1400.1ms/step | 14.3 samples/s
216
+ Step 9090/44000 | Loss: 3.1421 | LR: 4.56e-04 | 1401.9ms/step | 14.3 samples/s
217
+ Step 9100/44000 | Loss: 3.1232 | LR: 4.55e-04 | 1402.7ms/step | 14.3 samples/s
218
+ Step 9110/44000 | Loss: 3.1191 | LR: 4.55e-04 | 1405.5ms/step | 14.2 samples/s
219
+ Step 9120/44000 | Loss: 3.1411 | LR: 4.55e-04 | 1402.2ms/step | 14.3 samples/s
220
+ Step 9130/44000 | Loss: 3.1191 | LR: 4.55e-04 | 1403.6ms/step | 14.2 samples/s
221
+ Step 9140/44000 | Loss: 3.0964 | LR: 4.55e-04 | 1408.9ms/step | 14.2 samples/s
222
+ Step 9150/44000 | Loss: 3.1481 | LR: 4.55e-04 | 1409.7ms/step | 14.2 samples/s
223
+ Step 9160/44000 | Loss: 3.1581 | LR: 4.55e-04 | 1409.6ms/step | 14.2 samples/s
224
+ Step 9170/44000 | Loss: 3.1439 | LR: 4.55e-04 | 1410.3ms/step | 14.2 samples/s
225
+ Step 9180/44000 | Loss: 3.1059 | LR: 4.55e-04 | 1410.0ms/step | 14.2 samples/s
226
+ Step 9190/44000 | Loss: 3.1228 | LR: 4.54e-04 | 1409.9ms/step | 14.2 samples/s
227
+ Step 9200/44000 | Loss: 3.1262 | LR: 4.54e-04 | 1408.3ms/step | 14.2 samples/s
228
+ Step 9210/44000 | Loss: 3.1231 | LR: 4.54e-04 | 1410.7ms/step | 14.2 samples/s
229
+ Step 9220/44000 | Loss: 3.1166 | LR: 4.54e-04 | 1409.1ms/step | 14.2 samples/s
230
+ Step 9230/44000 | Loss: 3.1366 | LR: 4.54e-04 | 1408.1ms/step | 14.2 samples/s
231
+ Step 9240/44000 | Loss: 3.1300 | LR: 4.54e-04 | 1409.1ms/step | 14.2 samples/s
232
+ Step 9250/44000 | Loss: 3.1313 | LR: 4.54e-04 | 1409.7ms/step | 14.2 samples/s
233
+ Step 9260/44000 | Loss: 3.1668 | LR: 4.54e-04 | 1410.7ms/step | 14.2 samples/s
234
+ Step 9270/44000 | Loss: 3.1387 | LR: 4.54e-04 | 1412.4ms/step | 14.2 samples/s
235
+ Step 9280/44000 | Loss: 3.1438 | LR: 4.54e-04 | 1413.7ms/step | 14.1 samples/s
236
+ Step 9290/44000 | Loss: 3.1241 | LR: 4.53e-04 | 1414.8ms/step | 14.1 samples/s
237
+ Step 9300/44000 | Loss: 3.0910 | LR: 4.53e-04 | 1414.6ms/step | 14.1 samples/s
238
+ Step 9310/44000 | Loss: 3.1358 | LR: 4.53e-04 | 1411.7ms/step | 14.2 samples/s
239
+ Step 9320/44000 | Loss: 3.1396 | LR: 4.53e-04 | 1411.3ms/step | 14.2 samples/s
240
+ Step 9330/44000 | Loss: 3.1287 | LR: 4.53e-04 | 1410.0ms/step | 14.2 samples/s
241
+ Step 9340/44000 | Loss: 3.1039 | LR: 4.53e-04 | 1410.5ms/step | 14.2 samples/s
242
+ Step 9350/44000 | Loss: 3.1361 | LR: 4.53e-04 | 1411.1ms/step | 14.2 samples/s
243
+ Step 9360/44000 | Loss: 3.1379 | LR: 4.53e-04 | 1406.7ms/step | 14.2 samples/s
244
+ Step 9370/44000 | Loss: 3.1155 | LR: 4.53e-04 | 1409.3ms/step | 14.2 samples/s
245
+ Step 9380/44000 | Loss: 3.1364 | LR: 4.52e-04 | 1406.1ms/step | 14.2 samples/s
246
+ Step 9390/44000 | Loss: 3.1441 | LR: 4.52e-04 | 1403.9ms/step | 14.2 samples/s
247
+ Step 9400/44000 | Loss: 3.1410 | LR: 4.52e-04 | 1404.1ms/step | 14.2 samples/s
248
+ Step 9410/44000 | Loss: 3.1347 | LR: 4.52e-04 | 1408.2ms/step | 14.2 samples/s
249
+ Step 9420/44000 | Loss: 3.1305 | LR: 4.52e-04 | 1409.9ms/step | 14.2 samples/s
250
+ Step 9430/44000 | Loss: 3.1292 | LR: 4.52e-04 | 1410.6ms/step | 14.2 samples/s
251
+ Step 9440/44000 | Loss: 3.1208 | LR: 4.52e-04 | 1410.2ms/step | 14.2 samples/s
252
+ Step 9450/44000 | Loss: 3.1344 | LR: 4.52e-04 | 1409.5ms/step | 14.2 samples/s
253
+ Step 9460/44000 | Loss: 3.1122 | LR: 4.52e-04 | 1410.8ms/step | 14.2 samples/s
254
+ Step 9470/44000 | Loss: 3.1421 | LR: 4.52e-04 | 1403.0ms/step | 14.3 samples/s
255
+ Step 9480/44000 | Loss: 3.1245 | LR: 4.51e-04 | 1402.5ms/step | 14.3 samples/s
256
+ Step 9490/44000 | Loss: 3.1188 | LR: 4.51e-04 | 1405.5ms/step | 14.2 samples/s
257
+ Step 9500/44000 | Loss: 3.1198 | LR: 4.51e-04 | 1408.4ms/step | 14.2 samples/s
258
+ Step 9510/44000 | Loss: 3.1178 | LR: 4.51e-04 | 1408.0ms/step | 14.2 samples/s
259
+ Step 9520/44000 | Loss: 3.1111 | LR: 4.51e-04 | 1409.9ms/step | 14.2 samples/s
260
+ Step 9530/44000 | Loss: 3.1394 | LR: 4.51e-04 | 1409.8ms/step | 14.2 samples/s
261
+ Step 9540/44000 | Loss: 3.1264 | LR: 4.51e-04 | 1406.7ms/step | 14.2 samples/s
262
+ Step 9550/44000 | Loss: 3.0934 | LR: 4.51e-04 | 1402.8ms/step | 14.3 samples/s
263
+ Step 9560/44000 | Loss: 3.1112 | LR: 4.51e-04 | 1402.1ms/step | 14.3 samples/s
264
+ Step 9570/44000 | Loss: 3.1347 | LR: 4.50e-04 | 1403.9ms/step | 14.2 samples/s
265
+ Step 9580/44000 | Loss: 3.1207 | LR: 4.50e-04 | 1404.3ms/step | 14.2 samples/s
266
+ Step 9590/44000 | Loss: 3.1159 | LR: 4.50e-04 | 1403.8ms/step | 14.2 samples/s
267
+ Step 9600/44000 | Loss: 3.1566 | LR: 4.50e-04 | 1402.1ms/step | 14.3 samples/s
268
+ Step 9610/44000 | Loss: 3.1396 | LR: 4.50e-04 | 1402.1ms/step | 14.3 samples/s
269
+ Step 9620/44000 | Loss: 3.1584 | LR: 4.50e-04 | 1401.9ms/step | 14.3 samples/s
270
+ Step 9630/44000 | Loss: 3.0924 | LR: 4.50e-04 | 1399.9ms/step | 14.3 samples/s
271
+ Step 9640/44000 | Loss: 3.1430 | LR: 4.50e-04 | 1399.9ms/step | 14.3 samples/s
272
+ Step 9650/44000 | Loss: 3.1381 | LR: 4.50e-04 | 1400.4ms/step | 14.3 samples/s
273
+ Step 9660/44000 | Loss: 3.1325 | LR: 4.49e-04 | 1399.6ms/step | 14.3 samples/s
274
+ Step 9670/44000 | Loss: 3.0682 | LR: 4.49e-04 | 1400.1ms/step | 14.3 samples/s
275
+ Step 9680/44000 | Loss: 3.1254 | LR: 4.49e-04 | 1402.3ms/step | 14.3 samples/s
276
+ Step 9690/44000 | Loss: 3.1294 | LR: 4.49e-04 | 1402.7ms/step | 14.3 samples/s
277
+ Step 9700/44000 | Loss: 3.1663 | LR: 4.49e-04 | 1401.1ms/step | 14.3 samples/s
278
+ Step 9710/44000 | Loss: 3.1534 | LR: 4.49e-04 | 1402.4ms/step | 14.3 samples/s
279
+ Step 9720/44000 | Loss: 3.0960 | LR: 4.49e-04 | 1407.7ms/step | 14.2 samples/s
280
+ Step 9730/44000 | Loss: 3.1048 | LR: 4.49e-04 | 1405.9ms/step | 14.2 samples/s
281
+ Step 9740/44000 | Loss: 3.1167 | LR: 4.49e-04 | 1399.8ms/step | 14.3 samples/s
282
+ Step 9750/44000 | Loss: 3.1323 | LR: 4.48e-04 | 1400.9ms/step | 14.3 samples/s
283
+ Step 9760/44000 | Loss: 3.0747 | LR: 4.48e-04 | 1401.5ms/step | 14.3 samples/s
284
+ Step 9770/44000 | Loss: 3.1140 | LR: 4.48e-04 | 1399.9ms/step | 14.3 samples/s
285
+ Step 9780/44000 | Loss: 3.1137 | LR: 4.48e-04 | 1401.9ms/step | 14.3 samples/s
286
+ Step 9790/44000 | Loss: 3.1068 | LR: 4.48e-04 | 1406.6ms/step | 14.2 samples/s
287
+ Step 9800/44000 | Loss: 3.1330 | LR: 4.48e-04 | 1404.4ms/step | 14.2 samples/s
288
+ Step 9810/44000 | Loss: 3.1034 | LR: 4.48e-04 | 1405.3ms/step | 14.2 samples/s
289
+ Step 9820/44000 | Loss: 3.1293 | LR: 4.48e-04 | 1402.6ms/step | 14.3 samples/s
290
+ Step 9830/44000 | Loss: 3.1044 | LR: 4.48e-04 | 1402.5ms/step | 14.3 samples/s
291
+ Step 9840/44000 | Loss: 3.1018 | LR: 4.47e-04 | 1400.5ms/step | 14.3 samples/s
292
+ Step 9850/44000 | Loss: 3.1160 | LR: 4.47e-04 | 1400.4ms/step | 14.3 samples/s
293
+ Step 9860/44000 | Loss: 3.1473 | LR: 4.47e-04 | 1402.1ms/step | 14.3 samples/s
294
+ Step 9870/44000 | Loss: 3.1385 | LR: 4.47e-04 | 1402.1ms/step | 14.3 samples/s
295
+ Step 9880/44000 | Loss: 3.1358 | LR: 4.47e-04 | 1404.7ms/step | 14.2 samples/s
296
+ Step 9890/44000 | Loss: 3.1261 | LR: 4.47e-04 | 1406.9ms/step | 14.2 samples/s
297
+ Step 9900/44000 | Loss: 3.1528 | LR: 4.47e-04 | 1404.9ms/step | 14.2 samples/s
298
+ Step 9910/44000 | Loss: 3.1018 | LR: 4.47e-04 | 1407.4ms/step | 14.2 samples/s
299
+ Step 9920/44000 | Loss: 3.1114 | LR: 4.47e-04 | 1402.5ms/step | 14.3 samples/s
300
+ Step 9930/44000 | Loss: 3.1072 | LR: 4.46e-04 | 1405.7ms/step | 14.2 samples/s
301
+ Step 9940/44000 | Loss: 3.1244 | LR: 4.46e-04 | 1405.1ms/step | 14.2 samples/s
302
+ Step 9950/44000 | Loss: 3.1176 | LR: 4.46e-04 | 1405.7ms/step | 14.2 samples/s
303
+ Step 9960/44000 | Loss: 3.1344 | LR: 4.46e-04 | 1407.7ms/step | 14.2 samples/s
304
+ Step 9970/44000 | Loss: 3.1121 | LR: 4.46e-04 | 1408.1ms/step | 14.2 samples/s
305
+ Step 9980/44000 | Loss: 3.1522 | LR: 4.46e-04 | 1407.1ms/step | 14.2 samples/s
306
+ Step 9990/44000 | Loss: 3.1056 | LR: 4.46e-04 | 1402.3ms/step | 14.3 samples/s
307
+ Step 10000/44000 | Loss: 3.1291 | LR: 4.46e-04 | 1406.2ms/step | 14.2 samples/s
308
+ Validation at step 10000: Loss = 3.1027, Perplexity = 22.26
309
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-10000
310
+ Step 10010/44000 | Loss: 3.1139 | LR: 4.46e-04 | 3929.3ms/step | 5.1 samples/s
311
+ Step 10020/44000 | Loss: 3.0857 | LR: 4.45e-04 | 1400.5ms/step | 14.3 samples/s
312
+ Step 10030/44000 | Loss: 3.1113 | LR: 4.45e-04 | 1399.6ms/step | 14.3 samples/s
313
+ Step 10040/44000 | Loss: 3.1253 | LR: 4.45e-04 | 1402.4ms/step | 14.3 samples/s
314
+ Step 10050/44000 | Loss: 3.1238 | LR: 4.45e-04 | 1401.7ms/step | 14.3 samples/s
315
+ Step 10060/44000 | Loss: 3.1100 | LR: 4.45e-04 | 1400.2ms/step | 14.3 samples/s
316
+ Step 10070/44000 | Loss: 3.1107 | LR: 4.45e-04 | 1399.6ms/step | 14.3 samples/s
317
+ Step 10080/44000 | Loss: 3.1344 | LR: 4.45e-04 | 1400.7ms/step | 14.3 samples/s
318
+ Step 10090/44000 | Loss: 3.1042 | LR: 4.45e-04 | 1405.5ms/step | 14.2 samples/s
319
+ Step 10100/44000 | Loss: 3.1371 | LR: 4.45e-04 | 1406.7ms/step | 14.2 samples/s
320
+ Step 10110/44000 | Loss: 3.0871 | LR: 4.44e-04 | 1406.5ms/step | 14.2 samples/s
321
+ Step 10120/44000 | Loss: 3.0956 | LR: 4.44e-04 | 1407.3ms/step | 14.2 samples/s
322
+ Step 10130/44000 | Loss: 3.1282 | LR: 4.44e-04 | 1410.1ms/step | 14.2 samples/s
323
+ Step 10140/44000 | Loss: 3.0656 | LR: 4.44e-04 | 1410.1ms/step | 14.2 samples/s
324
+ Step 10150/44000 | Loss: 3.0713 | LR: 4.44e-04 | 1411.5ms/step | 14.2 samples/s
325
+ Step 10160/44000 | Loss: 3.0989 | LR: 4.44e-04 | 1410.6ms/step | 14.2 samples/s
326
+ Step 10170/44000 | Loss: 3.0823 | LR: 4.44e-04 | 1411.8ms/step | 14.2 samples/s
327
+ Step 10180/44000 | Loss: 3.1069 | LR: 4.44e-04 | 1411.8ms/step | 14.2 samples/s
328
+ Step 10190/44000 | Loss: 3.1158 | LR: 4.43e-04 | 1409.6ms/step | 14.2 samples/s
329
+ Step 10200/44000 | Loss: 3.0979 | LR: 4.43e-04 | 1410.8ms/step | 14.2 samples/s
330
+ Step 10210/44000 | Loss: 3.1027 | LR: 4.43e-04 | 1410.5ms/step | 14.2 samples/s
331
+ Step 10220/44000 | Loss: 3.0791 | LR: 4.43e-04 | 1408.4ms/step | 14.2 samples/s
332
+ Step 10230/44000 | Loss: 3.0968 | LR: 4.43e-04 | 1410.3ms/step | 14.2 samples/s
333
+ Step 10240/44000 | Loss: 3.1259 | LR: 4.43e-04 | 1410.7ms/step | 14.2 samples/s
334
+ Step 10250/44000 | Loss: 3.1145 | LR: 4.43e-04 | 1410.1ms/step | 14.2 samples/s
335
+ Step 10260/44000 | Loss: 3.0700 | LR: 4.43e-04 | 1412.6ms/step | 14.2 samples/s
336
+ Step 10270/44000 | Loss: 3.0649 | LR: 4.43e-04 | 1412.1ms/step | 14.2 samples/s
337
+ Step 10280/44000 | Loss: 3.1561 | LR: 4.42e-04 | 1412.3ms/step | 14.2 samples/s
338
+ Step 10290/44000 | Loss: 3.0974 | LR: 4.42e-04 | 1408.4ms/step | 14.2 samples/s
339
+ Step 10300/44000 | Loss: 3.1285 | LR: 4.42e-04 | 1406.4ms/step | 14.2 samples/s
340
+ Step 10310/44000 | Loss: 3.0942 | LR: 4.42e-04 | 1409.8ms/step | 14.2 samples/s
341
+ Step 10320/44000 | Loss: 3.1015 | LR: 4.42e-04 | 1410.3ms/step | 14.2 samples/s
342
+ Step 10330/44000 | Loss: 3.1418 | LR: 4.42e-04 | 1412.2ms/step | 14.2 samples/s
343
+ Step 10340/44000 | Loss: 3.0753 | LR: 4.42e-04 | 1409.3ms/step | 14.2 samples/s
344
+ Step 10350/44000 | Loss: 3.0967 | LR: 4.42e-04 | 1412.7ms/step | 14.2 samples/s
345
+ Step 10360/44000 | Loss: 3.0916 | LR: 4.42e-04 | 1411.4ms/step | 14.2 samples/s
346
+ Step 10370/44000 | Loss: 3.0797 | LR: 4.41e-04 | 1410.8ms/step | 14.2 samples/s
347
+ Step 10380/44000 | Loss: 3.0792 | LR: 4.41e-04 | 1410.1ms/step | 14.2 samples/s
348
+ Step 10390/44000 | Loss: 3.1106 | LR: 4.41e-04 | 1409.5ms/step | 14.2 samples/s
349
+ Step 10400/44000 | Loss: 3.0695 | LR: 4.41e-04 | 1409.6ms/step | 14.2 samples/s
350
+ Step 10410/44000 | Loss: 3.1261 | LR: 4.41e-04 | 1411.5ms/step | 14.2 samples/s
351
+ Step 10420/44000 | Loss: 3.1178 | LR: 4.41e-04 | 1413.4ms/step | 14.2 samples/s
352
+ Step 10430/44000 | Loss: 3.1224 | LR: 4.41e-04 | 1411.9ms/step | 14.2 samples/s
353
+ Step 10440/44000 | Loss: 3.1033 | LR: 4.41e-04 | 1409.6ms/step | 14.2 samples/s
354
+ Step 10450/44000 | Loss: 3.1016 | LR: 4.40e-04 | 1406.7ms/step | 14.2 samples/s
355
+ Step 10460/44000 | Loss: 3.0957 | LR: 4.40e-04 | 1405.4ms/step | 14.2 samples/s
356
+ Step 10470/44000 | Loss: 3.1374 | LR: 4.40e-04 | 1407.9ms/step | 14.2 samples/s
357
+ Step 10480/44000 | Loss: 3.1097 | LR: 4.40e-04 | 1405.5ms/step | 14.2 samples/s
358
+ Step 10490/44000 | Loss: 3.1210 | LR: 4.40e-04 | 1404.4ms/step | 14.2 samples/s
359
+ Step 10500/44000 | Loss: 3.1224 | LR: 4.40e-04 | 1406.3ms/step | 14.2 samples/s
360
+ Step 10510/44000 | Loss: 3.1040 | LR: 4.40e-04 | 1408.9ms/step | 14.2 samples/s
361
+ Step 10520/44000 | Loss: 3.0971 | LR: 4.40e-04 | 1407.6ms/step | 14.2 samples/s
362
+ Step 10530/44000 | Loss: 3.1116 | LR: 4.40e-04 | 1405.6ms/step | 14.2 samples/s
363
+ Step 10540/44000 | Loss: 3.0976 | LR: 4.39e-04 | 1409.7ms/step | 14.2 samples/s
364
+ Step 10550/44000 | Loss: 3.0950 | LR: 4.39e-04 | 1403.7ms/step | 14.2 samples/s
365
+ Step 10560/44000 | Loss: 3.1155 | LR: 4.39e-04 | 1400.0ms/step | 14.3 samples/s
366
+ Step 10570/44000 | Loss: 3.0834 | LR: 4.39e-04 | 1402.3ms/step | 14.3 samples/s
367
+ Step 10580/44000 | Loss: 3.1006 | LR: 4.39e-04 | 1403.6ms/step | 14.2 samples/s
368
+ Step 10590/44000 | Loss: 3.1374 | LR: 4.39e-04 | 1404.3ms/step | 14.2 samples/s
369
+ Step 10600/44000 | Loss: 3.0777 | LR: 4.39e-04 | 1399.8ms/step | 14.3 samples/s
370
+ Step 10610/44000 | Loss: 3.1259 | LR: 4.39e-04 | 1400.5ms/step | 14.3 samples/s
371
+ Step 10620/44000 | Loss: 3.1061 | LR: 4.38e-04 | 1402.2ms/step | 14.3 samples/s
372
+ Step 10630/44000 | Loss: 3.1307 | LR: 4.38e-04 | 1401.7ms/step | 14.3 samples/s
373
+ Step 10640/44000 | Loss: 3.1174 | LR: 4.38e-04 | 1403.9ms/step | 14.2 samples/s
374
+ Step 10650/44000 | Loss: 3.1028 | LR: 4.38e-04 | 1400.5ms/step | 14.3 samples/s
375
+ Step 10660/44000 | Loss: 3.0819 | LR: 4.38e-04 | 1404.1ms/step | 14.2 samples/s
376
+ Step 10670/44000 | Loss: 3.0769 | LR: 4.38e-04 | 1407.3ms/step | 14.2 samples/s
377
+ Step 10680/44000 | Loss: 3.0926 | LR: 4.38e-04 | 1406.3ms/step | 14.2 samples/s
378
+ Step 10690/44000 | Loss: 3.0938 | LR: 4.38e-04 | 1406.0ms/step | 14.2 samples/s
379
+ Step 10700/44000 | Loss: 3.1088 | LR: 4.37e-04 | 1407.7ms/step | 14.2 samples/s
380
+ Step 10710/44000 | Loss: 3.1150 | LR: 4.37e-04 | 1408.0ms/step | 14.2 samples/s
381
+ Step 10720/44000 | Loss: 3.0955 | LR: 4.37e-04 | 1406.6ms/step | 14.2 samples/s
382
+ Step 10730/44000 | Loss: 3.1439 | LR: 4.37e-04 | 1407.9ms/step | 14.2 samples/s
383
+ Step 10740/44000 | Loss: 3.1324 | LR: 4.37e-04 | 1405.7ms/step | 14.2 samples/s
384
+ Step 10750/44000 | Loss: 3.0899 | LR: 4.37e-04 | 1406.2ms/step | 14.2 samples/s
385
+ Step 10760/44000 | Loss: 3.0951 | LR: 4.37e-04 | 1401.0ms/step | 14.3 samples/s
386
+ Step 10770/44000 | Loss: 3.0855 | LR: 4.37e-04 | 1406.9ms/step | 14.2 samples/s
387
+ Step 10780/44000 | Loss: 3.1027 | LR: 4.37e-04 | 1407.2ms/step | 14.2 samples/s
388
+ Step 10790/44000 | Loss: 3.0676 | LR: 4.36e-04 | 1404.9ms/step | 14.2 samples/s
389
+ Step 10800/44000 | Loss: 3.0920 | LR: 4.36e-04 | 1405.0ms/step | 14.2 samples/s
390
+ Step 10810/44000 | Loss: 3.0940 | LR: 4.36e-04 | 1404.5ms/step | 14.2 samples/s
391
+ Step 10820/44000 | Loss: 3.1248 | LR: 4.36e-04 | 1404.3ms/step | 14.2 samples/s
392
+ Step 10830/44000 | Loss: 3.0871 | LR: 4.36e-04 | 1406.0ms/step | 14.2 samples/s
393
+ Step 10840/44000 | Loss: 3.0912 | LR: 4.36e-04 | 1401.7ms/step | 14.3 samples/s
394
+ Step 10850/44000 | Loss: 3.0636 | LR: 4.36e-04 | 1402.2ms/step | 14.3 samples/s
395
+ Step 10860/44000 | Loss: 3.1123 | LR: 4.36e-04 | 1404.1ms/step | 14.2 samples/s
396
+ Step 10870/44000 | Loss: 3.1204 | LR: 4.35e-04 | 1402.5ms/step | 14.3 samples/s
397
+ Step 10880/44000 | Loss: 3.0951 | LR: 4.35e-04 | 1401.9ms/step | 14.3 samples/s
398
+ Step 10890/44000 | Loss: 3.0830 | LR: 4.35e-04 | 1401.6ms/step | 14.3 samples/s
399
+ Step 10900/44000 | Loss: 3.0962 | LR: 4.35e-04 | 1402.0ms/step | 14.3 samples/s
400
+ Step 10910/44000 | Loss: 3.1161 | LR: 4.35e-04 | 1404.0ms/step | 14.2 samples/s
401
+ Step 10920/44000 | Loss: 3.0845 | LR: 4.35e-04 | 1403.9ms/step | 14.2 samples/s
402
+ Step 10930/44000 | Loss: 3.1032 | LR: 4.35e-04 | 1402.3ms/step | 14.3 samples/s
403
+ Step 10940/44000 | Loss: 3.1167 | LR: 4.35e-04 | 1407.8ms/step | 14.2 samples/s
404
+ Step 10950/44000 | Loss: 3.1012 | LR: 4.34e-04 | 1404.3ms/step | 14.2 samples/s
405
+ Step 10960/44000 | Loss: 3.0833 | LR: 4.34e-04 | 1406.1ms/step | 14.2 samples/s
406
+ Step 10970/44000 | Loss: 3.1097 | LR: 4.34e-04 | 1407.6ms/step | 14.2 samples/s
407
+ Step 10980/44000 | Loss: 3.0889 | LR: 4.34e-04 | 1405.9ms/step | 14.2 samples/s
408
+ Step 10990/44000 | Loss: 3.1079 | LR: 4.34e-04 | 1405.6ms/step | 14.2 samples/s
409
+ Step 11000/44000 | Loss: 3.1145 | LR: 4.34e-04 | 1404.7ms/step | 14.2 samples/s
410
+ Validation at step 11000: Loss = 3.0832, Perplexity = 21.83
411
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-11000
412
+ Step 11010/44000 | Loss: 3.1048 | LR: 4.34e-04 | 3923.8ms/step | 5.1 samples/s
413
+ Step 11020/44000 | Loss: 3.0973 | LR: 4.34e-04 | 1400.7ms/step | 14.3 samples/s
414
+ Step 11030/44000 | Loss: 3.1211 | LR: 4.33e-04 | 1405.1ms/step | 14.2 samples/s
415
+ Step 11040/44000 | Loss: 3.0705 | LR: 4.33e-04 | 1408.2ms/step | 14.2 samples/s
416
+ Step 11050/44000 | Loss: 3.0568 | LR: 4.33e-04 | 1404.2ms/step | 14.2 samples/s
417
+ Step 11060/44000 | Loss: 3.0985 | LR: 4.33e-04 | 1402.1ms/step | 14.3 samples/s
418
+ Step 11070/44000 | Loss: 3.1213 | LR: 4.33e-04 | 1407.4ms/step | 14.2 samples/s
419
+ Step 11080/44000 | Loss: 3.0798 | LR: 4.33e-04 | 1408.2ms/step | 14.2 samples/s
420
+ Step 11090/44000 | Loss: 3.0937 | LR: 4.33e-04 | 1411.0ms/step | 14.2 samples/s
421
+ Step 11100/44000 | Loss: 3.0921 | LR: 4.33e-04 | 1411.5ms/step | 14.2 samples/s
422
+ Step 11110/44000 | Loss: 3.0640 | LR: 4.32e-04 | 1412.3ms/step | 14.2 samples/s
423
+ Step 11120/44000 | Loss: 3.0772 | LR: 4.32e-04 | 1411.4ms/step | 14.2 samples/s
424
+ Step 11130/44000 | Loss: 3.0501 | LR: 4.32e-04 | 1408.8ms/step | 14.2 samples/s
425
+ Step 11140/44000 | Loss: 3.0983 | LR: 4.32e-04 | 1411.3ms/step | 14.2 samples/s
426
+ Step 11150/44000 | Loss: 3.0510 | LR: 4.32e-04 | 1407.8ms/step | 14.2 samples/s
427
+ Step 11160/44000 | Loss: 3.1102 | LR: 4.32e-04 | 1408.9ms/step | 14.2 samples/s
428
+ Step 11170/44000 | Loss: 3.1115 | LR: 4.32e-04 | 1409.6ms/step | 14.2 samples/s
429
+ Step 11180/44000 | Loss: 3.1016 | LR: 4.32e-04 | 1409.7ms/step | 14.2 samples/s
430
+ Step 11190/44000 | Loss: 3.0904 | LR: 4.31e-04 | 1410.6ms/step | 14.2 samples/s
431
+ Step 11200/44000 | Loss: 3.1061 | LR: 4.31e-04 | 1409.9ms/step | 14.2 samples/s
432
+ Step 11210/44000 | Loss: 3.0956 | LR: 4.31e-04 | 1412.3ms/step | 14.2 samples/s
433
+ Step 11220/44000 | Loss: 3.0955 | LR: 4.31e-04 | 1411.1ms/step | 14.2 samples/s
434
+ Step 11230/44000 | Loss: 3.0797 | LR: 4.31e-04 | 1412.1ms/step | 14.2 samples/s
435
+ Step 11240/44000 | Loss: 3.0769 | LR: 4.31e-04 | 1410.0ms/step | 14.2 samples/s
436
+ Step 11250/44000 | Loss: 3.0783 | LR: 4.31e-04 | 1410.4ms/step | 14.2 samples/s
437
+ Step 11260/44000 | Loss: 3.0971 | LR: 4.31e-04 | 1409.9ms/step | 14.2 samples/s
438
+ Step 11270/44000 | Loss: 3.0659 | LR: 4.30e-04 | 1409.7ms/step | 14.2 samples/s
439
+ Step 11280/44000 | Loss: 3.0995 | LR: 4.30e-04 | 1409.9ms/step | 14.2 samples/s
440
+ Step 11290/44000 | Loss: 3.0663 | LR: 4.30e-04 | 1409.8ms/step | 14.2 samples/s
441
+ Step 11300/44000 | Loss: 3.0665 | LR: 4.30e-04 | 1409.2ms/step | 14.2 samples/s
442
+ Step 11310/44000 | Loss: 3.0765 | LR: 4.30e-04 | 1409.3ms/step | 14.2 samples/s
443
+ Step 11320/44000 | Loss: 3.0624 | LR: 4.30e-04 | 1408.4ms/step | 14.2 samples/s
444
+ Step 11330/44000 | Loss: 3.1223 | LR: 4.30e-04 | 1409.0ms/step | 14.2 samples/s
445
+ Step 11340/44000 | Loss: 3.0610 | LR: 4.30e-04 | 1408.7ms/step | 14.2 samples/s
446
+ Step 11350/44000 | Loss: 3.1035 | LR: 4.29e-04 | 1406.5ms/step | 14.2 samples/s
447
+ Step 11360/44000 | Loss: 3.1070 | LR: 4.29e-04 | 1406.9ms/step | 14.2 samples/s
448
+ Step 11370/44000 | Loss: 3.0720 | LR: 4.29e-04 | 1402.0ms/step | 14.3 samples/s
449
+ Step 11380/44000 | Loss: 3.0813 | LR: 4.29e-04 | 1403.0ms/step | 14.3 samples/s
450
+ Step 11390/44000 | Loss: 3.0752 | LR: 4.29e-04 | 1403.7ms/step | 14.2 samples/s
451
+ Step 11400/44000 | Loss: 3.1073 | LR: 4.29e-04 | 1407.2ms/step | 14.2 samples/s
452
+ Step 11410/44000 | Loss: 3.1016 | LR: 4.29e-04 | 1408.4ms/step | 14.2 samples/s
453
+ Step 11420/44000 | Loss: 3.1016 | LR: 4.29e-04 | 1407.9ms/step | 14.2 samples/s
454
+ Step 11430/44000 | Loss: 3.0792 | LR: 4.28e-04 | 1406.9ms/step | 14.2 samples/s
455
+ Step 11440/44000 | Loss: 3.0864 | LR: 4.28e-04 | 1403.5ms/step | 14.3 samples/s
456
+ Step 11450/44000 | Loss: 3.0733 | LR: 4.28e-04 | 1405.6ms/step | 14.2 samples/s
457
+ Step 11460/44000 | Loss: 3.0638 | LR: 4.28e-04 | 1404.3ms/step | 14.2 samples/s
458
+ Step 11470/44000 | Loss: 3.0840 | LR: 4.28e-04 | 1408.0ms/step | 14.2 samples/s
459
+ Step 11480/44000 | Loss: 3.0510 | LR: 4.28e-04 | 1406.5ms/step | 14.2 samples/s
460
+ Step 11490/44000 | Loss: 3.0728 | LR: 4.28e-04 | 1407.0ms/step | 14.2 samples/s
461
+ Step 11500/44000 | Loss: 3.0773 | LR: 4.28e-04 | 1408.8ms/step | 14.2 samples/s
462
+ Step 11510/44000 | Loss: 3.0577 | LR: 4.27e-04 | 1407.5ms/step | 14.2 samples/s
463
+ Step 11520/44000 | Loss: 3.0858 | LR: 4.27e-04 | 1403.7ms/step | 14.2 samples/s
464
+ Step 11530/44000 | Loss: 3.0852 | LR: 4.27e-04 | 1400.3ms/step | 14.3 samples/s
465
+ Step 11540/44000 | Loss: 3.0854 | LR: 4.27e-04 | 1399.6ms/step | 14.3 samples/s
466
+ Step 11550/44000 | Loss: 3.0739 | LR: 4.27e-04 | 1400.0ms/step | 14.3 samples/s
467
+ Step 11560/44000 | Loss: 3.0739 | LR: 4.27e-04 | 1400.0ms/step | 14.3 samples/s
468
+ Step 11570/44000 | Loss: 3.0712 | LR: 4.27e-04 | 1402.0ms/step | 14.3 samples/s
469
+ Step 11580/44000 | Loss: 3.0592 | LR: 4.27e-04 | 1402.0ms/step | 14.3 samples/s
470
+ Step 11590/44000 | Loss: 3.0741 | LR: 4.26e-04 | 1404.2ms/step | 14.2 samples/s
471
+ Step 11600/44000 | Loss: 3.0738 | LR: 4.26e-04 | 1416.0ms/step | 14.1 samples/s
472
+ Step 11610/44000 | Loss: 3.1089 | LR: 4.26e-04 | 1417.6ms/step | 14.1 samples/s
473
+ Step 11620/44000 | Loss: 3.0818 | LR: 4.26e-04 | 1402.3ms/step | 14.3 samples/s
474
+ Step 11630/44000 | Loss: 3.0894 | LR: 4.26e-04 | 1404.8ms/step | 14.2 samples/s
475
+ Step 11640/44000 | Loss: 3.0970 | LR: 4.26e-04 | 1406.0ms/step | 14.2 samples/s
476
+ Step 11650/44000 | Loss: 3.1157 | LR: 4.26e-04 | 1403.6ms/step | 14.2 samples/s
477
+ Step 11660/44000 | Loss: 3.0692 | LR: 4.26e-04 | 1406.2ms/step | 14.2 samples/s
478
+ Step 11670/44000 | Loss: 3.0894 | LR: 4.25e-04 | 1405.4ms/step | 14.2 samples/s
479
+ Step 11680/44000 | Loss: 3.0619 | LR: 4.25e-04 | 1400.0ms/step | 14.3 samples/s
480
+ Step 11690/44000 | Loss: 3.0609 | LR: 4.25e-04 | 1400.0ms/step | 14.3 samples/s
481
+ Step 11700/44000 | Loss: 3.0820 | LR: 4.25e-04 | 1400.2ms/step | 14.3 samples/s
482
+ Step 11710/44000 | Loss: 3.0242 | LR: 4.25e-04 | 1400.1ms/step | 14.3 samples/s
483
+ Step 11720/44000 | Loss: 3.0831 | LR: 4.25e-04 | 1403.9ms/step | 14.2 samples/s
484
+ Step 11730/44000 | Loss: 3.0492 | LR: 4.25e-04 | 1405.7ms/step | 14.2 samples/s
485
+ Step 11740/44000 | Loss: 3.0424 | LR: 4.24e-04 | 1400.2ms/step | 14.3 samples/s
486
+ Step 11750/44000 | Loss: 3.0564 | LR: 4.24e-04 | 1402.7ms/step | 14.3 samples/s
487
+ Step 11760/44000 | Loss: 3.0838 | LR: 4.24e-04 | 1401.1ms/step | 14.3 samples/s
488
+ Step 11770/44000 | Loss: 3.0887 | LR: 4.24e-04 | 1400.1ms/step | 14.3 samples/s
489
+ Step 11780/44000 | Loss: 3.0900 | LR: 4.24e-04 | 1400.0ms/step | 14.3 samples/s
490
+ Step 11790/44000 | Loss: 3.0547 | LR: 4.24e-04 | 1399.9ms/step | 14.3 samples/s
491
+ Step 11800/44000 | Loss: 3.0719 | LR: 4.24e-04 | 1400.1ms/step | 14.3 samples/s
492
+ Step 11810/44000 | Loss: 3.0761 | LR: 4.24e-04 | 1400.4ms/step | 14.3 samples/s
493
+ Step 11820/44000 | Loss: 3.0725 | LR: 4.23e-04 | 1401.5ms/step | 14.3 samples/s
494
+ Step 11830/44000 | Loss: 3.1007 | LR: 4.23e-04 | 1400.0ms/step | 14.3 samples/s
495
+ Step 11840/44000 | Loss: 3.1051 | LR: 4.23e-04 | 1400.1ms/step | 14.3 samples/s
496
+ Step 11850/44000 | Loss: 3.0711 | LR: 4.23e-04 | 1401.8ms/step | 14.3 samples/s
497
+ Step 11860/44000 | Loss: 3.0656 | LR: 4.23e-04 | 1400.1ms/step | 14.3 samples/s
498
+ Step 11870/44000 | Loss: 3.0594 | LR: 4.23e-04 | 1400.0ms/step | 14.3 samples/s
499
+ Step 11880/44000 | Loss: 3.0579 | LR: 4.23e-04 | 1401.8ms/step | 14.3 samples/s
500
+ Step 11890/44000 | Loss: 3.0589 | LR: 4.23e-04 | 1404.5ms/step | 14.2 samples/s
501
+ Step 11900/44000 | Loss: 3.0828 | LR: 4.22e-04 | 1401.8ms/step | 14.3 samples/s
502
+ Step 11910/44000 | Loss: 3.0870 | LR: 4.22e-04 | 1405.9ms/step | 14.2 samples/s
503
+ Step 11920/44000 | Loss: 3.0685 | LR: 4.22e-04 | 1406.6ms/step | 14.2 samples/s
504
+ Step 11930/44000 | Loss: 3.0591 | LR: 4.22e-04 | 1405.8ms/step | 14.2 samples/s
505
+ Step 11940/44000 | Loss: 3.0915 | LR: 4.22e-04 | 1407.9ms/step | 14.2 samples/s
506
+ Step 11950/44000 | Loss: 3.0698 | LR: 4.22e-04 | 1406.7ms/step | 14.2 samples/s
507
+ Step 11960/44000 | Loss: 3.0657 | LR: 4.22e-04 | 1408.9ms/step | 14.2 samples/s
508
+ Step 11970/44000 | Loss: 3.0955 | LR: 4.21e-04 | 1404.9ms/step | 14.2 samples/s
509
+ Step 11980/44000 | Loss: 3.0901 | LR: 4.21e-04 | 1406.2ms/step | 14.2 samples/s
510
+ Step 11990/44000 | Loss: 3.0444 | LR: 4.21e-04 | 1401.3ms/step | 14.3 samples/s
511
+ Step 12000/44000 | Loss: 3.0649 | LR: 4.21e-04 | 1404.2ms/step | 14.2 samples/s
512
+ Validation at step 12000: Loss = 3.0646, Perplexity = 21.43
513
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-12000
514
+ Step 12010/44000 | Loss: 3.0627 | LR: 4.21e-04 | 3921.4ms/step | 5.1 samples/s
515
+ Step 12020/44000 | Loss: 3.1026 | LR: 4.21e-04 | 1400.3ms/step | 14.3 samples/s
516
+ Step 12030/44000 | Loss: 3.0989 | LR: 4.21e-04 | 1402.0ms/step | 14.3 samples/s
517
+ Step 12040/44000 | Loss: 3.0758 | LR: 4.21e-04 | 1406.9ms/step | 14.2 samples/s
518
+ Step 12050/44000 | Loss: 3.0446 | LR: 4.20e-04 | 1409.8ms/step | 14.2 samples/s
519
+ Step 12060/44000 | Loss: 3.0759 | LR: 4.20e-04 | 1410.0ms/step | 14.2 samples/s
520
+ Step 12070/44000 | Loss: 3.0751 | LR: 4.20e-04 | 1413.2ms/step | 14.2 samples/s
521
+ Step 12080/44000 | Loss: 3.0718 | LR: 4.20e-04 | 1411.0ms/step | 14.2 samples/s
522
+ Step 12090/44000 | Loss: 3.0596 | LR: 4.20e-04 | 1413.9ms/step | 14.1 samples/s
523
+ Step 12100/44000 | Loss: 3.0403 | LR: 4.20e-04 | 1412.7ms/step | 14.2 samples/s
524
+ Step 12110/44000 | Loss: 3.0812 | LR: 4.20e-04 | 1412.5ms/step | 14.2 samples/s
525
+ Step 12120/44000 | Loss: 3.0636 | LR: 4.19e-04 | 1408.6ms/step | 14.2 samples/s
526
+ Step 12130/44000 | Loss: 3.0582 | LR: 4.19e-04 | 1413.1ms/step | 14.2 samples/s
527
+ Step 12140/44000 | Loss: 3.0897 | LR: 4.19e-04 | 1411.0ms/step | 14.2 samples/s
528
+ Step 12150/44000 | Loss: 3.1058 | LR: 4.19e-04 | 1414.6ms/step | 14.1 samples/s
529
+ Step 12160/44000 | Loss: 3.0544 | LR: 4.19e-04 | 1412.5ms/step | 14.2 samples/s
530
+ Step 12170/44000 | Loss: 3.0819 | LR: 4.19e-04 | 1408.6ms/step | 14.2 samples/s
531
+ Step 12180/44000 | Loss: 3.0369 | LR: 4.19e-04 | 1410.5ms/step | 14.2 samples/s
532
+ Step 12190/44000 | Loss: 3.0773 | LR: 4.19e-04 | 1409.9ms/step | 14.2 samples/s
533
+ Step 12200/44000 | Loss: 3.0810 | LR: 4.18e-04 | 1408.9ms/step | 14.2 samples/s
534
+ Step 12210/44000 | Loss: 3.0895 | LR: 4.18e-04 | 1410.0ms/step | 14.2 samples/s
535
+ Step 12220/44000 | Loss: 3.0528 | LR: 4.18e-04 | 1410.8ms/step | 14.2 samples/s
536
+ Step 12230/44000 | Loss: 3.0559 | LR: 4.18e-04 | 1411.7ms/step | 14.2 samples/s
537
+ Step 12240/44000 | Loss: 3.0583 | LR: 4.18e-04 | 1413.3ms/step | 14.2 samples/s
538
+ Step 12250/44000 | Loss: 3.0711 | LR: 4.18e-04 | 1411.1ms/step | 14.2 samples/s
539
+ Step 12260/44000 | Loss: 3.1340 | LR: 4.18e-04 | 1409.9ms/step | 14.2 samples/s
540
+ Step 12270/44000 | Loss: 3.0843 | LR: 4.17e-04 | 1411.1ms/step | 14.2 samples/s
541
+ Step 12280/44000 | Loss: 3.0585 | LR: 4.17e-04 | 1412.4ms/step | 14.2 samples/s
542
+ Step 12290/44000 | Loss: 3.0507 | LR: 4.17e-04 | 1407.9ms/step | 14.2 samples/s
543
+ Step 12300/44000 | Loss: 3.0736 | LR: 4.17e-04 | 1405.9ms/step | 14.2 samples/s
544
+ Step 12310/44000 | Loss: 3.0615 | LR: 4.17e-04 | 1406.8ms/step | 14.2 samples/s
545
+ Step 12320/44000 | Loss: 3.0674 | LR: 4.17e-04 | 1407.6ms/step | 14.2 samples/s
546
+ Step 12330/44000 | Loss: 3.0688 | LR: 4.17e-04 | 1407.3ms/step | 14.2 samples/s
547
+ Step 12340/44000 | Loss: 3.0804 | LR: 4.17e-04 | 1403.9ms/step | 14.2 samples/s
548
+ Step 12350/44000 | Loss: 3.0697 | LR: 4.16e-04 | 1402.9ms/step | 14.3 samples/s
549
+ Step 12360/44000 | Loss: 3.0744 | LR: 4.16e-04 | 1406.4ms/step | 14.2 samples/s
550
+ Step 12370/44000 | Loss: 3.0824 | LR: 4.16e-04 | 1407.2ms/step | 14.2 samples/s
551
+ Step 12380/44000 | Loss: 3.0552 | LR: 4.16e-04 | 1410.3ms/step | 14.2 samples/s
552
+ Step 12390/44000 | Loss: 3.0683 | LR: 4.16e-04 | 1407.7ms/step | 14.2 samples/s
553
+ Step 12400/44000 | Loss: 3.0548 | LR: 4.16e-04 | 1407.9ms/step | 14.2 samples/s
554
+ Step 12410/44000 | Loss: 3.0833 | LR: 4.16e-04 | 1407.0ms/step | 14.2 samples/s
555
+ Step 12420/44000 | Loss: 3.0618 | LR: 4.15e-04 | 1408.7ms/step | 14.2 samples/s
556
+ Step 12430/44000 | Loss: 3.0875 | LR: 4.15e-04 | 1406.8ms/step | 14.2 samples/s
557
+ Step 12440/44000 | Loss: 3.0682 | LR: 4.15e-04 | 1405.5ms/step | 14.2 samples/s
558
+ Step 12450/44000 | Loss: 3.0752 | LR: 4.15e-04 | 1409.7ms/step | 14.2 samples/s
559
+ Step 12460/44000 | Loss: 3.0793 | LR: 4.15e-04 | 1402.1ms/step | 14.3 samples/s
560
+ Step 12470/44000 | Loss: 3.0682 | LR: 4.15e-04 | 1399.7ms/step | 14.3 samples/s
561
+ Step 12480/44000 | Loss: 3.0646 | LR: 4.15e-04 | 1400.1ms/step | 14.3 samples/s
562
+ Step 12490/44000 | Loss: 3.0717 | LR: 4.14e-04 | 1400.0ms/step | 14.3 samples/s
563
+ Step 12500/44000 | Loss: 3.0976 | LR: 4.14e-04 | 1400.0ms/step | 14.3 samples/s
564
+ Step 12510/44000 | Loss: 3.0680 | LR: 4.14e-04 | 1400.1ms/step | 14.3 samples/s
565
+ Step 12520/44000 | Loss: 3.0528 | LR: 4.14e-04 | 1400.0ms/step | 14.3 samples/s
566
+ Step 12530/44000 | Loss: 3.0819 | LR: 4.14e-04 | 1399.9ms/step | 14.3 samples/s
567
+ Step 12540/44000 | Loss: 3.0675 | LR: 4.14e-04 | 1400.5ms/step | 14.3 samples/s
568
+ Step 12550/44000 | Loss: 3.0520 | LR: 4.14e-04 | 1401.6ms/step | 14.3 samples/s
569
+ Step 12560/44000 | Loss: 3.0732 | LR: 4.14e-04 | 1399.8ms/step | 14.3 samples/s
570
+ Step 12570/44000 | Loss: 3.0631 | LR: 4.13e-04 | 1400.0ms/step | 14.3 samples/s
571
+ Step 12580/44000 | Loss: 3.0742 | LR: 4.13e-04 | 1400.3ms/step | 14.3 samples/s
572
+ Step 12590/44000 | Loss: 3.1108 | LR: 4.13e-04 | 1400.8ms/step | 14.3 samples/s
573
+ Step 12600/44000 | Loss: 3.0451 | LR: 4.13e-04 | 1403.3ms/step | 14.3 samples/s
574
+ Step 12610/44000 | Loss: 3.0410 | LR: 4.13e-04 | 1403.9ms/step | 14.2 samples/s
575
+ Step 12620/44000 | Loss: 3.0618 | LR: 4.13e-04 | 1404.8ms/step | 14.2 samples/s
576
+ Step 12630/44000 | Loss: 3.0638 | LR: 4.13e-04 | 1401.3ms/step | 14.3 samples/s
577
+ Step 12640/44000 | Loss: 3.0559 | LR: 4.12e-04 | 1404.8ms/step | 14.2 samples/s
578
+ Step 12650/44000 | Loss: 3.0699 | LR: 4.12e-04 | 1404.1ms/step | 14.2 samples/s
579
+ Step 12660/44000 | Loss: 3.0675 | LR: 4.12e-04 | 1401.6ms/step | 14.3 samples/s
580
+ Step 12670/44000 | Loss: 3.0641 | LR: 4.12e-04 | 1401.1ms/step | 14.3 samples/s
581
+ Step 12680/44000 | Loss: 3.0487 | LR: 4.12e-04 | 1400.1ms/step | 14.3 samples/s
582
+ Step 12690/44000 | Loss: 3.0641 | LR: 4.12e-04 | 1400.2ms/step | 14.3 samples/s
583
+ Step 12700/44000 | Loss: 3.0806 | LR: 4.12e-04 | 1399.9ms/step | 14.3 samples/s
584
+ Step 12710/44000 | Loss: 3.0648 | LR: 4.11e-04 | 1400.0ms/step | 14.3 samples/s
585
+ Step 12720/44000 | Loss: 3.0512 | LR: 4.11e-04 | 1400.0ms/step | 14.3 samples/s
586
+ Step 12730/44000 | Loss: 3.0933 | LR: 4.11e-04 | 1399.9ms/step | 14.3 samples/s
587
+ Step 12740/44000 | Loss: 3.0093 | LR: 4.11e-04 | 1400.4ms/step | 14.3 samples/s
588
+ Step 12750/44000 | Loss: 3.0656 | LR: 4.11e-04 | 1401.7ms/step | 14.3 samples/s
589
+ Step 12760/44000 | Loss: 3.0410 | LR: 4.11e-04 | 1400.3ms/step | 14.3 samples/s
590
+ Step 12770/44000 | Loss: 3.0597 | LR: 4.11e-04 | 1399.9ms/step | 14.3 samples/s
591
+ Step 12780/44000 | Loss: 3.0610 | LR: 4.10e-04 | 1399.7ms/step | 14.3 samples/s
592
+ Step 12790/44000 | Loss: 3.0594 | LR: 4.10e-04 | 1400.1ms/step | 14.3 samples/s
593
+ Step 12800/44000 | Loss: 3.0464 | LR: 4.10e-04 | 1400.2ms/step | 14.3 samples/s
594
+ Step 12810/44000 | Loss: 3.0682 | LR: 4.10e-04 | 1401.0ms/step | 14.3 samples/s
595
+ Step 12820/44000 | Loss: 3.0856 | LR: 4.10e-04 | 1403.5ms/step | 14.2 samples/s
596
+ Step 12830/44000 | Loss: 3.0819 | LR: 4.10e-04 | 1403.6ms/step | 14.2 samples/s
597
+ Step 12840/44000 | Loss: 3.0853 | LR: 4.10e-04 | 1400.0ms/step | 14.3 samples/s
598
+ Step 12850/44000 | Loss: 3.0428 | LR: 4.09e-04 | 1404.1ms/step | 14.2 samples/s
599
+ Step 12860/44000 | Loss: 3.0628 | LR: 4.09e-04 | 1405.7ms/step | 14.2 samples/s
600
+ Step 12870/44000 | Loss: 3.0766 | LR: 4.09e-04 | 1400.0ms/step | 14.3 samples/s
601
+ Step 12880/44000 | Loss: 3.0978 | LR: 4.09e-04 | 1399.8ms/step | 14.3 samples/s
602
+ Step 12890/44000 | Loss: 3.0582 | LR: 4.09e-04 | 1400.3ms/step | 14.3 samples/s
603
+ Step 12900/44000 | Loss: 3.0548 | LR: 4.09e-04 | 1400.0ms/step | 14.3 samples/s
604
+ Step 12910/44000 | Loss: 3.0661 | LR: 4.09e-04 | 1399.9ms/step | 14.3 samples/s
605
+ Step 12920/44000 | Loss: 3.0360 | LR: 4.09e-04 | 1400.7ms/step | 14.3 samples/s
606
+ Step 12930/44000 | Loss: 3.0695 | LR: 4.08e-04 | 1400.6ms/step | 14.3 samples/s
607
+ Step 12940/44000 | Loss: 3.0929 | LR: 4.08e-04 | 1403.5ms/step | 14.3 samples/s
608
+ Step 12950/44000 | Loss: 3.0696 | LR: 4.08e-04 | 1403.5ms/step | 14.2 samples/s
609
+ Step 12960/44000 | Loss: 3.0721 | LR: 4.08e-04 | 1404.6ms/step | 14.2 samples/s
610
+ Step 12970/44000 | Loss: 3.0563 | LR: 4.08e-04 | 1407.6ms/step | 14.2 samples/s
611
+ Step 12980/44000 | Loss: 3.0434 | LR: 4.08e-04 | 1401.6ms/step | 14.3 samples/s
612
+ Step 12990/44000 | Loss: 3.0289 | LR: 4.08e-04 | 1400.3ms/step | 14.3 samples/s
613
+ Step 13000/44000 | Loss: 3.0754 | LR: 4.07e-04 | 1403.7ms/step | 14.2 samples/s
614
+ Validation at step 13000: Loss = 3.0468, Perplexity = 21.05
615
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-13000
616
+ Step 13010/44000 | Loss: 3.0428 | LR: 4.07e-04 | 3936.1ms/step | 5.1 samples/s
617
+ Step 13020/44000 | Loss: 3.0560 | LR: 4.07e-04 | 1405.9ms/step | 14.2 samples/s
618
+ Step 13030/44000 | Loss: 3.0365 | LR: 4.07e-04 | 1408.3ms/step | 14.2 samples/s
619
+ Step 13040/44000 | Loss: 3.0766 | LR: 4.07e-04 | 1410.0ms/step | 14.2 samples/s
620
+ Step 13050/44000 | Loss: 3.0779 | LR: 4.07e-04 | 1410.2ms/step | 14.2 samples/s
621
+ Step 13060/44000 | Loss: 3.0487 | LR: 4.07e-04 | 1407.8ms/step | 14.2 samples/s
622
+ Step 13070/44000 | Loss: 3.0610 | LR: 4.06e-04 | 1407.8ms/step | 14.2 samples/s
623
+ Step 13080/44000 | Loss: 3.0886 | LR: 4.06e-04 | 1410.4ms/step | 14.2 samples/s
624
+ Step 13090/44000 | Loss: 3.0555 | LR: 4.06e-04 | 1410.6ms/step | 14.2 samples/s
625
+ Step 13100/44000 | Loss: 3.0537 | LR: 4.06e-04 | 1411.2ms/step | 14.2 samples/s
626
+ Step 13110/44000 | Loss: 3.0510 | LR: 4.06e-04 | 1408.7ms/step | 14.2 samples/s
627
+ Step 13120/44000 | Loss: 3.0669 | LR: 4.06e-04 | 1410.6ms/step | 14.2 samples/s
628
+ Step 13130/44000 | Loss: 3.0551 | LR: 4.06e-04 | 1410.8ms/step | 14.2 samples/s
629
+ Step 13140/44000 | Loss: 3.0557 | LR: 4.05e-04 | 1410.0ms/step | 14.2 samples/s
630
+ Step 13150/44000 | Loss: 3.0673 | LR: 4.05e-04 | 1410.0ms/step | 14.2 samples/s
631
+ Step 13160/44000 | Loss: 3.0161 | LR: 4.05e-04 | 1411.3ms/step | 14.2 samples/s
632
+ Step 13170/44000 | Loss: 3.0689 | LR: 4.05e-04 | 1412.9ms/step | 14.2 samples/s
633
+ Step 13180/44000 | Loss: 3.0458 | LR: 4.05e-04 | 1412.5ms/step | 14.2 samples/s
634
+ Step 13190/44000 | Loss: 3.0277 | LR: 4.05e-04 | 1413.7ms/step | 14.1 samples/s
635
+ Step 13200/44000 | Loss: 3.0752 | LR: 4.05e-04 | 1409.7ms/step | 14.2 samples/s
636
+ Step 13210/44000 | Loss: 3.0489 | LR: 4.04e-04 | 1410.0ms/step | 14.2 samples/s
637
+ Step 13220/44000 | Loss: 3.0476 | LR: 4.04e-04 | 1412.6ms/step | 14.2 samples/s
638
+ Step 13230/44000 | Loss: 3.0879 | LR: 4.04e-04 | 1410.0ms/step | 14.2 samples/s
639
+ Step 13240/44000 | Loss: 3.1028 | LR: 4.04e-04 | 1411.2ms/step | 14.2 samples/s
640
+ Step 13250/44000 | Loss: 3.0594 | LR: 4.04e-04 | 1411.1ms/step | 14.2 samples/s
641
+ Step 13260/44000 | Loss: 3.0662 | LR: 4.04e-04 | 1412.0ms/step | 14.2 samples/s
642
+ Step 13270/44000 | Loss: 3.0832 | LR: 4.04e-04 | 1411.4ms/step | 14.2 samples/s
643
+ Step 13280/44000 | Loss: 3.0850 | LR: 4.03e-04 | 1411.6ms/step | 14.2 samples/s
644
+ Step 13290/44000 | Loss: 3.0657 | LR: 4.03e-04 | 1408.4ms/step | 14.2 samples/s
645
+ Step 13300/44000 | Loss: 3.0562 | LR: 4.03e-04 | 1408.6ms/step | 14.2 samples/s
646
+ Step 13310/44000 | Loss: 3.0780 | LR: 4.03e-04 | 1411.2ms/step | 14.2 samples/s
647
+ Step 13320/44000 | Loss: 3.0437 | LR: 4.03e-04 | 1411.8ms/step | 14.2 samples/s
648
+ Step 13330/44000 | Loss: 3.0678 | LR: 4.03e-04 | 1410.4ms/step | 14.2 samples/s
649
+ Step 13340/44000 | Loss: 3.0267 | LR: 4.03e-04 | 1409.5ms/step | 14.2 samples/s
650
+ Step 13350/44000 | Loss: 3.0744 | LR: 4.02e-04 | 1410.3ms/step | 14.2 samples/s
651
+ Step 13360/44000 | Loss: 3.0552 | LR: 4.02e-04 | 1412.5ms/step | 14.2 samples/s
652
+ Step 13370/44000 | Loss: 3.0747 | LR: 4.02e-04 | 1413.4ms/step | 14.2 samples/s
653
+ Step 13380/44000 | Loss: 3.0651 | LR: 4.02e-04 | 1414.3ms/step | 14.1 samples/s
654
+ Step 13390/44000 | Loss: 3.0648 | LR: 4.02e-04 | 1412.2ms/step | 14.2 samples/s
655
+ Step 13400/44000 | Loss: 3.0421 | LR: 4.02e-04 | 1401.8ms/step | 14.3 samples/s
656
+ Step 13410/44000 | Loss: 3.0544 | LR: 4.02e-04 | 1399.8ms/step | 14.3 samples/s
657
+ Step 13420/44000 | Loss: 3.0155 | LR: 4.01e-04 | 1400.0ms/step | 14.3 samples/s
658
+ Step 13430/44000 | Loss: 3.0720 | LR: 4.01e-04 | 1399.9ms/step | 14.3 samples/s
659
+ Step 13440/44000 | Loss: 3.0407 | LR: 4.01e-04 | 1400.7ms/step | 14.3 samples/s
660
+ Step 13450/44000 | Loss: 3.0596 | LR: 4.01e-04 | 1401.4ms/step | 14.3 samples/s
661
+ Step 13460/44000 | Loss: 3.0534 | LR: 4.01e-04 | 1400.1ms/step | 14.3 samples/s
662
+ Step 13470/44000 | Loss: 3.0873 | LR: 4.01e-04 | 1402.3ms/step | 14.3 samples/s
663
+ Step 13480/44000 | Loss: 3.0607 | LR: 4.01e-04 | 1401.5ms/step | 14.3 samples/s
664
+ Step 13490/44000 | Loss: 3.0447 | LR: 4.00e-04 | 1400.5ms/step | 14.3 samples/s
665
+ Step 13500/44000 | Loss: 3.0687 | LR: 4.00e-04 | 1402.8ms/step | 14.3 samples/s
666
+ Step 13510/44000 | Loss: 3.0510 | LR: 4.00e-04 | 1405.3ms/step | 14.2 samples/s
667
+ Step 13520/44000 | Loss: 3.0207 | LR: 4.00e-04 | 1405.8ms/step | 14.2 samples/s
668
+ Step 13530/44000 | Loss: 3.0419 | LR: 4.00e-04 | 1406.6ms/step | 14.2 samples/s
669
+ Step 13540/44000 | Loss: 3.0400 | LR: 4.00e-04 | 1404.4ms/step | 14.2 samples/s
670
+ Step 13550/44000 | Loss: 3.0316 | LR: 4.00e-04 | 1400.7ms/step | 14.3 samples/s
671
+ Step 13560/44000 | Loss: 3.0495 | LR: 3.99e-04 | 1400.3ms/step | 14.3 samples/s
672
+ Step 13570/44000 | Loss: 3.0637 | LR: 3.99e-04 | 1402.0ms/step | 14.3 samples/s
673
+ Step 13580/44000 | Loss: 3.0523 | LR: 3.99e-04 | 1402.6ms/step | 14.3 samples/s
674
+ Step 13590/44000 | Loss: 3.0280 | LR: 3.99e-04 | 1403.5ms/step | 14.3 samples/s
675
+ Step 13600/44000 | Loss: 3.0588 | LR: 3.99e-04 | 1404.0ms/step | 14.2 samples/s
676
+ Step 13610/44000 | Loss: 3.0797 | LR: 3.99e-04 | 1406.4ms/step | 14.2 samples/s
677
+ Step 13620/44000 | Loss: 3.0541 | LR: 3.98e-04 | 1401.4ms/step | 14.3 samples/s
678
+ Step 13630/44000 | Loss: 3.0294 | LR: 3.98e-04 | 1402.4ms/step | 14.3 samples/s
679
+ Step 13640/44000 | Loss: 3.0598 | LR: 3.98e-04 | 1406.4ms/step | 14.2 samples/s
680
+ Step 13650/44000 | Loss: 3.0561 | LR: 3.98e-04 | 1405.0ms/step | 14.2 samples/s
681
+ Step 13660/44000 | Loss: 3.0666 | LR: 3.98e-04 | 1400.2ms/step | 14.3 samples/s
682
+ Step 13670/44000 | Loss: 3.0718 | LR: 3.98e-04 | 1402.6ms/step | 14.3 samples/s
683
+ Step 13680/44000 | Loss: 3.0343 | LR: 3.98e-04 | 1403.7ms/step | 14.2 samples/s
684
+ Step 13690/44000 | Loss: 3.0674 | LR: 3.97e-04 | 1401.6ms/step | 14.3 samples/s
685
+ Step 13700/44000 | Loss: 3.0214 | LR: 3.97e-04 | 1404.5ms/step | 14.2 samples/s
686
+ Step 13710/44000 | Loss: 3.0526 | LR: 3.97e-04 | 1405.5ms/step | 14.2 samples/s
687
+ Step 13720/44000 | Loss: 3.0501 | LR: 3.97e-04 | 1400.9ms/step | 14.3 samples/s
688
+ Step 13730/44000 | Loss: 3.0441 | LR: 3.97e-04 | 1403.1ms/step | 14.3 samples/s
689
+ Step 13740/44000 | Loss: 3.0598 | LR: 3.97e-04 | 1402.1ms/step | 14.3 samples/s
690
+ Step 13750/44000 | Loss: 3.0645 | LR: 3.97e-04 | 1400.0ms/step | 14.3 samples/s
691
+ Step 13760/44000 | Loss: 3.0413 | LR: 3.96e-04 | 1400.1ms/step | 14.3 samples/s
692
+ Step 13770/44000 | Loss: 3.0701 | LR: 3.96e-04 | 1401.8ms/step | 14.3 samples/s
693
+ Step 13780/44000 | Loss: 3.0331 | LR: 3.96e-04 | 1402.2ms/step | 14.3 samples/s
694
+ Step 13790/44000 | Loss: 3.0386 | LR: 3.96e-04 | 1399.9ms/step | 14.3 samples/s
695
+ Step 13800/44000 | Loss: 3.0633 | LR: 3.96e-04 | 1402.2ms/step | 14.3 samples/s
696
+ Step 13810/44000 | Loss: 2.9968 | LR: 3.96e-04 | 1402.5ms/step | 14.3 samples/s
697
+ Step 13820/44000 | Loss: 3.0581 | LR: 3.96e-04 | 1403.6ms/step | 14.2 samples/s
698
+ Step 13830/44000 | Loss: 3.0633 | LR: 3.95e-04 | 1404.1ms/step | 14.2 samples/s
699
+ Step 13840/44000 | Loss: 3.0076 | LR: 3.95e-04 | 1399.6ms/step | 14.3 samples/s
700
+ Step 13850/44000 | Loss: 3.0434 | LR: 3.95e-04 | 1400.0ms/step | 14.3 samples/s
701
+ Step 13860/44000 | Loss: 3.0491 | LR: 3.95e-04 | 1399.9ms/step | 14.3 samples/s
702
+ Step 13870/44000 | Loss: 3.0459 | LR: 3.95e-04 | 1400.2ms/step | 14.3 samples/s
703
+ Step 13880/44000 | Loss: 3.0587 | LR: 3.95e-04 | 1399.6ms/step | 14.3 samples/s
704
+ Step 13890/44000 | Loss: 3.0506 | LR: 3.95e-04 | 1400.4ms/step | 14.3 samples/s
705
+ Step 13900/44000 | Loss: 3.0759 | LR: 3.94e-04 | 1399.7ms/step | 14.3 samples/s
706
+ Step 13910/44000 | Loss: 3.0439 | LR: 3.94e-04 | 1400.2ms/step | 14.3 samples/s
707
+ Step 13920/44000 | Loss: 3.0446 | LR: 3.94e-04 | 1400.0ms/step | 14.3 samples/s
708
+ Step 13930/44000 | Loss: 3.0302 | LR: 3.94e-04 | 1401.1ms/step | 14.3 samples/s
709
+ Step 13940/44000 | Loss: 3.0302 | LR: 3.94e-04 | 1400.9ms/step | 14.3 samples/s
710
+ Step 13950/44000 | Loss: 3.0159 | LR: 3.94e-04 | 1400.3ms/step | 14.3 samples/s
711
+ Step 13960/44000 | Loss: 3.0230 | LR: 3.93e-04 | 1403.5ms/step | 14.2 samples/s
712
+ Step 13970/44000 | Loss: 3.0529 | LR: 3.93e-04 | 1400.2ms/step | 14.3 samples/s
713
+ Step 13980/44000 | Loss: 3.0109 | LR: 3.93e-04 | 1402.2ms/step | 14.3 samples/s
714
+ Step 13990/44000 | Loss: 3.0353 | LR: 3.93e-04 | 1406.4ms/step | 14.2 samples/s
715
+ Step 14000/44000 | Loss: 3.0585 | LR: 3.93e-04 | 1405.3ms/step | 14.2 samples/s
716
+ Validation at step 14000: Loss = 3.0315, Perplexity = 20.73
717
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-14000
epfl_final/SparseAtt/wandb/run-20260820_011128-ysa9iyov/files/requirements.txt ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pip==26.0.1
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.6.3
8
+ typing_extensions==4.15.0
9
+ triton==3.6.0
10
+ tqdm==4.67.3
11
+ torchinfo==1.8.0
12
+ sympy==1.14.0
13
+ smmap==5.0.3
14
+ six==1.17.0
15
+ shellingham==1.5.4
16
+ setuptools==81.0.0
17
+ safetensors==0.7.0
18
+ regex==2026.4.4
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.4.1
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.4.4
38
+ networkx==3.6.1
39
+ multidict==6.7.1
40
+ mdurl==0.1.2
41
+ MarkupSafe==3.0.3
42
+ idna==3.13
43
+ hf-xet==1.4.3
44
+ h11==0.16.0
45
+ fsspec==2026.2.0
46
+ frozenlist==1.8.0
47
+ filelock==3.29.0
48
+ dill==0.4.1
49
+ cuda-pathfinder==1.5.4
50
+ click==8.3.3
51
+ charset-normalizer==3.4.7
52
+ certifi==2026.4.22
53
+ attrs==26.1.0
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ yarl==1.23.0
58
+ typing-inspection==0.4.2
59
+ sentry-sdk==2.58.0
60
+ requests==2.33.1
61
+ python-dateutil==2.9.0.post0
62
+ pydantic_core==2.46.3
63
+ nvidia-cusparse==12.6.3.3
64
+ nvidia-cufft==12.0.0.61
65
+ nvidia-cudnn-cu13==9.19.0.56
66
+ multiprocess==0.70.19
67
+ markdown-it-py==4.0.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ cuda-bindings==13.2.0
72
+ anyio==4.13.0
73
+ aiosignal==1.4.0
74
+ rich==15.0.0
75
+ pydantic==2.13.3
76
+ pandas==3.0.2
77
+ nvidia-cusolver==12.0.4.66
78
+ httpx==0.28.1
79
+ GitPython==3.1.49
80
+ aiohttp==3.13.5
81
+ wandb==0.26.1
82
+ typer==0.25.1
83
+ torch==2.11.0
84
+ huggingface_hub==1.13.0
85
+ tokenizers==0.22.2
86
+ datasets==4.8.5
87
+ accelerate==1.13.0
88
+ transformers==5.7.0
89
+ word2number==1.1
90
+ sqlitedict==2.1.0
91
+ pytz==2026.2
92
+ threadpoolctl==3.6.0
93
+ tcolorpy==0.1.7
94
+ tabulate==0.10.0
95
+ scipy==1.17.1
96
+ portalocker==3.2.0
97
+ pathvalidate==3.3.1
98
+ narwhals==2.22.1
99
+ more-itertools==11.1.0
100
+ lxml==6.1.1
101
+ joblib==1.5.3
102
+ colorama==0.4.6
103
+ chardet==6.0.0.post1
104
+ absl-py==2.4.0
105
+ scikit-learn==1.9.0
106
+ sacrebleu==2.6.0
107
+ nltk==3.9.4
108
+ mbstrdecoder==1.1.5
109
+ typepy==1.3.5
110
+ rouge_score==0.1.2
111
+ DataProperty==1.1.1
112
+ tabledata==1.3.5
113
+ evaluate==0.4.6
114
+ pytablewriter==1.2.1
115
+ lm_eval==0.4.12
116
+ pluggy==1.6.0
117
+ iniconfig==2.3.0
118
+ pytest==9.1.0
119
+ py-spy==0.4.2
120
+ pillow==12.2.0
121
+ tifffile==2026.6.1
122
+ imagecodecs==2026.6.26
123
+ zipp==4.1.0
124
+ importlib_metadata==9.0.0
125
+ derivative==0.6.3
126
+ pysindy==2.1.0
127
+ pyparsing==3.3.2
128
+ kiwisolver==1.5.0
129
+ fonttools==4.63.0
130
+ cycler==0.12.1
131
+ contourpy==1.3.3
132
+ matplotlib==3.11.0
133
+ sortedcontainers==2.4.0
134
+ wcwidth==0.8.2
135
+ sigfig==1.3.19
136
+ importlib_resources==7.1.0
137
+ enum-tools==0.13.0
138
+ cramjam==2.11.0
139
+ python-snappy==0.7.3
140
+ compact-json==1.8.2
141
+ numbers-parser==4.18.5
142
+ entmax==1.3
epfl_final/SparseAtt/wandb/run-20260820_011128-ysa9iyov/files/wandb-metadata.json ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.13.13",
4
+ "startedAt": "2026-08-20T01:11:28.310122Z",
5
+ "args": [
6
+ "--config",
7
+ "/mloscratch/homes/navasard/sparse/configs_ropectl/resume_a1p0_wandb/seed43.json"
8
+ ],
9
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
10
+ "codePath": "train.py",
11
+ "codePathLocal": "train.py",
12
+ "git": {
13
+ "remote": "https://github.com/rubentium/TokenPrune.git",
14
+ "commit": "5637886f0cc760009989b14e68c6ae06d4827418"
15
+ },
16
+ "email": "ruben.navasardyan@mail.utoronto.ca",
17
+ "root": "/mloscratch/homes/navasard/sparse/SparseAtt",
18
+ "host": "h100-workspace-5-gpu-0-1",
19
+ "executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
20
+ "cpu_count": 96,
21
+ "cpu_count_logical": 192,
22
+ "gpu": "NVIDIA H100 80GB HBM3",
23
+ "gpu_count": 5,
24
+ "disk": {
25
+ "/": {
26
+ "total": "9374931091456",
27
+ "used": "6373417144320"
28
+ }
29
+ },
30
+ "memory": {
31
+ "total": "1623005732864"
32
+ },
33
+ "gpu_nvidia": [
34
+ {
35
+ "name": "NVIDIA H100 80GB HBM3",
36
+ "memoryTotal": "85520809984",
37
+ "cudaCores": 16896,
38
+ "architecture": "Hopper",
39
+ "uuid": "GPU-51f13867-a72c-7b65-ff60-661787f0c18a"
40
+ },
41
+ {
42
+ "name": "NVIDIA H100 80GB HBM3",
43
+ "memoryTotal": "85520809984",
44
+ "cudaCores": 16896,
45
+ "architecture": "Hopper",
46
+ "uuid": "GPU-a2122da0-1020-9f78-7e6f-a1d5011f694b"
47
+ },
48
+ {
49
+ "name": "NVIDIA H100 80GB HBM3",
50
+ "memoryTotal": "85520809984",
51
+ "cudaCores": 16896,
52
+ "architecture": "Hopper",
53
+ "uuid": "GPU-f65cda91-ff87-9bff-fb3d-3e1b3f9f8cf0"
54
+ },
55
+ {
56
+ "name": "NVIDIA H100 80GB HBM3",
57
+ "memoryTotal": "85520809984",
58
+ "cudaCores": 16896,
59
+ "architecture": "Hopper",
60
+ "uuid": "GPU-4c1d91e9-bf01-f8fd-1e8b-79aa8e0831df"
61
+ },
62
+ {
63
+ "name": "NVIDIA H100 80GB HBM3",
64
+ "memoryTotal": "85520809984",
65
+ "cudaCores": 16896,
66
+ "architecture": "Hopper",
67
+ "uuid": "GPU-bdb5e23e-0700-20a0-0b21-1ad0901f08b7"
68
+ }
69
+ ],
70
+ "cudaVersion": "13.0",
71
+ "writerId": "7gagez378twtaz7g830dk4jr4diov5jt"
72
+ }
epfl_final/SparseAtt/wandb/run-20260820_011128-ysa9iyov/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/SparseAtt/wandb/run-20260820_011128-ysa9iyov/logs/debug.log ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-08-20 01:11:28,334 INFO MainThread:9356 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-08-20 01:11:28,335 INFO MainThread:9356 [wandb_setup.py:_flush():81] Configure stats pid to 9356
3
+ 2026-08-20 01:11:28,335 INFO MainThread:9356 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-08-20 01:11:28,335 INFO MainThread:9356 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_011128-ysa9iyov/logs/debug.log
5
+ 2026-08-20 01:11:28,335 INFO MainThread:9356 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_011128-ysa9iyov/logs/debug-internal.log
6
+ 2026-08-20 01:11:28,335 INFO MainThread:9356 [wandb_init.py:init():850] calling init triggers
7
+ 2026-08-20 01:11:28,335 INFO MainThread:9356 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-7000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 1000, 'eval_batches': 50, 'logging_steps': 10, 'seed': 43, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'l8k365m_dense_ropectl_a1p0_seed43', 'wandb_tags': ['qwen3', '365m', '8k', 'dense', 'rope-control'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': 0.0, 'rope_train_position_scale_set': [1.0, 0.9896339488176222, 0.9692159692159692, 0.9420289855072463, 0.8730158730158729], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
9
+ 2026-08-20 01:11:28,335 INFO MainThread:9356 [wandb_init.py:init():898] starting backend
10
+ 2026-08-20 01:11:28,601 INFO MainThread:9356 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-08-20 01:11:28,981 INFO MainThread:9356 [wandb_init.py:init():918] backend started and connected
12
+ 2026-08-20 01:11:28,985 INFO MainThread:9356 [wandb_init.py:init():988] updated telemetry
13
+ 2026-08-20 01:11:29,004 INFO MainThread:9356 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-08-20 01:11:29,840 INFO MainThread:9356 [wandb_init.py:init():1056] starting run threads in backend
15
+ 2026-08-20 01:11:30,040 INFO MainThread:9356 [wandb_run.py:_console_start():2554] atexit reg
16
+ 2026-08-20 01:11:30,040 INFO MainThread:9356 [wandb_run.py:_redirect():2403] redirect: wrap_raw
17
+ 2026-08-20 01:11:30,040 INFO MainThread:9356 [wandb_run.py:_redirect():2472] Wrapping output streams.
18
+ 2026-08-20 01:11:30,040 INFO MainThread:9356 [wandb_run.py:_redirect():2495] Redirects installed.
19
+ 2026-08-20 01:11:30,044 INFO MainThread:9356 [wandb_init.py:init():1094] run started, returning control to user process
epfl_final/SparseAtt/wandb/run-20260820_011128-ysa9iyov/run-ysa9iyov.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8e68aa92130454ad458ad0a9592886368c2a99099a4ca96581e9051413307494
3
+ size 4227072
epfl_final/SparseAtt/wandb/run-20260820_031315-89kzpaot/files/output.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/SparseAtt/wandb/run-20260820_031315-89kzpaot/files/requirements.txt ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pip==26.0.1
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.6.3
8
+ typing_extensions==4.15.0
9
+ triton==3.6.0
10
+ tqdm==4.67.3
11
+ torchinfo==1.8.0
12
+ sympy==1.14.0
13
+ smmap==5.0.3
14
+ six==1.17.0
15
+ shellingham==1.5.4
16
+ setuptools==81.0.0
17
+ safetensors==0.7.0
18
+ regex==2026.4.4
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.4.1
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.4.4
38
+ networkx==3.6.1
39
+ multidict==6.7.1
40
+ mdurl==0.1.2
41
+ MarkupSafe==3.0.3
42
+ idna==3.13
43
+ hf-xet==1.4.3
44
+ h11==0.16.0
45
+ fsspec==2026.2.0
46
+ frozenlist==1.8.0
47
+ filelock==3.29.0
48
+ dill==0.4.1
49
+ cuda-pathfinder==1.5.4
50
+ click==8.3.3
51
+ charset-normalizer==3.4.7
52
+ certifi==2026.4.22
53
+ attrs==26.1.0
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ yarl==1.23.0
58
+ typing-inspection==0.4.2
59
+ sentry-sdk==2.58.0
60
+ requests==2.33.1
61
+ python-dateutil==2.9.0.post0
62
+ pydantic_core==2.46.3
63
+ nvidia-cusparse==12.6.3.3
64
+ nvidia-cufft==12.0.0.61
65
+ nvidia-cudnn-cu13==9.19.0.56
66
+ multiprocess==0.70.19
67
+ markdown-it-py==4.0.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ cuda-bindings==13.2.0
72
+ anyio==4.13.0
73
+ aiosignal==1.4.0
74
+ rich==15.0.0
75
+ pydantic==2.13.3
76
+ pandas==3.0.2
77
+ nvidia-cusolver==12.0.4.66
78
+ httpx==0.28.1
79
+ GitPython==3.1.49
80
+ aiohttp==3.13.5
81
+ wandb==0.26.1
82
+ typer==0.25.1
83
+ torch==2.11.0
84
+ huggingface_hub==1.13.0
85
+ tokenizers==0.22.2
86
+ datasets==4.8.5
87
+ accelerate==1.13.0
88
+ transformers==5.7.0
89
+ word2number==1.1
90
+ sqlitedict==2.1.0
91
+ pytz==2026.2
92
+ threadpoolctl==3.6.0
93
+ tcolorpy==0.1.7
94
+ tabulate==0.10.0
95
+ scipy==1.17.1
96
+ portalocker==3.2.0
97
+ pathvalidate==3.3.1
98
+ narwhals==2.22.1
99
+ more-itertools==11.1.0
100
+ lxml==6.1.1
101
+ joblib==1.5.3
102
+ colorama==0.4.6
103
+ chardet==6.0.0.post1
104
+ absl-py==2.4.0
105
+ scikit-learn==1.9.0
106
+ sacrebleu==2.6.0
107
+ nltk==3.9.4
108
+ mbstrdecoder==1.1.5
109
+ typepy==1.3.5
110
+ rouge_score==0.1.2
111
+ DataProperty==1.1.1
112
+ tabledata==1.3.5
113
+ evaluate==0.4.6
114
+ pytablewriter==1.2.1
115
+ lm_eval==0.4.12
116
+ pluggy==1.6.0
117
+ iniconfig==2.3.0
118
+ pytest==9.1.0
119
+ py-spy==0.4.2
120
+ pillow==12.2.0
121
+ tifffile==2026.6.1
122
+ imagecodecs==2026.6.26
123
+ zipp==4.1.0
124
+ importlib_metadata==9.0.0
125
+ derivative==0.6.3
126
+ pysindy==2.1.0
127
+ pyparsing==3.3.2
128
+ kiwisolver==1.5.0
129
+ fonttools==4.63.0
130
+ cycler==0.12.1
131
+ contourpy==1.3.3
132
+ matplotlib==3.11.0
133
+ sortedcontainers==2.4.0
134
+ wcwidth==0.8.2
135
+ sigfig==1.3.19
136
+ importlib_resources==7.1.0
137
+ enum-tools==0.13.0
138
+ cramjam==2.11.0
139
+ python-snappy==0.7.3
140
+ compact-json==1.8.2
141
+ numbers-parser==4.18.5
142
+ entmax==1.3
epfl_final/SparseAtt/wandb/run-20260820_031315-89kzpaot/files/wandb-metadata.json ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.13.13",
4
+ "startedAt": "2026-08-20T03:13:15.604057Z",
5
+ "args": [
6
+ "--config",
7
+ "/mloscratch/homes/navasard/sparse/configs_seed47_ddp2/dense_seed47_ddp2.json"
8
+ ],
9
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
10
+ "codePath": "train.py",
11
+ "codePathLocal": "train.py",
12
+ "git": {
13
+ "remote": "https://github.com/rubentium/TokenPrune.git",
14
+ "commit": "5637886f0cc760009989b14e68c6ae06d4827418"
15
+ },
16
+ "email": "ruben.navasardyan@mail.utoronto.ca",
17
+ "root": "/mloscratch/homes/navasard/sparse/SparseAtt",
18
+ "host": "a100-40gb-train-5-gpu-0-0",
19
+ "executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
20
+ "cpu_count": 32,
21
+ "cpu_count_logical": 64,
22
+ "gpu": "NVIDIA A100-SXM4-40GB",
23
+ "gpu_count": 5,
24
+ "disk": {
25
+ "/": {
26
+ "total": "7679361679360",
27
+ "used": "2981317980160"
28
+ }
29
+ },
30
+ "memory": {
31
+ "total": "540742438912"
32
+ },
33
+ "gpu_nvidia": [
34
+ {
35
+ "name": "NVIDIA A100-SXM4-40GB",
36
+ "memoryTotal": "42949672960",
37
+ "cudaCores": 6912,
38
+ "architecture": "Ampere",
39
+ "uuid": "GPU-d1fb9f59-34c2-ed7f-8e2f-be250e503c95"
40
+ },
41
+ {
42
+ "name": "NVIDIA A100-SXM4-40GB",
43
+ "memoryTotal": "42949672960",
44
+ "cudaCores": 6912,
45
+ "architecture": "Ampere",
46
+ "uuid": "GPU-0c470c94-21d0-bbe4-2457-d675efbf8ff1"
47
+ },
48
+ {
49
+ "name": "NVIDIA A100-SXM4-40GB",
50
+ "memoryTotal": "42949672960",
51
+ "cudaCores": 6912,
52
+ "architecture": "Ampere",
53
+ "uuid": "GPU-90b7600a-e1f3-d87d-d635-658a16a50235"
54
+ },
55
+ {
56
+ "name": "NVIDIA A100-SXM4-40GB",
57
+ "memoryTotal": "42949672960",
58
+ "cudaCores": 6912,
59
+ "architecture": "Ampere",
60
+ "uuid": "GPU-d173bebc-047d-78b1-1d9a-ed00f0531462"
61
+ },
62
+ {
63
+ "name": "NVIDIA A100-SXM4-40GB",
64
+ "memoryTotal": "42949672960",
65
+ "cudaCores": 6912,
66
+ "architecture": "Ampere",
67
+ "uuid": "GPU-027ae003-b23b-6506-f0f1-1e1a1f17682b"
68
+ }
69
+ ],
70
+ "cudaVersion": "13.0",
71
+ "writerId": "axmcvjtt50js3tjwzbf9fqa2nyl78fb4"
72
+ }
epfl_final/SparseAtt/wandb/run-20260820_031315-89kzpaot/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/SparseAtt/wandb/run-20260820_031315-89kzpaot/logs/debug.log ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-08-20 03:13:15,632 INFO MainThread:38961 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-08-20 03:13:15,632 INFO MainThread:38961 [wandb_setup.py:_flush():81] Configure stats pid to 38961
3
+ 2026-08-20 03:13:15,632 INFO MainThread:38961 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-08-20 03:13:15,633 INFO MainThread:38961 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_031315-89kzpaot/logs/debug.log
5
+ 2026-08-20 03:13:15,633 INFO MainThread:38961 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_031315-89kzpaot/logs/debug-internal.log
6
+ 2026-08-20 03:13:15,633 INFO MainThread:38961 [wandb_init.py:init():850] calling init triggers
7
+ 2026-08-20 03:13:15,633 INFO MainThread:38961 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47/run_20260820_031247', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 5, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 47, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m dense 8k seed47 ddp2', 'wandb_tags': ['qwen3', '365m', '8k', 'scaleladder', 'dense', 'multiseed', 'seed47', 'ddp2'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
9
+ 2026-08-20 03:13:15,633 INFO MainThread:38961 [wandb_init.py:init():898] starting backend
10
+ 2026-08-20 03:13:16,497 INFO MainThread:38961 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-08-20 03:13:17,038 INFO MainThread:38961 [wandb_init.py:init():918] backend started and connected
12
+ 2026-08-20 03:13:17,048 INFO MainThread:38961 [wandb_init.py:init():988] updated telemetry
13
+ 2026-08-20 03:13:17,079 INFO MainThread:38961 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-08-20 03:13:17,632 INFO MainThread:38961 [wandb_init.py:init():1056] starting run threads in backend
15
+ 2026-08-20 03:13:17,883 INFO MainThread:38961 [wandb_run.py:_console_start():2554] atexit reg
16
+ 2026-08-20 03:13:17,883 INFO MainThread:38961 [wandb_run.py:_redirect():2403] redirect: wrap_raw
17
+ 2026-08-20 03:13:17,883 INFO MainThread:38961 [wandb_run.py:_redirect():2472] Wrapping output streams.
18
+ 2026-08-20 03:13:17,883 INFO MainThread:38961 [wandb_run.py:_redirect():2495] Redirects installed.
19
+ 2026-08-20 03:13:17,893 INFO MainThread:38961 [wandb_init.py:init():1094] run started, returning control to user process
epfl_final/SparseAtt/wandb/run-20260820_031315-89kzpaot/run-89kzpaot.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:892abe9221215c7a31edbf4afddc0b172d3f6aa5feff4365cd43f82f47f40356
3
+ size 16809984
epfl_final/SparseAtt/wandb/run-20260820_032150-93ukqj9s/files/output.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/SparseAtt/wandb/run-20260820_032150-93ukqj9s/files/requirements.txt ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pip==26.0.1
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.6.3
8
+ typing_extensions==4.15.0
9
+ triton==3.6.0
10
+ tqdm==4.67.3
11
+ torchinfo==1.8.0
12
+ sympy==1.14.0
13
+ smmap==5.0.3
14
+ six==1.17.0
15
+ shellingham==1.5.4
16
+ setuptools==81.0.0
17
+ safetensors==0.7.0
18
+ regex==2026.4.4
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.4.1
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.4.4
38
+ networkx==3.6.1
39
+ multidict==6.7.1
40
+ mdurl==0.1.2
41
+ MarkupSafe==3.0.3
42
+ idna==3.13
43
+ hf-xet==1.4.3
44
+ h11==0.16.0
45
+ fsspec==2026.2.0
46
+ frozenlist==1.8.0
47
+ filelock==3.29.0
48
+ dill==0.4.1
49
+ cuda-pathfinder==1.5.4
50
+ click==8.3.3
51
+ charset-normalizer==3.4.7
52
+ certifi==2026.4.22
53
+ attrs==26.1.0
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ yarl==1.23.0
58
+ typing-inspection==0.4.2
59
+ sentry-sdk==2.58.0
60
+ requests==2.33.1
61
+ python-dateutil==2.9.0.post0
62
+ pydantic_core==2.46.3
63
+ nvidia-cusparse==12.6.3.3
64
+ nvidia-cufft==12.0.0.61
65
+ nvidia-cudnn-cu13==9.19.0.56
66
+ multiprocess==0.70.19
67
+ markdown-it-py==4.0.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ cuda-bindings==13.2.0
72
+ anyio==4.13.0
73
+ aiosignal==1.4.0
74
+ rich==15.0.0
75
+ pydantic==2.13.3
76
+ pandas==3.0.2
77
+ nvidia-cusolver==12.0.4.66
78
+ httpx==0.28.1
79
+ GitPython==3.1.49
80
+ aiohttp==3.13.5
81
+ wandb==0.26.1
82
+ typer==0.25.1
83
+ torch==2.11.0
84
+ huggingface_hub==1.13.0
85
+ tokenizers==0.22.2
86
+ datasets==4.8.5
87
+ accelerate==1.13.0
88
+ transformers==5.7.0
89
+ word2number==1.1
90
+ sqlitedict==2.1.0
91
+ pytz==2026.2
92
+ threadpoolctl==3.6.0
93
+ tcolorpy==0.1.7
94
+ tabulate==0.10.0
95
+ scipy==1.17.1
96
+ portalocker==3.2.0
97
+ pathvalidate==3.3.1
98
+ narwhals==2.22.1
99
+ more-itertools==11.1.0
100
+ lxml==6.1.1
101
+ joblib==1.5.3
102
+ colorama==0.4.6
103
+ chardet==6.0.0.post1
104
+ absl-py==2.4.0
105
+ scikit-learn==1.9.0
106
+ sacrebleu==2.6.0
107
+ nltk==3.9.4
108
+ mbstrdecoder==1.1.5
109
+ typepy==1.3.5
110
+ rouge_score==0.1.2
111
+ DataProperty==1.1.1
112
+ tabledata==1.3.5
113
+ evaluate==0.4.6
114
+ pytablewriter==1.2.1
115
+ lm_eval==0.4.12
116
+ pluggy==1.6.0
117
+ iniconfig==2.3.0
118
+ pytest==9.1.0
119
+ py-spy==0.4.2
120
+ pillow==12.2.0
121
+ tifffile==2026.6.1
122
+ imagecodecs==2026.6.26
123
+ zipp==4.1.0
124
+ importlib_metadata==9.0.0
125
+ derivative==0.6.3
126
+ pysindy==2.1.0
127
+ pyparsing==3.3.2
128
+ kiwisolver==1.5.0
129
+ fonttools==4.63.0
130
+ cycler==0.12.1
131
+ contourpy==1.3.3
132
+ matplotlib==3.11.0
133
+ sortedcontainers==2.4.0
134
+ wcwidth==0.8.2
135
+ sigfig==1.3.19
136
+ importlib_resources==7.1.0
137
+ enum-tools==0.13.0
138
+ cramjam==2.11.0
139
+ python-snappy==0.7.3
140
+ compact-json==1.8.2
141
+ numbers-parser==4.18.5
142
+ entmax==1.3
epfl_final/SparseAtt/wandb/run-20260820_032150-93ukqj9s/files/wandb-metadata.json ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.13.13",
4
+ "startedAt": "2026-08-20T03:21:50.900524Z",
5
+ "args": [
6
+ "--config",
7
+ "/mloscratch/homes/navasard/sparse/configs_seed47_ddp2/mixed_seed47_ddp2.json"
8
+ ],
9
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
10
+ "codePath": "train.py",
11
+ "codePathLocal": "train.py",
12
+ "git": {
13
+ "remote": "https://github.com/rubentium/TokenPrune.git",
14
+ "commit": "5637886f0cc760009989b14e68c6ae06d4827418"
15
+ },
16
+ "email": "ruben.navasardyan@mail.utoronto.ca",
17
+ "root": "/mloscratch/homes/navasard/sparse/SparseAtt",
18
+ "host": "a100-40gb-train-5-gpu-0-0",
19
+ "executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
20
+ "cpu_count": 32,
21
+ "cpu_count_logical": 64,
22
+ "gpu": "NVIDIA A100-SXM4-40GB",
23
+ "gpu_count": 5,
24
+ "disk": {
25
+ "/": {
26
+ "total": "7679361679360",
27
+ "used": "2981606805504"
28
+ }
29
+ },
30
+ "memory": {
31
+ "total": "540742438912"
32
+ },
33
+ "gpu_nvidia": [
34
+ {
35
+ "name": "NVIDIA A100-SXM4-40GB",
36
+ "memoryTotal": "42949672960",
37
+ "cudaCores": 6912,
38
+ "architecture": "Ampere",
39
+ "uuid": "GPU-d1fb9f59-34c2-ed7f-8e2f-be250e503c95"
40
+ },
41
+ {
42
+ "name": "NVIDIA A100-SXM4-40GB",
43
+ "memoryTotal": "42949672960",
44
+ "cudaCores": 6912,
45
+ "architecture": "Ampere",
46
+ "uuid": "GPU-0c470c94-21d0-bbe4-2457-d675efbf8ff1"
47
+ },
48
+ {
49
+ "name": "NVIDIA A100-SXM4-40GB",
50
+ "memoryTotal": "42949672960",
51
+ "cudaCores": 6912,
52
+ "architecture": "Ampere",
53
+ "uuid": "GPU-90b7600a-e1f3-d87d-d635-658a16a50235"
54
+ },
55
+ {
56
+ "name": "NVIDIA A100-SXM4-40GB",
57
+ "memoryTotal": "42949672960",
58
+ "cudaCores": 6912,
59
+ "architecture": "Ampere",
60
+ "uuid": "GPU-d173bebc-047d-78b1-1d9a-ed00f0531462"
61
+ },
62
+ {
63
+ "name": "NVIDIA A100-SXM4-40GB",
64
+ "memoryTotal": "42949672960",
65
+ "cudaCores": 6912,
66
+ "architecture": "Ampere",
67
+ "uuid": "GPU-027ae003-b23b-6506-f0f1-1e1a1f17682b"
68
+ }
69
+ ],
70
+ "cudaVersion": "13.0",
71
+ "writerId": "5uc0cbvm7jgzt3kbm7e1nxa3nuxqcdrk"
72
+ }
epfl_final/SparseAtt/wandb/run-20260820_032150-93ukqj9s/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/SparseAtt/wandb/run-20260820_032150-93ukqj9s/logs/debug.log ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-08-20 03:21:50,929 INFO MainThread:40442 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-08-20 03:21:50,929 INFO MainThread:40442 [wandb_setup.py:_flush():81] Configure stats pid to 40442
3
+ 2026-08-20 03:21:50,929 INFO MainThread:40442 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-08-20 03:21:50,929 INFO MainThread:40442 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_032150-93ukqj9s/logs/debug.log
5
+ 2026-08-20 03:21:50,929 INFO MainThread:40442 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_032150-93ukqj9s/logs/debug-internal.log
6
+ 2026-08-20 03:21:50,929 INFO MainThread:40442 [wandb_init.py:init():850] calling init triggers
7
+ 2026-08-20 03:21:50,929 INFO MainThread:40442 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47/run_20260820_031812', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 5, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 47, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '365m mixed 8k seed47 ddp2', 'wandb_tags': ['qwen3', '365m', '8k', 'mixed', 'multiseed', 'seed47', 'ddp2'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed', 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': [63, 47, 31, 21, 11], 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
9
+ 2026-08-20 03:21:50,929 INFO MainThread:40442 [wandb_init.py:init():898] starting backend
10
+ 2026-08-20 03:21:51,248 INFO MainThread:40442 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-08-20 03:21:51,550 INFO MainThread:40442 [wandb_init.py:init():918] backend started and connected
12
+ 2026-08-20 03:21:51,552 INFO MainThread:40442 [wandb_init.py:init():988] updated telemetry
13
+ 2026-08-20 03:21:51,587 INFO MainThread:40442 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-08-20 03:21:52,303 INFO MainThread:40442 [wandb_init.py:init():1056] starting run threads in backend
15
+ 2026-08-20 03:21:52,716 INFO MainThread:40442 [wandb_run.py:_console_start():2554] atexit reg
16
+ 2026-08-20 03:21:52,716 INFO MainThread:40442 [wandb_run.py:_redirect():2403] redirect: wrap_raw
17
+ 2026-08-20 03:21:52,716 INFO MainThread:40442 [wandb_run.py:_redirect():2472] Wrapping output streams.
18
+ 2026-08-20 03:21:52,716 INFO MainThread:40442 [wandb_run.py:_redirect():2495] Redirects installed.
19
+ 2026-08-20 03:21:52,723 INFO MainThread:40442 [wandb_init.py:init():1094] run started, returning control to user process
epfl_final/SparseAtt/wandb/run-20260820_032150-93ukqj9s/run-93ukqj9s.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c65408a4d38aa9ccf920540612b71421ffe35eba0c3109ec4e367288d6654891
3
+ size 15958016
epfl_final/SparseAtt/wandb/run-20260820_041558-bajm6b15/files/output.log ADDED
@@ -0,0 +1,482 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Starting training...
2
+ Validation at step 13000: Loss = 3.0696, Perplexity = 21.53
3
+ wandb: WARNING Tried to log to step 13000 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
4
+ Switching attention mode for step 13001: dense (dense step interval)
5
+ Step 13010/44000 | Loss: 3.0583 | LR: 4.07e-04 | 30145.6ms/step | 0.7 samples/s
6
+ wandb: WARNING Tried to log to step 13010 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
7
+ Step 13020/44000 | Loss: 3.0539 | LR: 4.07e-04 | 1368.0ms/step | 14.6 samples/s
8
+ wandb: WARNING Tried to log to step 13020 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
9
+ Step 13030/44000 | Loss: 3.0645 | LR: 4.07e-04 | 1371.0ms/step | 14.6 samples/s
10
+ wandb: WARNING Tried to log to step 13030 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
11
+ Step 13040/44000 | Loss: 3.0438 | LR: 4.07e-04 | 1375.5ms/step | 14.5 samples/s
12
+ wandb: WARNING Tried to log to step 13040 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
13
+ Step 13050/44000 | Loss: 3.0602 | LR: 4.07e-04 | 1378.6ms/step | 14.5 samples/s
14
+ wandb: WARNING Tried to log to step 13050 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
15
+ Step 13060/44000 | Loss: 3.0601 | LR: 4.07e-04 | 1376.2ms/step | 14.5 samples/s
16
+ wandb: WARNING Tried to log to step 13060 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
17
+ Step 13070/44000 | Loss: 3.0745 | LR: 4.06e-04 | 1380.3ms/step | 14.5 samples/s
18
+ wandb: WARNING Tried to log to step 13070 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
19
+ Step 13080/44000 | Loss: 3.0356 | LR: 4.06e-04 | 1379.5ms/step | 14.5 samples/s
20
+ wandb: WARNING Tried to log to step 13080 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
21
+ Step 13090/44000 | Loss: 3.0677 | LR: 4.06e-04 | 1377.7ms/step | 14.5 samples/s
22
+ wandb: WARNING Tried to log to step 13090 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
23
+ Step 13100/44000 | Loss: 3.0657 | LR: 4.06e-04 | 1379.9ms/step | 14.5 samples/s
24
+ wandb: WARNING Tried to log to step 13100 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
25
+ Step 13110/44000 | Loss: 3.0377 | LR: 4.06e-04 | 1382.3ms/step | 14.5 samples/s
26
+ wandb: WARNING Tried to log to step 13110 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
27
+ Step 13120/44000 | Loss: 3.0419 | LR: 4.06e-04 | 1377.8ms/step | 14.5 samples/s
28
+ wandb: WARNING Tried to log to step 13120 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
29
+ Step 13130/44000 | Loss: 3.0623 | LR: 4.06e-04 | 1374.8ms/step | 14.5 samples/s
30
+ wandb: WARNING Tried to log to step 13130 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
31
+ Step 13140/44000 | Loss: 3.0456 | LR: 4.05e-04 | 1373.6ms/step | 14.6 samples/s
32
+ wandb: WARNING Tried to log to step 13140 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
33
+ Step 13150/44000 | Loss: 3.0531 | LR: 4.05e-04 | 1375.9ms/step | 14.5 samples/s
34
+ wandb: WARNING Tried to log to step 13150 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
35
+ Step 13160/44000 | Loss: 3.0258 | LR: 4.05e-04 | 1375.7ms/step | 14.5 samples/s
36
+ wandb: WARNING Tried to log to step 13160 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
37
+ Step 13170/44000 | Loss: 3.0970 | LR: 4.05e-04 | 1373.9ms/step | 14.6 samples/s
38
+ wandb: WARNING Tried to log to step 13170 that is less than the current step 13171. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
39
+ Step 13180/44000 | Loss: 3.0733 | LR: 4.05e-04 | 1372.2ms/step | 14.6 samples/s
40
+ Step 13190/44000 | Loss: 3.0473 | LR: 4.05e-04 | 1373.9ms/step | 14.6 samples/s
41
+ Step 13200/44000 | Loss: 3.0934 | LR: 4.05e-04 | 1375.0ms/step | 14.5 samples/s
42
+ Step 13210/44000 | Loss: 3.0429 | LR: 4.04e-04 | 1372.5ms/step | 14.6 samples/s
43
+ Step 13220/44000 | Loss: 3.0658 | LR: 4.04e-04 | 1372.0ms/step | 14.6 samples/s
44
+ Step 13230/44000 | Loss: 3.0785 | LR: 4.04e-04 | 1373.3ms/step | 14.6 samples/s
45
+ Step 13240/44000 | Loss: 3.0605 | LR: 4.04e-04 | 1373.4ms/step | 14.6 samples/s
46
+ Step 13250/44000 | Loss: 3.0669 | LR: 4.04e-04 | 1373.5ms/step | 14.6 samples/s
47
+ Step 13260/44000 | Loss: 3.0628 | LR: 4.04e-04 | 1372.4ms/step | 14.6 samples/s
48
+ Step 13270/44000 | Loss: 3.0632 | LR: 4.04e-04 | 1373.9ms/step | 14.6 samples/s
49
+ Step 13280/44000 | Loss: 3.0612 | LR: 4.03e-04 | 1376.4ms/step | 14.5 samples/s
50
+ Step 13290/44000 | Loss: 3.0461 | LR: 4.03e-04 | 1373.5ms/step | 14.6 samples/s
51
+ Step 13300/44000 | Loss: 3.0573 | LR: 4.03e-04 | 1374.0ms/step | 14.6 samples/s
52
+ Step 13310/44000 | Loss: 3.0515 | LR: 4.03e-04 | 1373.8ms/step | 14.6 samples/s
53
+ Step 13320/44000 | Loss: 3.0342 | LR: 4.03e-04 | 1374.6ms/step | 14.5 samples/s
54
+ Step 13330/44000 | Loss: 3.0614 | LR: 4.03e-04 | 1372.6ms/step | 14.6 samples/s
55
+ Step 13340/44000 | Loss: 3.0749 | LR: 4.03e-04 | 1372.3ms/step | 14.6 samples/s
56
+ Step 13350/44000 | Loss: 3.0320 | LR: 4.02e-04 | 1372.7ms/step | 14.6 samples/s
57
+ Step 13360/44000 | Loss: 3.1017 | LR: 4.02e-04 | 1374.6ms/step | 14.6 samples/s
58
+ Step 13370/44000 | Loss: 3.0585 | LR: 4.02e-04 | 1373.5ms/step | 14.6 samples/s
59
+ Step 13380/44000 | Loss: 3.0661 | LR: 4.02e-04 | 1373.5ms/step | 14.6 samples/s
60
+ Step 13390/44000 | Loss: 3.0359 | LR: 4.02e-04 | 1376.2ms/step | 14.5 samples/s
61
+ Step 13400/44000 | Loss: 3.0451 | LR: 4.02e-04 | 1377.0ms/step | 14.5 samples/s
62
+ Step 13410/44000 | Loss: 3.0756 | LR: 4.02e-04 | 1375.1ms/step | 14.5 samples/s
63
+ Step 13420/44000 | Loss: 3.0443 | LR: 4.01e-04 | 1376.3ms/step | 14.5 samples/s
64
+ Step 13430/44000 | Loss: 3.0581 | LR: 4.01e-04 | 1375.4ms/step | 14.5 samples/s
65
+ Step 13440/44000 | Loss: 3.0353 | LR: 4.01e-04 | 1375.8ms/step | 14.5 samples/s
66
+ Step 13450/44000 | Loss: 3.0295 | LR: 4.01e-04 | 1372.9ms/step | 14.6 samples/s
67
+ Step 13460/44000 | Loss: 3.0325 | LR: 4.01e-04 | 1373.6ms/step | 14.6 samples/s
68
+ Step 13470/44000 | Loss: 3.0720 | LR: 4.01e-04 | 1374.7ms/step | 14.5 samples/s
69
+ Step 13480/44000 | Loss: 3.0366 | LR: 4.01e-04 | 1375.3ms/step | 14.5 samples/s
70
+ Step 13490/44000 | Loss: 3.0641 | LR: 4.00e-04 | 1374.6ms/step | 14.6 samples/s
71
+ Step 13500/44000 | Loss: 3.0567 | LR: 4.00e-04 | 1375.5ms/step | 14.5 samples/s
72
+ Step 13510/44000 | Loss: 3.0549 | LR: 4.00e-04 | 1374.9ms/step | 14.5 samples/s
73
+ Step 13520/44000 | Loss: 3.0648 | LR: 4.00e-04 | 1376.7ms/step | 14.5 samples/s
74
+ Step 13530/44000 | Loss: 3.0518 | LR: 4.00e-04 | 1374.2ms/step | 14.6 samples/s
75
+ Step 13540/44000 | Loss: 3.0569 | LR: 4.00e-04 | 1374.2ms/step | 14.6 samples/s
76
+ Step 13550/44000 | Loss: 3.0538 | LR: 4.00e-04 | 1374.4ms/step | 14.6 samples/s
77
+ Step 13560/44000 | Loss: 3.0502 | LR: 3.99e-04 | 1374.0ms/step | 14.6 samples/s
78
+ Step 13570/44000 | Loss: 3.0712 | LR: 3.99e-04 | 1373.0ms/step | 14.6 samples/s
79
+ Step 13580/44000 | Loss: 3.0557 | LR: 3.99e-04 | 1372.4ms/step | 14.6 samples/s
80
+ Step 13590/44000 | Loss: 3.0451 | LR: 3.99e-04 | 1376.1ms/step | 14.5 samples/s
81
+ Step 13600/44000 | Loss: 3.0511 | LR: 3.99e-04 | 1374.8ms/step | 14.5 samples/s
82
+ Step 13610/44000 | Loss: 3.0764 | LR: 3.99e-04 | 1373.4ms/step | 14.6 samples/s
83
+ Step 13620/44000 | Loss: 3.0381 | LR: 3.98e-04 | 1373.0ms/step | 14.6 samples/s
84
+ Step 13630/44000 | Loss: 3.0307 | LR: 3.98e-04 | 1374.2ms/step | 14.6 samples/s
85
+ Step 13640/44000 | Loss: 3.0425 | LR: 3.98e-04 | 1376.8ms/step | 14.5 samples/s
86
+ Step 13650/44000 | Loss: 3.0684 | LR: 3.98e-04 | 1372.6ms/step | 14.6 samples/s
87
+ Step 13660/44000 | Loss: 3.0299 | LR: 3.98e-04 | 1371.7ms/step | 14.6 samples/s
88
+ Step 13670/44000 | Loss: 3.0322 | LR: 3.98e-04 | 1374.8ms/step | 14.5 samples/s
89
+ Step 13680/44000 | Loss: 3.0632 | LR: 3.98e-04 | 1375.3ms/step | 14.5 samples/s
90
+ Step 13690/44000 | Loss: 3.0209 | LR: 3.97e-04 | 1372.8ms/step | 14.6 samples/s
91
+ Step 13700/44000 | Loss: 3.0666 | LR: 3.97e-04 | 1372.7ms/step | 14.6 samples/s
92
+ Step 13710/44000 | Loss: 3.0531 | LR: 3.97e-04 | 1375.3ms/step | 14.5 samples/s
93
+ Step 13720/44000 | Loss: 3.0627 | LR: 3.97e-04 | 1374.7ms/step | 14.5 samples/s
94
+ Step 13730/44000 | Loss: 3.0650 | LR: 3.97e-04 | 1373.7ms/step | 14.6 samples/s
95
+ Step 13740/44000 | Loss: 3.0528 | LR: 3.97e-04 | 1372.0ms/step | 14.6 samples/s
96
+ Step 13750/44000 | Loss: 3.0296 | LR: 3.97e-04 | 1377.0ms/step | 14.5 samples/s
97
+ Step 13760/44000 | Loss: 3.0609 | LR: 3.96e-04 | 1377.0ms/step | 14.5 samples/s
98
+ Step 13770/44000 | Loss: 3.0541 | LR: 3.96e-04 | 1373.5ms/step | 14.6 samples/s
99
+ Step 13780/44000 | Loss: 3.0327 | LR: 3.96e-04 | 1372.9ms/step | 14.6 samples/s
100
+ Step 13790/44000 | Loss: 3.0562 | LR: 3.96e-04 | 1372.0ms/step | 14.6 samples/s
101
+ Step 13800/44000 | Loss: 3.0464 | LR: 3.96e-04 | 1373.8ms/step | 14.6 samples/s
102
+ Step 13810/44000 | Loss: 3.0374 | LR: 3.96e-04 | 1373.6ms/step | 14.6 samples/s
103
+ Step 13820/44000 | Loss: 3.0165 | LR: 3.96e-04 | 1372.0ms/step | 14.6 samples/s
104
+ Step 13830/44000 | Loss: 3.0255 | LR: 3.95e-04 | 1373.2ms/step | 14.6 samples/s
105
+ Step 13840/44000 | Loss: 3.0070 | LR: 3.95e-04 | 1374.4ms/step | 14.6 samples/s
106
+ Step 13850/44000 | Loss: 3.0490 | LR: 3.95e-04 | 1371.8ms/step | 14.6 samples/s
107
+ Step 13860/44000 | Loss: 3.0623 | LR: 3.95e-04 | 1372.3ms/step | 14.6 samples/s
108
+ Step 13870/44000 | Loss: 3.0529 | LR: 3.95e-04 | 1375.4ms/step | 14.5 samples/s
109
+ Step 13880/44000 | Loss: 3.0477 | LR: 3.95e-04 | 1370.7ms/step | 14.6 samples/s
110
+ Step 13890/44000 | Loss: 3.0355 | LR: 3.95e-04 | 1371.6ms/step | 14.6 samples/s
111
+ Step 13900/44000 | Loss: 3.0340 | LR: 3.94e-04 | 1371.9ms/step | 14.6 samples/s
112
+ Step 13910/44000 | Loss: 3.0398 | LR: 3.94e-04 | 1375.2ms/step | 14.5 samples/s
113
+ Step 13920/44000 | Loss: 3.0894 | LR: 3.94e-04 | 1373.9ms/step | 14.6 samples/s
114
+ Step 13930/44000 | Loss: 3.0537 | LR: 3.94e-04 | 1373.3ms/step | 14.6 samples/s
115
+ Step 13940/44000 | Loss: 3.0667 | LR: 3.94e-04 | 1374.1ms/step | 14.6 samples/s
116
+ Step 13950/44000 | Loss: 3.0609 | LR: 3.94e-04 | 1374.8ms/step | 14.5 samples/s
117
+ Step 13960/44000 | Loss: 3.0364 | LR: 3.93e-04 | 1374.3ms/step | 14.6 samples/s
118
+ Step 13970/44000 | Loss: 3.0370 | LR: 3.93e-04 | 1373.4ms/step | 14.6 samples/s
119
+ Step 13980/44000 | Loss: 3.0801 | LR: 3.93e-04 | 1375.5ms/step | 14.5 samples/s
120
+ Step 13990/44000 | Loss: 3.0125 | LR: 3.93e-04 | 1377.0ms/step | 14.5 samples/s
121
+ Step 14000/44000 | Loss: 3.0749 | LR: 3.93e-04 | 1375.5ms/step | 14.5 samples/s
122
+ Validation at step 14000: Loss = 3.0537, Perplexity = 21.19
123
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-14000
124
+ Step 14010/44000 | Loss: 3.0348 | LR: 3.93e-04 | 3894.1ms/step | 5.1 samples/s
125
+ Step 14020/44000 | Loss: 3.0500 | LR: 3.93e-04 | 1385.9ms/step | 14.4 samples/s
126
+ Step 14030/44000 | Loss: 3.0532 | LR: 3.92e-04 | 1377.6ms/step | 14.5 samples/s
127
+ Step 14040/44000 | Loss: 3.0203 | LR: 3.92e-04 | 1378.8ms/step | 14.5 samples/s
128
+ Step 14050/44000 | Loss: 3.0323 | LR: 3.92e-04 | 1381.5ms/step | 14.5 samples/s
129
+ Step 14060/44000 | Loss: 3.0529 | LR: 3.92e-04 | 1381.6ms/step | 14.5 samples/s
130
+ Step 14070/44000 | Loss: 3.0459 | LR: 3.92e-04 | 1384.4ms/step | 14.4 samples/s
131
+ Step 14080/44000 | Loss: 3.0383 | LR: 3.92e-04 | 1385.9ms/step | 14.4 samples/s
132
+ Step 14090/44000 | Loss: 3.0212 | LR: 3.92e-04 | 1382.8ms/step | 14.5 samples/s
133
+ Step 14100/44000 | Loss: 3.0382 | LR: 3.91e-04 | 1382.0ms/step | 14.5 samples/s
134
+ Step 14110/44000 | Loss: 3.0276 | LR: 3.91e-04 | 1385.4ms/step | 14.4 samples/s
135
+ Step 14120/44000 | Loss: 3.0614 | LR: 3.91e-04 | 1386.5ms/step | 14.4 samples/s
136
+ Step 14130/44000 | Loss: 3.0314 | LR: 3.91e-04 | 1385.7ms/step | 14.4 samples/s
137
+ Step 14140/44000 | Loss: 3.0562 | LR: 3.91e-04 | 1384.5ms/step | 14.4 samples/s
138
+ Step 14150/44000 | Loss: 3.0508 | LR: 3.91e-04 | 1387.3ms/step | 14.4 samples/s
139
+ Step 14160/44000 | Loss: 3.0385 | LR: 3.90e-04 | 1384.0ms/step | 14.5 samples/s
140
+ Step 14170/44000 | Loss: 3.0346 | LR: 3.90e-04 | 1387.1ms/step | 14.4 samples/s
141
+ Step 14180/44000 | Loss: 3.0018 | LR: 3.90e-04 | 1388.9ms/step | 14.4 samples/s
142
+ Step 14190/44000 | Loss: 3.0360 | LR: 3.90e-04 | 1384.1ms/step | 14.4 samples/s
143
+ Step 14200/44000 | Loss: 3.0175 | LR: 3.90e-04 | 1385.5ms/step | 14.4 samples/s
144
+ Step 14210/44000 | Loss: 3.0325 | LR: 3.90e-04 | 1385.8ms/step | 14.4 samples/s
145
+ Step 14220/44000 | Loss: 3.0249 | LR: 3.90e-04 | 1384.6ms/step | 14.4 samples/s
146
+ Step 14230/44000 | Loss: 3.0705 | LR: 3.89e-04 | 1385.9ms/step | 14.4 samples/s
147
+ Step 14240/44000 | Loss: 3.0322 | LR: 3.89e-04 | 1385.6ms/step | 14.4 samples/s
148
+ Step 14250/44000 | Loss: 3.0173 | LR: 3.89e-04 | 1383.4ms/step | 14.5 samples/s
149
+ Step 14260/44000 | Loss: 3.0340 | LR: 3.89e-04 | 1384.7ms/step | 14.4 samples/s
150
+ Step 14270/44000 | Loss: 3.0348 | LR: 3.89e-04 | 1385.8ms/step | 14.4 samples/s
151
+ Step 14280/44000 | Loss: 3.0256 | LR: 3.89e-04 | 1385.2ms/step | 14.4 samples/s
152
+ Step 14290/44000 | Loss: 3.0314 | LR: 3.89e-04 | 1382.8ms/step | 14.5 samples/s
153
+ Step 14300/44000 | Loss: 3.0434 | LR: 3.88e-04 | 1383.6ms/step | 14.5 samples/s
154
+ Step 14310/44000 | Loss: 3.0538 | LR: 3.88e-04 | 1384.8ms/step | 14.4 samples/s
155
+ Step 14320/44000 | Loss: 3.0534 | LR: 3.88e-04 | 1383.7ms/step | 14.5 samples/s
156
+ Step 14330/44000 | Loss: 3.0495 | LR: 3.88e-04 | 1383.4ms/step | 14.5 samples/s
157
+ Step 14340/44000 | Loss: 3.0320 | LR: 3.88e-04 | 1384.7ms/step | 14.4 samples/s
158
+ Step 14350/44000 | Loss: 3.0422 | LR: 3.88e-04 | 1382.5ms/step | 14.5 samples/s
159
+ Step 14360/44000 | Loss: 2.9975 | LR: 3.87e-04 | 1383.7ms/step | 14.5 samples/s
160
+ Step 14370/44000 | Loss: 3.0624 | LR: 3.87e-04 | 1385.1ms/step | 14.4 samples/s
161
+ Step 14380/44000 | Loss: 3.0577 | LR: 3.87e-04 | 1386.6ms/step | 14.4 samples/s
162
+ Step 14390/44000 | Loss: 3.0378 | LR: 3.87e-04 | 1388.2ms/step | 14.4 samples/s
163
+ Step 14400/44000 | Loss: 3.0463 | LR: 3.87e-04 | 1389.3ms/step | 14.4 samples/s
164
+ Step 14410/44000 | Loss: 3.0376 | LR: 3.87e-04 | 1381.8ms/step | 14.5 samples/s
165
+ Step 14420/44000 | Loss: 3.0484 | LR: 3.87e-04 | 1384.2ms/step | 14.4 samples/s
166
+ Step 14430/44000 | Loss: 3.0090 | LR: 3.86e-04 | 1384.3ms/step | 14.4 samples/s
167
+ Step 14440/44000 | Loss: 3.0306 | LR: 3.86e-04 | 1387.3ms/step | 14.4 samples/s
168
+ Step 14450/44000 | Loss: 3.0626 | LR: 3.86e-04 | 1384.5ms/step | 14.4 samples/s
169
+ Step 14460/44000 | Loss: 3.0276 | LR: 3.86e-04 | 1383.1ms/step | 14.5 samples/s
170
+ Step 14470/44000 | Loss: 3.0396 | LR: 3.86e-04 | 1385.3ms/step | 14.4 samples/s
171
+ Step 14480/44000 | Loss: 3.0202 | LR: 3.86e-04 | 1385.1ms/step | 14.4 samples/s
172
+ Step 14490/44000 | Loss: 3.0549 | LR: 3.85e-04 | 1385.3ms/step | 14.4 samples/s
173
+ Step 14500/44000 | Loss: 3.0393 | LR: 3.85e-04 | 1390.2ms/step | 14.4 samples/s
174
+ Step 14510/44000 | Loss: 3.0490 | LR: 3.85e-04 | 1388.0ms/step | 14.4 samples/s
175
+ Step 14520/44000 | Loss: 3.0261 | LR: 3.85e-04 | 1386.4ms/step | 14.4 samples/s
176
+ Step 14530/44000 | Loss: 3.0523 | LR: 3.85e-04 | 1384.2ms/step | 14.4 samples/s
177
+ Step 14540/44000 | Loss: 3.0644 | LR: 3.85e-04 | 1386.2ms/step | 14.4 samples/s
178
+ Step 14550/44000 | Loss: 3.0110 | LR: 3.85e-04 | 1381.8ms/step | 14.5 samples/s
179
+ Step 14560/44000 | Loss: 3.0658 | LR: 3.84e-04 | 1386.2ms/step | 14.4 samples/s
180
+ Step 14570/44000 | Loss: 3.0399 | LR: 3.84e-04 | 1383.6ms/step | 14.5 samples/s
181
+ Step 14580/44000 | Loss: 3.0060 | LR: 3.84e-04 | 1382.1ms/step | 14.5 samples/s
182
+ Step 14590/44000 | Loss: 3.0186 | LR: 3.84e-04 | 1384.6ms/step | 14.4 samples/s
183
+ Step 14600/44000 | Loss: 3.0539 | LR: 3.84e-04 | 1383.9ms/step | 14.5 samples/s
184
+ Step 14610/44000 | Loss: 3.0583 | LR: 3.84e-04 | 1382.9ms/step | 14.5 samples/s
185
+ Step 14620/44000 | Loss: 3.0253 | LR: 3.83e-04 | 1385.6ms/step | 14.4 samples/s
186
+ Step 14630/44000 | Loss: 3.0322 | LR: 3.83e-04 | 1383.8ms/step | 14.5 samples/s
187
+ Step 14640/44000 | Loss: 3.0508 | LR: 3.83e-04 | 1384.9ms/step | 14.4 samples/s
188
+ Step 14650/44000 | Loss: 3.0502 | LR: 3.83e-04 | 1388.3ms/step | 14.4 samples/s
189
+ Step 14660/44000 | Loss: 3.0512 | LR: 3.83e-04 | 1385.5ms/step | 14.4 samples/s
190
+ Step 14670/44000 | Loss: 3.0824 | LR: 3.83e-04 | 1384.4ms/step | 14.4 samples/s
191
+ Step 14680/44000 | Loss: 3.0295 | LR: 3.83e-04 | 1385.3ms/step | 14.4 samples/s
192
+ Step 14690/44000 | Loss: 3.0247 | LR: 3.82e-04 | 1384.1ms/step | 14.4 samples/s
193
+ Step 14700/44000 | Loss: 3.0529 | LR: 3.82e-04 | 1385.0ms/step | 14.4 samples/s
194
+ Step 14710/44000 | Loss: 3.0174 | LR: 3.82e-04 | 1382.8ms/step | 14.5 samples/s
195
+ Step 14720/44000 | Loss: 3.0467 | LR: 3.82e-04 | 1384.0ms/step | 14.5 samples/s
196
+ Step 14730/44000 | Loss: 3.0135 | LR: 3.82e-04 | 1383.8ms/step | 14.5 samples/s
197
+ Step 14740/44000 | Loss: 3.0235 | LR: 3.82e-04 | 1384.4ms/step | 14.4 samples/s
198
+ Step 14750/44000 | Loss: 3.0217 | LR: 3.81e-04 | 1385.0ms/step | 14.4 samples/s
199
+ Step 14760/44000 | Loss: 3.0297 | LR: 3.81e-04 | 1383.8ms/step | 14.5 samples/s
200
+ Step 14770/44000 | Loss: 3.0435 | LR: 3.81e-04 | 1385.4ms/step | 14.4 samples/s
201
+ Step 14780/44000 | Loss: 3.0377 | LR: 3.81e-04 | 1388.8ms/step | 14.4 samples/s
202
+ Step 14790/44000 | Loss: 3.0492 | LR: 3.81e-04 | 1383.7ms/step | 14.5 samples/s
203
+ Step 14800/44000 | Loss: 3.0522 | LR: 3.81e-04 | 1385.4ms/step | 14.4 samples/s
204
+ Step 14810/44000 | Loss: 3.0323 | LR: 3.81e-04 | 1385.8ms/step | 14.4 samples/s
205
+ Step 14820/44000 | Loss: 3.0518 | LR: 3.80e-04 | 1384.0ms/step | 14.5 samples/s
206
+ Step 14830/44000 | Loss: 3.0578 | LR: 3.80e-04 | 1385.3ms/step | 14.4 samples/s
207
+ Step 14840/44000 | Loss: 3.0104 | LR: 3.80e-04 | 1388.0ms/step | 14.4 samples/s
208
+ Step 14850/44000 | Loss: 3.0387 | LR: 3.80e-04 | 1384.8ms/step | 14.4 samples/s
209
+ Step 14860/44000 | Loss: 3.0327 | LR: 3.80e-04 | 1385.9ms/step | 14.4 samples/s
210
+ Step 14870/44000 | Loss: 3.0378 | LR: 3.80e-04 | 1388.7ms/step | 14.4 samples/s
211
+ Step 14880/44000 | Loss: 3.0459 | LR: 3.79e-04 | 1385.7ms/step | 14.4 samples/s
212
+ Step 14890/44000 | Loss: 3.0603 | LR: 3.79e-04 | 1383.2ms/step | 14.5 samples/s
213
+ Step 14900/44000 | Loss: 3.0527 | LR: 3.79e-04 | 1384.5ms/step | 14.4 samples/s
214
+ Step 14910/44000 | Loss: 3.0334 | LR: 3.79e-04 | 1387.1ms/step | 14.4 samples/s
215
+ Step 14920/44000 | Loss: 3.0379 | LR: 3.79e-04 | 1384.0ms/step | 14.5 samples/s
216
+ Step 14930/44000 | Loss: 3.0499 | LR: 3.79e-04 | 1383.6ms/step | 14.5 samples/s
217
+ Step 14940/44000 | Loss: 3.0472 | LR: 3.79e-04 | 1386.1ms/step | 14.4 samples/s
218
+ Step 14950/44000 | Loss: 3.0311 | LR: 3.78e-04 | 1382.3ms/step | 14.5 samples/s
219
+ Step 14960/44000 | Loss: 3.0361 | LR: 3.78e-04 | 1384.8ms/step | 14.4 samples/s
220
+ Step 14970/44000 | Loss: 3.0268 | LR: 3.78e-04 | 1384.9ms/step | 14.4 samples/s
221
+ Step 14980/44000 | Loss: 3.0461 | LR: 3.78e-04 | 1382.4ms/step | 14.5 samples/s
222
+ Step 14990/44000 | Loss: 3.0686 | LR: 3.78e-04 | 1382.4ms/step | 14.5 samples/s
223
+ Step 15000/44000 | Loss: 3.0209 | LR: 3.78e-04 | 1384.0ms/step | 14.5 samples/s
224
+ Validation at step 15000: Loss = 3.0394, Perplexity = 20.89
225
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-15000
226
+ Step 15010/44000 | Loss: 3.0339 | LR: 3.77e-04 | 3917.6ms/step | 5.1 samples/s
227
+ Step 15020/44000 | Loss: 3.0625 | LR: 3.77e-04 | 1381.9ms/step | 14.5 samples/s
228
+ Step 15030/44000 | Loss: 3.0331 | LR: 3.77e-04 | 1378.7ms/step | 14.5 samples/s
229
+ Step 15040/44000 | Loss: 3.0055 | LR: 3.77e-04 | 1380.4ms/step | 14.5 samples/s
230
+ Step 15050/44000 | Loss: 3.0099 | LR: 3.77e-04 | 1380.6ms/step | 14.5 samples/s
231
+ Step 15060/44000 | Loss: 3.0344 | LR: 3.77e-04 | 1381.2ms/step | 14.5 samples/s
232
+ Step 15070/44000 | Loss: 3.0197 | LR: 3.77e-04 | 1381.9ms/step | 14.5 samples/s
233
+ Step 15080/44000 | Loss: 3.0658 | LR: 3.76e-04 | 1382.8ms/step | 14.5 samples/s
234
+ Step 15090/44000 | Loss: 3.0302 | LR: 3.76e-04 | 1383.3ms/step | 14.5 samples/s
235
+ Step 15100/44000 | Loss: 3.0547 | LR: 3.76e-04 | 1383.9ms/step | 14.5 samples/s
236
+ Step 15110/44000 | Loss: 3.0684 | LR: 3.76e-04 | 1382.7ms/step | 14.5 samples/s
237
+ Step 15120/44000 | Loss: 3.0258 | LR: 3.76e-04 | 1384.3ms/step | 14.4 samples/s
238
+ Step 15130/44000 | Loss: 3.0248 | LR: 3.76e-04 | 1385.1ms/step | 14.4 samples/s
239
+ Step 15140/44000 | Loss: 3.0363 | LR: 3.75e-04 | 1387.2ms/step | 14.4 samples/s
240
+ Step 15150/44000 | Loss: 3.0476 | LR: 3.75e-04 | 1384.4ms/step | 14.4 samples/s
241
+ Step 15160/44000 | Loss: 3.0110 | LR: 3.75e-04 | 1383.2ms/step | 14.5 samples/s
242
+ Step 15170/44000 | Loss: 3.0182 | LR: 3.75e-04 | 1383.7ms/step | 14.5 samples/s
243
+ Step 15180/44000 | Loss: 3.0260 | LR: 3.75e-04 | 1382.6ms/step | 14.5 samples/s
244
+ Step 15190/44000 | Loss: 3.0572 | LR: 3.75e-04 | 1384.5ms/step | 14.4 samples/s
245
+ Step 15200/44000 | Loss: 3.0297 | LR: 3.74e-04 | 1384.2ms/step | 14.4 samples/s
246
+ Step 15210/44000 | Loss: 3.0048 | LR: 3.74e-04 | 1384.0ms/step | 14.5 samples/s
247
+ Step 15220/44000 | Loss: 3.0268 | LR: 3.74e-04 | 1385.3ms/step | 14.4 samples/s
248
+ Step 15230/44000 | Loss: 3.0351 | LR: 3.74e-04 | 1385.4ms/step | 14.4 samples/s
249
+ Step 15240/44000 | Loss: 3.0413 | LR: 3.74e-04 | 1384.1ms/step | 14.4 samples/s
250
+ Step 15250/44000 | Loss: 3.0270 | LR: 3.74e-04 | 1384.8ms/step | 14.4 samples/s
251
+ Step 15260/44000 | Loss: 2.9857 | LR: 3.74e-04 | 1385.5ms/step | 14.4 samples/s
252
+ Step 15270/44000 | Loss: 3.0281 | LR: 3.73e-04 | 1383.4ms/step | 14.5 samples/s
253
+ Step 15280/44000 | Loss: 3.0324 | LR: 3.73e-04 | 1384.1ms/step | 14.4 samples/s
254
+ Step 15290/44000 | Loss: 3.0523 | LR: 3.73e-04 | 1387.1ms/step | 14.4 samples/s
255
+ Step 15300/44000 | Loss: 3.0513 | LR: 3.73e-04 | 1388.0ms/step | 14.4 samples/s
256
+ Step 15310/44000 | Loss: 3.0282 | LR: 3.73e-04 | 1387.5ms/step | 14.4 samples/s
257
+ Step 15320/44000 | Loss: 3.0610 | LR: 3.73e-04 | 1384.0ms/step | 14.5 samples/s
258
+ Step 15330/44000 | Loss: 3.0312 | LR: 3.72e-04 | 1385.6ms/step | 14.4 samples/s
259
+ Step 15340/44000 | Loss: 3.0255 | LR: 3.72e-04 | 1384.6ms/step | 14.4 samples/s
260
+ Step 15350/44000 | Loss: 3.0519 | LR: 3.72e-04 | 1384.7ms/step | 14.4 samples/s
261
+ Step 15360/44000 | Loss: 3.0079 | LR: 3.72e-04 | 1385.1ms/step | 14.4 samples/s
262
+ Step 15370/44000 | Loss: 3.0117 | LR: 3.72e-04 | 1387.6ms/step | 14.4 samples/s
263
+ Step 15380/44000 | Loss: 3.0427 | LR: 3.72e-04 | 1385.4ms/step | 14.4 samples/s
264
+ Step 15390/44000 | Loss: 3.0269 | LR: 3.71e-04 | 1384.9ms/step | 14.4 samples/s
265
+ Step 15400/44000 | Loss: 3.0333 | LR: 3.71e-04 | 1382.6ms/step | 14.5 samples/s
266
+ Step 15410/44000 | Loss: 3.0421 | LR: 3.71e-04 | 1383.0ms/step | 14.5 samples/s
267
+ Step 15420/44000 | Loss: 3.0108 | LR: 3.71e-04 | 1384.4ms/step | 14.4 samples/s
268
+ Step 15430/44000 | Loss: 3.0731 | LR: 3.71e-04 | 1385.8ms/step | 14.4 samples/s
269
+ Step 15440/44000 | Loss: 3.0163 | LR: 3.71e-04 | 1383.8ms/step | 14.5 samples/s
270
+ Step 15450/44000 | Loss: 3.0307 | LR: 3.71e-04 | 1387.6ms/step | 14.4 samples/s
271
+ Step 15460/44000 | Loss: 3.0323 | LR: 3.70e-04 | 1385.2ms/step | 14.4 samples/s
272
+ Step 15470/44000 | Loss: 3.0277 | LR: 3.70e-04 | 1386.1ms/step | 14.4 samples/s
273
+ Step 15480/44000 | Loss: 3.0237 | LR: 3.70e-04 | 1385.1ms/step | 14.4 samples/s
274
+ Step 15490/44000 | Loss: 3.0137 | LR: 3.70e-04 | 1385.3ms/step | 14.4 samples/s
275
+ Step 15500/44000 | Loss: 3.0038 | LR: 3.70e-04 | 1384.8ms/step | 14.4 samples/s
276
+ Step 15510/44000 | Loss: 3.0320 | LR: 3.70e-04 | 1386.4ms/step | 14.4 samples/s
277
+ Step 15520/44000 | Loss: 3.0344 | LR: 3.69e-04 | 1386.7ms/step | 14.4 samples/s
278
+ Step 15530/44000 | Loss: 3.0344 | LR: 3.69e-04 | 1385.9ms/step | 14.4 samples/s
279
+ Step 15540/44000 | Loss: 3.0105 | LR: 3.69e-04 | 1384.8ms/step | 14.4 samples/s
280
+ Step 15550/44000 | Loss: 3.0442 | LR: 3.69e-04 | 1386.5ms/step | 14.4 samples/s
281
+ Step 15560/44000 | Loss: 3.0318 | LR: 3.69e-04 | 1384.8ms/step | 14.4 samples/s
282
+ Step 15570/44000 | Loss: 3.0118 | LR: 3.69e-04 | 1385.8ms/step | 14.4 samples/s
283
+ Step 15580/44000 | Loss: 3.0386 | LR: 3.68e-04 | 1386.3ms/step | 14.4 samples/s
284
+ Step 15590/44000 | Loss: 2.9988 | LR: 3.68e-04 | 1385.2ms/step | 14.4 samples/s
285
+ Step 15600/44000 | Loss: 3.0204 | LR: 3.68e-04 | 1386.0ms/step | 14.4 samples/s
286
+ Step 15610/44000 | Loss: 3.0008 | LR: 3.68e-04 | 1387.0ms/step | 14.4 samples/s
287
+ Step 15620/44000 | Loss: 3.0103 | LR: 3.68e-04 | 1385.4ms/step | 14.4 samples/s
288
+ Step 15630/44000 | Loss: 3.0302 | LR: 3.68e-04 | 1386.4ms/step | 14.4 samples/s
289
+ Step 15640/44000 | Loss: 3.0298 | LR: 3.67e-04 | 1386.8ms/step | 14.4 samples/s
290
+ Step 15650/44000 | Loss: 3.0513 | LR: 3.67e-04 | 1385.2ms/step | 14.4 samples/s
291
+ Step 15660/44000 | Loss: 3.0410 | LR: 3.67e-04 | 1386.8ms/step | 14.4 samples/s
292
+ Step 15670/44000 | Loss: 3.0181 | LR: 3.67e-04 | 1385.4ms/step | 14.4 samples/s
293
+ Step 15680/44000 | Loss: 3.0058 | LR: 3.67e-04 | 1387.5ms/step | 14.4 samples/s
294
+ Step 15690/44000 | Loss: 3.0076 | LR: 3.67e-04 | 1386.5ms/step | 14.4 samples/s
295
+ Step 15700/44000 | Loss: 3.0186 | LR: 3.67e-04 | 1385.2ms/step | 14.4 samples/s
296
+ Step 15710/44000 | Loss: 3.0103 | LR: 3.66e-04 | 1384.5ms/step | 14.4 samples/s
297
+ Step 15720/44000 | Loss: 3.0278 | LR: 3.66e-04 | 1385.5ms/step | 14.4 samples/s
298
+ Step 15730/44000 | Loss: 3.0192 | LR: 3.66e-04 | 1386.3ms/step | 14.4 samples/s
299
+ Step 15740/44000 | Loss: 3.0188 | LR: 3.66e-04 | 1384.6ms/step | 14.4 samples/s
300
+ Step 15750/44000 | Loss: 3.0381 | LR: 3.66e-04 | 1383.5ms/step | 14.5 samples/s
301
+ Step 15760/44000 | Loss: 3.0392 | LR: 3.66e-04 | 1384.4ms/step | 14.4 samples/s
302
+ Step 15770/44000 | Loss: 3.0306 | LR: 3.65e-04 | 1383.2ms/step | 14.5 samples/s
303
+ Step 15780/44000 | Loss: 3.0264 | LR: 3.65e-04 | 1384.6ms/step | 14.4 samples/s
304
+ Step 15790/44000 | Loss: 3.0023 | LR: 3.65e-04 | 1383.5ms/step | 14.5 samples/s
305
+ Step 15800/44000 | Loss: 3.0266 | LR: 3.65e-04 | 1384.8ms/step | 14.4 samples/s
306
+ Step 15810/44000 | Loss: 3.0322 | LR: 3.65e-04 | 1384.9ms/step | 14.4 samples/s
307
+ Step 15820/44000 | Loss: 3.0017 | LR: 3.65e-04 | 1385.5ms/step | 14.4 samples/s
308
+ Step 15830/44000 | Loss: 3.0294 | LR: 3.64e-04 | 1382.9ms/step | 14.5 samples/s
309
+ Step 15840/44000 | Loss: 3.0042 | LR: 3.64e-04 | 1384.3ms/step | 14.4 samples/s
310
+ Step 15850/44000 | Loss: 3.0312 | LR: 3.64e-04 | 1387.2ms/step | 14.4 samples/s
311
+ Step 15860/44000 | Loss: 3.0034 | LR: 3.64e-04 | 1383.6ms/step | 14.5 samples/s
312
+ Step 15870/44000 | Loss: 3.0205 | LR: 3.64e-04 | 1385.2ms/step | 14.4 samples/s
313
+ Step 15880/44000 | Loss: 3.0478 | LR: 3.64e-04 | 1384.7ms/step | 14.4 samples/s
314
+ Step 15890/44000 | Loss: 3.0021 | LR: 3.63e-04 | 1383.1ms/step | 14.5 samples/s
315
+ Step 15900/44000 | Loss: 3.0343 | LR: 3.63e-04 | 1385.2ms/step | 14.4 samples/s
316
+ Step 15910/44000 | Loss: 2.9988 | LR: 3.63e-04 | 1384.2ms/step | 14.4 samples/s
317
+ Step 15920/44000 | Loss: 2.9947 | LR: 3.63e-04 | 1384.3ms/step | 14.4 samples/s
318
+ Step 15930/44000 | Loss: 3.0073 | LR: 3.63e-04 | 1384.4ms/step | 14.4 samples/s
319
+ Step 15940/44000 | Loss: 3.0401 | LR: 3.63e-04 | 1383.6ms/step | 14.5 samples/s
320
+ Step 15950/44000 | Loss: 3.0311 | LR: 3.62e-04 | 1383.2ms/step | 14.5 samples/s
321
+ Step 15960/44000 | Loss: 3.0153 | LR: 3.62e-04 | 1383.6ms/step | 14.5 samples/s
322
+ Step 15970/44000 | Loss: 3.0067 | LR: 3.62e-04 | 1384.7ms/step | 14.4 samples/s
323
+ Step 15980/44000 | Loss: 3.0466 | LR: 3.62e-04 | 1386.7ms/step | 14.4 samples/s
324
+ Step 15990/44000 | Loss: 3.0005 | LR: 3.62e-04 | 1386.6ms/step | 14.4 samples/s
325
+ Step 16000/44000 | Loss: 3.0193 | LR: 3.62e-04 | 1385.5ms/step | 14.4 samples/s
326
+ Validation at step 16000: Loss = 3.0266, Perplexity = 20.63
327
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-16000
328
+ Step 16010/44000 | Loss: 3.0097 | LR: 3.62e-04 | 3895.0ms/step | 5.1 samples/s
329
+ Step 16020/44000 | Loss: 3.0518 | LR: 3.61e-04 | 1379.8ms/step | 14.5 samples/s
330
+ Step 16030/44000 | Loss: 3.0150 | LR: 3.61e-04 | 1380.5ms/step | 14.5 samples/s
331
+ Step 16040/44000 | Loss: 3.0480 | LR: 3.61e-04 | 1381.2ms/step | 14.5 samples/s
332
+ Step 16050/44000 | Loss: 2.9980 | LR: 3.61e-04 | 1381.9ms/step | 14.5 samples/s
333
+ Step 16060/44000 | Loss: 3.0403 | LR: 3.61e-04 | 1382.0ms/step | 14.5 samples/s
334
+ Step 16070/44000 | Loss: 3.0418 | LR: 3.61e-04 | 1379.8ms/step | 14.5 samples/s
335
+ Step 16080/44000 | Loss: 3.0072 | LR: 3.60e-04 | 1379.8ms/step | 14.5 samples/s
336
+ Step 16090/44000 | Loss: 3.0479 | LR: 3.60e-04 | 1380.1ms/step | 14.5 samples/s
337
+ Step 16100/44000 | Loss: 3.0067 | LR: 3.60e-04 | 1378.8ms/step | 14.5 samples/s
338
+ Step 16110/44000 | Loss: 3.0202 | LR: 3.60e-04 | 1381.4ms/step | 14.5 samples/s
339
+ Step 16120/44000 | Loss: 2.9743 | LR: 3.60e-04 | 1381.9ms/step | 14.5 samples/s
340
+ Step 16130/44000 | Loss: 3.0282 | LR: 3.60e-04 | 1382.7ms/step | 14.5 samples/s
341
+ Step 16140/44000 | Loss: 3.0275 | LR: 3.59e-04 | 1382.2ms/step | 14.5 samples/s
342
+ Step 16150/44000 | Loss: 2.9930 | LR: 3.59e-04 | 1383.8ms/step | 14.5 samples/s
343
+ Step 16160/44000 | Loss: 2.9986 | LR: 3.59e-04 | 1382.8ms/step | 14.5 samples/s
344
+ Step 16170/44000 | Loss: 2.9994 | LR: 3.59e-04 | 1383.0ms/step | 14.5 samples/s
345
+ Step 16180/44000 | Loss: 3.0058 | LR: 3.59e-04 | 1381.9ms/step | 14.5 samples/s
346
+ Step 16190/44000 | Loss: 3.0110 | LR: 3.59e-04 | 1382.4ms/step | 14.5 samples/s
347
+ Step 16200/44000 | Loss: 3.0368 | LR: 3.58e-04 | 1382.9ms/step | 14.5 samples/s
348
+ Step 16210/44000 | Loss: 3.0315 | LR: 3.58e-04 | 1383.2ms/step | 14.5 samples/s
349
+ Step 16220/44000 | Loss: 2.9890 | LR: 3.58e-04 | 1381.7ms/step | 14.5 samples/s
350
+ Step 16230/44000 | Loss: 3.0031 | LR: 3.58e-04 | 1383.6ms/step | 14.5 samples/s
351
+ Step 16240/44000 | Loss: 3.0225 | LR: 3.58e-04 | 1382.5ms/step | 14.5 samples/s
352
+ Step 16250/44000 | Loss: 3.0390 | LR: 3.58e-04 | 1384.8ms/step | 14.4 samples/s
353
+ Step 16260/44000 | Loss: 3.0144 | LR: 3.57e-04 | 1382.8ms/step | 14.5 samples/s
354
+ Step 16270/44000 | Loss: 2.9970 | LR: 3.57e-04 | 1384.3ms/step | 14.4 samples/s
355
+ Step 16280/44000 | Loss: 3.0295 | LR: 3.57e-04 | 1385.6ms/step | 14.4 samples/s
356
+ Step 16290/44000 | Loss: 3.0052 | LR: 3.57e-04 | 1383.5ms/step | 14.5 samples/s
357
+ Step 16300/44000 | Loss: 3.0128 | LR: 3.57e-04 | 1381.1ms/step | 14.5 samples/s
358
+ Step 16310/44000 | Loss: 3.0132 | LR: 3.57e-04 | 1382.0ms/step | 14.5 samples/s
359
+ Step 16320/44000 | Loss: 3.0091 | LR: 3.56e-04 | 1383.8ms/step | 14.5 samples/s
360
+ Step 16330/44000 | Loss: 3.0360 | LR: 3.56e-04 | 1381.8ms/step | 14.5 samples/s
361
+ Step 16340/44000 | Loss: 2.9995 | LR: 3.56e-04 | 1385.2ms/step | 14.4 samples/s
362
+ Step 16350/44000 | Loss: 3.0521 | LR: 3.56e-04 | 1384.5ms/step | 14.4 samples/s
363
+ Step 16360/44000 | Loss: 2.9818 | LR: 3.56e-04 | 1384.4ms/step | 14.4 samples/s
364
+ Step 16370/44000 | Loss: 3.0304 | LR: 3.56e-04 | 1385.3ms/step | 14.4 samples/s
365
+ Step 16380/44000 | Loss: 3.0303 | LR: 3.55e-04 | 1382.0ms/step | 14.5 samples/s
366
+ Step 16390/44000 | Loss: 3.0041 | LR: 3.55e-04 | 1385.5ms/step | 14.4 samples/s
367
+ Step 16400/44000 | Loss: 2.9996 | LR: 3.55e-04 | 1382.7ms/step | 14.5 samples/s
368
+ Step 16410/44000 | Loss: 2.9975 | LR: 3.55e-04 | 1382.9ms/step | 14.5 samples/s
369
+ Step 16420/44000 | Loss: 2.9944 | LR: 3.55e-04 | 1383.2ms/step | 14.5 samples/s
370
+ Step 16430/44000 | Loss: 3.0136 | LR: 3.55e-04 | 1383.7ms/step | 14.5 samples/s
371
+ Step 16440/44000 | Loss: 3.0332 | LR: 3.54e-04 | 1382.2ms/step | 14.5 samples/s
372
+ Step 16450/44000 | Loss: 2.9992 | LR: 3.54e-04 | 1382.3ms/step | 14.5 samples/s
373
+ Step 16460/44000 | Loss: 3.0007 | LR: 3.54e-04 | 1381.6ms/step | 14.5 samples/s
374
+ Step 16470/44000 | Loss: 3.0298 | LR: 3.54e-04 | 1384.0ms/step | 14.5 samples/s
375
+ Step 16480/44000 | Loss: 2.9728 | LR: 3.54e-04 | 1383.8ms/step | 14.5 samples/s
376
+ Step 16490/44000 | Loss: 3.0055 | LR: 3.54e-04 | 1383.4ms/step | 14.5 samples/s
377
+ Step 16500/44000 | Loss: 3.0486 | LR: 3.53e-04 | 1383.6ms/step | 14.5 samples/s
378
+ Step 16510/44000 | Loss: 2.9800 | LR: 3.53e-04 | 1383.6ms/step | 14.5 samples/s
379
+ Step 16520/44000 | Loss: 3.0139 | LR: 3.53e-04 | 1380.3ms/step | 14.5 samples/s
380
+ Step 16530/44000 | Loss: 3.0321 | LR: 3.53e-04 | 1383.2ms/step | 14.5 samples/s
381
+ Step 16540/44000 | Loss: 3.0359 | LR: 3.53e-04 | 1385.6ms/step | 14.4 samples/s
382
+ Step 16550/44000 | Loss: 3.0076 | LR: 3.53e-04 | 1382.2ms/step | 14.5 samples/s
383
+ Step 16560/44000 | Loss: 3.0044 | LR: 3.52e-04 | 1382.8ms/step | 14.5 samples/s
384
+ Step 16570/44000 | Loss: 3.0331 | LR: 3.52e-04 | 1383.6ms/step | 14.5 samples/s
385
+ Step 16580/44000 | Loss: 3.0068 | LR: 3.52e-04 | 1383.4ms/step | 14.5 samples/s
386
+ Step 16590/44000 | Loss: 2.9892 | LR: 3.52e-04 | 1380.8ms/step | 14.5 samples/s
387
+ Step 16600/44000 | Loss: 2.9902 | LR: 3.52e-04 | 1385.8ms/step | 14.4 samples/s
388
+ Step 16610/44000 | Loss: 2.9925 | LR: 3.52e-04 | 1383.0ms/step | 14.5 samples/s
389
+ Step 16620/44000 | Loss: 3.0049 | LR: 3.51e-04 | 1384.8ms/step | 14.4 samples/s
390
+ Step 16630/44000 | Loss: 2.9694 | LR: 3.51e-04 | 1381.8ms/step | 14.5 samples/s
391
+ Step 16640/44000 | Loss: 2.9884 | LR: 3.51e-04 | 1384.6ms/step | 14.4 samples/s
392
+ Step 16650/44000 | Loss: 3.0005 | LR: 3.51e-04 | 1383.6ms/step | 14.5 samples/s
393
+ Step 16660/44000 | Loss: 3.0015 | LR: 3.51e-04 | 1382.6ms/step | 14.5 samples/s
394
+ Step 16670/44000 | Loss: 3.0329 | LR: 3.51e-04 | 1383.8ms/step | 14.5 samples/s
395
+ Step 16680/44000 | Loss: 3.0019 | LR: 3.50e-04 | 1382.3ms/step | 14.5 samples/s
396
+ Step 16690/44000 | Loss: 3.0203 | LR: 3.50e-04 | 1382.4ms/step | 14.5 samples/s
397
+ Step 16700/44000 | Loss: 2.9827 | LR: 3.50e-04 | 1382.8ms/step | 14.5 samples/s
398
+ Step 16710/44000 | Loss: 3.0042 | LR: 3.50e-04 | 1382.6ms/step | 14.5 samples/s
399
+ Step 16720/44000 | Loss: 3.0251 | LR: 3.50e-04 | 1381.6ms/step | 14.5 samples/s
400
+ Step 16730/44000 | Loss: 3.0141 | LR: 3.50e-04 | 1384.9ms/step | 14.4 samples/s
401
+ Step 16740/44000 | Loss: 3.0411 | LR: 3.49e-04 | 1382.9ms/step | 14.5 samples/s
402
+ Step 16750/44000 | Loss: 3.0324 | LR: 3.49e-04 | 1385.6ms/step | 14.4 samples/s
403
+ Step 16760/44000 | Loss: 2.9866 | LR: 3.49e-04 | 1383.3ms/step | 14.5 samples/s
404
+ Step 16770/44000 | Loss: 3.0003 | LR: 3.49e-04 | 1383.6ms/step | 14.5 samples/s
405
+ Step 16780/44000 | Loss: 3.0134 | LR: 3.49e-04 | 1386.1ms/step | 14.4 samples/s
406
+ Step 16790/44000 | Loss: 3.0487 | LR: 3.49e-04 | 1385.9ms/step | 14.4 samples/s
407
+ Step 16800/44000 | Loss: 2.9927 | LR: 3.48e-04 | 1384.0ms/step | 14.5 samples/s
408
+ Step 16810/44000 | Loss: 3.0002 | LR: 3.48e-04 | 1382.5ms/step | 14.5 samples/s
409
+ Step 16820/44000 | Loss: 3.0196 | LR: 3.48e-04 | 1386.5ms/step | 14.4 samples/s
410
+ Step 16830/44000 | Loss: 3.0025 | LR: 3.48e-04 | 1384.0ms/step | 14.5 samples/s
411
+ Step 16840/44000 | Loss: 3.0079 | LR: 3.48e-04 | 1383.4ms/step | 14.5 samples/s
412
+ Step 16850/44000 | Loss: 2.9754 | LR: 3.48e-04 | 1386.6ms/step | 14.4 samples/s
413
+ Step 16860/44000 | Loss: 3.0265 | LR: 3.47e-04 | 1384.1ms/step | 14.5 samples/s
414
+ Step 16870/44000 | Loss: 2.9931 | LR: 3.47e-04 | 1383.1ms/step | 14.5 samples/s
415
+ Step 16880/44000 | Loss: 2.9883 | LR: 3.47e-04 | 1385.4ms/step | 14.4 samples/s
416
+ Step 16890/44000 | Loss: 3.0091 | LR: 3.47e-04 | 1384.9ms/step | 14.4 samples/s
417
+ Step 16900/44000 | Loss: 3.0087 | LR: 3.47e-04 | 1385.6ms/step | 14.4 samples/s
418
+ Step 16910/44000 | Loss: 3.0072 | LR: 3.47e-04 | 1384.5ms/step | 14.4 samples/s
419
+ Step 16920/44000 | Loss: 3.0145 | LR: 3.46e-04 | 1383.8ms/step | 14.5 samples/s
420
+ Step 16930/44000 | Loss: 2.9838 | LR: 3.46e-04 | 1384.8ms/step | 14.4 samples/s
421
+ Step 16940/44000 | Loss: 2.9971 | LR: 3.46e-04 | 1385.1ms/step | 14.4 samples/s
422
+ Step 16950/44000 | Loss: 2.9736 | LR: 3.46e-04 | 1385.7ms/step | 14.4 samples/s
423
+ Step 16960/44000 | Loss: 2.9987 | LR: 3.46e-04 | 1387.3ms/step | 14.4 samples/s
424
+ Step 16970/44000 | Loss: 2.9549 | LR: 3.46e-04 | 1387.4ms/step | 14.4 samples/s
425
+ Step 16980/44000 | Loss: 3.0334 | LR: 3.45e-04 | 1387.1ms/step | 14.4 samples/s
426
+ Step 16990/44000 | Loss: 3.0090 | LR: 3.45e-04 | 1386.3ms/step | 14.4 samples/s
427
+ Step 17000/44000 | Loss: 3.0116 | LR: 3.45e-04 | 1384.1ms/step | 14.4 samples/s
428
+ Validation at step 17000: Loss = 3.0135, Perplexity = 20.36
429
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-17000
430
+ Step 17010/44000 | Loss: 3.0070 | LR: 3.45e-04 | 3948.1ms/step | 5.1 samples/s
431
+ Step 17020/44000 | Loss: 2.9870 | LR: 3.45e-04 | 1378.0ms/step | 14.5 samples/s
432
+ Step 17030/44000 | Loss: 3.0102 | LR: 3.45e-04 | 1378.9ms/step | 14.5 samples/s
433
+ Step 17040/44000 | Loss: 3.0028 | LR: 3.44e-04 | 1378.5ms/step | 14.5 samples/s
434
+ Step 17050/44000 | Loss: 2.9953 | LR: 3.44e-04 | 1380.3ms/step | 14.5 samples/s
435
+ Step 17060/44000 | Loss: 2.9630 | LR: 3.44e-04 | 1379.8ms/step | 14.5 samples/s
436
+ Step 17070/44000 | Loss: 3.0029 | LR: 3.44e-04 | 1382.4ms/step | 14.5 samples/s
437
+ Step 17080/44000 | Loss: 2.9980 | LR: 3.44e-04 | 1381.6ms/step | 14.5 samples/s
438
+ Step 17090/44000 | Loss: 3.0183 | LR: 3.44e-04 | 1382.4ms/step | 14.5 samples/s
439
+ Step 17100/44000 | Loss: 3.0024 | LR: 3.43e-04 | 1385.2ms/step | 14.4 samples/s
440
+ Step 17110/44000 | Loss: 2.9932 | LR: 3.43e-04 | 1380.4ms/step | 14.5 samples/s
441
+ Step 17120/44000 | Loss: 3.0063 | LR: 3.43e-04 | 1378.4ms/step | 14.5 samples/s
442
+ Step 17130/44000 | Loss: 3.0088 | LR: 3.43e-04 | 1376.5ms/step | 14.5 samples/s
443
+ Step 17140/44000 | Loss: 2.9916 | LR: 3.43e-04 | 1374.7ms/step | 14.5 samples/s
444
+ Step 17150/44000 | Loss: 3.0089 | LR: 3.43e-04 | 1375.9ms/step | 14.5 samples/s
445
+ Step 17160/44000 | Loss: 3.0026 | LR: 3.42e-04 | 1374.2ms/step | 14.6 samples/s
446
+ Step 17170/44000 | Loss: 3.0051 | LR: 3.42e-04 | 1377.0ms/step | 14.5 samples/s
447
+ Step 17180/44000 | Loss: 3.0175 | LR: 3.42e-04 | 1376.1ms/step | 14.5 samples/s
448
+ Step 17190/44000 | Loss: 3.0205 | LR: 3.42e-04 | 1376.5ms/step | 14.5 samples/s
449
+ Step 17200/44000 | Loss: 2.9861 | LR: 3.42e-04 | 1377.6ms/step | 14.5 samples/s
450
+ Step 17210/44000 | Loss: 3.0223 | LR: 3.42e-04 | 1376.6ms/step | 14.5 samples/s
451
+ Step 17220/44000 | Loss: 3.0380 | LR: 3.41e-04 | 1376.9ms/step | 14.5 samples/s
452
+ Step 17230/44000 | Loss: 3.0040 | LR: 3.41e-04 | 1376.9ms/step | 14.5 samples/s
453
+ Step 17240/44000 | Loss: 3.0171 | LR: 3.41e-04 | 1376.8ms/step | 14.5 samples/s
454
+ Step 17250/44000 | Loss: 3.0174 | LR: 3.41e-04 | 1376.8ms/step | 14.5 samples/s
455
+ Step 17260/44000 | Loss: 3.0344 | LR: 3.41e-04 | 1376.1ms/step | 14.5 samples/s
456
+ Step 17270/44000 | Loss: 3.0230 | LR: 3.41e-04 | 1374.9ms/step | 14.5 samples/s
457
+ Step 17280/44000 | Loss: 2.9788 | LR: 3.40e-04 | 1376.9ms/step | 14.5 samples/s
458
+ Step 17290/44000 | Loss: 2.9969 | LR: 3.40e-04 | 1376.4ms/step | 14.5 samples/s
459
+ Step 17300/44000 | Loss: 3.0321 | LR: 3.40e-04 | 1378.2ms/step | 14.5 samples/s
460
+ Step 17310/44000 | Loss: 2.9923 | LR: 3.40e-04 | 1377.3ms/step | 14.5 samples/s
461
+ Step 17320/44000 | Loss: 3.0062 | LR: 3.40e-04 | 1374.5ms/step | 14.6 samples/s
462
+ Step 17330/44000 | Loss: 2.9933 | LR: 3.40e-04 | 1376.8ms/step | 14.5 samples/s
463
+ Step 17340/44000 | Loss: 3.0236 | LR: 3.39e-04 | 1376.7ms/step | 14.5 samples/s
464
+ Step 17350/44000 | Loss: 3.0148 | LR: 3.39e-04 | 1377.2ms/step | 14.5 samples/s
465
+ Step 17360/44000 | Loss: 2.9887 | LR: 3.39e-04 | 1380.1ms/step | 14.5 samples/s
466
+ Step 17370/44000 | Loss: 3.0001 | LR: 3.39e-04 | 1379.5ms/step | 14.5 samples/s
467
+ Step 17380/44000 | Loss: 2.9878 | LR: 3.39e-04 | 1380.4ms/step | 14.5 samples/s
468
+ Step 17390/44000 | Loss: 3.0254 | LR: 3.39e-04 | 1380.3ms/step | 14.5 samples/s
469
+ Step 17400/44000 | Loss: 2.9928 | LR: 3.38e-04 | 1377.3ms/step | 14.5 samples/s
470
+ Step 17410/44000 | Loss: 2.9962 | LR: 3.38e-04 | 1374.9ms/step | 14.5 samples/s
471
+ Step 17420/44000 | Loss: 2.9696 | LR: 3.38e-04 | 1377.0ms/step | 14.5 samples/s
472
+ Step 17430/44000 | Loss: 2.9984 | LR: 3.38e-04 | 1377.8ms/step | 14.5 samples/s
473
+ Step 17440/44000 | Loss: 3.0245 | LR: 3.38e-04 | 1378.0ms/step | 14.5 samples/s
474
+ Step 17450/44000 | Loss: 3.0262 | LR: 3.38e-04 | 1376.4ms/step | 14.5 samples/s
475
+ Step 17460/44000 | Loss: 2.9876 | LR: 3.37e-04 | 1375.7ms/step | 14.5 samples/s
476
+ Step 17470/44000 | Loss: 2.9730 | LR: 3.37e-04 | 1376.5ms/step | 14.5 samples/s
477
+ Step 17480/44000 | Loss: 3.0102 | LR: 3.37e-04 | 1377.3ms/step | 14.5 samples/s
478
+ Step 17490/44000 | Loss: 2.9959 | LR: 3.37e-04 | 1376.7ms/step | 14.5 samples/s
479
+ Step 17500/44000 | Loss: 3.0135 | LR: 3.37e-04 | 1377.7ms/step | 14.5 samples/s
480
+ Step 17510/44000 | Loss: 2.9908 | LR: 3.37e-04 | 1377.6ms/step | 14.5 samples/s
481
+ Step 17520/44000 | Loss: 3.0079 | LR: 3.36e-04 | 1376.3ms/step | 14.5 samples/s
482
+ Step 17530/44000 | Loss: 3.0292 | LR: 3.36e-04 | 1377.5ms/step | 14.5 samples/s
epfl_final/SparseAtt/wandb/run-20260820_041558-bajm6b15/files/requirements.txt ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pip==26.0.1
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.6.3
8
+ typing_extensions==4.15.0
9
+ triton==3.6.0
10
+ tqdm==4.67.3
11
+ torchinfo==1.8.0
12
+ sympy==1.14.0
13
+ smmap==5.0.3
14
+ six==1.17.0
15
+ shellingham==1.5.4
16
+ setuptools==81.0.0
17
+ safetensors==0.7.0
18
+ regex==2026.4.4
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.4.1
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.4.4
38
+ networkx==3.6.1
39
+ multidict==6.7.1
40
+ mdurl==0.1.2
41
+ MarkupSafe==3.0.3
42
+ idna==3.13
43
+ hf-xet==1.4.3
44
+ h11==0.16.0
45
+ fsspec==2026.2.0
46
+ frozenlist==1.8.0
47
+ filelock==3.29.0
48
+ dill==0.4.1
49
+ cuda-pathfinder==1.5.4
50
+ click==8.3.3
51
+ charset-normalizer==3.4.7
52
+ certifi==2026.4.22
53
+ attrs==26.1.0
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ yarl==1.23.0
58
+ typing-inspection==0.4.2
59
+ sentry-sdk==2.58.0
60
+ requests==2.33.1
61
+ python-dateutil==2.9.0.post0
62
+ pydantic_core==2.46.3
63
+ nvidia-cusparse==12.6.3.3
64
+ nvidia-cufft==12.0.0.61
65
+ nvidia-cudnn-cu13==9.19.0.56
66
+ multiprocess==0.70.19
67
+ markdown-it-py==4.0.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ cuda-bindings==13.2.0
72
+ anyio==4.13.0
73
+ aiosignal==1.4.0
74
+ rich==15.0.0
75
+ pydantic==2.13.3
76
+ pandas==3.0.2
77
+ nvidia-cusolver==12.0.4.66
78
+ httpx==0.28.1
79
+ GitPython==3.1.49
80
+ aiohttp==3.13.5
81
+ wandb==0.26.1
82
+ typer==0.25.1
83
+ torch==2.11.0
84
+ huggingface_hub==1.13.0
85
+ tokenizers==0.22.2
86
+ datasets==4.8.5
87
+ accelerate==1.13.0
88
+ transformers==5.7.0
89
+ word2number==1.1
90
+ sqlitedict==2.1.0
91
+ pytz==2026.2
92
+ threadpoolctl==3.6.0
93
+ tcolorpy==0.1.7
94
+ tabulate==0.10.0
95
+ scipy==1.17.1
96
+ portalocker==3.2.0
97
+ pathvalidate==3.3.1
98
+ narwhals==2.22.1
99
+ more-itertools==11.1.0
100
+ lxml==6.1.1
101
+ joblib==1.5.3
102
+ colorama==0.4.6
103
+ chardet==6.0.0.post1
104
+ absl-py==2.4.0
105
+ scikit-learn==1.9.0
106
+ sacrebleu==2.6.0
107
+ nltk==3.9.4
108
+ mbstrdecoder==1.1.5
109
+ typepy==1.3.5
110
+ rouge_score==0.1.2
111
+ DataProperty==1.1.1
112
+ tabledata==1.3.5
113
+ evaluate==0.4.6
114
+ pytablewriter==1.2.1
115
+ lm_eval==0.4.12
116
+ pluggy==1.6.0
117
+ iniconfig==2.3.0
118
+ pytest==9.1.0
119
+ py-spy==0.4.2
120
+ pillow==12.2.0
121
+ tifffile==2026.6.1
122
+ imagecodecs==2026.6.26
123
+ zipp==4.1.0
124
+ importlib_metadata==9.0.0
125
+ derivative==0.6.3
126
+ pysindy==2.1.0
127
+ pyparsing==3.3.2
128
+ kiwisolver==1.5.0
129
+ fonttools==4.63.0
130
+ cycler==0.12.1
131
+ contourpy==1.3.3
132
+ matplotlib==3.11.0
133
+ sortedcontainers==2.4.0
134
+ wcwidth==0.8.2
135
+ sigfig==1.3.19
136
+ importlib_resources==7.1.0
137
+ enum-tools==0.13.0
138
+ cramjam==2.11.0
139
+ python-snappy==0.7.3
140
+ compact-json==1.8.2
141
+ numbers-parser==4.18.5
142
+ entmax==1.3
epfl_final/SparseAtt/wandb/run-20260820_041558-bajm6b15/files/wandb-metadata.json ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.13.13",
4
+ "startedAt": "2026-08-20T04:15:58.875522Z",
5
+ "args": [
6
+ "--config",
7
+ "/mloscratch/homes/navasard/sparse/configs_ropectl/resume_a1p0_wandb/seed44.json"
8
+ ],
9
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
10
+ "codePath": "train.py",
11
+ "codePathLocal": "train.py",
12
+ "git": {
13
+ "remote": "https://github.com/rubentium/TokenPrune.git",
14
+ "commit": "5637886f0cc760009989b14e68c6ae06d4827418"
15
+ },
16
+ "email": "ruben.navasardyan@mail.utoronto.ca",
17
+ "root": "/mloscratch/homes/navasard/sparse/SparseAtt",
18
+ "host": "h100-workspace-5-gpu-0-2",
19
+ "executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
20
+ "cpu_count": 96,
21
+ "cpu_count_logical": 192,
22
+ "gpu": "NVIDIA H100 80GB HBM3",
23
+ "gpu_count": 5,
24
+ "disk": {
25
+ "/": {
26
+ "total": "9374931091456",
27
+ "used": "5285341044736"
28
+ }
29
+ },
30
+ "memory": {
31
+ "total": "1623005626368"
32
+ },
33
+ "gpu_nvidia": [
34
+ {
35
+ "name": "NVIDIA H100 80GB HBM3",
36
+ "memoryTotal": "85520809984",
37
+ "cudaCores": 16896,
38
+ "architecture": "Hopper",
39
+ "uuid": "GPU-6fb12770-6b51-9595-91c9-0d20801e3896"
40
+ },
41
+ {
42
+ "name": "NVIDIA H100 80GB HBM3",
43
+ "memoryTotal": "85520809984",
44
+ "cudaCores": 16896,
45
+ "architecture": "Hopper",
46
+ "uuid": "GPU-5cb9f8b7-4a13-4f85-d73e-6bcbebebefb8"
47
+ },
48
+ {
49
+ "name": "NVIDIA H100 80GB HBM3",
50
+ "memoryTotal": "85520809984",
51
+ "cudaCores": 16896,
52
+ "architecture": "Hopper",
53
+ "uuid": "GPU-3512802a-d501-0a3c-9812-4ab9deb3a3a9"
54
+ },
55
+ {
56
+ "name": "NVIDIA H100 80GB HBM3",
57
+ "memoryTotal": "85520809984",
58
+ "cudaCores": 16896,
59
+ "architecture": "Hopper",
60
+ "uuid": "GPU-24436def-beee-c9f1-bd05-c97df556de08"
61
+ },
62
+ {
63
+ "name": "NVIDIA H100 80GB HBM3",
64
+ "memoryTotal": "85520809984",
65
+ "cudaCores": 16896,
66
+ "architecture": "Hopper",
67
+ "uuid": "GPU-7b827c86-854a-af51-b5d3-6aaa43307d4a"
68
+ }
69
+ ],
70
+ "cudaVersion": "13.0",
71
+ "writerId": "s2f260p7x7qa0p82j4omu95st4p5qxud"
72
+ }
epfl_final/SparseAtt/wandb/run-20260820_041558-bajm6b15/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/SparseAtt/wandb/run-20260820_041558-bajm6b15/logs/debug.log ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-08-20 04:15:58,922 INFO MainThread:371 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-08-20 04:15:58,922 INFO MainThread:371 [wandb_setup.py:_flush():81] Configure stats pid to 371
3
+ 2026-08-20 04:15:58,922 INFO MainThread:371 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-08-20 04:15:58,922 INFO MainThread:371 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_041558-bajm6b15/logs/debug.log
5
+ 2026-08-20 04:15:58,922 INFO MainThread:371 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_041558-bajm6b15/logs/debug-internal.log
6
+ 2026-08-20 04:15:58,922 INFO MainThread:371 [wandb_init.py:init():850] calling init triggers
7
+ 2026-08-20 04:15:58,922 INFO MainThread:371 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44/run_20260819_153858/checkpoint-13000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 1000, 'eval_batches': 50, 'logging_steps': 10, 'seed': 44, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'l8k365m_dense_ropectl_a1p0_seed44', 'wandb_tags': ['qwen3', '365m', '8k', 'dense', 'rope-control'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': 0.0, 'rope_train_position_scale_set': [1.0, 0.9896339488176222, 0.9692159692159692, 0.9420289855072463, 0.8730158730158729], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
9
+ 2026-08-20 04:15:58,922 INFO MainThread:371 [wandb_init.py:init():898] starting backend
10
+ 2026-08-20 04:15:59,636 INFO MainThread:371 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-08-20 04:16:00,162 INFO MainThread:371 [wandb_init.py:init():918] backend started and connected
12
+ 2026-08-20 04:16:00,165 INFO MainThread:371 [wandb_init.py:init():988] updated telemetry
13
+ 2026-08-20 04:16:00,183 INFO MainThread:371 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-08-20 04:16:00,723 INFO MainThread:371 [wandb_init.py:init():1051] run resumed
15
+ 2026-08-20 04:16:00,725 INFO MainThread:371 [wandb_init.py:init():1056] starting run threads in backend
16
+ 2026-08-20 04:16:00,877 INFO MainThread:371 [wandb_run.py:_console_start():2554] atexit reg
17
+ 2026-08-20 04:16:00,877 INFO MainThread:371 [wandb_run.py:_redirect():2403] redirect: wrap_raw
18
+ 2026-08-20 04:16:00,877 INFO MainThread:371 [wandb_run.py:_redirect():2472] Wrapping output streams.
19
+ 2026-08-20 04:16:00,877 INFO MainThread:371 [wandb_run.py:_redirect():2495] Redirects installed.
20
+ 2026-08-20 04:16:00,883 INFO MainThread:371 [wandb_init.py:init():1094] run started, returning control to user process
epfl_final/SparseAtt/wandb/run-20260820_041558-bajm6b15/run-bajm6b15.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c94d9e1b4dd88865be11980241d456236e641582b6c88cecf7dff7fb36b88a31
3
+ size 2785280
epfl_final/SparseAtt/wandb/run-20260820_041559-jqstnvej/files/output.log ADDED
@@ -0,0 +1,463 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Starting training...
2
+ Validation at step 13000: Loss = 3.0665, Perplexity = 21.47
3
+ wandb: WARNING Tried to log to step 13000 that is less than the current step 13021. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
4
+ Switching attention mode for step 13001: dense (dense step interval)
5
+ Step 13010/44000 | Loss: 3.0287 | LR: 4.07e-04 | 30165.3ms/step | 0.7 samples/s
6
+ wandb: WARNING Tried to log to step 13010 that is less than the current step 13021. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
7
+ Step 13020/44000 | Loss: 3.0440 | LR: 4.07e-04 | 1375.2ms/step | 14.5 samples/s
8
+ wandb: WARNING Tried to log to step 13020 that is less than the current step 13021. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
9
+ Step 13030/44000 | Loss: 3.0257 | LR: 4.07e-04 | 1378.1ms/step | 14.5 samples/s
10
+ Step 13040/44000 | Loss: 3.0862 | LR: 4.07e-04 | 1380.1ms/step | 14.5 samples/s
11
+ Step 13050/44000 | Loss: 3.0640 | LR: 4.07e-04 | 1380.4ms/step | 14.5 samples/s
12
+ Step 13060/44000 | Loss: 3.0764 | LR: 4.07e-04 | 1379.9ms/step | 14.5 samples/s
13
+ Step 13070/44000 | Loss: 3.0550 | LR: 4.06e-04 | 1384.0ms/step | 14.5 samples/s
14
+ Step 13080/44000 | Loss: 3.0384 | LR: 4.06e-04 | 1386.1ms/step | 14.4 samples/s
15
+ Step 13090/44000 | Loss: 3.0527 | LR: 4.06e-04 | 1385.1ms/step | 14.4 samples/s
16
+ Step 13100/44000 | Loss: 3.0500 | LR: 4.06e-04 | 1386.4ms/step | 14.4 samples/s
17
+ Step 13110/44000 | Loss: 3.0585 | LR: 4.06e-04 | 1385.3ms/step | 14.4 samples/s
18
+ Step 13120/44000 | Loss: 3.0356 | LR: 4.06e-04 | 1385.5ms/step | 14.4 samples/s
19
+ Step 13130/44000 | Loss: 3.0393 | LR: 4.06e-04 | 1383.5ms/step | 14.5 samples/s
20
+ Step 13140/44000 | Loss: 3.0439 | LR: 4.05e-04 | 1385.2ms/step | 14.4 samples/s
21
+ Step 13150/44000 | Loss: 3.0256 | LR: 4.05e-04 | 1388.8ms/step | 14.4 samples/s
22
+ Step 13160/44000 | Loss: 3.0225 | LR: 4.05e-04 | 1388.1ms/step | 14.4 samples/s
23
+ Step 13170/44000 | Loss: 3.0341 | LR: 4.05e-04 | 1387.4ms/step | 14.4 samples/s
24
+ Step 13180/44000 | Loss: 3.0508 | LR: 4.05e-04 | 1387.3ms/step | 14.4 samples/s
25
+ Step 13190/44000 | Loss: 3.0768 | LR: 4.05e-04 | 1387.4ms/step | 14.4 samples/s
26
+ Step 13200/44000 | Loss: 3.0454 | LR: 4.05e-04 | 1388.1ms/step | 14.4 samples/s
27
+ Step 13210/44000 | Loss: 3.0508 | LR: 4.04e-04 | 1387.9ms/step | 14.4 samples/s
28
+ Step 13220/44000 | Loss: 3.0491 | LR: 4.04e-04 | 1389.3ms/step | 14.4 samples/s
29
+ Step 13230/44000 | Loss: 3.0392 | LR: 4.04e-04 | 1390.9ms/step | 14.4 samples/s
30
+ Step 13240/44000 | Loss: 3.0731 | LR: 4.04e-04 | 1387.6ms/step | 14.4 samples/s
31
+ Step 13250/44000 | Loss: 3.0547 | LR: 4.04e-04 | 1386.2ms/step | 14.4 samples/s
32
+ Step 13260/44000 | Loss: 3.0456 | LR: 4.04e-04 | 1387.3ms/step | 14.4 samples/s
33
+ Step 13270/44000 | Loss: 3.0193 | LR: 4.04e-04 | 1386.3ms/step | 14.4 samples/s
34
+ Step 13280/44000 | Loss: 3.0294 | LR: 4.03e-04 | 1388.8ms/step | 14.4 samples/s
35
+ Step 13290/44000 | Loss: 3.0476 | LR: 4.03e-04 | 1386.3ms/step | 14.4 samples/s
36
+ Step 13300/44000 | Loss: 3.0134 | LR: 4.03e-04 | 1387.1ms/step | 14.4 samples/s
37
+ Step 13310/44000 | Loss: 3.0769 | LR: 4.03e-04 | 1388.9ms/step | 14.4 samples/s
38
+ Step 13320/44000 | Loss: 3.0242 | LR: 4.03e-04 | 1391.2ms/step | 14.4 samples/s
39
+ Step 13330/44000 | Loss: 3.0419 | LR: 4.03e-04 | 1390.2ms/step | 14.4 samples/s
40
+ Step 13340/44000 | Loss: 3.0476 | LR: 4.03e-04 | 1389.4ms/step | 14.4 samples/s
41
+ Step 13350/44000 | Loss: 3.0307 | LR: 4.02e-04 | 1387.8ms/step | 14.4 samples/s
42
+ Step 13360/44000 | Loss: 3.0297 | LR: 4.02e-04 | 1388.4ms/step | 14.4 samples/s
43
+ Step 13370/44000 | Loss: 3.0360 | LR: 4.02e-04 | 1386.4ms/step | 14.4 samples/s
44
+ Step 13380/44000 | Loss: 3.0373 | LR: 4.02e-04 | 1387.7ms/step | 14.4 samples/s
45
+ Step 13390/44000 | Loss: 3.0441 | LR: 4.02e-04 | 1389.9ms/step | 14.4 samples/s
46
+ Step 13400/44000 | Loss: 3.0488 | LR: 4.02e-04 | 1388.0ms/step | 14.4 samples/s
47
+ Step 13410/44000 | Loss: 3.0572 | LR: 4.02e-04 | 1384.0ms/step | 14.5 samples/s
48
+ Step 13420/44000 | Loss: 3.0663 | LR: 4.01e-04 | 1384.1ms/step | 14.5 samples/s
49
+ Step 13430/44000 | Loss: 3.0234 | LR: 4.01e-04 | 1385.7ms/step | 14.4 samples/s
50
+ Step 13440/44000 | Loss: 3.0172 | LR: 4.01e-04 | 1389.9ms/step | 14.4 samples/s
51
+ Step 13450/44000 | Loss: 3.0330 | LR: 4.01e-04 | 1386.2ms/step | 14.4 samples/s
52
+ Step 13460/44000 | Loss: 3.0628 | LR: 4.01e-04 | 1386.8ms/step | 14.4 samples/s
53
+ Step 13470/44000 | Loss: 3.0356 | LR: 4.01e-04 | 1389.4ms/step | 14.4 samples/s
54
+ Step 13480/44000 | Loss: 3.0476 | LR: 4.01e-04 | 1392.8ms/step | 14.4 samples/s
55
+ Step 13490/44000 | Loss: 3.0474 | LR: 4.00e-04 | 1387.1ms/step | 14.4 samples/s
56
+ Step 13500/44000 | Loss: 3.0593 | LR: 4.00e-04 | 1387.7ms/step | 14.4 samples/s
57
+ Step 13510/44000 | Loss: 3.0408 | LR: 4.00e-04 | 1386.8ms/step | 14.4 samples/s
58
+ Step 13520/44000 | Loss: 3.0588 | LR: 4.00e-04 | 1386.9ms/step | 14.4 samples/s
59
+ Step 13530/44000 | Loss: 3.0276 | LR: 4.00e-04 | 1387.7ms/step | 14.4 samples/s
60
+ Step 13540/44000 | Loss: 3.0556 | LR: 4.00e-04 | 1385.9ms/step | 14.4 samples/s
61
+ Step 13550/44000 | Loss: 3.0772 | LR: 4.00e-04 | 1387.1ms/step | 14.4 samples/s
62
+ Step 13560/44000 | Loss: 3.0487 | LR: 3.99e-04 | 1384.6ms/step | 14.4 samples/s
63
+ Step 13570/44000 | Loss: 3.0246 | LR: 3.99e-04 | 1385.4ms/step | 14.4 samples/s
64
+ Step 13580/44000 | Loss: 3.0007 | LR: 3.99e-04 | 1387.3ms/step | 14.4 samples/s
65
+ Step 13590/44000 | Loss: 3.0154 | LR: 3.99e-04 | 1387.3ms/step | 14.4 samples/s
66
+ Step 13600/44000 | Loss: 3.0443 | LR: 3.99e-04 | 1387.9ms/step | 14.4 samples/s
67
+ Step 13610/44000 | Loss: 3.0471 | LR: 3.99e-04 | 1388.1ms/step | 14.4 samples/s
68
+ Step 13620/44000 | Loss: 3.0051 | LR: 3.98e-04 | 1389.2ms/step | 14.4 samples/s
69
+ Step 13630/44000 | Loss: 3.0344 | LR: 3.98e-04 | 1389.6ms/step | 14.4 samples/s
70
+ Step 13640/44000 | Loss: 3.0839 | LR: 3.98e-04 | 1386.4ms/step | 14.4 samples/s
71
+ Step 13650/44000 | Loss: 3.0618 | LR: 3.98e-04 | 1385.2ms/step | 14.4 samples/s
72
+ Step 13660/44000 | Loss: 3.0412 | LR: 3.98e-04 | 1385.2ms/step | 14.4 samples/s
73
+ Step 13670/44000 | Loss: 3.0370 | LR: 3.98e-04 | 1386.4ms/step | 14.4 samples/s
74
+ Step 13680/44000 | Loss: 3.0431 | LR: 3.98e-04 | 1387.5ms/step | 14.4 samples/s
75
+ Step 13690/44000 | Loss: 3.0439 | LR: 3.97e-04 | 1387.2ms/step | 14.4 samples/s
76
+ Step 13700/44000 | Loss: 3.0576 | LR: 3.97e-04 | 1386.3ms/step | 14.4 samples/s
77
+ Step 13710/44000 | Loss: 3.0240 | LR: 3.97e-04 | 1388.1ms/step | 14.4 samples/s
78
+ Step 13720/44000 | Loss: 3.0602 | LR: 3.97e-04 | 1389.9ms/step | 14.4 samples/s
79
+ Step 13730/44000 | Loss: 3.0537 | LR: 3.97e-04 | 1387.3ms/step | 14.4 samples/s
80
+ Step 13740/44000 | Loss: 3.0484 | LR: 3.97e-04 | 1387.1ms/step | 14.4 samples/s
81
+ Step 13750/44000 | Loss: 3.0405 | LR: 3.97e-04 | 1388.1ms/step | 14.4 samples/s
82
+ Step 13760/44000 | Loss: 3.0393 | LR: 3.96e-04 | 1389.5ms/step | 14.4 samples/s
83
+ Step 13770/44000 | Loss: 3.0233 | LR: 3.96e-04 | 1386.4ms/step | 14.4 samples/s
84
+ Step 13780/44000 | Loss: 3.0353 | LR: 3.96e-04 | 1390.2ms/step | 14.4 samples/s
85
+ Step 13790/44000 | Loss: 3.0609 | LR: 3.96e-04 | 1388.8ms/step | 14.4 samples/s
86
+ Step 13800/44000 | Loss: 3.0408 | LR: 3.96e-04 | 1384.2ms/step | 14.4 samples/s
87
+ Step 13810/44000 | Loss: 3.0345 | LR: 3.96e-04 | 1385.0ms/step | 14.4 samples/s
88
+ Step 13820/44000 | Loss: 3.0194 | LR: 3.96e-04 | 1387.9ms/step | 14.4 samples/s
89
+ Step 13830/44000 | Loss: 3.0231 | LR: 3.95e-04 | 1386.2ms/step | 14.4 samples/s
90
+ Step 13840/44000 | Loss: 3.0142 | LR: 3.95e-04 | 1387.7ms/step | 14.4 samples/s
91
+ Step 13850/44000 | Loss: 3.0426 | LR: 3.95e-04 | 1388.0ms/step | 14.4 samples/s
92
+ Step 13860/44000 | Loss: 3.0148 | LR: 3.95e-04 | 1385.9ms/step | 14.4 samples/s
93
+ Step 13870/44000 | Loss: 3.0317 | LR: 3.95e-04 | 1389.2ms/step | 14.4 samples/s
94
+ Step 13880/44000 | Loss: 3.0121 | LR: 3.95e-04 | 1384.5ms/step | 14.4 samples/s
95
+ Step 13890/44000 | Loss: 3.0297 | LR: 3.95e-04 | 1385.9ms/step | 14.4 samples/s
96
+ Step 13900/44000 | Loss: 3.0451 | LR: 3.94e-04 | 1387.3ms/step | 14.4 samples/s
97
+ Step 13910/44000 | Loss: 2.9991 | LR: 3.94e-04 | 1388.2ms/step | 14.4 samples/s
98
+ Step 13920/44000 | Loss: 3.0000 | LR: 3.94e-04 | 1387.5ms/step | 14.4 samples/s
99
+ Step 13930/44000 | Loss: 3.0256 | LR: 3.94e-04 | 1387.4ms/step | 14.4 samples/s
100
+ Step 13940/44000 | Loss: 3.0589 | LR: 3.94e-04 | 1390.6ms/step | 14.4 samples/s
101
+ Step 13950/44000 | Loss: 3.0292 | LR: 3.94e-04 | 1387.7ms/step | 14.4 samples/s
102
+ Step 13960/44000 | Loss: 3.0183 | LR: 3.93e-04 | 1388.6ms/step | 14.4 samples/s
103
+ Step 13970/44000 | Loss: 3.0254 | LR: 3.93e-04 | 1386.7ms/step | 14.4 samples/s
104
+ Step 13980/44000 | Loss: 3.0400 | LR: 3.93e-04 | 1388.6ms/step | 14.4 samples/s
105
+ Step 13990/44000 | Loss: 3.0382 | LR: 3.93e-04 | 1389.5ms/step | 14.4 samples/s
106
+ Step 14000/44000 | Loss: 3.0055 | LR: 3.93e-04 | 1386.5ms/step | 14.4 samples/s
107
+ Validation at step 14000: Loss = 3.0514, Perplexity = 21.14
108
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-14000
109
+ Step 14010/44000 | Loss: 3.0195 | LR: 3.93e-04 | 3910.8ms/step | 5.1 samples/s
110
+ Step 14020/44000 | Loss: 3.0264 | LR: 3.93e-04 | 1377.6ms/step | 14.5 samples/s
111
+ Step 14030/44000 | Loss: 3.0339 | LR: 3.92e-04 | 1379.4ms/step | 14.5 samples/s
112
+ Step 14040/44000 | Loss: 3.0399 | LR: 3.92e-04 | 1384.5ms/step | 14.4 samples/s
113
+ Step 14050/44000 | Loss: 3.0264 | LR: 3.92e-04 | 1383.6ms/step | 14.5 samples/s
114
+ Step 14060/44000 | Loss: 3.0510 | LR: 3.92e-04 | 1388.0ms/step | 14.4 samples/s
115
+ Step 14070/44000 | Loss: 3.0416 | LR: 3.92e-04 | 1388.5ms/step | 14.4 samples/s
116
+ Step 14080/44000 | Loss: 2.9989 | LR: 3.92e-04 | 1392.9ms/step | 14.4 samples/s
117
+ Step 14090/44000 | Loss: 3.0299 | LR: 3.92e-04 | 1390.4ms/step | 14.4 samples/s
118
+ Step 14100/44000 | Loss: 3.0058 | LR: 3.91e-04 | 1394.3ms/step | 14.3 samples/s
119
+ Step 14110/44000 | Loss: 3.0750 | LR: 3.91e-04 | 1393.8ms/step | 14.3 samples/s
120
+ Step 14120/44000 | Loss: 3.0350 | LR: 3.91e-04 | 1395.7ms/step | 14.3 samples/s
121
+ Step 14130/44000 | Loss: 3.0346 | LR: 3.91e-04 | 1391.6ms/step | 14.4 samples/s
122
+ Step 14140/44000 | Loss: 3.0181 | LR: 3.91e-04 | 1394.6ms/step | 14.3 samples/s
123
+ Step 14150/44000 | Loss: 3.0178 | LR: 3.91e-04 | 1392.2ms/step | 14.4 samples/s
124
+ Step 14160/44000 | Loss: 3.0493 | LR: 3.90e-04 | 1393.9ms/step | 14.3 samples/s
125
+ Step 14170/44000 | Loss: 3.0609 | LR: 3.90e-04 | 1398.1ms/step | 14.3 samples/s
126
+ Step 14180/44000 | Loss: 3.0742 | LR: 3.90e-04 | 1399.6ms/step | 14.3 samples/s
127
+ Step 14190/44000 | Loss: 3.0194 | LR: 3.90e-04 | 1394.1ms/step | 14.3 samples/s
128
+ Step 14200/44000 | Loss: 3.0512 | LR: 3.90e-04 | 1400.2ms/step | 14.3 samples/s
129
+ Step 14210/44000 | Loss: 3.0552 | LR: 3.90e-04 | 1399.9ms/step | 14.3 samples/s
130
+ Step 14220/44000 | Loss: 3.0104 | LR: 3.90e-04 | 1400.0ms/step | 14.3 samples/s
131
+ Step 14230/44000 | Loss: 3.0535 | LR: 3.89e-04 | 1400.0ms/step | 14.3 samples/s
132
+ Step 14240/44000 | Loss: 3.0235 | LR: 3.89e-04 | 1398.5ms/step | 14.3 samples/s
133
+ Step 14250/44000 | Loss: 3.0587 | LR: 3.89e-04 | 1397.3ms/step | 14.3 samples/s
134
+ Step 14260/44000 | Loss: 3.0575 | LR: 3.89e-04 | 1396.4ms/step | 14.3 samples/s
135
+ Step 14270/44000 | Loss: 2.9848 | LR: 3.89e-04 | 1397.7ms/step | 14.3 samples/s
136
+ Step 14280/44000 | Loss: 3.0336 | LR: 3.89e-04 | 1396.2ms/step | 14.3 samples/s
137
+ Step 14290/44000 | Loss: 3.0208 | LR: 3.89e-04 | 1398.0ms/step | 14.3 samples/s
138
+ Step 14300/44000 | Loss: 3.0309 | LR: 3.88e-04 | 1399.8ms/step | 14.3 samples/s
139
+ Step 14310/44000 | Loss: 3.0246 | LR: 3.88e-04 | 1397.8ms/step | 14.3 samples/s
140
+ Step 14320/44000 | Loss: 3.0096 | LR: 3.88e-04 | 1400.3ms/step | 14.3 samples/s
141
+ Step 14330/44000 | Loss: 3.0129 | LR: 3.88e-04 | 1400.0ms/step | 14.3 samples/s
142
+ Step 14340/44000 | Loss: 3.0275 | LR: 3.88e-04 | 1399.5ms/step | 14.3 samples/s
143
+ Step 14350/44000 | Loss: 3.0316 | LR: 3.88e-04 | 1396.6ms/step | 14.3 samples/s
144
+ Step 14360/44000 | Loss: 3.0323 | LR: 3.87e-04 | 1399.7ms/step | 14.3 samples/s
145
+ Step 14370/44000 | Loss: 3.0338 | LR: 3.87e-04 | 1394.3ms/step | 14.3 samples/s
146
+ Step 14380/44000 | Loss: 3.0403 | LR: 3.87e-04 | 1393.4ms/step | 14.4 samples/s
147
+ Step 14390/44000 | Loss: 3.0455 | LR: 3.87e-04 | 1398.5ms/step | 14.3 samples/s
148
+ Step 14400/44000 | Loss: 3.0131 | LR: 3.87e-04 | 1399.9ms/step | 14.3 samples/s
149
+ Step 14410/44000 | Loss: 3.0276 | LR: 3.87e-04 | 1396.1ms/step | 14.3 samples/s
150
+ Step 14420/44000 | Loss: 3.0622 | LR: 3.87e-04 | 1398.2ms/step | 14.3 samples/s
151
+ Step 14430/44000 | Loss: 3.0231 | LR: 3.86e-04 | 1397.6ms/step | 14.3 samples/s
152
+ Step 14440/44000 | Loss: 3.0176 | LR: 3.86e-04 | 1396.1ms/step | 14.3 samples/s
153
+ Step 14450/44000 | Loss: 3.0216 | LR: 3.86e-04 | 1396.5ms/step | 14.3 samples/s
154
+ Step 14460/44000 | Loss: 2.9962 | LR: 3.86e-04 | 1399.7ms/step | 14.3 samples/s
155
+ Step 14470/44000 | Loss: 2.9985 | LR: 3.86e-04 | 1397.9ms/step | 14.3 samples/s
156
+ Step 14480/44000 | Loss: 3.0100 | LR: 3.86e-04 | 1400.2ms/step | 14.3 samples/s
157
+ Step 14490/44000 | Loss: 3.0250 | LR: 3.85e-04 | 1399.3ms/step | 14.3 samples/s
158
+ Step 14500/44000 | Loss: 3.0494 | LR: 3.85e-04 | 1398.6ms/step | 14.3 samples/s
159
+ Step 14510/44000 | Loss: 3.0211 | LR: 3.85e-04 | 1400.1ms/step | 14.3 samples/s
160
+ Step 14520/44000 | Loss: 3.0304 | LR: 3.85e-04 | 1399.8ms/step | 14.3 samples/s
161
+ Step 14530/44000 | Loss: 3.0245 | LR: 3.85e-04 | 1398.1ms/step | 14.3 samples/s
162
+ Step 14540/44000 | Loss: 3.0155 | LR: 3.85e-04 | 1397.9ms/step | 14.3 samples/s
163
+ Step 14550/44000 | Loss: 3.0204 | LR: 3.85e-04 | 1400.1ms/step | 14.3 samples/s
164
+ Step 14560/44000 | Loss: 3.0063 | LR: 3.84e-04 | 1397.6ms/step | 14.3 samples/s
165
+ Step 14570/44000 | Loss: 3.0307 | LR: 3.84e-04 | 1398.2ms/step | 14.3 samples/s
166
+ Step 14580/44000 | Loss: 3.0312 | LR: 3.84e-04 | 1397.9ms/step | 14.3 samples/s
167
+ Step 14590/44000 | Loss: 3.0091 | LR: 3.84e-04 | 1398.2ms/step | 14.3 samples/s
168
+ Step 14600/44000 | Loss: 3.0276 | LR: 3.84e-04 | 1398.7ms/step | 14.3 samples/s
169
+ Step 14610/44000 | Loss: 3.0123 | LR: 3.84e-04 | 1399.6ms/step | 14.3 samples/s
170
+ Step 14620/44000 | Loss: 3.0115 | LR: 3.83e-04 | 1397.8ms/step | 14.3 samples/s
171
+ Step 14630/44000 | Loss: 3.0249 | LR: 3.83e-04 | 1399.8ms/step | 14.3 samples/s
172
+ Step 14640/44000 | Loss: 3.0200 | LR: 3.83e-04 | 1399.6ms/step | 14.3 samples/s
173
+ Step 14650/44000 | Loss: 3.0181 | LR: 3.83e-04 | 1395.6ms/step | 14.3 samples/s
174
+ Step 14660/44000 | Loss: 2.9862 | LR: 3.83e-04 | 1398.8ms/step | 14.3 samples/s
175
+ Step 14670/44000 | Loss: 3.0156 | LR: 3.83e-04 | 1398.1ms/step | 14.3 samples/s
176
+ Step 14680/44000 | Loss: 3.0400 | LR: 3.83e-04 | 1394.5ms/step | 14.3 samples/s
177
+ Step 14690/44000 | Loss: 3.0197 | LR: 3.82e-04 | 1397.5ms/step | 14.3 samples/s
178
+ Step 14700/44000 | Loss: 3.0294 | LR: 3.82e-04 | 1398.0ms/step | 14.3 samples/s
179
+ Step 14710/44000 | Loss: 3.0228 | LR: 3.82e-04 | 1400.3ms/step | 14.3 samples/s
180
+ Step 14720/44000 | Loss: 3.0467 | LR: 3.82e-04 | 1395.8ms/step | 14.3 samples/s
181
+ Step 14730/44000 | Loss: 2.9995 | LR: 3.82e-04 | 1400.1ms/step | 14.3 samples/s
182
+ Step 14740/44000 | Loss: 3.0405 | LR: 3.82e-04 | 1399.7ms/step | 14.3 samples/s
183
+ Step 14750/44000 | Loss: 3.0521 | LR: 3.81e-04 | 1398.2ms/step | 14.3 samples/s
184
+ Step 14760/44000 | Loss: 3.0057 | LR: 3.81e-04 | 1397.8ms/step | 14.3 samples/s
185
+ Step 14770/44000 | Loss: 3.0283 | LR: 3.81e-04 | 1398.2ms/step | 14.3 samples/s
186
+ Step 14780/44000 | Loss: 2.9995 | LR: 3.81e-04 | 1398.1ms/step | 14.3 samples/s
187
+ Step 14790/44000 | Loss: 3.0225 | LR: 3.81e-04 | 1395.9ms/step | 14.3 samples/s
188
+ Step 14800/44000 | Loss: 2.9918 | LR: 3.81e-04 | 1398.2ms/step | 14.3 samples/s
189
+ Step 14810/44000 | Loss: 3.0154 | LR: 3.81e-04 | 1400.2ms/step | 14.3 samples/s
190
+ Step 14820/44000 | Loss: 3.0140 | LR: 3.80e-04 | 1398.7ms/step | 14.3 samples/s
191
+ Step 14830/44000 | Loss: 3.0100 | LR: 3.80e-04 | 1398.9ms/step | 14.3 samples/s
192
+ Step 14840/44000 | Loss: 2.9869 | LR: 3.80e-04 | 1398.1ms/step | 14.3 samples/s
193
+ Step 14850/44000 | Loss: 3.0370 | LR: 3.80e-04 | 1399.9ms/step | 14.3 samples/s
194
+ Step 14860/44000 | Loss: 3.0687 | LR: 3.80e-04 | 1400.1ms/step | 14.3 samples/s
195
+ Step 14870/44000 | Loss: 3.0336 | LR: 3.80e-04 | 1395.7ms/step | 14.3 samples/s
196
+ Step 14880/44000 | Loss: 3.0418 | LR: 3.79e-04 | 1396.1ms/step | 14.3 samples/s
197
+ Step 14890/44000 | Loss: 2.9973 | LR: 3.79e-04 | 1398.5ms/step | 14.3 samples/s
198
+ Step 14900/44000 | Loss: 3.0477 | LR: 3.79e-04 | 1397.7ms/step | 14.3 samples/s
199
+ Step 14910/44000 | Loss: 3.0226 | LR: 3.79e-04 | 1400.0ms/step | 14.3 samples/s
200
+ Step 14920/44000 | Loss: 3.0062 | LR: 3.79e-04 | 1397.9ms/step | 14.3 samples/s
201
+ Step 14930/44000 | Loss: 3.0351 | LR: 3.79e-04 | 1395.7ms/step | 14.3 samples/s
202
+ Step 14940/44000 | Loss: 2.9710 | LR: 3.79e-04 | 1400.3ms/step | 14.3 samples/s
203
+ Step 14950/44000 | Loss: 3.0007 | LR: 3.78e-04 | 1398.0ms/step | 14.3 samples/s
204
+ Step 14960/44000 | Loss: 3.0032 | LR: 3.78e-04 | 1398.0ms/step | 14.3 samples/s
205
+ Step 14970/44000 | Loss: 3.0145 | LR: 3.78e-04 | 1395.5ms/step | 14.3 samples/s
206
+ Step 14980/44000 | Loss: 2.9904 | LR: 3.78e-04 | 1398.3ms/step | 14.3 samples/s
207
+ Step 14990/44000 | Loss: 3.0434 | LR: 3.78e-04 | 1400.0ms/step | 14.3 samples/s
208
+ Step 15000/44000 | Loss: 3.0264 | LR: 3.78e-04 | 1400.3ms/step | 14.3 samples/s
209
+ Validation at step 15000: Loss = 3.0379, Perplexity = 20.86
210
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-15000
211
+ Step 15010/44000 | Loss: 3.0395 | LR: 3.77e-04 | 3905.4ms/step | 5.1 samples/s
212
+ Step 15020/44000 | Loss: 2.9784 | LR: 3.77e-04 | 1383.4ms/step | 14.5 samples/s
213
+ Step 15030/44000 | Loss: 3.0472 | LR: 3.77e-04 | 1383.1ms/step | 14.5 samples/s
214
+ Step 15040/44000 | Loss: 3.0234 | LR: 3.77e-04 | 1381.6ms/step | 14.5 samples/s
215
+ Step 15050/44000 | Loss: 3.0335 | LR: 3.77e-04 | 1384.9ms/step | 14.4 samples/s
216
+ Step 15060/44000 | Loss: 3.0408 | LR: 3.77e-04 | 1388.4ms/step | 14.4 samples/s
217
+ Step 15070/44000 | Loss: 3.0389 | LR: 3.77e-04 | 1391.9ms/step | 14.4 samples/s
218
+ Step 15080/44000 | Loss: 3.0075 | LR: 3.76e-04 | 1397.3ms/step | 14.3 samples/s
219
+ Step 15090/44000 | Loss: 3.0260 | LR: 3.76e-04 | 1396.1ms/step | 14.3 samples/s
220
+ Step 15100/44000 | Loss: 3.0122 | LR: 3.76e-04 | 1399.7ms/step | 14.3 samples/s
221
+ Step 15110/44000 | Loss: 3.0386 | LR: 3.76e-04 | 1398.0ms/step | 14.3 samples/s
222
+ Step 15120/44000 | Loss: 3.0303 | LR: 3.76e-04 | 1396.0ms/step | 14.3 samples/s
223
+ Step 15130/44000 | Loss: 3.0005 | LR: 3.76e-04 | 1396.0ms/step | 14.3 samples/s
224
+ Step 15140/44000 | Loss: 3.0191 | LR: 3.75e-04 | 1396.2ms/step | 14.3 samples/s
225
+ Step 15150/44000 | Loss: 3.0069 | LR: 3.75e-04 | 1396.5ms/step | 14.3 samples/s
226
+ Step 15160/44000 | Loss: 3.0318 | LR: 3.75e-04 | 1393.4ms/step | 14.4 samples/s
227
+ Step 15170/44000 | Loss: 3.0635 | LR: 3.75e-04 | 1394.1ms/step | 14.3 samples/s
228
+ Step 15180/44000 | Loss: 2.9948 | LR: 3.75e-04 | 1394.7ms/step | 14.3 samples/s
229
+ Step 15190/44000 | Loss: 3.0487 | LR: 3.75e-04 | 1396.7ms/step | 14.3 samples/s
230
+ Step 15200/44000 | Loss: 3.0296 | LR: 3.74e-04 | 1394.4ms/step | 14.3 samples/s
231
+ Step 15210/44000 | Loss: 3.0246 | LR: 3.74e-04 | 1395.4ms/step | 14.3 samples/s
232
+ Step 15220/44000 | Loss: 2.9780 | LR: 3.74e-04 | 1396.0ms/step | 14.3 samples/s
233
+ Step 15230/44000 | Loss: 3.0285 | LR: 3.74e-04 | 1396.4ms/step | 14.3 samples/s
234
+ Step 15240/44000 | Loss: 3.0217 | LR: 3.74e-04 | 1398.1ms/step | 14.3 samples/s
235
+ Step 15250/44000 | Loss: 3.0170 | LR: 3.74e-04 | 1400.0ms/step | 14.3 samples/s
236
+ Step 15260/44000 | Loss: 3.0567 | LR: 3.74e-04 | 1398.2ms/step | 14.3 samples/s
237
+ Step 15270/44000 | Loss: 3.0121 | LR: 3.73e-04 | 1400.0ms/step | 14.3 samples/s
238
+ Step 15280/44000 | Loss: 2.9865 | LR: 3.73e-04 | 1400.2ms/step | 14.3 samples/s
239
+ Step 15290/44000 | Loss: 3.0229 | LR: 3.73e-04 | 1399.9ms/step | 14.3 samples/s
240
+ Step 15300/44000 | Loss: 2.9984 | LR: 3.73e-04 | 1399.8ms/step | 14.3 samples/s
241
+ Step 15310/44000 | Loss: 3.0261 | LR: 3.73e-04 | 1397.9ms/step | 14.3 samples/s
242
+ Step 15320/44000 | Loss: 2.9886 | LR: 3.73e-04 | 1396.9ms/step | 14.3 samples/s
243
+ Step 15330/44000 | Loss: 2.9936 | LR: 3.72e-04 | 1396.5ms/step | 14.3 samples/s
244
+ Step 15340/44000 | Loss: 3.0132 | LR: 3.72e-04 | 1394.6ms/step | 14.3 samples/s
245
+ Step 15350/44000 | Loss: 3.0146 | LR: 3.72e-04 | 1394.9ms/step | 14.3 samples/s
246
+ Step 15360/44000 | Loss: 3.0296 | LR: 3.72e-04 | 1393.0ms/step | 14.4 samples/s
247
+ Step 15370/44000 | Loss: 3.0021 | LR: 3.72e-04 | 1395.6ms/step | 14.3 samples/s
248
+ Step 15380/44000 | Loss: 2.9956 | LR: 3.72e-04 | 1392.9ms/step | 14.4 samples/s
249
+ Step 15390/44000 | Loss: 2.9975 | LR: 3.71e-04 | 1395.1ms/step | 14.3 samples/s
250
+ Step 15400/44000 | Loss: 3.0158 | LR: 3.71e-04 | 1398.6ms/step | 14.3 samples/s
251
+ Step 15410/44000 | Loss: 3.0168 | LR: 3.71e-04 | 1395.9ms/step | 14.3 samples/s
252
+ Step 15420/44000 | Loss: 3.0116 | LR: 3.71e-04 | 1398.4ms/step | 14.3 samples/s
253
+ Step 15430/44000 | Loss: 3.0107 | LR: 3.71e-04 | 1398.0ms/step | 14.3 samples/s
254
+ Step 15440/44000 | Loss: 2.9948 | LR: 3.71e-04 | 1396.0ms/step | 14.3 samples/s
255
+ Step 15450/44000 | Loss: 3.0260 | LR: 3.71e-04 | 1397.8ms/step | 14.3 samples/s
256
+ Step 15460/44000 | Loss: 3.0192 | LR: 3.70e-04 | 1396.0ms/step | 14.3 samples/s
257
+ Step 15470/44000 | Loss: 2.9916 | LR: 3.70e-04 | 1400.0ms/step | 14.3 samples/s
258
+ Step 15480/44000 | Loss: 3.0190 | LR: 3.70e-04 | 1399.9ms/step | 14.3 samples/s
259
+ Step 15490/44000 | Loss: 3.0406 | LR: 3.70e-04 | 1398.2ms/step | 14.3 samples/s
260
+ Step 15500/44000 | Loss: 3.0192 | LR: 3.70e-04 | 1399.9ms/step | 14.3 samples/s
261
+ Step 15510/44000 | Loss: 2.9820 | LR: 3.70e-04 | 1400.0ms/step | 14.3 samples/s
262
+ Step 15520/44000 | Loss: 3.0317 | LR: 3.69e-04 | 1400.0ms/step | 14.3 samples/s
263
+ Step 15530/44000 | Loss: 2.9941 | LR: 3.69e-04 | 1397.6ms/step | 14.3 samples/s
264
+ Step 15540/44000 | Loss: 3.0270 | LR: 3.69e-04 | 1398.6ms/step | 14.3 samples/s
265
+ Step 15550/44000 | Loss: 3.0610 | LR: 3.69e-04 | 1398.1ms/step | 14.3 samples/s
266
+ Step 15560/44000 | Loss: 3.0169 | LR: 3.69e-04 | 1399.8ms/step | 14.3 samples/s
267
+ Step 15570/44000 | Loss: 3.0076 | LR: 3.69e-04 | 1400.2ms/step | 14.3 samples/s
268
+ Step 15580/44000 | Loss: 2.9940 | LR: 3.68e-04 | 1393.8ms/step | 14.3 samples/s
269
+ Step 15590/44000 | Loss: 3.0269 | LR: 3.68e-04 | 1398.6ms/step | 14.3 samples/s
270
+ Step 15600/44000 | Loss: 3.0211 | LR: 3.68e-04 | 1397.5ms/step | 14.3 samples/s
271
+ Step 15610/44000 | Loss: 3.0502 | LR: 3.68e-04 | 1399.8ms/step | 14.3 samples/s
272
+ Step 15620/44000 | Loss: 3.0169 | LR: 3.68e-04 | 1393.8ms/step | 14.3 samples/s
273
+ Step 15630/44000 | Loss: 3.0036 | LR: 3.68e-04 | 1394.1ms/step | 14.3 samples/s
274
+ Step 15640/44000 | Loss: 3.0207 | LR: 3.67e-04 | 1394.0ms/step | 14.3 samples/s
275
+ Step 15650/44000 | Loss: 3.0254 | LR: 3.67e-04 | 1396.2ms/step | 14.3 samples/s
276
+ Step 15660/44000 | Loss: 3.0146 | LR: 3.67e-04 | 1400.1ms/step | 14.3 samples/s
277
+ Step 15670/44000 | Loss: 3.0043 | LR: 3.67e-04 | 1397.8ms/step | 14.3 samples/s
278
+ Step 15680/44000 | Loss: 2.9986 | LR: 3.67e-04 | 1400.4ms/step | 14.3 samples/s
279
+ Step 15690/44000 | Loss: 3.0438 | LR: 3.67e-04 | 1397.4ms/step | 14.3 samples/s
280
+ Step 15700/44000 | Loss: 2.9976 | LR: 3.67e-04 | 1398.1ms/step | 14.3 samples/s
281
+ Step 15710/44000 | Loss: 2.9913 | LR: 3.66e-04 | 1396.1ms/step | 14.3 samples/s
282
+ Step 15720/44000 | Loss: 3.0242 | LR: 3.66e-04 | 1396.5ms/step | 14.3 samples/s
283
+ Step 15730/44000 | Loss: 3.0170 | LR: 3.66e-04 | 1395.9ms/step | 14.3 samples/s
284
+ Step 15740/44000 | Loss: 3.0116 | LR: 3.66e-04 | 1397.7ms/step | 14.3 samples/s
285
+ Step 15750/44000 | Loss: 3.0043 | LR: 3.66e-04 | 1396.1ms/step | 14.3 samples/s
286
+ Step 15760/44000 | Loss: 2.9945 | LR: 3.66e-04 | 1399.8ms/step | 14.3 samples/s
287
+ Step 15770/44000 | Loss: 3.0071 | LR: 3.65e-04 | 1398.2ms/step | 14.3 samples/s
288
+ Step 15780/44000 | Loss: 3.0274 | LR: 3.65e-04 | 1400.2ms/step | 14.3 samples/s
289
+ Step 15790/44000 | Loss: 3.0277 | LR: 3.65e-04 | 1397.5ms/step | 14.3 samples/s
290
+ Step 15800/44000 | Loss: 2.9978 | LR: 3.65e-04 | 1400.1ms/step | 14.3 samples/s
291
+ Step 15810/44000 | Loss: 3.0099 | LR: 3.65e-04 | 1397.7ms/step | 14.3 samples/s
292
+ Step 15820/44000 | Loss: 3.0154 | LR: 3.65e-04 | 1397.9ms/step | 14.3 samples/s
293
+ Step 15830/44000 | Loss: 3.0436 | LR: 3.64e-04 | 1394.5ms/step | 14.3 samples/s
294
+ Step 15840/44000 | Loss: 2.9938 | LR: 3.64e-04 | 1397.8ms/step | 14.3 samples/s
295
+ Step 15850/44000 | Loss: 2.9820 | LR: 3.64e-04 | 1397.6ms/step | 14.3 samples/s
296
+ Step 15860/44000 | Loss: 3.0010 | LR: 3.64e-04 | 1396.4ms/step | 14.3 samples/s
297
+ Step 15870/44000 | Loss: 3.0045 | LR: 3.64e-04 | 1394.4ms/step | 14.3 samples/s
298
+ Step 15880/44000 | Loss: 3.0196 | LR: 3.64e-04 | 1399.9ms/step | 14.3 samples/s
299
+ Step 15890/44000 | Loss: 3.0452 | LR: 3.63e-04 | 1397.9ms/step | 14.3 samples/s
300
+ Step 15900/44000 | Loss: 2.9920 | LR: 3.63e-04 | 1397.0ms/step | 14.3 samples/s
301
+ Step 15910/44000 | Loss: 3.0221 | LR: 3.63e-04 | 1393.3ms/step | 14.4 samples/s
302
+ Step 15920/44000 | Loss: 3.0163 | LR: 3.63e-04 | 1399.8ms/step | 14.3 samples/s
303
+ Step 15930/44000 | Loss: 3.0266 | LR: 3.63e-04 | 1400.0ms/step | 14.3 samples/s
304
+ Step 15940/44000 | Loss: 3.0067 | LR: 3.63e-04 | 1393.8ms/step | 14.3 samples/s
305
+ Step 15950/44000 | Loss: 3.0263 | LR: 3.62e-04 | 1396.5ms/step | 14.3 samples/s
306
+ Step 15960/44000 | Loss: 3.0358 | LR: 3.62e-04 | 1395.6ms/step | 14.3 samples/s
307
+ Step 15970/44000 | Loss: 3.0408 | LR: 3.62e-04 | 1398.1ms/step | 14.3 samples/s
308
+ Step 15980/44000 | Loss: 2.9845 | LR: 3.62e-04 | 1398.0ms/step | 14.3 samples/s
309
+ Step 15990/44000 | Loss: 3.0149 | LR: 3.62e-04 | 1398.1ms/step | 14.3 samples/s
310
+ Step 16000/44000 | Loss: 3.0248 | LR: 3.62e-04 | 1399.4ms/step | 14.3 samples/s
311
+ Validation at step 16000: Loss = 3.0281, Perplexity = 20.66
312
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-16000
313
+ Step 16010/44000 | Loss: 2.9864 | LR: 3.62e-04 | 3888.2ms/step | 5.1 samples/s
314
+ Step 16020/44000 | Loss: 2.9857 | LR: 3.61e-04 | 1379.9ms/step | 14.5 samples/s
315
+ Step 16030/44000 | Loss: 3.0181 | LR: 3.61e-04 | 1384.4ms/step | 14.4 samples/s
316
+ Step 16040/44000 | Loss: 3.0149 | LR: 3.61e-04 | 1386.6ms/step | 14.4 samples/s
317
+ Step 16050/44000 | Loss: 3.0004 | LR: 3.61e-04 | 1382.8ms/step | 14.5 samples/s
318
+ Step 16060/44000 | Loss: 3.0185 | LR: 3.61e-04 | 1386.5ms/step | 14.4 samples/s
319
+ Step 16070/44000 | Loss: 3.0414 | LR: 3.61e-04 | 1387.2ms/step | 14.4 samples/s
320
+ Step 16080/44000 | Loss: 3.0259 | LR: 3.60e-04 | 1392.0ms/step | 14.4 samples/s
321
+ Step 16090/44000 | Loss: 2.9867 | LR: 3.60e-04 | 1393.1ms/step | 14.4 samples/s
322
+ Step 16100/44000 | Loss: 3.0156 | LR: 3.60e-04 | 1391.6ms/step | 14.4 samples/s
323
+ Step 16110/44000 | Loss: 2.9859 | LR: 3.60e-04 | 1393.9ms/step | 14.3 samples/s
324
+ Step 16120/44000 | Loss: 2.9940 | LR: 3.60e-04 | 1394.4ms/step | 14.3 samples/s
325
+ Step 16130/44000 | Loss: 3.0248 | LR: 3.60e-04 | 1393.7ms/step | 14.4 samples/s
326
+ Step 16140/44000 | Loss: 2.9950 | LR: 3.59e-04 | 1394.0ms/step | 14.3 samples/s
327
+ Step 16150/44000 | Loss: 2.9980 | LR: 3.59e-04 | 1392.2ms/step | 14.4 samples/s
328
+ Step 16160/44000 | Loss: 3.0314 | LR: 3.59e-04 | 1395.5ms/step | 14.3 samples/s
329
+ Step 16170/44000 | Loss: 3.0048 | LR: 3.59e-04 | 1395.9ms/step | 14.3 samples/s
330
+ Step 16180/44000 | Loss: 2.9694 | LR: 3.59e-04 | 1391.4ms/step | 14.4 samples/s
331
+ Step 16190/44000 | Loss: 3.0218 | LR: 3.59e-04 | 1395.1ms/step | 14.3 samples/s
332
+ Step 16200/44000 | Loss: 3.0063 | LR: 3.58e-04 | 1400.2ms/step | 14.3 samples/s
333
+ Step 16210/44000 | Loss: 3.0114 | LR: 3.58e-04 | 1395.8ms/step | 14.3 samples/s
334
+ Step 16220/44000 | Loss: 2.9844 | LR: 3.58e-04 | 1394.1ms/step | 14.3 samples/s
335
+ Step 16230/44000 | Loss: 3.0263 | LR: 3.58e-04 | 1395.9ms/step | 14.3 samples/s
336
+ Step 16240/44000 | Loss: 3.0202 | LR: 3.58e-04 | 1396.3ms/step | 14.3 samples/s
337
+ Step 16250/44000 | Loss: 2.9751 | LR: 3.58e-04 | 1397.7ms/step | 14.3 samples/s
338
+ Step 16260/44000 | Loss: 3.0133 | LR: 3.57e-04 | 1394.7ms/step | 14.3 samples/s
339
+ Step 16270/44000 | Loss: 3.0227 | LR: 3.57e-04 | 1395.3ms/step | 14.3 samples/s
340
+ Step 16280/44000 | Loss: 3.0159 | LR: 3.57e-04 | 1398.3ms/step | 14.3 samples/s
341
+ Step 16290/44000 | Loss: 3.0043 | LR: 3.57e-04 | 1395.9ms/step | 14.3 samples/s
342
+ Step 16300/44000 | Loss: 3.0060 | LR: 3.57e-04 | 1395.7ms/step | 14.3 samples/s
343
+ Step 16310/44000 | Loss: 2.9945 | LR: 3.57e-04 | 1398.3ms/step | 14.3 samples/s
344
+ Step 16320/44000 | Loss: 2.9799 | LR: 3.56e-04 | 1396.1ms/step | 14.3 samples/s
345
+ Step 16330/44000 | Loss: 3.0184 | LR: 3.56e-04 | 1397.7ms/step | 14.3 samples/s
346
+ Step 16340/44000 | Loss: 3.0043 | LR: 3.56e-04 | 1396.3ms/step | 14.3 samples/s
347
+ Step 16350/44000 | Loss: 3.0138 | LR: 3.56e-04 | 1393.5ms/step | 14.4 samples/s
348
+ Step 16360/44000 | Loss: 2.9752 | LR: 3.56e-04 | 1394.3ms/step | 14.3 samples/s
349
+ Step 16370/44000 | Loss: 2.9994 | LR: 3.56e-04 | 1398.1ms/step | 14.3 samples/s
350
+ Step 16380/44000 | Loss: 2.9924 | LR: 3.55e-04 | 1397.9ms/step | 14.3 samples/s
351
+ Step 16390/44000 | Loss: 2.9954 | LR: 3.55e-04 | 1397.2ms/step | 14.3 samples/s
352
+ Step 16400/44000 | Loss: 2.9960 | LR: 3.55e-04 | 1398.7ms/step | 14.3 samples/s
353
+ Step 16410/44000 | Loss: 2.9906 | LR: 3.55e-04 | 1396.2ms/step | 14.3 samples/s
354
+ Step 16420/44000 | Loss: 3.0041 | LR: 3.55e-04 | 1399.7ms/step | 14.3 samples/s
355
+ Step 16430/44000 | Loss: 3.0090 | LR: 3.55e-04 | 1400.4ms/step | 14.3 samples/s
356
+ Step 16440/44000 | Loss: 3.0280 | LR: 3.54e-04 | 1395.8ms/step | 14.3 samples/s
357
+ Step 16450/44000 | Loss: 3.0094 | LR: 3.54e-04 | 1395.9ms/step | 14.3 samples/s
358
+ Step 16460/44000 | Loss: 2.9559 | LR: 3.54e-04 | 1396.1ms/step | 14.3 samples/s
359
+ Step 16470/44000 | Loss: 2.9743 | LR: 3.54e-04 | 1396.1ms/step | 14.3 samples/s
360
+ Step 16480/44000 | Loss: 3.0436 | LR: 3.54e-04 | 1396.0ms/step | 14.3 samples/s
361
+ Step 16490/44000 | Loss: 2.9995 | LR: 3.54e-04 | 1398.2ms/step | 14.3 samples/s
362
+ Step 16500/44000 | Loss: 3.0170 | LR: 3.53e-04 | 1393.9ms/step | 14.3 samples/s
363
+ Step 16510/44000 | Loss: 3.0246 | LR: 3.53e-04 | 1395.8ms/step | 14.3 samples/s
364
+ Step 16520/44000 | Loss: 3.0180 | LR: 3.53e-04 | 1392.1ms/step | 14.4 samples/s
365
+ Step 16530/44000 | Loss: 2.9973 | LR: 3.53e-04 | 1400.1ms/step | 14.3 samples/s
366
+ Step 16540/44000 | Loss: 3.0152 | LR: 3.53e-04 | 1400.2ms/step | 14.3 samples/s
367
+ Step 16550/44000 | Loss: 2.9887 | LR: 3.53e-04 | 1397.9ms/step | 14.3 samples/s
368
+ Step 16560/44000 | Loss: 2.9889 | LR: 3.52e-04 | 1399.9ms/step | 14.3 samples/s
369
+ Step 16570/44000 | Loss: 3.0022 | LR: 3.52e-04 | 1395.7ms/step | 14.3 samples/s
370
+ Step 16580/44000 | Loss: 2.9782 | LR: 3.52e-04 | 1395.7ms/step | 14.3 samples/s
371
+ Step 16590/44000 | Loss: 2.9683 | LR: 3.52e-04 | 1396.8ms/step | 14.3 samples/s
372
+ Step 16600/44000 | Loss: 3.0078 | LR: 3.52e-04 | 1399.9ms/step | 14.3 samples/s
373
+ Step 16610/44000 | Loss: 2.9827 | LR: 3.52e-04 | 1398.1ms/step | 14.3 samples/s
374
+ Step 16620/44000 | Loss: 2.9881 | LR: 3.51e-04 | 1396.6ms/step | 14.3 samples/s
375
+ Step 16630/44000 | Loss: 2.9986 | LR: 3.51e-04 | 1391.3ms/step | 14.4 samples/s
376
+ Step 16640/44000 | Loss: 3.0085 | LR: 3.51e-04 | 1399.7ms/step | 14.3 samples/s
377
+ Step 16650/44000 | Loss: 3.0182 | LR: 3.51e-04 | 1399.6ms/step | 14.3 samples/s
378
+ Step 16660/44000 | Loss: 3.0038 | LR: 3.51e-04 | 1392.8ms/step | 14.4 samples/s
379
+ Step 16670/44000 | Loss: 3.0267 | LR: 3.51e-04 | 1397.5ms/step | 14.3 samples/s
380
+ Step 16680/44000 | Loss: 3.0212 | LR: 3.50e-04 | 1400.3ms/step | 14.3 samples/s
381
+ Step 16690/44000 | Loss: 3.0109 | LR: 3.50e-04 | 1398.2ms/step | 14.3 samples/s
382
+ Step 16700/44000 | Loss: 2.9904 | LR: 3.50e-04 | 1397.8ms/step | 14.3 samples/s
383
+ Step 16710/44000 | Loss: 2.9792 | LR: 3.50e-04 | 1398.0ms/step | 14.3 samples/s
384
+ Step 16720/44000 | Loss: 2.9861 | LR: 3.50e-04 | 1394.3ms/step | 14.3 samples/s
385
+ Step 16730/44000 | Loss: 3.0105 | LR: 3.50e-04 | 1396.1ms/step | 14.3 samples/s
386
+ Step 16740/44000 | Loss: 3.0149 | LR: 3.49e-04 | 1399.5ms/step | 14.3 samples/s
387
+ Step 16750/44000 | Loss: 3.0014 | LR: 3.49e-04 | 1396.1ms/step | 14.3 samples/s
388
+ Step 16760/44000 | Loss: 3.0067 | LR: 3.49e-04 | 1394.1ms/step | 14.3 samples/s
389
+ Step 16770/44000 | Loss: 2.9674 | LR: 3.49e-04 | 1397.6ms/step | 14.3 samples/s
390
+ Step 16780/44000 | Loss: 3.0223 | LR: 3.49e-04 | 1398.1ms/step | 14.3 samples/s
391
+ Step 16790/44000 | Loss: 2.9809 | LR: 3.49e-04 | 1398.1ms/step | 14.3 samples/s
392
+ Step 16800/44000 | Loss: 3.0072 | LR: 3.48e-04 | 1395.4ms/step | 14.3 samples/s
393
+ Step 16810/44000 | Loss: 2.9969 | LR: 3.48e-04 | 1396.0ms/step | 14.3 samples/s
394
+ Step 16820/44000 | Loss: 2.9723 | LR: 3.48e-04 | 1390.7ms/step | 14.4 samples/s
395
+ Step 16830/44000 | Loss: 3.0121 | LR: 3.48e-04 | 1395.6ms/step | 14.3 samples/s
396
+ Step 16840/44000 | Loss: 2.9837 | LR: 3.48e-04 | 1395.5ms/step | 14.3 samples/s
397
+ Step 16850/44000 | Loss: 3.0045 | LR: 3.48e-04 | 1395.0ms/step | 14.3 samples/s
398
+ Step 16860/44000 | Loss: 3.0111 | LR: 3.47e-04 | 1399.9ms/step | 14.3 samples/s
399
+ Step 16870/44000 | Loss: 2.9940 | LR: 3.47e-04 | 1396.0ms/step | 14.3 samples/s
400
+ Step 16880/44000 | Loss: 2.9770 | LR: 3.47e-04 | 1399.6ms/step | 14.3 samples/s
401
+ Step 16890/44000 | Loss: 2.9884 | LR: 3.47e-04 | 1396.5ms/step | 14.3 samples/s
402
+ Step 16900/44000 | Loss: 2.9774 | LR: 3.47e-04 | 1397.8ms/step | 14.3 samples/s
403
+ Step 16910/44000 | Loss: 3.0140 | LR: 3.47e-04 | 1398.3ms/step | 14.3 samples/s
404
+ Step 16920/44000 | Loss: 3.0135 | LR: 3.46e-04 | 1397.7ms/step | 14.3 samples/s
405
+ Step 16930/44000 | Loss: 3.0102 | LR: 3.46e-04 | 1400.4ms/step | 14.3 samples/s
406
+ Step 16940/44000 | Loss: 3.0144 | LR: 3.46e-04 | 1399.9ms/step | 14.3 samples/s
407
+ Step 16950/44000 | Loss: 2.9817 | LR: 3.46e-04 | 1397.9ms/step | 14.3 samples/s
408
+ Step 16960/44000 | Loss: 3.0258 | LR: 3.46e-04 | 1395.6ms/step | 14.3 samples/s
409
+ Step 16970/44000 | Loss: 2.9987 | LR: 3.46e-04 | 1398.4ms/step | 14.3 samples/s
410
+ Step 16980/44000 | Loss: 2.9798 | LR: 3.45e-04 | 1399.7ms/step | 14.3 samples/s
411
+ Step 16990/44000 | Loss: 3.0147 | LR: 3.45e-04 | 1398.4ms/step | 14.3 samples/s
412
+ Step 17000/44000 | Loss: 3.0201 | LR: 3.45e-04 | 1391.1ms/step | 14.4 samples/s
413
+ Validation at step 17000: Loss = 3.0121, Perplexity = 20.33
414
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-17000
415
+ Step 17010/44000 | Loss: 2.9965 | LR: 3.45e-04 | 3968.6ms/step | 5.0 samples/s
416
+ Step 17020/44000 | Loss: 2.9838 | LR: 3.45e-04 | 1384.7ms/step | 14.4 samples/s
417
+ Step 17030/44000 | Loss: 2.9773 | LR: 3.45e-04 | 1383.2ms/step | 14.5 samples/s
418
+ Step 17040/44000 | Loss: 2.9716 | LR: 3.44e-04 | 1386.5ms/step | 14.4 samples/s
419
+ Step 17050/44000 | Loss: 3.0080 | LR: 3.44e-04 | 1384.9ms/step | 14.4 samples/s
420
+ Step 17060/44000 | Loss: 3.0121 | LR: 3.44e-04 | 1387.2ms/step | 14.4 samples/s
421
+ Step 17070/44000 | Loss: 2.9939 | LR: 3.44e-04 | 1389.7ms/step | 14.4 samples/s
422
+ Step 17080/44000 | Loss: 3.0381 | LR: 3.44e-04 | 1391.7ms/step | 14.4 samples/s
423
+ Step 17090/44000 | Loss: 2.9735 | LR: 3.44e-04 | 1390.1ms/step | 14.4 samples/s
424
+ Step 17100/44000 | Loss: 2.9871 | LR: 3.43e-04 | 1390.1ms/step | 14.4 samples/s
425
+ Step 17110/44000 | Loss: 3.0153 | LR: 3.43e-04 | 1393.7ms/step | 14.4 samples/s
426
+ Step 17120/44000 | Loss: 3.0041 | LR: 3.43e-04 | 1389.3ms/step | 14.4 samples/s
427
+ Step 17130/44000 | Loss: 2.9881 | LR: 3.43e-04 | 1391.1ms/step | 14.4 samples/s
428
+ Step 17140/44000 | Loss: 3.0179 | LR: 3.43e-04 | 1389.9ms/step | 14.4 samples/s
429
+ Step 17150/44000 | Loss: 2.9896 | LR: 3.43e-04 | 1390.6ms/step | 14.4 samples/s
430
+ Step 17160/44000 | Loss: 3.0055 | LR: 3.42e-04 | 1391.1ms/step | 14.4 samples/s
431
+ Step 17170/44000 | Loss: 3.0043 | LR: 3.42e-04 | 1389.0ms/step | 14.4 samples/s
432
+ Step 17180/44000 | Loss: 3.0208 | LR: 3.42e-04 | 1388.5ms/step | 14.4 samples/s
433
+ Step 17190/44000 | Loss: 2.9791 | LR: 3.42e-04 | 1390.5ms/step | 14.4 samples/s
434
+ Step 17200/44000 | Loss: 2.9915 | LR: 3.42e-04 | 1391.8ms/step | 14.4 samples/s
435
+ Step 17210/44000 | Loss: 2.9879 | LR: 3.42e-04 | 1389.7ms/step | 14.4 samples/s
436
+ Step 17220/44000 | Loss: 2.9841 | LR: 3.41e-04 | 1389.6ms/step | 14.4 samples/s
437
+ Step 17230/44000 | Loss: 2.9995 | LR: 3.41e-04 | 1391.6ms/step | 14.4 samples/s
438
+ Step 17240/44000 | Loss: 2.9744 | LR: 3.41e-04 | 1393.3ms/step | 14.4 samples/s
439
+ Step 17250/44000 | Loss: 2.9778 | LR: 3.41e-04 | 1392.0ms/step | 14.4 samples/s
440
+ Step 17260/44000 | Loss: 2.9987 | LR: 3.41e-04 | 1392.1ms/step | 14.4 samples/s
441
+ Step 17270/44000 | Loss: 2.9892 | LR: 3.41e-04 | 1392.3ms/step | 14.4 samples/s
442
+ Step 17280/44000 | Loss: 3.0014 | LR: 3.40e-04 | 1395.5ms/step | 14.3 samples/s
443
+ Step 17290/44000 | Loss: 3.0176 | LR: 3.40e-04 | 1393.5ms/step | 14.4 samples/s
444
+ Step 17300/44000 | Loss: 2.9963 | LR: 3.40e-04 | 1390.0ms/step | 14.4 samples/s
445
+ Step 17310/44000 | Loss: 3.0070 | LR: 3.40e-04 | 1391.2ms/step | 14.4 samples/s
446
+ Step 17320/44000 | Loss: 3.0012 | LR: 3.40e-04 | 1389.7ms/step | 14.4 samples/s
447
+ Step 17330/44000 | Loss: 3.0012 | LR: 3.40e-04 | 1395.8ms/step | 14.3 samples/s
448
+ Step 17340/44000 | Loss: 2.9895 | LR: 3.39e-04 | 1392.9ms/step | 14.4 samples/s
449
+ Step 17350/44000 | Loss: 2.9804 | LR: 3.39e-04 | 1391.0ms/step | 14.4 samples/s
450
+ Step 17360/44000 | Loss: 2.9887 | LR: 3.39e-04 | 1392.4ms/step | 14.4 samples/s
451
+ Step 17370/44000 | Loss: 2.9760 | LR: 3.39e-04 | 1393.4ms/step | 14.4 samples/s
452
+ Step 17380/44000 | Loss: 2.9965 | LR: 3.39e-04 | 1396.1ms/step | 14.3 samples/s
453
+ Step 17390/44000 | Loss: 2.9764 | LR: 3.39e-04 | 1392.5ms/step | 14.4 samples/s
454
+ Step 17400/44000 | Loss: 2.9813 | LR: 3.38e-04 | 1392.2ms/step | 14.4 samples/s
455
+ Step 17410/44000 | Loss: 2.9788 | LR: 3.38e-04 | 1391.5ms/step | 14.4 samples/s
456
+ Step 17420/44000 | Loss: 2.9823 | LR: 3.38e-04 | 1392.0ms/step | 14.4 samples/s
457
+ Step 17430/44000 | Loss: 3.0083 | LR: 3.38e-04 | 1393.0ms/step | 14.4 samples/s
458
+ Step 17440/44000 | Loss: 2.9813 | LR: 3.38e-04 | 1393.6ms/step | 14.4 samples/s
459
+ Step 17450/44000 | Loss: 3.0100 | LR: 3.38e-04 | 1394.9ms/step | 14.3 samples/s
460
+ Step 17460/44000 | Loss: 2.9793 | LR: 3.37e-04 | 1392.2ms/step | 14.4 samples/s
461
+ Step 17470/44000 | Loss: 2.9795 | LR: 3.37e-04 | 1388.9ms/step | 14.4 samples/s
462
+ Step 17480/44000 | Loss: 2.9965 | LR: 3.37e-04 | 1391.6ms/step | 14.4 samples/s
463
+ Step 17490/44000 | Loss: 2.9569 | LR: 3.37e-04 | 1391.1ms/step | 14.4 samples/s
epfl_final/SparseAtt/wandb/run-20260820_041559-jqstnvej/files/requirements.txt ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pip==26.0.1
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.6.3
8
+ typing_extensions==4.15.0
9
+ triton==3.6.0
10
+ tqdm==4.67.3
11
+ torchinfo==1.8.0
12
+ sympy==1.14.0
13
+ smmap==5.0.3
14
+ six==1.17.0
15
+ shellingham==1.5.4
16
+ setuptools==81.0.0
17
+ safetensors==0.7.0
18
+ regex==2026.4.4
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.4.1
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.4.4
38
+ networkx==3.6.1
39
+ multidict==6.7.1
40
+ mdurl==0.1.2
41
+ MarkupSafe==3.0.3
42
+ idna==3.13
43
+ hf-xet==1.4.3
44
+ h11==0.16.0
45
+ fsspec==2026.2.0
46
+ frozenlist==1.8.0
47
+ filelock==3.29.0
48
+ dill==0.4.1
49
+ cuda-pathfinder==1.5.4
50
+ click==8.3.3
51
+ charset-normalizer==3.4.7
52
+ certifi==2026.4.22
53
+ attrs==26.1.0
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ yarl==1.23.0
58
+ typing-inspection==0.4.2
59
+ sentry-sdk==2.58.0
60
+ requests==2.33.1
61
+ python-dateutil==2.9.0.post0
62
+ pydantic_core==2.46.3
63
+ nvidia-cusparse==12.6.3.3
64
+ nvidia-cufft==12.0.0.61
65
+ nvidia-cudnn-cu13==9.19.0.56
66
+ multiprocess==0.70.19
67
+ markdown-it-py==4.0.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ cuda-bindings==13.2.0
72
+ anyio==4.13.0
73
+ aiosignal==1.4.0
74
+ rich==15.0.0
75
+ pydantic==2.13.3
76
+ pandas==3.0.2
77
+ nvidia-cusolver==12.0.4.66
78
+ httpx==0.28.1
79
+ GitPython==3.1.49
80
+ aiohttp==3.13.5
81
+ wandb==0.26.1
82
+ typer==0.25.1
83
+ torch==2.11.0
84
+ huggingface_hub==1.13.0
85
+ tokenizers==0.22.2
86
+ datasets==4.8.5
87
+ accelerate==1.13.0
88
+ transformers==5.7.0
89
+ word2number==1.1
90
+ sqlitedict==2.1.0
91
+ pytz==2026.2
92
+ threadpoolctl==3.6.0
93
+ tcolorpy==0.1.7
94
+ tabulate==0.10.0
95
+ scipy==1.17.1
96
+ portalocker==3.2.0
97
+ pathvalidate==3.3.1
98
+ narwhals==2.22.1
99
+ more-itertools==11.1.0
100
+ lxml==6.1.1
101
+ joblib==1.5.3
102
+ colorama==0.4.6
103
+ chardet==6.0.0.post1
104
+ absl-py==2.4.0
105
+ scikit-learn==1.9.0
106
+ sacrebleu==2.6.0
107
+ nltk==3.9.4
108
+ mbstrdecoder==1.1.5
109
+ typepy==1.3.5
110
+ rouge_score==0.1.2
111
+ DataProperty==1.1.1
112
+ tabledata==1.3.5
113
+ evaluate==0.4.6
114
+ pytablewriter==1.2.1
115
+ lm_eval==0.4.12
116
+ pluggy==1.6.0
117
+ iniconfig==2.3.0
118
+ pytest==9.1.0
119
+ py-spy==0.4.2
120
+ pillow==12.2.0
121
+ tifffile==2026.6.1
122
+ imagecodecs==2026.6.26
123
+ zipp==4.1.0
124
+ importlib_metadata==9.0.0
125
+ derivative==0.6.3
126
+ pysindy==2.1.0
127
+ pyparsing==3.3.2
128
+ kiwisolver==1.5.0
129
+ fonttools==4.63.0
130
+ cycler==0.12.1
131
+ contourpy==1.3.3
132
+ matplotlib==3.11.0
133
+ sortedcontainers==2.4.0
134
+ wcwidth==0.8.2
135
+ sigfig==1.3.19
136
+ importlib_resources==7.1.0
137
+ enum-tools==0.13.0
138
+ cramjam==2.11.0
139
+ python-snappy==0.7.3
140
+ compact-json==1.8.2
141
+ numbers-parser==4.18.5
142
+ entmax==1.3
epfl_final/SparseAtt/wandb/run-20260820_041559-jqstnvej/files/wandb-metadata.json ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.13.13",
4
+ "startedAt": "2026-08-20T04:15:59.198192Z",
5
+ "args": [
6
+ "--config",
7
+ "/mloscratch/homes/navasard/sparse/configs_ropectl/resume_a1p0_wandb/seed42.json"
8
+ ],
9
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
10
+ "codePath": "train.py",
11
+ "codePathLocal": "train.py",
12
+ "git": {
13
+ "remote": "https://github.com/rubentium/TokenPrune.git",
14
+ "commit": "5637886f0cc760009989b14e68c6ae06d4827418"
15
+ },
16
+ "email": "ruben.navasardyan@mail.utoronto.ca",
17
+ "root": "/mloscratch/homes/navasard/sparse/SparseAtt",
18
+ "host": "h100-workspace-5-gpu-0-2",
19
+ "executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
20
+ "cpu_count": 96,
21
+ "cpu_count_logical": 192,
22
+ "gpu": "NVIDIA H100 80GB HBM3",
23
+ "gpu_count": 5,
24
+ "disk": {
25
+ "/": {
26
+ "total": "9374931091456",
27
+ "used": "5285341044736"
28
+ }
29
+ },
30
+ "memory": {
31
+ "total": "1623005626368"
32
+ },
33
+ "gpu_nvidia": [
34
+ {
35
+ "name": "NVIDIA H100 80GB HBM3",
36
+ "memoryTotal": "85520809984",
37
+ "cudaCores": 16896,
38
+ "architecture": "Hopper",
39
+ "uuid": "GPU-6fb12770-6b51-9595-91c9-0d20801e3896"
40
+ },
41
+ {
42
+ "name": "NVIDIA H100 80GB HBM3",
43
+ "memoryTotal": "85520809984",
44
+ "cudaCores": 16896,
45
+ "architecture": "Hopper",
46
+ "uuid": "GPU-5cb9f8b7-4a13-4f85-d73e-6bcbebebefb8"
47
+ },
48
+ {
49
+ "name": "NVIDIA H100 80GB HBM3",
50
+ "memoryTotal": "85520809984",
51
+ "cudaCores": 16896,
52
+ "architecture": "Hopper",
53
+ "uuid": "GPU-3512802a-d501-0a3c-9812-4ab9deb3a3a9"
54
+ },
55
+ {
56
+ "name": "NVIDIA H100 80GB HBM3",
57
+ "memoryTotal": "85520809984",
58
+ "cudaCores": 16896,
59
+ "architecture": "Hopper",
60
+ "uuid": "GPU-24436def-beee-c9f1-bd05-c97df556de08"
61
+ },
62
+ {
63
+ "name": "NVIDIA H100 80GB HBM3",
64
+ "memoryTotal": "85520809984",
65
+ "cudaCores": 16896,
66
+ "architecture": "Hopper",
67
+ "uuid": "GPU-7b827c86-854a-af51-b5d3-6aaa43307d4a"
68
+ }
69
+ ],
70
+ "cudaVersion": "13.0",
71
+ "writerId": "kubps3g8hm42dnwny2ivywim7vqvuo12"
72
+ }
epfl_final/SparseAtt/wandb/run-20260820_041559-jqstnvej/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/SparseAtt/wandb/run-20260820_041559-jqstnvej/logs/debug.log ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-08-20 04:15:59,227 INFO MainThread:369 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-08-20 04:15:59,227 INFO MainThread:369 [wandb_setup.py:_flush():81] Configure stats pid to 369
3
+ 2026-08-20 04:15:59,227 INFO MainThread:369 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-08-20 04:15:59,227 INFO MainThread:369 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_041559-jqstnvej/logs/debug.log
5
+ 2026-08-20 04:15:59,227 INFO MainThread:369 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_041559-jqstnvej/logs/debug-internal.log
6
+ 2026-08-20 04:15:59,228 INFO MainThread:369 [wandb_init.py:init():850] calling init triggers
7
+ 2026-08-20 04:15:59,228 INFO MainThread:369 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42/run_20260819_153735/checkpoint-13000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 1000, 'eval_batches': 50, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'l8k365m_dense_ropectl_a1p0_seed42', 'wandb_tags': ['qwen3', '365m', '8k', 'dense', 'rope-control'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': 0.0, 'rope_train_position_scale_set': [1.0, 0.9896339488176222, 0.9692159692159692, 0.9420289855072463, 0.8730158730158729], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
9
+ 2026-08-20 04:15:59,228 INFO MainThread:369 [wandb_init.py:init():898] starting backend
10
+ 2026-08-20 04:15:59,711 INFO MainThread:369 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-08-20 04:16:00,128 INFO MainThread:369 [wandb_init.py:init():918] backend started and connected
12
+ 2026-08-20 04:16:00,131 INFO MainThread:369 [wandb_init.py:init():988] updated telemetry
13
+ 2026-08-20 04:16:00,150 INFO MainThread:369 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-08-20 04:16:00,777 INFO MainThread:369 [wandb_init.py:init():1051] run resumed
15
+ 2026-08-20 04:16:00,779 INFO MainThread:369 [wandb_init.py:init():1056] starting run threads in backend
16
+ 2026-08-20 04:16:00,945 INFO MainThread:369 [wandb_run.py:_console_start():2554] atexit reg
17
+ 2026-08-20 04:16:00,945 INFO MainThread:369 [wandb_run.py:_redirect():2403] redirect: wrap_raw
18
+ 2026-08-20 04:16:00,945 INFO MainThread:369 [wandb_run.py:_redirect():2472] Wrapping output streams.
19
+ 2026-08-20 04:16:00,945 INFO MainThread:369 [wandb_run.py:_redirect():2495] Redirects installed.
20
+ 2026-08-20 04:16:00,948 INFO MainThread:369 [wandb_init.py:init():1094] run started, returning control to user process
epfl_final/SparseAtt/wandb/run-20260820_041559-jqstnvej/run-jqstnvej.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c1cb27106354aa11bd60e6bf6f15db904e5f62bd2c5f8158defdadeb8f1901a9
3
+ size 2785280
epfl_final/SparseAtt/wandb/run-20260820_041559-xfnxdn24/files/output.log ADDED
@@ -0,0 +1,472 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Starting training...
2
+ Validation at step 13000: Loss = 3.0601, Perplexity = 21.33
3
+ wandb: WARNING Tried to log to step 13000 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
4
+ Switching attention mode for step 13001: dense (dense step interval)
5
+ Step 13010/44000 | Loss: 3.0530 | LR: 4.07e-04 | 30157.5ms/step | 0.7 samples/s
6
+ wandb: WARNING Tried to log to step 13010 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
7
+ Step 13020/44000 | Loss: 3.0363 | LR: 4.07e-04 | 1397.9ms/step | 14.3 samples/s
8
+ wandb: WARNING Tried to log to step 13020 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
9
+ Step 13030/44000 | Loss: 3.0282 | LR: 4.07e-04 | 1395.5ms/step | 14.3 samples/s
10
+ wandb: WARNING Tried to log to step 13030 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
11
+ Step 13040/44000 | Loss: 3.0404 | LR: 4.07e-04 | 1398.6ms/step | 14.3 samples/s
12
+ wandb: WARNING Tried to log to step 13040 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
13
+ Step 13050/44000 | Loss: 3.0329 | LR: 4.07e-04 | 1400.2ms/step | 14.3 samples/s
14
+ wandb: WARNING Tried to log to step 13050 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
15
+ Step 13060/44000 | Loss: 3.0708 | LR: 4.07e-04 | 1400.0ms/step | 14.3 samples/s
16
+ wandb: WARNING Tried to log to step 13060 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
17
+ Step 13070/44000 | Loss: 3.0480 | LR: 4.06e-04 | 1399.8ms/step | 14.3 samples/s
18
+ wandb: WARNING Tried to log to step 13070 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
19
+ Step 13080/44000 | Loss: 3.0513 | LR: 4.06e-04 | 1400.2ms/step | 14.3 samples/s
20
+ wandb: WARNING Tried to log to step 13080 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
21
+ Step 13090/44000 | Loss: 3.0571 | LR: 4.06e-04 | 1399.9ms/step | 14.3 samples/s
22
+ wandb: WARNING Tried to log to step 13090 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
23
+ Step 13100/44000 | Loss: 3.0298 | LR: 4.06e-04 | 1400.0ms/step | 14.3 samples/s
24
+ wandb: WARNING Tried to log to step 13100 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
25
+ Step 13110/44000 | Loss: 3.0451 | LR: 4.06e-04 | 1400.1ms/step | 14.3 samples/s
26
+ wandb: WARNING Tried to log to step 13110 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
27
+ Step 13120/44000 | Loss: 3.0519 | LR: 4.06e-04 | 1402.0ms/step | 14.3 samples/s
28
+ wandb: WARNING Tried to log to step 13120 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
29
+ Step 13130/44000 | Loss: 3.0832 | LR: 4.06e-04 | 1400.0ms/step | 14.3 samples/s
30
+ wandb: WARNING Tried to log to step 13130 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
31
+ Step 13140/44000 | Loss: 3.0502 | LR: 4.05e-04 | 1400.3ms/step | 14.3 samples/s
32
+ wandb: WARNING Tried to log to step 13140 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
33
+ Step 13150/44000 | Loss: 3.0396 | LR: 4.05e-04 | 1401.6ms/step | 14.3 samples/s
34
+ wandb: WARNING Tried to log to step 13150 that is less than the current step 13151. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
35
+ Step 13160/44000 | Loss: 3.0522 | LR: 4.05e-04 | 1400.8ms/step | 14.3 samples/s
36
+ Step 13170/44000 | Loss: 3.0706 | LR: 4.05e-04 | 1399.2ms/step | 14.3 samples/s
37
+ Step 13180/44000 | Loss: 3.0776 | LR: 4.05e-04 | 1400.2ms/step | 14.3 samples/s
38
+ Step 13190/44000 | Loss: 3.0760 | LR: 4.05e-04 | 1401.8ms/step | 14.3 samples/s
39
+ Step 13200/44000 | Loss: 3.0560 | LR: 4.05e-04 | 1402.3ms/step | 14.3 samples/s
40
+ Step 13210/44000 | Loss: 3.0916 | LR: 4.04e-04 | 1400.5ms/step | 14.3 samples/s
41
+ Step 13220/44000 | Loss: 3.0245 | LR: 4.04e-04 | 1399.6ms/step | 14.3 samples/s
42
+ Step 13230/44000 | Loss: 3.1008 | LR: 4.04e-04 | 1401.6ms/step | 14.3 samples/s
43
+ Step 13240/44000 | Loss: 3.0804 | LR: 4.04e-04 | 1402.0ms/step | 14.3 samples/s
44
+ Step 13250/44000 | Loss: 3.0645 | LR: 4.04e-04 | 1400.1ms/step | 14.3 samples/s
45
+ Step 13260/44000 | Loss: 3.0688 | LR: 4.04e-04 | 1400.1ms/step | 14.3 samples/s
46
+ Step 13270/44000 | Loss: 3.0776 | LR: 4.04e-04 | 1399.8ms/step | 14.3 samples/s
47
+ Step 13280/44000 | Loss: 3.0387 | LR: 4.03e-04 | 1400.2ms/step | 14.3 samples/s
48
+ Step 13290/44000 | Loss: 3.0463 | LR: 4.03e-04 | 1402.2ms/step | 14.3 samples/s
49
+ Step 13300/44000 | Loss: 3.0579 | LR: 4.03e-04 | 1402.0ms/step | 14.3 samples/s
50
+ Step 13310/44000 | Loss: 3.0696 | LR: 4.03e-04 | 1399.9ms/step | 14.3 samples/s
51
+ Step 13320/44000 | Loss: 3.0288 | LR: 4.03e-04 | 1401.8ms/step | 14.3 samples/s
52
+ Step 13330/44000 | Loss: 3.0232 | LR: 4.03e-04 | 1402.2ms/step | 14.3 samples/s
53
+ Step 13340/44000 | Loss: 3.0448 | LR: 4.03e-04 | 1400.1ms/step | 14.3 samples/s
54
+ Step 13350/44000 | Loss: 3.0312 | LR: 4.02e-04 | 1404.5ms/step | 14.2 samples/s
55
+ Step 13360/44000 | Loss: 3.0378 | LR: 4.02e-04 | 1405.4ms/step | 14.2 samples/s
56
+ Step 13370/44000 | Loss: 3.0470 | LR: 4.02e-04 | 1402.0ms/step | 14.3 samples/s
57
+ Step 13380/44000 | Loss: 3.0633 | LR: 4.02e-04 | 1403.9ms/step | 14.2 samples/s
58
+ Step 13390/44000 | Loss: 3.0842 | LR: 4.02e-04 | 1404.6ms/step | 14.2 samples/s
59
+ Step 13400/44000 | Loss: 3.0427 | LR: 4.02e-04 | 1403.9ms/step | 14.2 samples/s
60
+ Step 13410/44000 | Loss: 3.0296 | LR: 4.02e-04 | 1399.2ms/step | 14.3 samples/s
61
+ Step 13420/44000 | Loss: 3.0456 | LR: 4.01e-04 | 1401.5ms/step | 14.3 samples/s
62
+ Step 13430/44000 | Loss: 3.0520 | LR: 4.01e-04 | 1404.5ms/step | 14.2 samples/s
63
+ Step 13440/44000 | Loss: 3.0475 | LR: 4.01e-04 | 1400.1ms/step | 14.3 samples/s
64
+ Step 13450/44000 | Loss: 3.0465 | LR: 4.01e-04 | 1400.2ms/step | 14.3 samples/s
65
+ Step 13460/44000 | Loss: 3.0584 | LR: 4.01e-04 | 1400.5ms/step | 14.3 samples/s
66
+ Step 13470/44000 | Loss: 3.0524 | LR: 4.01e-04 | 1399.5ms/step | 14.3 samples/s
67
+ Step 13480/44000 | Loss: 3.0477 | LR: 4.01e-04 | 1399.7ms/step | 14.3 samples/s
68
+ Step 13490/44000 | Loss: 3.0477 | LR: 4.00e-04 | 1400.2ms/step | 14.3 samples/s
69
+ Step 13500/44000 | Loss: 3.0386 | LR: 4.00e-04 | 1399.9ms/step | 14.3 samples/s
70
+ Step 13510/44000 | Loss: 3.0418 | LR: 4.00e-04 | 1402.4ms/step | 14.3 samples/s
71
+ Step 13520/44000 | Loss: 3.0534 | LR: 4.00e-04 | 1399.8ms/step | 14.3 samples/s
72
+ Step 13530/44000 | Loss: 3.0395 | LR: 4.00e-04 | 1399.7ms/step | 14.3 samples/s
73
+ Step 13540/44000 | Loss: 3.0473 | LR: 4.00e-04 | 1400.3ms/step | 14.3 samples/s
74
+ Step 13550/44000 | Loss: 3.0782 | LR: 4.00e-04 | 1401.6ms/step | 14.3 samples/s
75
+ Step 13560/44000 | Loss: 3.0321 | LR: 3.99e-04 | 1400.3ms/step | 14.3 samples/s
76
+ Step 13570/44000 | Loss: 3.0586 | LR: 3.99e-04 | 1399.8ms/step | 14.3 samples/s
77
+ Step 13580/44000 | Loss: 3.0491 | LR: 3.99e-04 | 1400.7ms/step | 14.3 samples/s
78
+ Step 13590/44000 | Loss: 3.0513 | LR: 3.99e-04 | 1401.3ms/step | 14.3 samples/s
79
+ Step 13600/44000 | Loss: 3.0597 | LR: 3.99e-04 | 1400.1ms/step | 14.3 samples/s
80
+ Step 13610/44000 | Loss: 3.0593 | LR: 3.99e-04 | 1399.8ms/step | 14.3 samples/s
81
+ Step 13620/44000 | Loss: 3.0857 | LR: 3.98e-04 | 1402.3ms/step | 14.3 samples/s
82
+ Step 13630/44000 | Loss: 3.0691 | LR: 3.98e-04 | 1400.3ms/step | 14.3 samples/s
83
+ Step 13640/44000 | Loss: 3.0180 | LR: 3.98e-04 | 1401.5ms/step | 14.3 samples/s
84
+ Step 13650/44000 | Loss: 3.0189 | LR: 3.98e-04 | 1400.1ms/step | 14.3 samples/s
85
+ Step 13660/44000 | Loss: 3.0850 | LR: 3.98e-04 | 1404.3ms/step | 14.2 samples/s
86
+ Step 13670/44000 | Loss: 3.0420 | LR: 3.98e-04 | 1399.5ms/step | 14.3 samples/s
87
+ Step 13680/44000 | Loss: 3.0472 | LR: 3.98e-04 | 1400.0ms/step | 14.3 samples/s
88
+ Step 13690/44000 | Loss: 3.0447 | LR: 3.97e-04 | 1400.2ms/step | 14.3 samples/s
89
+ Step 13700/44000 | Loss: 3.0604 | LR: 3.97e-04 | 1404.4ms/step | 14.2 samples/s
90
+ Step 13710/44000 | Loss: 3.0303 | LR: 3.97e-04 | 1401.4ms/step | 14.3 samples/s
91
+ Step 13720/44000 | Loss: 3.0465 | LR: 3.97e-04 | 1400.6ms/step | 14.3 samples/s
92
+ Step 13730/44000 | Loss: 3.0249 | LR: 3.97e-04 | 1401.8ms/step | 14.3 samples/s
93
+ Step 13740/44000 | Loss: 3.0263 | LR: 3.97e-04 | 1403.8ms/step | 14.2 samples/s
94
+ Step 13750/44000 | Loss: 3.0443 | LR: 3.97e-04 | 1401.8ms/step | 14.3 samples/s
95
+ Step 13760/44000 | Loss: 3.0645 | LR: 3.96e-04 | 1402.2ms/step | 14.3 samples/s
96
+ Step 13770/44000 | Loss: 3.0402 | LR: 3.96e-04 | 1402.5ms/step | 14.3 samples/s
97
+ Step 13780/44000 | Loss: 3.0526 | LR: 3.96e-04 | 1403.6ms/step | 14.2 samples/s
98
+ Step 13790/44000 | Loss: 3.0623 | LR: 3.96e-04 | 1401.8ms/step | 14.3 samples/s
99
+ Step 13800/44000 | Loss: 3.0307 | LR: 3.96e-04 | 1400.5ms/step | 14.3 samples/s
100
+ Step 13810/44000 | Loss: 3.0090 | LR: 3.96e-04 | 1401.4ms/step | 14.3 samples/s
101
+ Step 13820/44000 | Loss: 3.0181 | LR: 3.96e-04 | 1399.7ms/step | 14.3 samples/s
102
+ Step 13830/44000 | Loss: 3.0469 | LR: 3.95e-04 | 1400.1ms/step | 14.3 samples/s
103
+ Step 13840/44000 | Loss: 3.0570 | LR: 3.95e-04 | 1400.1ms/step | 14.3 samples/s
104
+ Step 13850/44000 | Loss: 3.0570 | LR: 3.95e-04 | 1400.1ms/step | 14.3 samples/s
105
+ Step 13860/44000 | Loss: 3.0614 | LR: 3.95e-04 | 1399.6ms/step | 14.3 samples/s
106
+ Step 13870/44000 | Loss: 3.0264 | LR: 3.95e-04 | 1400.1ms/step | 14.3 samples/s
107
+ Step 13880/44000 | Loss: 3.0517 | LR: 3.95e-04 | 1399.9ms/step | 14.3 samples/s
108
+ Step 13890/44000 | Loss: 3.0714 | LR: 3.95e-04 | 1402.1ms/step | 14.3 samples/s
109
+ Step 13900/44000 | Loss: 3.0571 | LR: 3.94e-04 | 1400.3ms/step | 14.3 samples/s
110
+ Step 13910/44000 | Loss: 3.0328 | LR: 3.94e-04 | 1399.7ms/step | 14.3 samples/s
111
+ Step 13920/44000 | Loss: 3.0541 | LR: 3.94e-04 | 1402.1ms/step | 14.3 samples/s
112
+ Step 13930/44000 | Loss: 3.0549 | LR: 3.94e-04 | 1402.9ms/step | 14.3 samples/s
113
+ Step 13940/44000 | Loss: 3.0528 | LR: 3.94e-04 | 1402.9ms/step | 14.3 samples/s
114
+ Step 13950/44000 | Loss: 3.0231 | LR: 3.94e-04 | 1400.7ms/step | 14.3 samples/s
115
+ Step 13960/44000 | Loss: 3.0538 | LR: 3.93e-04 | 1401.3ms/step | 14.3 samples/s
116
+ Step 13970/44000 | Loss: 3.0205 | LR: 3.93e-04 | 1402.2ms/step | 14.3 samples/s
117
+ Step 13980/44000 | Loss: 3.0674 | LR: 3.93e-04 | 1399.9ms/step | 14.3 samples/s
118
+ Step 13990/44000 | Loss: 3.0573 | LR: 3.93e-04 | 1400.3ms/step | 14.3 samples/s
119
+ Step 14000/44000 | Loss: 3.0497 | LR: 3.93e-04 | 1404.1ms/step | 14.2 samples/s
120
+ Validation at step 14000: Loss = 3.0457, Perplexity = 21.02
121
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-14000
122
+ Step 14010/44000 | Loss: 3.0269 | LR: 3.93e-04 | 3933.2ms/step | 5.1 samples/s
123
+ Step 14020/44000 | Loss: 3.0386 | LR: 3.93e-04 | 1399.9ms/step | 14.3 samples/s
124
+ Step 14030/44000 | Loss: 3.0397 | LR: 3.92e-04 | 1400.2ms/step | 14.3 samples/s
125
+ Step 14040/44000 | Loss: 3.0568 | LR: 3.92e-04 | 1399.9ms/step | 14.3 samples/s
126
+ Step 14050/44000 | Loss: 3.0401 | LR: 3.92e-04 | 1400.3ms/step | 14.3 samples/s
127
+ Step 14060/44000 | Loss: 3.0412 | LR: 3.92e-04 | 1400.2ms/step | 14.3 samples/s
128
+ Step 14070/44000 | Loss: 3.0348 | LR: 3.92e-04 | 1399.8ms/step | 14.3 samples/s
129
+ Step 14080/44000 | Loss: 3.0322 | LR: 3.92e-04 | 1402.5ms/step | 14.3 samples/s
130
+ Step 14090/44000 | Loss: 3.0133 | LR: 3.92e-04 | 1406.3ms/step | 14.2 samples/s
131
+ Step 14100/44000 | Loss: 3.0499 | LR: 3.91e-04 | 1403.7ms/step | 14.2 samples/s
132
+ Step 14110/44000 | Loss: 3.0379 | LR: 3.91e-04 | 1403.4ms/step | 14.3 samples/s
133
+ Step 14120/44000 | Loss: 3.0224 | LR: 3.91e-04 | 1402.5ms/step | 14.3 samples/s
134
+ Step 14130/44000 | Loss: 3.0570 | LR: 3.91e-04 | 1403.8ms/step | 14.2 samples/s
135
+ Step 14140/44000 | Loss: 3.0246 | LR: 3.91e-04 | 1404.0ms/step | 14.2 samples/s
136
+ Step 14150/44000 | Loss: 3.0318 | LR: 3.91e-04 | 1404.3ms/step | 14.2 samples/s
137
+ Step 14160/44000 | Loss: 3.0360 | LR: 3.90e-04 | 1406.8ms/step | 14.2 samples/s
138
+ Step 14170/44000 | Loss: 3.0725 | LR: 3.90e-04 | 1403.5ms/step | 14.3 samples/s
139
+ Step 14180/44000 | Loss: 3.0542 | LR: 3.90e-04 | 1404.4ms/step | 14.2 samples/s
140
+ Step 14190/44000 | Loss: 3.0367 | LR: 3.90e-04 | 1407.1ms/step | 14.2 samples/s
141
+ Step 14200/44000 | Loss: 3.0304 | LR: 3.90e-04 | 1405.9ms/step | 14.2 samples/s
142
+ Step 14210/44000 | Loss: 3.0421 | LR: 3.90e-04 | 1402.6ms/step | 14.3 samples/s
143
+ Step 14220/44000 | Loss: 3.0308 | LR: 3.90e-04 | 1406.5ms/step | 14.2 samples/s
144
+ Step 14230/44000 | Loss: 3.0301 | LR: 3.89e-04 | 1405.2ms/step | 14.2 samples/s
145
+ Step 14240/44000 | Loss: 3.0277 | LR: 3.89e-04 | 1405.7ms/step | 14.2 samples/s
146
+ Step 14250/44000 | Loss: 3.0392 | LR: 3.89e-04 | 1405.0ms/step | 14.2 samples/s
147
+ Step 14260/44000 | Loss: 3.0446 | LR: 3.89e-04 | 1405.2ms/step | 14.2 samples/s
148
+ Step 14270/44000 | Loss: 3.0438 | LR: 3.89e-04 | 1405.5ms/step | 14.2 samples/s
149
+ Step 14280/44000 | Loss: 3.0680 | LR: 3.89e-04 | 1405.0ms/step | 14.2 samples/s
150
+ Step 14290/44000 | Loss: 3.0635 | LR: 3.89e-04 | 1407.4ms/step | 14.2 samples/s
151
+ Step 14300/44000 | Loss: 3.0529 | LR: 3.88e-04 | 1406.1ms/step | 14.2 samples/s
152
+ Step 14310/44000 | Loss: 3.0404 | LR: 3.88e-04 | 1406.5ms/step | 14.2 samples/s
153
+ Step 14320/44000 | Loss: 3.0086 | LR: 3.88e-04 | 1409.2ms/step | 14.2 samples/s
154
+ Step 14330/44000 | Loss: 3.0353 | LR: 3.88e-04 | 1403.7ms/step | 14.2 samples/s
155
+ Step 14340/44000 | Loss: 3.0280 | LR: 3.88e-04 | 1408.1ms/step | 14.2 samples/s
156
+ Step 14350/44000 | Loss: 3.0095 | LR: 3.88e-04 | 1408.6ms/step | 14.2 samples/s
157
+ Step 14360/44000 | Loss: 3.0805 | LR: 3.87e-04 | 1405.8ms/step | 14.2 samples/s
158
+ Step 14370/44000 | Loss: 3.0334 | LR: 3.87e-04 | 1406.0ms/step | 14.2 samples/s
159
+ Step 14380/44000 | Loss: 3.0244 | LR: 3.87e-04 | 1408.0ms/step | 14.2 samples/s
160
+ Step 14390/44000 | Loss: 3.0272 | LR: 3.87e-04 | 1407.2ms/step | 14.2 samples/s
161
+ Step 14400/44000 | Loss: 3.0292 | LR: 3.87e-04 | 1404.3ms/step | 14.2 samples/s
162
+ Step 14410/44000 | Loss: 3.0409 | LR: 3.87e-04 | 1407.9ms/step | 14.2 samples/s
163
+ Step 14420/44000 | Loss: 3.0291 | LR: 3.87e-04 | 1404.7ms/step | 14.2 samples/s
164
+ Step 14430/44000 | Loss: 3.0118 | LR: 3.86e-04 | 1405.9ms/step | 14.2 samples/s
165
+ Step 14440/44000 | Loss: 3.0532 | LR: 3.86e-04 | 1411.6ms/step | 14.2 samples/s
166
+ Step 14450/44000 | Loss: 3.0091 | LR: 3.86e-04 | 1407.8ms/step | 14.2 samples/s
167
+ Step 14460/44000 | Loss: 3.0106 | LR: 3.86e-04 | 1404.1ms/step | 14.2 samples/s
168
+ Step 14470/44000 | Loss: 3.0369 | LR: 3.86e-04 | 1405.5ms/step | 14.2 samples/s
169
+ Step 14480/44000 | Loss: 3.0082 | LR: 3.86e-04 | 1406.6ms/step | 14.2 samples/s
170
+ Step 14490/44000 | Loss: 3.0267 | LR: 3.85e-04 | 1406.0ms/step | 14.2 samples/s
171
+ Step 14500/44000 | Loss: 3.0286 | LR: 3.85e-04 | 1407.7ms/step | 14.2 samples/s
172
+ Step 14510/44000 | Loss: 3.0183 | LR: 3.85e-04 | 1404.8ms/step | 14.2 samples/s
173
+ Step 14520/44000 | Loss: 3.0552 | LR: 3.85e-04 | 1405.8ms/step | 14.2 samples/s
174
+ Step 14530/44000 | Loss: 3.0307 | LR: 3.85e-04 | 1407.5ms/step | 14.2 samples/s
175
+ Step 14540/44000 | Loss: 3.0493 | LR: 3.85e-04 | 1408.3ms/step | 14.2 samples/s
176
+ Step 14550/44000 | Loss: 3.0265 | LR: 3.85e-04 | 1407.8ms/step | 14.2 samples/s
177
+ Step 14560/44000 | Loss: 3.0358 | LR: 3.84e-04 | 1408.7ms/step | 14.2 samples/s
178
+ Step 14570/44000 | Loss: 3.0553 | LR: 3.84e-04 | 1407.2ms/step | 14.2 samples/s
179
+ Step 14580/44000 | Loss: 3.0542 | LR: 3.84e-04 | 1405.7ms/step | 14.2 samples/s
180
+ Step 14590/44000 | Loss: 3.0441 | LR: 3.84e-04 | 1407.9ms/step | 14.2 samples/s
181
+ Step 14600/44000 | Loss: 3.0100 | LR: 3.84e-04 | 1406.5ms/step | 14.2 samples/s
182
+ Step 14610/44000 | Loss: 3.0265 | LR: 3.84e-04 | 1412.5ms/step | 14.2 samples/s
183
+ Step 14620/44000 | Loss: 3.0501 | LR: 3.83e-04 | 1409.7ms/step | 14.2 samples/s
184
+ Step 14630/44000 | Loss: 3.0324 | LR: 3.83e-04 | 1406.2ms/step | 14.2 samples/s
185
+ Step 14640/44000 | Loss: 2.9894 | LR: 3.83e-04 | 1405.6ms/step | 14.2 samples/s
186
+ Step 14650/44000 | Loss: 3.0245 | LR: 3.83e-04 | 1409.3ms/step | 14.2 samples/s
187
+ Step 14660/44000 | Loss: 3.0410 | LR: 3.83e-04 | 1406.0ms/step | 14.2 samples/s
188
+ Step 14670/44000 | Loss: 3.0240 | LR: 3.83e-04 | 1404.7ms/step | 14.2 samples/s
189
+ Step 14680/44000 | Loss: 3.0410 | LR: 3.83e-04 | 1407.7ms/step | 14.2 samples/s
190
+ Step 14690/44000 | Loss: 3.0278 | LR: 3.82e-04 | 1407.8ms/step | 14.2 samples/s
191
+ Step 14700/44000 | Loss: 2.9932 | LR: 3.82e-04 | 1404.6ms/step | 14.2 samples/s
192
+ Step 14710/44000 | Loss: 3.0281 | LR: 3.82e-04 | 1406.0ms/step | 14.2 samples/s
193
+ Step 14720/44000 | Loss: 3.0153 | LR: 3.82e-04 | 1411.6ms/step | 14.2 samples/s
194
+ Step 14730/44000 | Loss: 3.0230 | LR: 3.82e-04 | 1406.4ms/step | 14.2 samples/s
195
+ Step 14740/44000 | Loss: 3.0272 | LR: 3.82e-04 | 1406.3ms/step | 14.2 samples/s
196
+ Step 14750/44000 | Loss: 3.0469 | LR: 3.81e-04 | 1407.6ms/step | 14.2 samples/s
197
+ Step 14760/44000 | Loss: 3.0199 | LR: 3.81e-04 | 1409.7ms/step | 14.2 samples/s
198
+ Step 14770/44000 | Loss: 3.0266 | LR: 3.81e-04 | 1406.0ms/step | 14.2 samples/s
199
+ Step 14780/44000 | Loss: 3.0151 | LR: 3.81e-04 | 1404.4ms/step | 14.2 samples/s
200
+ Step 14790/44000 | Loss: 3.0456 | LR: 3.81e-04 | 1403.9ms/step | 14.2 samples/s
201
+ Step 14800/44000 | Loss: 3.0314 | LR: 3.81e-04 | 1406.3ms/step | 14.2 samples/s
202
+ Step 14810/44000 | Loss: 3.0352 | LR: 3.81e-04 | 1407.0ms/step | 14.2 samples/s
203
+ Step 14820/44000 | Loss: 3.0393 | LR: 3.80e-04 | 1404.2ms/step | 14.2 samples/s
204
+ Step 14830/44000 | Loss: 2.9928 | LR: 3.80e-04 | 1409.9ms/step | 14.2 samples/s
205
+ Step 14840/44000 | Loss: 3.0206 | LR: 3.80e-04 | 1407.5ms/step | 14.2 samples/s
206
+ Step 14850/44000 | Loss: 3.0378 | LR: 3.80e-04 | 1407.4ms/step | 14.2 samples/s
207
+ Step 14860/44000 | Loss: 3.0264 | LR: 3.80e-04 | 1405.5ms/step | 14.2 samples/s
208
+ Step 14870/44000 | Loss: 3.0054 | LR: 3.80e-04 | 1404.2ms/step | 14.2 samples/s
209
+ Step 14880/44000 | Loss: 3.0470 | LR: 3.79e-04 | 1407.6ms/step | 14.2 samples/s
210
+ Step 14890/44000 | Loss: 3.0423 | LR: 3.79e-04 | 1404.8ms/step | 14.2 samples/s
211
+ Step 14900/44000 | Loss: 3.0194 | LR: 3.79e-04 | 1405.0ms/step | 14.2 samples/s
212
+ Step 14910/44000 | Loss: 3.0062 | LR: 3.79e-04 | 1408.3ms/step | 14.2 samples/s
213
+ Step 14920/44000 | Loss: 3.0421 | LR: 3.79e-04 | 1405.9ms/step | 14.2 samples/s
214
+ Step 14930/44000 | Loss: 3.0251 | LR: 3.79e-04 | 1405.8ms/step | 14.2 samples/s
215
+ Step 14940/44000 | Loss: 3.0564 | LR: 3.79e-04 | 1408.3ms/step | 14.2 samples/s
216
+ Step 14950/44000 | Loss: 3.0071 | LR: 3.78e-04 | 1405.8ms/step | 14.2 samples/s
217
+ Step 14960/44000 | Loss: 3.0480 | LR: 3.78e-04 | 1404.5ms/step | 14.2 samples/s
218
+ Step 14970/44000 | Loss: 3.0143 | LR: 3.78e-04 | 1405.7ms/step | 14.2 samples/s
219
+ Step 14980/44000 | Loss: 3.0089 | LR: 3.78e-04 | 1406.1ms/step | 14.2 samples/s
220
+ Step 14990/44000 | Loss: 3.0396 | LR: 3.78e-04 | 1407.0ms/step | 14.2 samples/s
221
+ Step 15000/44000 | Loss: 3.0423 | LR: 3.78e-04 | 1405.0ms/step | 14.2 samples/s
222
+ Validation at step 15000: Loss = 3.0311, Perplexity = 20.72
223
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-15000
224
+ Step 15010/44000 | Loss: 3.0371 | LR: 3.77e-04 | 3929.6ms/step | 5.1 samples/s
225
+ Step 15020/44000 | Loss: 3.0466 | LR: 3.77e-04 | 1400.1ms/step | 14.3 samples/s
226
+ Step 15030/44000 | Loss: 3.0087 | LR: 3.77e-04 | 1400.1ms/step | 14.3 samples/s
227
+ Step 15040/44000 | Loss: 3.0522 | LR: 3.77e-04 | 1402.1ms/step | 14.3 samples/s
228
+ Step 15050/44000 | Loss: 3.0278 | LR: 3.77e-04 | 1401.8ms/step | 14.3 samples/s
229
+ Step 15060/44000 | Loss: 3.0584 | LR: 3.77e-04 | 1404.2ms/step | 14.2 samples/s
230
+ Step 15070/44000 | Loss: 3.0250 | LR: 3.77e-04 | 1406.1ms/step | 14.2 samples/s
231
+ Step 15080/44000 | Loss: 3.0197 | LR: 3.76e-04 | 1403.7ms/step | 14.2 samples/s
232
+ Step 15090/44000 | Loss: 3.0212 | LR: 3.76e-04 | 1404.6ms/step | 14.2 samples/s
233
+ Step 15100/44000 | Loss: 3.0223 | LR: 3.76e-04 | 1401.9ms/step | 14.3 samples/s
234
+ Step 15110/44000 | Loss: 3.0271 | LR: 3.76e-04 | 1403.0ms/step | 14.3 samples/s
235
+ Step 15120/44000 | Loss: 3.0168 | LR: 3.76e-04 | 1402.9ms/step | 14.3 samples/s
236
+ Step 15130/44000 | Loss: 3.0303 | LR: 3.76e-04 | 1404.0ms/step | 14.2 samples/s
237
+ Step 15140/44000 | Loss: 3.0492 | LR: 3.75e-04 | 1407.5ms/step | 14.2 samples/s
238
+ Step 15150/44000 | Loss: 2.9950 | LR: 3.75e-04 | 1404.1ms/step | 14.2 samples/s
239
+ Step 15160/44000 | Loss: 3.0116 | LR: 3.75e-04 | 1405.6ms/step | 14.2 samples/s
240
+ Step 15170/44000 | Loss: 3.0240 | LR: 3.75e-04 | 1404.6ms/step | 14.2 samples/s
241
+ Step 15180/44000 | Loss: 3.0313 | LR: 3.75e-04 | 1405.3ms/step | 14.2 samples/s
242
+ Step 15190/44000 | Loss: 3.0341 | LR: 3.75e-04 | 1403.4ms/step | 14.3 samples/s
243
+ Step 15200/44000 | Loss: 3.0550 | LR: 3.74e-04 | 1404.8ms/step | 14.2 samples/s
244
+ Step 15210/44000 | Loss: 3.0267 | LR: 3.74e-04 | 1405.4ms/step | 14.2 samples/s
245
+ Step 15220/44000 | Loss: 3.0085 | LR: 3.74e-04 | 1408.8ms/step | 14.2 samples/s
246
+ Step 15230/44000 | Loss: 3.0154 | LR: 3.74e-04 | 1408.6ms/step | 14.2 samples/s
247
+ Step 15240/44000 | Loss: 3.0402 | LR: 3.74e-04 | 1405.5ms/step | 14.2 samples/s
248
+ Step 15250/44000 | Loss: 3.0188 | LR: 3.74e-04 | 1404.4ms/step | 14.2 samples/s
249
+ Step 15260/44000 | Loss: 3.0388 | LR: 3.74e-04 | 1406.5ms/step | 14.2 samples/s
250
+ Step 15270/44000 | Loss: 3.0418 | LR: 3.73e-04 | 1408.5ms/step | 14.2 samples/s
251
+ Step 15280/44000 | Loss: 3.0311 | LR: 3.73e-04 | 1408.9ms/step | 14.2 samples/s
252
+ Step 15290/44000 | Loss: 3.0193 | LR: 3.73e-04 | 1408.9ms/step | 14.2 samples/s
253
+ Step 15300/44000 | Loss: 3.0369 | LR: 3.73e-04 | 1404.6ms/step | 14.2 samples/s
254
+ Step 15310/44000 | Loss: 2.9981 | LR: 3.73e-04 | 1406.1ms/step | 14.2 samples/s
255
+ Step 15320/44000 | Loss: 3.0409 | LR: 3.73e-04 | 1406.1ms/step | 14.2 samples/s
256
+ Step 15330/44000 | Loss: 3.0352 | LR: 3.72e-04 | 1406.4ms/step | 14.2 samples/s
257
+ Step 15340/44000 | Loss: 3.0447 | LR: 3.72e-04 | 1406.2ms/step | 14.2 samples/s
258
+ Step 15350/44000 | Loss: 3.0216 | LR: 3.72e-04 | 1407.1ms/step | 14.2 samples/s
259
+ Step 15360/44000 | Loss: 3.0293 | LR: 3.72e-04 | 1402.5ms/step | 14.3 samples/s
260
+ Step 15370/44000 | Loss: 3.0100 | LR: 3.72e-04 | 1405.8ms/step | 14.2 samples/s
261
+ Step 15380/44000 | Loss: 3.0581 | LR: 3.72e-04 | 1404.8ms/step | 14.2 samples/s
262
+ Step 15390/44000 | Loss: 3.0117 | LR: 3.71e-04 | 1403.8ms/step | 14.2 samples/s
263
+ Step 15400/44000 | Loss: 3.0380 | LR: 3.71e-04 | 1405.7ms/step | 14.2 samples/s
264
+ Step 15410/44000 | Loss: 3.0102 | LR: 3.71e-04 | 1408.0ms/step | 14.2 samples/s
265
+ Step 15420/44000 | Loss: 3.0234 | LR: 3.71e-04 | 1405.6ms/step | 14.2 samples/s
266
+ Step 15430/44000 | Loss: 3.0101 | LR: 3.71e-04 | 1406.1ms/step | 14.2 samples/s
267
+ Step 15440/44000 | Loss: 3.0203 | LR: 3.71e-04 | 1405.4ms/step | 14.2 samples/s
268
+ Step 15450/44000 | Loss: 3.0209 | LR: 3.71e-04 | 1404.7ms/step | 14.2 samples/s
269
+ Step 15460/44000 | Loss: 3.0301 | LR: 3.70e-04 | 1406.3ms/step | 14.2 samples/s
270
+ Step 15470/44000 | Loss: 3.0357 | LR: 3.70e-04 | 1405.3ms/step | 14.2 samples/s
271
+ Step 15480/44000 | Loss: 3.0228 | LR: 3.70e-04 | 1404.1ms/step | 14.2 samples/s
272
+ Step 15490/44000 | Loss: 3.0229 | LR: 3.70e-04 | 1406.1ms/step | 14.2 samples/s
273
+ Step 15500/44000 | Loss: 3.0138 | LR: 3.70e-04 | 1405.9ms/step | 14.2 samples/s
274
+ Step 15510/44000 | Loss: 3.0343 | LR: 3.70e-04 | 1408.6ms/step | 14.2 samples/s
275
+ Step 15520/44000 | Loss: 3.0124 | LR: 3.69e-04 | 1407.6ms/step | 14.2 samples/s
276
+ Step 15530/44000 | Loss: 3.0176 | LR: 3.69e-04 | 1406.2ms/step | 14.2 samples/s
277
+ Step 15540/44000 | Loss: 3.0415 | LR: 3.69e-04 | 1407.5ms/step | 14.2 samples/s
278
+ Step 15550/44000 | Loss: 2.9999 | LR: 3.69e-04 | 1405.8ms/step | 14.2 samples/s
279
+ Step 15560/44000 | Loss: 3.0004 | LR: 3.69e-04 | 1406.5ms/step | 14.2 samples/s
280
+ Step 15570/44000 | Loss: 3.0474 | LR: 3.69e-04 | 1404.4ms/step | 14.2 samples/s
281
+ Step 15580/44000 | Loss: 2.9992 | LR: 3.68e-04 | 1406.0ms/step | 14.2 samples/s
282
+ Step 15590/44000 | Loss: 3.0146 | LR: 3.68e-04 | 1405.3ms/step | 14.2 samples/s
283
+ Step 15600/44000 | Loss: 3.0301 | LR: 3.68e-04 | 1404.5ms/step | 14.2 samples/s
284
+ Step 15610/44000 | Loss: 2.9960 | LR: 3.68e-04 | 1405.7ms/step | 14.2 samples/s
285
+ Step 15620/44000 | Loss: 2.9921 | LR: 3.68e-04 | 1407.9ms/step | 14.2 samples/s
286
+ Step 15630/44000 | Loss: 3.0463 | LR: 3.68e-04 | 1406.7ms/step | 14.2 samples/s
287
+ Step 15640/44000 | Loss: 3.0026 | LR: 3.67e-04 | 1407.9ms/step | 14.2 samples/s
288
+ Step 15650/44000 | Loss: 3.0406 | LR: 3.67e-04 | 1405.9ms/step | 14.2 samples/s
289
+ Step 15660/44000 | Loss: 3.0052 | LR: 3.67e-04 | 1407.0ms/step | 14.2 samples/s
290
+ Step 15670/44000 | Loss: 3.0266 | LR: 3.67e-04 | 1404.4ms/step | 14.2 samples/s
291
+ Step 15680/44000 | Loss: 3.0003 | LR: 3.67e-04 | 1408.0ms/step | 14.2 samples/s
292
+ Step 15690/44000 | Loss: 3.0145 | LR: 3.67e-04 | 1410.5ms/step | 14.2 samples/s
293
+ Step 15700/44000 | Loss: 3.0214 | LR: 3.67e-04 | 1407.5ms/step | 14.2 samples/s
294
+ Step 15710/44000 | Loss: 3.0644 | LR: 3.66e-04 | 1405.8ms/step | 14.2 samples/s
295
+ Step 15720/44000 | Loss: 2.9972 | LR: 3.66e-04 | 1405.9ms/step | 14.2 samples/s
296
+ Step 15730/44000 | Loss: 3.0144 | LR: 3.66e-04 | 1410.0ms/step | 14.2 samples/s
297
+ Step 15740/44000 | Loss: 3.0384 | LR: 3.66e-04 | 1410.4ms/step | 14.2 samples/s
298
+ Step 15750/44000 | Loss: 3.0303 | LR: 3.66e-04 | 1410.7ms/step | 14.2 samples/s
299
+ Step 15760/44000 | Loss: 3.0438 | LR: 3.66e-04 | 1406.7ms/step | 14.2 samples/s
300
+ Step 15770/44000 | Loss: 3.0205 | LR: 3.65e-04 | 1404.9ms/step | 14.2 samples/s
301
+ Step 15780/44000 | Loss: 3.0505 | LR: 3.65e-04 | 1407.6ms/step | 14.2 samples/s
302
+ Step 15790/44000 | Loss: 2.9942 | LR: 3.65e-04 | 1405.6ms/step | 14.2 samples/s
303
+ Step 15800/44000 | Loss: 3.0126 | LR: 3.65e-04 | 1405.8ms/step | 14.2 samples/s
304
+ Step 15810/44000 | Loss: 3.0108 | LR: 3.65e-04 | 1405.2ms/step | 14.2 samples/s
305
+ Step 15820/44000 | Loss: 2.9895 | LR: 3.65e-04 | 1405.7ms/step | 14.2 samples/s
306
+ Step 15830/44000 | Loss: 3.0383 | LR: 3.64e-04 | 1405.6ms/step | 14.2 samples/s
307
+ Step 15840/44000 | Loss: 3.0233 | LR: 3.64e-04 | 1406.5ms/step | 14.2 samples/s
308
+ Step 15850/44000 | Loss: 3.0265 | LR: 3.64e-04 | 1406.5ms/step | 14.2 samples/s
309
+ Step 15860/44000 | Loss: 2.9929 | LR: 3.64e-04 | 1406.4ms/step | 14.2 samples/s
310
+ Step 15870/44000 | Loss: 3.0140 | LR: 3.64e-04 | 1404.6ms/step | 14.2 samples/s
311
+ Step 15880/44000 | Loss: 3.0298 | LR: 3.64e-04 | 1408.2ms/step | 14.2 samples/s
312
+ Step 15890/44000 | Loss: 3.0192 | LR: 3.63e-04 | 1408.4ms/step | 14.2 samples/s
313
+ Step 15900/44000 | Loss: 3.0388 | LR: 3.63e-04 | 1409.1ms/step | 14.2 samples/s
314
+ Step 15910/44000 | Loss: 3.0177 | LR: 3.63e-04 | 1406.7ms/step | 14.2 samples/s
315
+ Step 15920/44000 | Loss: 3.0049 | LR: 3.63e-04 | 1407.4ms/step | 14.2 samples/s
316
+ Step 15930/44000 | Loss: 3.0327 | LR: 3.63e-04 | 1406.3ms/step | 14.2 samples/s
317
+ Step 15940/44000 | Loss: 3.0011 | LR: 3.63e-04 | 1404.4ms/step | 14.2 samples/s
318
+ Step 15950/44000 | Loss: 3.0167 | LR: 3.62e-04 | 1405.1ms/step | 14.2 samples/s
319
+ Step 15960/44000 | Loss: 2.9896 | LR: 3.62e-04 | 1406.8ms/step | 14.2 samples/s
320
+ Step 15970/44000 | Loss: 3.0153 | LR: 3.62e-04 | 1406.0ms/step | 14.2 samples/s
321
+ Step 15980/44000 | Loss: 3.0171 | LR: 3.62e-04 | 1407.1ms/step | 14.2 samples/s
322
+ Step 15990/44000 | Loss: 3.0024 | LR: 3.62e-04 | 1400.0ms/step | 14.3 samples/s
323
+ Step 16000/44000 | Loss: 3.0164 | LR: 3.62e-04 | 1400.2ms/step | 14.3 samples/s
324
+ Validation at step 16000: Loss = 3.0182, Perplexity = 20.45
325
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-16000
326
+ Step 16010/44000 | Loss: 2.9981 | LR: 3.62e-04 | 3945.8ms/step | 5.1 samples/s
327
+ Step 16020/44000 | Loss: 2.9873 | LR: 3.61e-04 | 1399.8ms/step | 14.3 samples/s
328
+ Step 16030/44000 | Loss: 2.9954 | LR: 3.61e-04 | 1400.0ms/step | 14.3 samples/s
329
+ Step 16040/44000 | Loss: 3.0271 | LR: 3.61e-04 | 1400.3ms/step | 14.3 samples/s
330
+ Step 16050/44000 | Loss: 3.0306 | LR: 3.61e-04 | 1399.8ms/step | 14.3 samples/s
331
+ Step 16060/44000 | Loss: 3.0192 | LR: 3.61e-04 | 1400.3ms/step | 14.3 samples/s
332
+ Step 16070/44000 | Loss: 3.0197 | LR: 3.61e-04 | 1400.3ms/step | 14.3 samples/s
333
+ Step 16080/44000 | Loss: 3.0014 | LR: 3.60e-04 | 1401.9ms/step | 14.3 samples/s
334
+ Step 16090/44000 | Loss: 2.9861 | LR: 3.60e-04 | 1403.8ms/step | 14.2 samples/s
335
+ Step 16100/44000 | Loss: 3.0215 | LR: 3.60e-04 | 1401.8ms/step | 14.3 samples/s
336
+ Step 16110/44000 | Loss: 3.0163 | LR: 3.60e-04 | 1402.9ms/step | 14.3 samples/s
337
+ Step 16120/44000 | Loss: 2.9848 | LR: 3.60e-04 | 1405.4ms/step | 14.2 samples/s
338
+ Step 16130/44000 | Loss: 3.0070 | LR: 3.60e-04 | 1404.1ms/step | 14.2 samples/s
339
+ Step 16140/44000 | Loss: 3.0463 | LR: 3.59e-04 | 1405.9ms/step | 14.2 samples/s
340
+ Step 16150/44000 | Loss: 3.0108 | LR: 3.59e-04 | 1403.7ms/step | 14.2 samples/s
341
+ Step 16160/44000 | Loss: 3.0066 | LR: 3.59e-04 | 1404.4ms/step | 14.2 samples/s
342
+ Step 16170/44000 | Loss: 3.0117 | LR: 3.59e-04 | 1402.0ms/step | 14.3 samples/s
343
+ Step 16180/44000 | Loss: 3.0272 | LR: 3.59e-04 | 1404.3ms/step | 14.2 samples/s
344
+ Step 16190/44000 | Loss: 3.0013 | LR: 3.59e-04 | 1404.4ms/step | 14.2 samples/s
345
+ Step 16200/44000 | Loss: 2.9888 | LR: 3.58e-04 | 1407.0ms/step | 14.2 samples/s
346
+ Step 16210/44000 | Loss: 3.0218 | LR: 3.58e-04 | 1404.4ms/step | 14.2 samples/s
347
+ Step 16220/44000 | Loss: 3.0274 | LR: 3.58e-04 | 1402.2ms/step | 14.3 samples/s
348
+ Step 16230/44000 | Loss: 2.9845 | LR: 3.58e-04 | 1403.6ms/step | 14.2 samples/s
349
+ Step 16240/44000 | Loss: 3.0240 | LR: 3.58e-04 | 1403.7ms/step | 14.2 samples/s
350
+ Step 16250/44000 | Loss: 3.0101 | LR: 3.58e-04 | 1402.2ms/step | 14.3 samples/s
351
+ Step 16260/44000 | Loss: 2.9791 | LR: 3.57e-04 | 1401.7ms/step | 14.3 samples/s
352
+ Step 16270/44000 | Loss: 3.0358 | LR: 3.57e-04 | 1402.1ms/step | 14.3 samples/s
353
+ Step 16280/44000 | Loss: 3.0304 | LR: 3.57e-04 | 1406.9ms/step | 14.2 samples/s
354
+ Step 16290/44000 | Loss: 3.0055 | LR: 3.57e-04 | 1406.8ms/step | 14.2 samples/s
355
+ Step 16300/44000 | Loss: 3.0490 | LR: 3.57e-04 | 1406.7ms/step | 14.2 samples/s
356
+ Step 16310/44000 | Loss: 2.9993 | LR: 3.57e-04 | 1403.7ms/step | 14.2 samples/s
357
+ Step 16320/44000 | Loss: 3.0338 | LR: 3.56e-04 | 1405.5ms/step | 14.2 samples/s
358
+ Step 16330/44000 | Loss: 3.0121 | LR: 3.56e-04 | 1402.6ms/step | 14.3 samples/s
359
+ Step 16340/44000 | Loss: 3.0107 | LR: 3.56e-04 | 1404.7ms/step | 14.2 samples/s
360
+ Step 16350/44000 | Loss: 3.0077 | LR: 3.56e-04 | 1403.5ms/step | 14.3 samples/s
361
+ Step 16360/44000 | Loss: 3.0100 | LR: 3.56e-04 | 1406.1ms/step | 14.2 samples/s
362
+ Step 16370/44000 | Loss: 3.0233 | LR: 3.56e-04 | 1406.8ms/step | 14.2 samples/s
363
+ Step 16380/44000 | Loss: 2.9960 | LR: 3.55e-04 | 1407.5ms/step | 14.2 samples/s
364
+ Step 16390/44000 | Loss: 3.0204 | LR: 3.55e-04 | 1407.0ms/step | 14.2 samples/s
365
+ Step 16400/44000 | Loss: 3.0122 | LR: 3.55e-04 | 1404.4ms/step | 14.2 samples/s
366
+ Step 16410/44000 | Loss: 3.0322 | LR: 3.55e-04 | 1403.5ms/step | 14.2 samples/s
367
+ Step 16420/44000 | Loss: 3.0079 | LR: 3.55e-04 | 1402.1ms/step | 14.3 samples/s
368
+ Step 16430/44000 | Loss: 2.9612 | LR: 3.55e-04 | 1402.6ms/step | 14.3 samples/s
369
+ Step 16440/44000 | Loss: 3.0090 | LR: 3.54e-04 | 1405.5ms/step | 14.2 samples/s
370
+ Step 16450/44000 | Loss: 3.0478 | LR: 3.54e-04 | 1404.1ms/step | 14.2 samples/s
371
+ Step 16460/44000 | Loss: 2.9971 | LR: 3.54e-04 | 1402.5ms/step | 14.3 samples/s
372
+ Step 16470/44000 | Loss: 3.0192 | LR: 3.54e-04 | 1403.2ms/step | 14.3 samples/s
373
+ Step 16480/44000 | Loss: 2.9986 | LR: 3.54e-04 | 1400.5ms/step | 14.3 samples/s
374
+ Step 16490/44000 | Loss: 3.0094 | LR: 3.54e-04 | 1403.8ms/step | 14.2 samples/s
375
+ Step 16500/44000 | Loss: 2.9890 | LR: 3.53e-04 | 1404.4ms/step | 14.2 samples/s
376
+ Step 16510/44000 | Loss: 2.9981 | LR: 3.53e-04 | 1403.8ms/step | 14.2 samples/s
377
+ Step 16520/44000 | Loss: 3.0088 | LR: 3.53e-04 | 1406.4ms/step | 14.2 samples/s
378
+ Step 16530/44000 | Loss: 3.0235 | LR: 3.53e-04 | 1406.9ms/step | 14.2 samples/s
379
+ Step 16540/44000 | Loss: 3.0084 | LR: 3.53e-04 | 1404.3ms/step | 14.2 samples/s
380
+ Step 16550/44000 | Loss: 3.0437 | LR: 3.53e-04 | 1406.2ms/step | 14.2 samples/s
381
+ Step 16560/44000 | Loss: 3.0119 | LR: 3.52e-04 | 1407.5ms/step | 14.2 samples/s
382
+ Step 16570/44000 | Loss: 3.0102 | LR: 3.52e-04 | 1407.9ms/step | 14.2 samples/s
383
+ Step 16580/44000 | Loss: 2.9882 | LR: 3.52e-04 | 1404.5ms/step | 14.2 samples/s
384
+ Step 16590/44000 | Loss: 2.9727 | LR: 3.52e-04 | 1407.3ms/step | 14.2 samples/s
385
+ Step 16600/44000 | Loss: 3.0048 | LR: 3.52e-04 | 1404.1ms/step | 14.2 samples/s
386
+ Step 16610/44000 | Loss: 2.9878 | LR: 3.52e-04 | 1406.3ms/step | 14.2 samples/s
387
+ Step 16620/44000 | Loss: 3.0154 | LR: 3.51e-04 | 1408.2ms/step | 14.2 samples/s
388
+ Step 16630/44000 | Loss: 2.9833 | LR: 3.51e-04 | 1408.2ms/step | 14.2 samples/s
389
+ Step 16640/44000 | Loss: 3.0043 | LR: 3.51e-04 | 1408.2ms/step | 14.2 samples/s
390
+ Step 16650/44000 | Loss: 3.0141 | LR: 3.51e-04 | 1407.1ms/step | 14.2 samples/s
391
+ Step 16660/44000 | Loss: 3.0043 | LR: 3.51e-04 | 1406.2ms/step | 14.2 samples/s
392
+ Step 16670/44000 | Loss: 3.0042 | LR: 3.51e-04 | 1406.0ms/step | 14.2 samples/s
393
+ Step 16680/44000 | Loss: 3.0202 | LR: 3.50e-04 | 1404.1ms/step | 14.2 samples/s
394
+ Step 16690/44000 | Loss: 2.9715 | LR: 3.50e-04 | 1404.1ms/step | 14.2 samples/s
395
+ Step 16700/44000 | Loss: 2.9954 | LR: 3.50e-04 | 1403.1ms/step | 14.3 samples/s
396
+ Step 16710/44000 | Loss: 2.9902 | LR: 3.50e-04 | 1404.2ms/step | 14.2 samples/s
397
+ Step 16720/44000 | Loss: 3.0234 | LR: 3.50e-04 | 1400.8ms/step | 14.3 samples/s
398
+ Step 16730/44000 | Loss: 3.0029 | LR: 3.50e-04 | 1405.7ms/step | 14.2 samples/s
399
+ Step 16740/44000 | Loss: 3.0064 | LR: 3.49e-04 | 1407.8ms/step | 14.2 samples/s
400
+ Step 16750/44000 | Loss: 2.9984 | LR: 3.49e-04 | 1406.1ms/step | 14.2 samples/s
401
+ Step 16760/44000 | Loss: 3.0193 | LR: 3.49e-04 | 1404.6ms/step | 14.2 samples/s
402
+ Step 16770/44000 | Loss: 3.0108 | LR: 3.49e-04 | 1407.1ms/step | 14.2 samples/s
403
+ Step 16780/44000 | Loss: 3.0096 | LR: 3.49e-04 | 1406.3ms/step | 14.2 samples/s
404
+ Step 16790/44000 | Loss: 2.9972 | LR: 3.49e-04 | 1406.9ms/step | 14.2 samples/s
405
+ Step 16800/44000 | Loss: 3.0061 | LR: 3.48e-04 | 1405.3ms/step | 14.2 samples/s
406
+ Step 16810/44000 | Loss: 3.0070 | LR: 3.48e-04 | 1406.7ms/step | 14.2 samples/s
407
+ Step 16820/44000 | Loss: 3.0215 | LR: 3.48e-04 | 1407.0ms/step | 14.2 samples/s
408
+ Step 16830/44000 | Loss: 2.9923 | LR: 3.48e-04 | 1405.9ms/step | 14.2 samples/s
409
+ Step 16840/44000 | Loss: 3.0203 | LR: 3.48e-04 | 1406.4ms/step | 14.2 samples/s
410
+ Step 16850/44000 | Loss: 3.0243 | LR: 3.48e-04 | 1405.9ms/step | 14.2 samples/s
411
+ Step 16860/44000 | Loss: 3.0320 | LR: 3.47e-04 | 1407.7ms/step | 14.2 samples/s
412
+ Step 16870/44000 | Loss: 3.0149 | LR: 3.47e-04 | 1407.5ms/step | 14.2 samples/s
413
+ Step 16880/44000 | Loss: 3.0218 | LR: 3.47e-04 | 1406.8ms/step | 14.2 samples/s
414
+ Step 16890/44000 | Loss: 2.9908 | LR: 3.47e-04 | 1406.9ms/step | 14.2 samples/s
415
+ Step 16900/44000 | Loss: 2.9648 | LR: 3.47e-04 | 1407.7ms/step | 14.2 samples/s
416
+ Step 16910/44000 | Loss: 3.0129 | LR: 3.47e-04 | 1411.1ms/step | 14.2 samples/s
417
+ Step 16920/44000 | Loss: 2.9968 | LR: 3.46e-04 | 1408.1ms/step | 14.2 samples/s
418
+ Step 16930/44000 | Loss: 3.0167 | LR: 3.46e-04 | 1406.3ms/step | 14.2 samples/s
419
+ Step 16940/44000 | Loss: 3.0183 | LR: 3.46e-04 | 1402.4ms/step | 14.3 samples/s
420
+ Step 16950/44000 | Loss: 3.0071 | LR: 3.46e-04 | 1402.8ms/step | 14.3 samples/s
421
+ Step 16960/44000 | Loss: 3.0012 | LR: 3.46e-04 | 1405.0ms/step | 14.2 samples/s
422
+ Step 16970/44000 | Loss: 3.0226 | LR: 3.46e-04 | 1401.1ms/step | 14.3 samples/s
423
+ Step 16980/44000 | Loss: 3.0106 | LR: 3.45e-04 | 1400.0ms/step | 14.3 samples/s
424
+ Step 16990/44000 | Loss: 2.9798 | LR: 3.45e-04 | 1399.9ms/step | 14.3 samples/s
425
+ Step 17000/44000 | Loss: 3.0017 | LR: 3.45e-04 | 1400.2ms/step | 14.3 samples/s
426
+ Validation at step 17000: Loss = 3.0056, Perplexity = 20.20
427
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-17000
428
+ Step 17010/44000 | Loss: 3.0082 | LR: 3.45e-04 | 4014.1ms/step | 5.0 samples/s
429
+ Step 17020/44000 | Loss: 2.9929 | LR: 3.45e-04 | 1399.8ms/step | 14.3 samples/s
430
+ Step 17030/44000 | Loss: 2.9992 | LR: 3.45e-04 | 1401.5ms/step | 14.3 samples/s
431
+ Step 17040/44000 | Loss: 3.0050 | LR: 3.44e-04 | 1402.9ms/step | 14.3 samples/s
432
+ Step 17050/44000 | Loss: 3.0076 | LR: 3.44e-04 | 1402.4ms/step | 14.3 samples/s
433
+ Step 17060/44000 | Loss: 2.9949 | LR: 3.44e-04 | 1403.8ms/step | 14.2 samples/s
434
+ Step 17070/44000 | Loss: 3.0166 | LR: 3.44e-04 | 1402.1ms/step | 14.3 samples/s
435
+ Step 17080/44000 | Loss: 3.0174 | LR: 3.44e-04 | 1406.3ms/step | 14.2 samples/s
436
+ Step 17090/44000 | Loss: 3.0093 | LR: 3.44e-04 | 1406.6ms/step | 14.2 samples/s
437
+ Step 17100/44000 | Loss: 3.0086 | LR: 3.43e-04 | 1404.8ms/step | 14.2 samples/s
438
+ Step 17110/44000 | Loss: 2.9833 | LR: 3.43e-04 | 1406.1ms/step | 14.2 samples/s
439
+ Step 17120/44000 | Loss: 3.0130 | LR: 3.43e-04 | 1406.0ms/step | 14.2 samples/s
440
+ Step 17130/44000 | Loss: 3.0143 | LR: 3.43e-04 | 1406.8ms/step | 14.2 samples/s
441
+ Step 17140/44000 | Loss: 2.9852 | LR: 3.43e-04 | 1409.3ms/step | 14.2 samples/s
442
+ Step 17150/44000 | Loss: 2.9984 | LR: 3.43e-04 | 1408.4ms/step | 14.2 samples/s
443
+ Step 17160/44000 | Loss: 2.9864 | LR: 3.42e-04 | 1407.1ms/step | 14.2 samples/s
444
+ Step 17170/44000 | Loss: 3.0419 | LR: 3.42e-04 | 1406.2ms/step | 14.2 samples/s
445
+ Step 17180/44000 | Loss: 2.9945 | LR: 3.42e-04 | 1406.3ms/step | 14.2 samples/s
446
+ Step 17190/44000 | Loss: 3.0120 | LR: 3.42e-04 | 1406.6ms/step | 14.2 samples/s
447
+ Step 17200/44000 | Loss: 3.0314 | LR: 3.42e-04 | 1409.0ms/step | 14.2 samples/s
448
+ Step 17210/44000 | Loss: 3.0222 | LR: 3.42e-04 | 1407.1ms/step | 14.2 samples/s
449
+ Step 17220/44000 | Loss: 3.0067 | LR: 3.41e-04 | 1407.3ms/step | 14.2 samples/s
450
+ Step 17230/44000 | Loss: 2.9997 | LR: 3.41e-04 | 1408.2ms/step | 14.2 samples/s
451
+ Step 17240/44000 | Loss: 3.0174 | LR: 3.41e-04 | 1409.3ms/step | 14.2 samples/s
452
+ Step 17250/44000 | Loss: 2.9681 | LR: 3.41e-04 | 1406.5ms/step | 14.2 samples/s
453
+ Step 17260/44000 | Loss: 2.9908 | LR: 3.41e-04 | 1408.4ms/step | 14.2 samples/s
454
+ Step 17270/44000 | Loss: 2.9869 | LR: 3.41e-04 | 1409.2ms/step | 14.2 samples/s
455
+ Step 17280/44000 | Loss: 2.9766 | LR: 3.40e-04 | 1407.6ms/step | 14.2 samples/s
456
+ Step 17290/44000 | Loss: 2.9601 | LR: 3.40e-04 | 1408.6ms/step | 14.2 samples/s
457
+ Step 17300/44000 | Loss: 2.9960 | LR: 3.40e-04 | 1408.3ms/step | 14.2 samples/s
458
+ Step 17310/44000 | Loss: 3.0084 | LR: 3.40e-04 | 1409.2ms/step | 14.2 samples/s
459
+ Step 17320/44000 | Loss: 3.0018 | LR: 3.40e-04 | 1409.6ms/step | 14.2 samples/s
460
+ Step 17330/44000 | Loss: 2.9974 | LR: 3.40e-04 | 1410.4ms/step | 14.2 samples/s
461
+ Step 17340/44000 | Loss: 3.0035 | LR: 3.39e-04 | 1408.6ms/step | 14.2 samples/s
462
+ Step 17350/44000 | Loss: 3.0107 | LR: 3.39e-04 | 1409.7ms/step | 14.2 samples/s
463
+ Step 17360/44000 | Loss: 3.0203 | LR: 3.39e-04 | 1410.7ms/step | 14.2 samples/s
464
+ Step 17370/44000 | Loss: 3.0077 | LR: 3.39e-04 | 1407.5ms/step | 14.2 samples/s
465
+ Step 17380/44000 | Loss: 3.0024 | LR: 3.39e-04 | 1408.4ms/step | 14.2 samples/s
466
+ Step 17390/44000 | Loss: 2.9851 | LR: 3.39e-04 | 1406.1ms/step | 14.2 samples/s
467
+ Step 17400/44000 | Loss: 3.0201 | LR: 3.38e-04 | 1408.2ms/step | 14.2 samples/s
468
+ Step 17410/44000 | Loss: 2.9925 | LR: 3.38e-04 | 1409.0ms/step | 14.2 samples/s
469
+ Step 17420/44000 | Loss: 2.9969 | LR: 3.38e-04 | 1408.1ms/step | 14.2 samples/s
470
+ Step 17430/44000 | Loss: 2.9662 | LR: 3.38e-04 | 1403.5ms/step | 14.3 samples/s
471
+ Step 17440/44000 | Loss: 2.9928 | LR: 3.38e-04 | 1408.4ms/step | 14.2 samples/s
472
+ Step 17450/44000 | Loss: 2.9656 | LR: 3.38e-04 | 1407.9ms/step | 14.2 samples/s
epfl_final/SparseAtt/wandb/run-20260820_041559-xfnxdn24/files/requirements.txt ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pip==26.0.1
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.6.3
8
+ typing_extensions==4.15.0
9
+ triton==3.6.0
10
+ tqdm==4.67.3
11
+ torchinfo==1.8.0
12
+ sympy==1.14.0
13
+ smmap==5.0.3
14
+ six==1.17.0
15
+ shellingham==1.5.4
16
+ setuptools==81.0.0
17
+ safetensors==0.7.0
18
+ regex==2026.4.4
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.4.1
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.4.4
38
+ networkx==3.6.1
39
+ multidict==6.7.1
40
+ mdurl==0.1.2
41
+ MarkupSafe==3.0.3
42
+ idna==3.13
43
+ hf-xet==1.4.3
44
+ h11==0.16.0
45
+ fsspec==2026.2.0
46
+ frozenlist==1.8.0
47
+ filelock==3.29.0
48
+ dill==0.4.1
49
+ cuda-pathfinder==1.5.4
50
+ click==8.3.3
51
+ charset-normalizer==3.4.7
52
+ certifi==2026.4.22
53
+ attrs==26.1.0
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ yarl==1.23.0
58
+ typing-inspection==0.4.2
59
+ sentry-sdk==2.58.0
60
+ requests==2.33.1
61
+ python-dateutil==2.9.0.post0
62
+ pydantic_core==2.46.3
63
+ nvidia-cusparse==12.6.3.3
64
+ nvidia-cufft==12.0.0.61
65
+ nvidia-cudnn-cu13==9.19.0.56
66
+ multiprocess==0.70.19
67
+ markdown-it-py==4.0.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ cuda-bindings==13.2.0
72
+ anyio==4.13.0
73
+ aiosignal==1.4.0
74
+ rich==15.0.0
75
+ pydantic==2.13.3
76
+ pandas==3.0.2
77
+ nvidia-cusolver==12.0.4.66
78
+ httpx==0.28.1
79
+ GitPython==3.1.49
80
+ aiohttp==3.13.5
81
+ wandb==0.26.1
82
+ typer==0.25.1
83
+ torch==2.11.0
84
+ huggingface_hub==1.13.0
85
+ tokenizers==0.22.2
86
+ datasets==4.8.5
87
+ accelerate==1.13.0
88
+ transformers==5.7.0
89
+ word2number==1.1
90
+ sqlitedict==2.1.0
91
+ pytz==2026.2
92
+ threadpoolctl==3.6.0
93
+ tcolorpy==0.1.7
94
+ tabulate==0.10.0
95
+ scipy==1.17.1
96
+ portalocker==3.2.0
97
+ pathvalidate==3.3.1
98
+ narwhals==2.22.1
99
+ more-itertools==11.1.0
100
+ lxml==6.1.1
101
+ joblib==1.5.3
102
+ colorama==0.4.6
103
+ chardet==6.0.0.post1
104
+ absl-py==2.4.0
105
+ scikit-learn==1.9.0
106
+ sacrebleu==2.6.0
107
+ nltk==3.9.4
108
+ mbstrdecoder==1.1.5
109
+ typepy==1.3.5
110
+ rouge_score==0.1.2
111
+ DataProperty==1.1.1
112
+ tabledata==1.3.5
113
+ evaluate==0.4.6
114
+ pytablewriter==1.2.1
115
+ lm_eval==0.4.12
116
+ pluggy==1.6.0
117
+ iniconfig==2.3.0
118
+ pytest==9.1.0
119
+ py-spy==0.4.2
120
+ pillow==12.2.0
121
+ tifffile==2026.6.1
122
+ imagecodecs==2026.6.26
123
+ zipp==4.1.0
124
+ importlib_metadata==9.0.0
125
+ derivative==0.6.3
126
+ pysindy==2.1.0
127
+ pyparsing==3.3.2
128
+ kiwisolver==1.5.0
129
+ fonttools==4.63.0
130
+ cycler==0.12.1
131
+ contourpy==1.3.3
132
+ matplotlib==3.11.0
133
+ sortedcontainers==2.4.0
134
+ wcwidth==0.8.2
135
+ sigfig==1.3.19
136
+ importlib_resources==7.1.0
137
+ enum-tools==0.13.0
138
+ cramjam==2.11.0
139
+ python-snappy==0.7.3
140
+ compact-json==1.8.2
141
+ numbers-parser==4.18.5
142
+ entmax==1.3
epfl_final/SparseAtt/wandb/run-20260820_041559-xfnxdn24/files/wandb-metadata.json ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.13.13",
4
+ "startedAt": "2026-08-20T04:15:59.115513Z",
5
+ "args": [
6
+ "--config",
7
+ "/mloscratch/homes/navasard/sparse/configs_ropectl/resume_a1p0_wandb/seed45.json"
8
+ ],
9
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
10
+ "codePath": "train.py",
11
+ "codePathLocal": "train.py",
12
+ "git": {
13
+ "remote": "https://github.com/rubentium/TokenPrune.git",
14
+ "commit": "5637886f0cc760009989b14e68c6ae06d4827418"
15
+ },
16
+ "email": "ruben.navasardyan@mail.utoronto.ca",
17
+ "root": "/mloscratch/homes/navasard/sparse/SparseAtt",
18
+ "host": "h100-workspace-5-gpu-0-2",
19
+ "executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
20
+ "cpu_count": 96,
21
+ "cpu_count_logical": 192,
22
+ "gpu": "NVIDIA H100 80GB HBM3",
23
+ "gpu_count": 5,
24
+ "disk": {
25
+ "/": {
26
+ "total": "9374931091456",
27
+ "used": "5285341044736"
28
+ }
29
+ },
30
+ "memory": {
31
+ "total": "1623005626368"
32
+ },
33
+ "gpu_nvidia": [
34
+ {
35
+ "name": "NVIDIA H100 80GB HBM3",
36
+ "memoryTotal": "85520809984",
37
+ "cudaCores": 16896,
38
+ "architecture": "Hopper",
39
+ "uuid": "GPU-6fb12770-6b51-9595-91c9-0d20801e3896"
40
+ },
41
+ {
42
+ "name": "NVIDIA H100 80GB HBM3",
43
+ "memoryTotal": "85520809984",
44
+ "cudaCores": 16896,
45
+ "architecture": "Hopper",
46
+ "uuid": "GPU-5cb9f8b7-4a13-4f85-d73e-6bcbebebefb8"
47
+ },
48
+ {
49
+ "name": "NVIDIA H100 80GB HBM3",
50
+ "memoryTotal": "85520809984",
51
+ "cudaCores": 16896,
52
+ "architecture": "Hopper",
53
+ "uuid": "GPU-3512802a-d501-0a3c-9812-4ab9deb3a3a9"
54
+ },
55
+ {
56
+ "name": "NVIDIA H100 80GB HBM3",
57
+ "memoryTotal": "85520809984",
58
+ "cudaCores": 16896,
59
+ "architecture": "Hopper",
60
+ "uuid": "GPU-24436def-beee-c9f1-bd05-c97df556de08"
61
+ },
62
+ {
63
+ "name": "NVIDIA H100 80GB HBM3",
64
+ "memoryTotal": "85520809984",
65
+ "cudaCores": 16896,
66
+ "architecture": "Hopper",
67
+ "uuid": "GPU-7b827c86-854a-af51-b5d3-6aaa43307d4a"
68
+ }
69
+ ],
70
+ "cudaVersion": "13.0",
71
+ "writerId": "gdnpslttvtc2iy2l4b460kjmdhx9n7nk"
72
+ }
epfl_final/SparseAtt/wandb/run-20260820_041559-xfnxdn24/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/SparseAtt/wandb/run-20260820_041559-xfnxdn24/logs/debug.log ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-08-20 04:15:59,149 INFO MainThread:372 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-08-20 04:15:59,149 INFO MainThread:372 [wandb_setup.py:_flush():81] Configure stats pid to 372
3
+ 2026-08-20 04:15:59,149 INFO MainThread:372 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-08-20 04:15:59,149 INFO MainThread:372 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_041559-xfnxdn24/logs/debug.log
5
+ 2026-08-20 04:15:59,149 INFO MainThread:372 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_041559-xfnxdn24/logs/debug-internal.log
6
+ 2026-08-20 04:15:59,149 INFO MainThread:372 [wandb_init.py:init():850] calling init triggers
7
+ 2026-08-20 04:15:59,149 INFO MainThread:372 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45/run_20260819_153735/checkpoint-13000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 1000, 'eval_batches': 50, 'logging_steps': 10, 'seed': 45, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'l8k365m_dense_ropectl_a1p0_seed45', 'wandb_tags': ['qwen3', '365m', '8k', 'dense', 'rope-control'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': 0.0, 'rope_train_position_scale_set': [1.0, 0.9896339488176222, 0.9692159692159692, 0.9420289855072463, 0.8730158730158729], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
9
+ 2026-08-20 04:15:59,149 INFO MainThread:372 [wandb_init.py:init():898] starting backend
10
+ 2026-08-20 04:15:59,636 INFO MainThread:372 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-08-20 04:16:00,159 INFO MainThread:372 [wandb_init.py:init():918] backend started and connected
12
+ 2026-08-20 04:16:00,162 INFO MainThread:372 [wandb_init.py:init():988] updated telemetry
13
+ 2026-08-20 04:16:00,186 INFO MainThread:372 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-08-20 04:16:00,794 INFO MainThread:372 [wandb_init.py:init():1051] run resumed
15
+ 2026-08-20 04:16:00,795 INFO MainThread:372 [wandb_init.py:init():1056] starting run threads in backend
16
+ 2026-08-20 04:16:00,957 INFO MainThread:372 [wandb_run.py:_console_start():2554] atexit reg
17
+ 2026-08-20 04:16:00,957 INFO MainThread:372 [wandb_run.py:_redirect():2403] redirect: wrap_raw
18
+ 2026-08-20 04:16:00,957 INFO MainThread:372 [wandb_run.py:_redirect():2472] Wrapping output streams.
19
+ 2026-08-20 04:16:00,957 INFO MainThread:372 [wandb_run.py:_redirect():2495] Redirects installed.
20
+ 2026-08-20 04:16:00,961 INFO MainThread:372 [wandb_init.py:init():1094] run started, returning control to user process
epfl_final/SparseAtt/wandb/run-20260820_041559-xfnxdn24/run-xfnxdn24.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2204dd73b4f25bc2846a4c68443f0ec8d9f1194e4df210da5cee68873e9fa4e0
3
+ size 2785280
epfl_final/SparseAtt/wandb/run-20260820_041559-ysa9iyov/logs/debug.log ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-08-20 04:15:59,535 INFO MainThread:370 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-08-20 04:15:59,535 INFO MainThread:370 [wandb_setup.py:_flush():81] Configure stats pid to 370
3
+ 2026-08-20 04:15:59,535 INFO MainThread:370 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-08-20 04:15:59,536 INFO MainThread:370 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_041559-ysa9iyov/logs/debug.log
5
+ 2026-08-20 04:15:59,536 INFO MainThread:370 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_041559-ysa9iyov/logs/debug-internal.log
6
+ 2026-08-20 04:15:59,536 INFO MainThread:370 [wandb_init.py:init():850] calling init triggers
7
+ 2026-08-20 04:15:59,536 INFO MainThread:370 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43/run_20260819_153735/checkpoint-14000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 1000, 'eval_batches': 50, 'logging_steps': 10, 'seed': 43, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'l8k365m_dense_ropectl_a1p0_seed43', 'wandb_tags': ['qwen3', '365m', '8k', 'dense', 'rope-control'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': 0.0, 'rope_train_position_scale_set': [1.0, 0.9896339488176222, 0.9692159692159692, 0.9420289855072463, 0.8730158730158729], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
9
+ 2026-08-20 04:15:59,536 INFO MainThread:370 [wandb_init.py:init():898] starting backend
10
+ 2026-08-20 04:15:59,768 INFO MainThread:370 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-08-20 04:16:00,070 INFO MainThread:370 [wandb_init.py:init():918] backend started and connected
12
+ 2026-08-20 04:16:00,074 INFO MainThread:370 [wandb_init.py:init():988] updated telemetry
13
+ 2026-08-20 04:16:00,102 INFO MainThread:370 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-08-20 04:16:00,648 INFO MainThread:370 [wandb_init.py:init():1051] run resumed
15
+ 2026-08-20 04:16:00,649 INFO MainThread:370 [wandb_init.py:init():1056] starting run threads in backend
16
+ 2026-08-20 04:16:00,830 INFO MainThread:370 [wandb_run.py:_console_start():2554] atexit reg
17
+ 2026-08-20 04:16:00,830 INFO MainThread:370 [wandb_run.py:_redirect():2403] redirect: wrap_raw
18
+ 2026-08-20 04:16:00,830 INFO MainThread:370 [wandb_run.py:_redirect():2472] Wrapping output streams.
19
+ 2026-08-20 04:16:00,830 INFO MainThread:370 [wandb_run.py:_redirect():2495] Redirects installed.
20
+ 2026-08-20 04:16:00,838 INFO MainThread:370 [wandb_init.py:init():1094] run started, returning control to user process
epfl_final/SparseAtt/wandb/run-20260820_041559-zovy63wq/files/output.log ADDED
@@ -0,0 +1,469 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Starting training...
2
+ Validation at step 13000: Loss = 3.0598, Perplexity = 21.32
3
+ wandb: WARNING Tried to log to step 13000 that is less than the current step 13051. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
4
+ Switching attention mode for step 13001: dense (dense step interval)
5
+ Step 13010/44000 | Loss: 3.0481 | LR: 4.07e-04 | 30389.3ms/step | 0.7 samples/s
6
+ wandb: WARNING Tried to log to step 13010 that is less than the current step 13051. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
7
+ Step 13020/44000 | Loss: 3.0472 | LR: 4.07e-04 | 1376.9ms/step | 14.5 samples/s
8
+ wandb: WARNING Tried to log to step 13020 that is less than the current step 13051. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
9
+ Step 13030/44000 | Loss: 3.0740 | LR: 4.07e-04 | 1380.9ms/step | 14.5 samples/s
10
+ wandb: WARNING Tried to log to step 13030 that is less than the current step 13051. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
11
+ Step 13040/44000 | Loss: 3.0868 | LR: 4.07e-04 | 1375.5ms/step | 14.5 samples/s
12
+ wandb: WARNING Tried to log to step 13040 that is less than the current step 13051. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
13
+ Step 13050/44000 | Loss: 3.0517 | LR: 4.07e-04 | 1374.0ms/step | 14.6 samples/s
14
+ wandb: WARNING Tried to log to step 13050 that is less than the current step 13051. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
15
+ Step 13060/44000 | Loss: 3.0409 | LR: 4.07e-04 | 1377.3ms/step | 14.5 samples/s
16
+ Step 13070/44000 | Loss: 3.0491 | LR: 4.06e-04 | 1378.9ms/step | 14.5 samples/s
17
+ Step 13080/44000 | Loss: 3.0805 | LR: 4.06e-04 | 1379.0ms/step | 14.5 samples/s
18
+ Step 13090/44000 | Loss: 3.0620 | LR: 4.06e-04 | 1378.4ms/step | 14.5 samples/s
19
+ Step 13100/44000 | Loss: 3.0623 | LR: 4.06e-04 | 1379.6ms/step | 14.5 samples/s
20
+ Step 13110/44000 | Loss: 3.0679 | LR: 4.06e-04 | 1376.4ms/step | 14.5 samples/s
21
+ Step 13120/44000 | Loss: 3.0415 | LR: 4.06e-04 | 1378.6ms/step | 14.5 samples/s
22
+ Step 13130/44000 | Loss: 3.0552 | LR: 4.06e-04 | 1377.3ms/step | 14.5 samples/s
23
+ Step 13140/44000 | Loss: 3.0384 | LR: 4.05e-04 | 1376.3ms/step | 14.5 samples/s
24
+ Step 13150/44000 | Loss: 3.0496 | LR: 4.05e-04 | 1379.6ms/step | 14.5 samples/s
25
+ Step 13160/44000 | Loss: 3.0589 | LR: 4.05e-04 | 1380.1ms/step | 14.5 samples/s
26
+ Step 13170/44000 | Loss: 3.0586 | LR: 4.05e-04 | 1377.6ms/step | 14.5 samples/s
27
+ Step 13180/44000 | Loss: 3.0326 | LR: 4.05e-04 | 1377.6ms/step | 14.5 samples/s
28
+ Step 13190/44000 | Loss: 3.0617 | LR: 4.05e-04 | 1378.9ms/step | 14.5 samples/s
29
+ Step 13200/44000 | Loss: 3.0821 | LR: 4.05e-04 | 1380.6ms/step | 14.5 samples/s
30
+ Step 13210/44000 | Loss: 3.0834 | LR: 4.04e-04 | 1376.7ms/step | 14.5 samples/s
31
+ Step 13220/44000 | Loss: 3.0198 | LR: 4.04e-04 | 1376.4ms/step | 14.5 samples/s
32
+ Step 13230/44000 | Loss: 3.0559 | LR: 4.04e-04 | 1378.8ms/step | 14.5 samples/s
33
+ Step 13240/44000 | Loss: 3.0391 | LR: 4.04e-04 | 1379.2ms/step | 14.5 samples/s
34
+ Step 13250/44000 | Loss: 3.0350 | LR: 4.04e-04 | 1378.3ms/step | 14.5 samples/s
35
+ Step 13260/44000 | Loss: 3.0151 | LR: 4.04e-04 | 1376.3ms/step | 14.5 samples/s
36
+ Step 13270/44000 | Loss: 3.0577 | LR: 4.04e-04 | 1384.5ms/step | 14.4 samples/s
37
+ Step 13280/44000 | Loss: 3.0382 | LR: 4.03e-04 | 1384.1ms/step | 14.5 samples/s
38
+ Step 13290/44000 | Loss: 3.0494 | LR: 4.03e-04 | 1383.8ms/step | 14.5 samples/s
39
+ Step 13300/44000 | Loss: 3.0422 | LR: 4.03e-04 | 1384.0ms/step | 14.5 samples/s
40
+ Step 13310/44000 | Loss: 3.0291 | LR: 4.03e-04 | 1385.1ms/step | 14.4 samples/s
41
+ Step 13320/44000 | Loss: 3.0818 | LR: 4.03e-04 | 1383.8ms/step | 14.5 samples/s
42
+ Step 13330/44000 | Loss: 3.0389 | LR: 4.03e-04 | 1381.9ms/step | 14.5 samples/s
43
+ Step 13340/44000 | Loss: 3.0393 | LR: 4.03e-04 | 1381.5ms/step | 14.5 samples/s
44
+ Step 13350/44000 | Loss: 3.0494 | LR: 4.02e-04 | 1378.8ms/step | 14.5 samples/s
45
+ Step 13360/44000 | Loss: 2.9992 | LR: 4.02e-04 | 1379.8ms/step | 14.5 samples/s
46
+ Step 13370/44000 | Loss: 3.0336 | LR: 4.02e-04 | 1377.1ms/step | 14.5 samples/s
47
+ Step 13380/44000 | Loss: 3.0688 | LR: 4.02e-04 | 1377.2ms/step | 14.5 samples/s
48
+ Step 13390/44000 | Loss: 3.0480 | LR: 4.02e-04 | 1377.6ms/step | 14.5 samples/s
49
+ Step 13400/44000 | Loss: 3.0450 | LR: 4.02e-04 | 1378.3ms/step | 14.5 samples/s
50
+ Step 13410/44000 | Loss: 3.0689 | LR: 4.02e-04 | 1376.3ms/step | 14.5 samples/s
51
+ Step 13420/44000 | Loss: 3.0442 | LR: 4.01e-04 | 1377.1ms/step | 14.5 samples/s
52
+ Step 13430/44000 | Loss: 3.0597 | LR: 4.01e-04 | 1378.5ms/step | 14.5 samples/s
53
+ Step 13440/44000 | Loss: 3.0440 | LR: 4.01e-04 | 1379.3ms/step | 14.5 samples/s
54
+ Step 13450/44000 | Loss: 3.0382 | LR: 4.01e-04 | 1375.8ms/step | 14.5 samples/s
55
+ Step 13460/44000 | Loss: 3.0512 | LR: 4.01e-04 | 1377.6ms/step | 14.5 samples/s
56
+ Step 13470/44000 | Loss: 3.0468 | LR: 4.01e-04 | 1379.4ms/step | 14.5 samples/s
57
+ Step 13480/44000 | Loss: 3.0806 | LR: 4.01e-04 | 1378.4ms/step | 14.5 samples/s
58
+ Step 13490/44000 | Loss: 3.0723 | LR: 4.00e-04 | 1375.8ms/step | 14.5 samples/s
59
+ Step 13500/44000 | Loss: 3.0526 | LR: 4.00e-04 | 1377.0ms/step | 14.5 samples/s
60
+ Step 13510/44000 | Loss: 3.0405 | LR: 4.00e-04 | 1377.5ms/step | 14.5 samples/s
61
+ Step 13520/44000 | Loss: 3.0453 | LR: 4.00e-04 | 1379.3ms/step | 14.5 samples/s
62
+ Step 13530/44000 | Loss: 3.0552 | LR: 4.00e-04 | 1378.0ms/step | 14.5 samples/s
63
+ Step 13540/44000 | Loss: 3.0549 | LR: 4.00e-04 | 1379.1ms/step | 14.5 samples/s
64
+ Step 13550/44000 | Loss: 3.0258 | LR: 4.00e-04 | 1380.2ms/step | 14.5 samples/s
65
+ Step 13560/44000 | Loss: 3.0542 | LR: 3.99e-04 | 1379.2ms/step | 14.5 samples/s
66
+ Step 13570/44000 | Loss: 3.0473 | LR: 3.99e-04 | 1376.8ms/step | 14.5 samples/s
67
+ Step 13580/44000 | Loss: 3.0352 | LR: 3.99e-04 | 1376.5ms/step | 14.5 samples/s
68
+ Step 13590/44000 | Loss: 3.0440 | LR: 3.99e-04 | 1380.5ms/step | 14.5 samples/s
69
+ Step 13600/44000 | Loss: 3.0300 | LR: 3.99e-04 | 1381.3ms/step | 14.5 samples/s
70
+ Step 13610/44000 | Loss: 3.0538 | LR: 3.99e-04 | 1380.1ms/step | 14.5 samples/s
71
+ Step 13620/44000 | Loss: 3.0784 | LR: 3.98e-04 | 1378.8ms/step | 14.5 samples/s
72
+ Step 13630/44000 | Loss: 3.0545 | LR: 3.98e-04 | 1382.0ms/step | 14.5 samples/s
73
+ Step 13640/44000 | Loss: 3.0056 | LR: 3.98e-04 | 1381.7ms/step | 14.5 samples/s
74
+ Step 13650/44000 | Loss: 3.0601 | LR: 3.98e-04 | 1378.9ms/step | 14.5 samples/s
75
+ Step 13660/44000 | Loss: 3.0521 | LR: 3.98e-04 | 1378.3ms/step | 14.5 samples/s
76
+ Step 13670/44000 | Loss: 3.0288 | LR: 3.98e-04 | 1380.8ms/step | 14.5 samples/s
77
+ Step 13680/44000 | Loss: 3.0444 | LR: 3.98e-04 | 1379.5ms/step | 14.5 samples/s
78
+ Step 13690/44000 | Loss: 3.0248 | LR: 3.97e-04 | 1378.0ms/step | 14.5 samples/s
79
+ Step 13700/44000 | Loss: 3.0686 | LR: 3.97e-04 | 1376.4ms/step | 14.5 samples/s
80
+ Step 13710/44000 | Loss: 3.0330 | LR: 3.97e-04 | 1378.9ms/step | 14.5 samples/s
81
+ Step 13720/44000 | Loss: 3.0380 | LR: 3.97e-04 | 1377.1ms/step | 14.5 samples/s
82
+ Step 13730/44000 | Loss: 2.9996 | LR: 3.97e-04 | 1377.7ms/step | 14.5 samples/s
83
+ Step 13740/44000 | Loss: 3.0181 | LR: 3.97e-04 | 1378.0ms/step | 14.5 samples/s
84
+ Step 13750/44000 | Loss: 3.0539 | LR: 3.97e-04 | 1379.1ms/step | 14.5 samples/s
85
+ Step 13760/44000 | Loss: 3.0448 | LR: 3.96e-04 | 1378.6ms/step | 14.5 samples/s
86
+ Step 13770/44000 | Loss: 3.0532 | LR: 3.96e-04 | 1378.2ms/step | 14.5 samples/s
87
+ Step 13780/44000 | Loss: 3.0652 | LR: 3.96e-04 | 1376.1ms/step | 14.5 samples/s
88
+ Step 13790/44000 | Loss: 3.0325 | LR: 3.96e-04 | 1378.2ms/step | 14.5 samples/s
89
+ Step 13800/44000 | Loss: 3.0507 | LR: 3.96e-04 | 1377.3ms/step | 14.5 samples/s
90
+ Step 13810/44000 | Loss: 3.0212 | LR: 3.96e-04 | 1376.1ms/step | 14.5 samples/s
91
+ Step 13820/44000 | Loss: 3.0419 | LR: 3.96e-04 | 1377.8ms/step | 14.5 samples/s
92
+ Step 13830/44000 | Loss: 3.0317 | LR: 3.95e-04 | 1376.6ms/step | 14.5 samples/s
93
+ Step 13840/44000 | Loss: 3.0505 | LR: 3.95e-04 | 1376.4ms/step | 14.5 samples/s
94
+ Step 13850/44000 | Loss: 3.0452 | LR: 3.95e-04 | 1376.3ms/step | 14.5 samples/s
95
+ Step 13860/44000 | Loss: 3.0308 | LR: 3.95e-04 | 1381.0ms/step | 14.5 samples/s
96
+ Step 13870/44000 | Loss: 3.0502 | LR: 3.95e-04 | 1380.3ms/step | 14.5 samples/s
97
+ Step 13880/44000 | Loss: 3.0730 | LR: 3.95e-04 | 1375.8ms/step | 14.5 samples/s
98
+ Step 13890/44000 | Loss: 3.0543 | LR: 3.95e-04 | 1377.9ms/step | 14.5 samples/s
99
+ Step 13900/44000 | Loss: 3.0624 | LR: 3.94e-04 | 1377.4ms/step | 14.5 samples/s
100
+ Step 13910/44000 | Loss: 3.0401 | LR: 3.94e-04 | 1377.8ms/step | 14.5 samples/s
101
+ Step 13920/44000 | Loss: 3.0559 | LR: 3.94e-04 | 1376.7ms/step | 14.5 samples/s
102
+ Step 13930/44000 | Loss: 3.0415 | LR: 3.94e-04 | 1377.2ms/step | 14.5 samples/s
103
+ Step 13940/44000 | Loss: 3.0456 | LR: 3.94e-04 | 1377.8ms/step | 14.5 samples/s
104
+ Step 13950/44000 | Loss: 3.0764 | LR: 3.94e-04 | 1381.1ms/step | 14.5 samples/s
105
+ Step 13960/44000 | Loss: 3.0517 | LR: 3.93e-04 | 1379.6ms/step | 14.5 samples/s
106
+ Step 13970/44000 | Loss: 3.0275 | LR: 3.93e-04 | 1378.0ms/step | 14.5 samples/s
107
+ Step 13980/44000 | Loss: 3.0508 | LR: 3.93e-04 | 1378.6ms/step | 14.5 samples/s
108
+ Step 13990/44000 | Loss: 3.0339 | LR: 3.93e-04 | 1379.9ms/step | 14.5 samples/s
109
+ Step 14000/44000 | Loss: 3.0430 | LR: 3.93e-04 | 1376.7ms/step | 14.5 samples/s
110
+ Validation at step 14000: Loss = 3.0458, Perplexity = 21.03
111
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-14000
112
+ Step 14010/44000 | Loss: 3.0636 | LR: 3.93e-04 | 3907.6ms/step | 5.1 samples/s
113
+ Step 14020/44000 | Loss: 3.0477 | LR: 3.93e-04 | 1390.0ms/step | 14.4 samples/s
114
+ Step 14030/44000 | Loss: 3.0135 | LR: 3.92e-04 | 1390.0ms/step | 14.4 samples/s
115
+ Step 14040/44000 | Loss: 3.0325 | LR: 3.92e-04 | 1389.8ms/step | 14.4 samples/s
116
+ Step 14050/44000 | Loss: 3.0606 | LR: 3.92e-04 | 1393.8ms/step | 14.3 samples/s
117
+ Step 14060/44000 | Loss: 3.0371 | LR: 3.92e-04 | 1396.4ms/step | 14.3 samples/s
118
+ Step 14070/44000 | Loss: 3.0377 | LR: 3.92e-04 | 1398.0ms/step | 14.3 samples/s
119
+ Step 14080/44000 | Loss: 3.0223 | LR: 3.92e-04 | 1396.2ms/step | 14.3 samples/s
120
+ Step 14090/44000 | Loss: 3.0452 | LR: 3.92e-04 | 1393.4ms/step | 14.4 samples/s
121
+ Step 14100/44000 | Loss: 3.0447 | LR: 3.91e-04 | 1393.9ms/step | 14.3 samples/s
122
+ Step 14110/44000 | Loss: 3.0483 | LR: 3.91e-04 | 1394.4ms/step | 14.3 samples/s
123
+ Step 14120/44000 | Loss: 3.0587 | LR: 3.91e-04 | 1395.8ms/step | 14.3 samples/s
124
+ Step 14130/44000 | Loss: 3.0422 | LR: 3.91e-04 | 1394.2ms/step | 14.3 samples/s
125
+ Step 14140/44000 | Loss: 3.0681 | LR: 3.91e-04 | 1397.8ms/step | 14.3 samples/s
126
+ Step 14150/44000 | Loss: 3.0426 | LR: 3.91e-04 | 1386.5ms/step | 14.4 samples/s
127
+ Step 14160/44000 | Loss: 3.0381 | LR: 3.90e-04 | 1382.2ms/step | 14.5 samples/s
128
+ Step 14170/44000 | Loss: 3.0104 | LR: 3.90e-04 | 1382.5ms/step | 14.5 samples/s
129
+ Step 14180/44000 | Loss: 3.0441 | LR: 3.90e-04 | 1382.3ms/step | 14.5 samples/s
130
+ Step 14190/44000 | Loss: 3.0137 | LR: 3.90e-04 | 1380.7ms/step | 14.5 samples/s
131
+ Step 14200/44000 | Loss: 3.0176 | LR: 3.90e-04 | 1381.7ms/step | 14.5 samples/s
132
+ Step 14210/44000 | Loss: 3.0291 | LR: 3.90e-04 | 1383.4ms/step | 14.5 samples/s
133
+ Step 14220/44000 | Loss: 3.0395 | LR: 3.90e-04 | 1381.0ms/step | 14.5 samples/s
134
+ Step 14230/44000 | Loss: 3.0509 | LR: 3.89e-04 | 1380.3ms/step | 14.5 samples/s
135
+ Step 14240/44000 | Loss: 3.0615 | LR: 3.89e-04 | 1379.5ms/step | 14.5 samples/s
136
+ Step 14250/44000 | Loss: 3.0315 | LR: 3.89e-04 | 1380.4ms/step | 14.5 samples/s
137
+ Step 14260/44000 | Loss: 3.0270 | LR: 3.89e-04 | 1383.1ms/step | 14.5 samples/s
138
+ Step 14270/44000 | Loss: 3.0274 | LR: 3.89e-04 | 1383.9ms/step | 14.5 samples/s
139
+ Step 14280/44000 | Loss: 3.0264 | LR: 3.89e-04 | 1381.2ms/step | 14.5 samples/s
140
+ Step 14290/44000 | Loss: 3.0398 | LR: 3.89e-04 | 1380.7ms/step | 14.5 samples/s
141
+ Step 14300/44000 | Loss: 3.0369 | LR: 3.88e-04 | 1385.3ms/step | 14.4 samples/s
142
+ Step 14310/44000 | Loss: 3.0450 | LR: 3.88e-04 | 1383.8ms/step | 14.5 samples/s
143
+ Step 14320/44000 | Loss: 3.0471 | LR: 3.88e-04 | 1382.7ms/step | 14.5 samples/s
144
+ Step 14330/44000 | Loss: 3.0381 | LR: 3.88e-04 | 1381.8ms/step | 14.5 samples/s
145
+ Step 14340/44000 | Loss: 3.0571 | LR: 3.88e-04 | 1381.5ms/step | 14.5 samples/s
146
+ Step 14350/44000 | Loss: 3.0220 | LR: 3.88e-04 | 1383.6ms/step | 14.5 samples/s
147
+ Step 14360/44000 | Loss: 3.0370 | LR: 3.87e-04 | 1383.9ms/step | 14.5 samples/s
148
+ Step 14370/44000 | Loss: 3.0404 | LR: 3.87e-04 | 1384.0ms/step | 14.5 samples/s
149
+ Step 14380/44000 | Loss: 3.0496 | LR: 3.87e-04 | 1382.5ms/step | 14.5 samples/s
150
+ Step 14390/44000 | Loss: 3.0281 | LR: 3.87e-04 | 1383.2ms/step | 14.5 samples/s
151
+ Step 14400/44000 | Loss: 3.0186 | LR: 3.87e-04 | 1385.0ms/step | 14.4 samples/s
152
+ Step 14410/44000 | Loss: 3.0406 | LR: 3.87e-04 | 1378.9ms/step | 14.5 samples/s
153
+ Step 14420/44000 | Loss: 3.0558 | LR: 3.87e-04 | 1379.1ms/step | 14.5 samples/s
154
+ Step 14430/44000 | Loss: 3.0387 | LR: 3.86e-04 | 1383.4ms/step | 14.5 samples/s
155
+ Step 14440/44000 | Loss: 3.0316 | LR: 3.86e-04 | 1382.2ms/step | 14.5 samples/s
156
+ Step 14450/44000 | Loss: 3.0545 | LR: 3.86e-04 | 1382.6ms/step | 14.5 samples/s
157
+ Step 14460/44000 | Loss: 3.0381 | LR: 3.86e-04 | 1384.0ms/step | 14.5 samples/s
158
+ Step 14470/44000 | Loss: 3.0713 | LR: 3.86e-04 | 1381.9ms/step | 14.5 samples/s
159
+ Step 14480/44000 | Loss: 3.0358 | LR: 3.86e-04 | 1382.3ms/step | 14.5 samples/s
160
+ Step 14490/44000 | Loss: 3.0262 | LR: 3.85e-04 | 1382.9ms/step | 14.5 samples/s
161
+ Step 14500/44000 | Loss: 3.0348 | LR: 3.85e-04 | 1384.2ms/step | 14.4 samples/s
162
+ Step 14510/44000 | Loss: 3.0211 | LR: 3.85e-04 | 1382.2ms/step | 14.5 samples/s
163
+ Step 14520/44000 | Loss: 3.0274 | LR: 3.85e-04 | 1380.5ms/step | 14.5 samples/s
164
+ Step 14530/44000 | Loss: 3.0248 | LR: 3.85e-04 | 1381.3ms/step | 14.5 samples/s
165
+ Step 14540/44000 | Loss: 3.0323 | LR: 3.85e-04 | 1381.5ms/step | 14.5 samples/s
166
+ Step 14550/44000 | Loss: 3.0271 | LR: 3.85e-04 | 1382.8ms/step | 14.5 samples/s
167
+ Step 14560/44000 | Loss: 3.0052 | LR: 3.84e-04 | 1383.2ms/step | 14.5 samples/s
168
+ Step 14570/44000 | Loss: 3.0016 | LR: 3.84e-04 | 1381.1ms/step | 14.5 samples/s
169
+ Step 14580/44000 | Loss: 3.0015 | LR: 3.84e-04 | 1383.1ms/step | 14.5 samples/s
170
+ Step 14590/44000 | Loss: 3.0262 | LR: 3.84e-04 | 1381.2ms/step | 14.5 samples/s
171
+ Step 14600/44000 | Loss: 3.0262 | LR: 3.84e-04 | 1381.1ms/step | 14.5 samples/s
172
+ Step 14610/44000 | Loss: 3.0369 | LR: 3.84e-04 | 1381.3ms/step | 14.5 samples/s
173
+ Step 14620/44000 | Loss: 3.0198 | LR: 3.83e-04 | 1383.5ms/step | 14.5 samples/s
174
+ Step 14630/44000 | Loss: 3.0277 | LR: 3.83e-04 | 1381.6ms/step | 14.5 samples/s
175
+ Step 14640/44000 | Loss: 3.0424 | LR: 3.83e-04 | 1382.7ms/step | 14.5 samples/s
176
+ Step 14650/44000 | Loss: 3.0167 | LR: 3.83e-04 | 1386.3ms/step | 14.4 samples/s
177
+ Step 14660/44000 | Loss: 3.0428 | LR: 3.83e-04 | 1382.3ms/step | 14.5 samples/s
178
+ Step 14670/44000 | Loss: 3.0339 | LR: 3.83e-04 | 1383.4ms/step | 14.5 samples/s
179
+ Step 14680/44000 | Loss: 3.0237 | LR: 3.83e-04 | 1383.9ms/step | 14.5 samples/s
180
+ Step 14690/44000 | Loss: 3.0304 | LR: 3.82e-04 | 1382.4ms/step | 14.5 samples/s
181
+ Step 14700/44000 | Loss: 3.0316 | LR: 3.82e-04 | 1382.5ms/step | 14.5 samples/s
182
+ Step 14710/44000 | Loss: 3.0364 | LR: 3.82e-04 | 1383.0ms/step | 14.5 samples/s
183
+ Step 14720/44000 | Loss: 3.0285 | LR: 3.82e-04 | 1383.0ms/step | 14.5 samples/s
184
+ Step 14730/44000 | Loss: 3.0419 | LR: 3.82e-04 | 1383.7ms/step | 14.5 samples/s
185
+ Step 14740/44000 | Loss: 3.0382 | LR: 3.82e-04 | 1383.9ms/step | 14.5 samples/s
186
+ Step 14750/44000 | Loss: 2.9881 | LR: 3.81e-04 | 1383.9ms/step | 14.5 samples/s
187
+ Step 14760/44000 | Loss: 3.0553 | LR: 3.81e-04 | 1380.1ms/step | 14.5 samples/s
188
+ Step 14770/44000 | Loss: 3.0577 | LR: 3.81e-04 | 1382.1ms/step | 14.5 samples/s
189
+ Step 14780/44000 | Loss: 3.0383 | LR: 3.81e-04 | 1381.9ms/step | 14.5 samples/s
190
+ Step 14790/44000 | Loss: 3.0158 | LR: 3.81e-04 | 1382.7ms/step | 14.5 samples/s
191
+ Step 14800/44000 | Loss: 3.0125 | LR: 3.81e-04 | 1383.1ms/step | 14.5 samples/s
192
+ Step 14810/44000 | Loss: 3.0505 | LR: 3.81e-04 | 1384.5ms/step | 14.4 samples/s
193
+ Step 14820/44000 | Loss: 3.0259 | LR: 3.80e-04 | 1382.8ms/step | 14.5 samples/s
194
+ Step 14830/44000 | Loss: 3.0332 | LR: 3.80e-04 | 1383.2ms/step | 14.5 samples/s
195
+ Step 14840/44000 | Loss: 3.0196 | LR: 3.80e-04 | 1383.7ms/step | 14.5 samples/s
196
+ Step 14850/44000 | Loss: 3.0481 | LR: 3.80e-04 | 1380.0ms/step | 14.5 samples/s
197
+ Step 14860/44000 | Loss: 3.0521 | LR: 3.80e-04 | 1381.9ms/step | 14.5 samples/s
198
+ Step 14870/44000 | Loss: 3.0343 | LR: 3.80e-04 | 1383.7ms/step | 14.5 samples/s
199
+ Step 14880/44000 | Loss: 3.0264 | LR: 3.79e-04 | 1381.9ms/step | 14.5 samples/s
200
+ Step 14890/44000 | Loss: 3.0287 | LR: 3.79e-04 | 1382.4ms/step | 14.5 samples/s
201
+ Step 14900/44000 | Loss: 3.0374 | LR: 3.79e-04 | 1382.5ms/step | 14.5 samples/s
202
+ Step 14910/44000 | Loss: 3.0206 | LR: 3.79e-04 | 1380.1ms/step | 14.5 samples/s
203
+ Step 14920/44000 | Loss: 3.0437 | LR: 3.79e-04 | 1382.1ms/step | 14.5 samples/s
204
+ Step 14930/44000 | Loss: 3.0468 | LR: 3.79e-04 | 1383.9ms/step | 14.5 samples/s
205
+ Step 14940/44000 | Loss: 3.0035 | LR: 3.79e-04 | 1383.5ms/step | 14.5 samples/s
206
+ Step 14950/44000 | Loss: 3.0613 | LR: 3.78e-04 | 1383.9ms/step | 14.5 samples/s
207
+ Step 14960/44000 | Loss: 3.0190 | LR: 3.78e-04 | 1380.5ms/step | 14.5 samples/s
208
+ Step 14970/44000 | Loss: 3.0510 | LR: 3.78e-04 | 1380.8ms/step | 14.5 samples/s
209
+ Step 14980/44000 | Loss: 3.0112 | LR: 3.78e-04 | 1380.9ms/step | 14.5 samples/s
210
+ Step 14990/44000 | Loss: 3.0359 | LR: 3.78e-04 | 1383.6ms/step | 14.5 samples/s
211
+ Step 15000/44000 | Loss: 3.0024 | LR: 3.78e-04 | 1382.7ms/step | 14.5 samples/s
212
+ Validation at step 15000: Loss = 3.0311, Perplexity = 20.72
213
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-15000
214
+ Step 15010/44000 | Loss: 3.0154 | LR: 3.77e-04 | 3905.7ms/step | 5.1 samples/s
215
+ Step 15020/44000 | Loss: 3.0459 | LR: 3.77e-04 | 1389.1ms/step | 14.4 samples/s
216
+ Step 15030/44000 | Loss: 3.0015 | LR: 3.77e-04 | 1387.0ms/step | 14.4 samples/s
217
+ Step 15040/44000 | Loss: 3.0281 | LR: 3.77e-04 | 1395.4ms/step | 14.3 samples/s
218
+ Step 15050/44000 | Loss: 3.0318 | LR: 3.77e-04 | 1387.9ms/step | 14.4 samples/s
219
+ Step 15060/44000 | Loss: 3.0187 | LR: 3.77e-04 | 1388.2ms/step | 14.4 samples/s
220
+ Step 15070/44000 | Loss: 3.0174 | LR: 3.77e-04 | 1392.2ms/step | 14.4 samples/s
221
+ Step 15080/44000 | Loss: 3.0162 | LR: 3.76e-04 | 1393.8ms/step | 14.3 samples/s
222
+ Step 15090/44000 | Loss: 3.0184 | LR: 3.76e-04 | 1393.9ms/step | 14.3 samples/s
223
+ Step 15100/44000 | Loss: 3.0379 | LR: 3.76e-04 | 1395.5ms/step | 14.3 samples/s
224
+ Step 15110/44000 | Loss: 3.0052 | LR: 3.76e-04 | 1396.6ms/step | 14.3 samples/s
225
+ Step 15120/44000 | Loss: 3.0370 | LR: 3.76e-04 | 1394.0ms/step | 14.3 samples/s
226
+ Step 15130/44000 | Loss: 3.0509 | LR: 3.76e-04 | 1398.1ms/step | 14.3 samples/s
227
+ Step 15140/44000 | Loss: 3.0366 | LR: 3.75e-04 | 1393.8ms/step | 14.3 samples/s
228
+ Step 15150/44000 | Loss: 3.0408 | LR: 3.75e-04 | 1397.1ms/step | 14.3 samples/s
229
+ Step 15160/44000 | Loss: 3.0499 | LR: 3.75e-04 | 1398.9ms/step | 14.3 samples/s
230
+ Step 15170/44000 | Loss: 3.0475 | LR: 3.75e-04 | 1395.9ms/step | 14.3 samples/s
231
+ Step 15180/44000 | Loss: 3.0006 | LR: 3.75e-04 | 1399.6ms/step | 14.3 samples/s
232
+ Step 15190/44000 | Loss: 3.0402 | LR: 3.75e-04 | 1394.7ms/step | 14.3 samples/s
233
+ Step 15200/44000 | Loss: 3.0069 | LR: 3.74e-04 | 1400.4ms/step | 14.3 samples/s
234
+ Step 15210/44000 | Loss: 2.9984 | LR: 3.74e-04 | 1397.4ms/step | 14.3 samples/s
235
+ Step 15220/44000 | Loss: 3.0251 | LR: 3.74e-04 | 1394.2ms/step | 14.3 samples/s
236
+ Step 15230/44000 | Loss: 3.0220 | LR: 3.74e-04 | 1393.9ms/step | 14.3 samples/s
237
+ Step 15240/44000 | Loss: 3.0309 | LR: 3.74e-04 | 1391.8ms/step | 14.4 samples/s
238
+ Step 15250/44000 | Loss: 3.0045 | LR: 3.74e-04 | 1392.3ms/step | 14.4 samples/s
239
+ Step 15260/44000 | Loss: 3.0354 | LR: 3.74e-04 | 1396.0ms/step | 14.3 samples/s
240
+ Step 15270/44000 | Loss: 3.0393 | LR: 3.73e-04 | 1398.1ms/step | 14.3 samples/s
241
+ Step 15280/44000 | Loss: 3.0223 | LR: 3.73e-04 | 1397.8ms/step | 14.3 samples/s
242
+ Step 15290/44000 | Loss: 3.0307 | LR: 3.73e-04 | 1397.5ms/step | 14.3 samples/s
243
+ Step 15300/44000 | Loss: 3.0057 | LR: 3.73e-04 | 1396.4ms/step | 14.3 samples/s
244
+ Step 15310/44000 | Loss: 3.0119 | LR: 3.73e-04 | 1395.9ms/step | 14.3 samples/s
245
+ Step 15320/44000 | Loss: 3.0289 | LR: 3.73e-04 | 1396.3ms/step | 14.3 samples/s
246
+ Step 15330/44000 | Loss: 3.0405 | LR: 3.72e-04 | 1392.0ms/step | 14.4 samples/s
247
+ Step 15340/44000 | Loss: 3.0404 | LR: 3.72e-04 | 1396.4ms/step | 14.3 samples/s
248
+ Step 15350/44000 | Loss: 3.0310 | LR: 3.72e-04 | 1391.7ms/step | 14.4 samples/s
249
+ Step 15360/44000 | Loss: 3.0393 | LR: 3.72e-04 | 1393.6ms/step | 14.4 samples/s
250
+ Step 15370/44000 | Loss: 3.0226 | LR: 3.72e-04 | 1397.8ms/step | 14.3 samples/s
251
+ Step 15380/44000 | Loss: 3.0479 | LR: 3.72e-04 | 1398.5ms/step | 14.3 samples/s
252
+ Step 15390/44000 | Loss: 3.0179 | LR: 3.71e-04 | 1397.7ms/step | 14.3 samples/s
253
+ Step 15400/44000 | Loss: 3.0449 | LR: 3.71e-04 | 1398.1ms/step | 14.3 samples/s
254
+ Step 15410/44000 | Loss: 3.0025 | LR: 3.71e-04 | 1394.2ms/step | 14.3 samples/s
255
+ Step 15420/44000 | Loss: 3.0394 | LR: 3.71e-04 | 1393.8ms/step | 14.3 samples/s
256
+ Step 15430/44000 | Loss: 3.0173 | LR: 3.71e-04 | 1396.0ms/step | 14.3 samples/s
257
+ Step 15440/44000 | Loss: 3.0320 | LR: 3.71e-04 | 1397.8ms/step | 14.3 samples/s
258
+ Step 15450/44000 | Loss: 3.0389 | LR: 3.71e-04 | 1386.7ms/step | 14.4 samples/s
259
+ Step 15460/44000 | Loss: 3.0033 | LR: 3.70e-04 | 1383.2ms/step | 14.5 samples/s
260
+ Step 15470/44000 | Loss: 3.0039 | LR: 3.70e-04 | 1380.8ms/step | 14.5 samples/s
261
+ Step 15480/44000 | Loss: 3.0047 | LR: 3.70e-04 | 1381.7ms/step | 14.5 samples/s
262
+ Step 15490/44000 | Loss: 3.0137 | LR: 3.70e-04 | 1385.5ms/step | 14.4 samples/s
263
+ Step 15500/44000 | Loss: 3.0237 | LR: 3.70e-04 | 1382.8ms/step | 14.5 samples/s
264
+ Step 15510/44000 | Loss: 3.0418 | LR: 3.70e-04 | 1383.0ms/step | 14.5 samples/s
265
+ Step 15520/44000 | Loss: 3.0105 | LR: 3.69e-04 | 1382.6ms/step | 14.5 samples/s
266
+ Step 15530/44000 | Loss: 3.0415 | LR: 3.69e-04 | 1384.1ms/step | 14.5 samples/s
267
+ Step 15540/44000 | Loss: 3.0484 | LR: 3.69e-04 | 1384.1ms/step | 14.4 samples/s
268
+ Step 15550/44000 | Loss: 3.0181 | LR: 3.69e-04 | 1381.8ms/step | 14.5 samples/s
269
+ Step 15560/44000 | Loss: 3.0018 | LR: 3.69e-04 | 1381.8ms/step | 14.5 samples/s
270
+ Step 15570/44000 | Loss: 3.0029 | LR: 3.69e-04 | 1383.0ms/step | 14.5 samples/s
271
+ Step 15580/44000 | Loss: 3.0316 | LR: 3.68e-04 | 1381.7ms/step | 14.5 samples/s
272
+ Step 15590/44000 | Loss: 3.0127 | LR: 3.68e-04 | 1381.3ms/step | 14.5 samples/s
273
+ Step 15600/44000 | Loss: 3.0162 | LR: 3.68e-04 | 1382.6ms/step | 14.5 samples/s
274
+ Step 15610/44000 | Loss: 3.0102 | LR: 3.68e-04 | 1385.5ms/step | 14.4 samples/s
275
+ Step 15620/44000 | Loss: 3.0454 | LR: 3.68e-04 | 1385.0ms/step | 14.4 samples/s
276
+ Step 15630/44000 | Loss: 3.0184 | LR: 3.68e-04 | 1383.9ms/step | 14.5 samples/s
277
+ Step 15640/44000 | Loss: 2.9881 | LR: 3.67e-04 | 1383.0ms/step | 14.5 samples/s
278
+ Step 15650/44000 | Loss: 3.0266 | LR: 3.67e-04 | 1383.6ms/step | 14.5 samples/s
279
+ Step 15660/44000 | Loss: 2.9721 | LR: 3.67e-04 | 1381.1ms/step | 14.5 samples/s
280
+ Step 15670/44000 | Loss: 3.0388 | LR: 3.67e-04 | 1379.6ms/step | 14.5 samples/s
281
+ Step 15680/44000 | Loss: 3.0331 | LR: 3.67e-04 | 1382.7ms/step | 14.5 samples/s
282
+ Step 15690/44000 | Loss: 3.0214 | LR: 3.67e-04 | 1381.8ms/step | 14.5 samples/s
283
+ Step 15700/44000 | Loss: 3.0060 | LR: 3.67e-04 | 1385.2ms/step | 14.4 samples/s
284
+ Step 15710/44000 | Loss: 3.0150 | LR: 3.66e-04 | 1381.9ms/step | 14.5 samples/s
285
+ Step 15720/44000 | Loss: 3.0232 | LR: 3.66e-04 | 1380.9ms/step | 14.5 samples/s
286
+ Step 15730/44000 | Loss: 3.0138 | LR: 3.66e-04 | 1383.2ms/step | 14.5 samples/s
287
+ Step 15740/44000 | Loss: 3.0213 | LR: 3.66e-04 | 1382.0ms/step | 14.5 samples/s
288
+ Step 15750/44000 | Loss: 3.0062 | LR: 3.66e-04 | 1380.2ms/step | 14.5 samples/s
289
+ Step 15760/44000 | Loss: 2.9919 | LR: 3.66e-04 | 1383.9ms/step | 14.5 samples/s
290
+ Step 15770/44000 | Loss: 2.9964 | LR: 3.65e-04 | 1384.2ms/step | 14.4 samples/s
291
+ Step 15780/44000 | Loss: 3.0229 | LR: 3.65e-04 | 1383.7ms/step | 14.5 samples/s
292
+ Step 15790/44000 | Loss: 3.0417 | LR: 3.65e-04 | 1382.2ms/step | 14.5 samples/s
293
+ Step 15800/44000 | Loss: 3.0243 | LR: 3.65e-04 | 1383.0ms/step | 14.5 samples/s
294
+ Step 15810/44000 | Loss: 3.0098 | LR: 3.65e-04 | 1381.5ms/step | 14.5 samples/s
295
+ Step 15820/44000 | Loss: 3.0059 | LR: 3.65e-04 | 1382.3ms/step | 14.5 samples/s
296
+ Step 15830/44000 | Loss: 3.0036 | LR: 3.64e-04 | 1381.4ms/step | 14.5 samples/s
297
+ Step 15840/44000 | Loss: 3.0112 | LR: 3.64e-04 | 1382.7ms/step | 14.5 samples/s
298
+ Step 15850/44000 | Loss: 2.9943 | LR: 3.64e-04 | 1381.7ms/step | 14.5 samples/s
299
+ Step 15860/44000 | Loss: 2.9963 | LR: 3.64e-04 | 1381.1ms/step | 14.5 samples/s
300
+ Step 15870/44000 | Loss: 3.0264 | LR: 3.64e-04 | 1382.9ms/step | 14.5 samples/s
301
+ Step 15880/44000 | Loss: 3.0167 | LR: 3.64e-04 | 1379.6ms/step | 14.5 samples/s
302
+ Step 15890/44000 | Loss: 3.0253 | LR: 3.63e-04 | 1383.2ms/step | 14.5 samples/s
303
+ Step 15900/44000 | Loss: 3.0185 | LR: 3.63e-04 | 1380.4ms/step | 14.5 samples/s
304
+ Step 15910/44000 | Loss: 3.0075 | LR: 3.63e-04 | 1380.6ms/step | 14.5 samples/s
305
+ Step 15920/44000 | Loss: 3.0237 | LR: 3.63e-04 | 1383.7ms/step | 14.5 samples/s
306
+ Step 15930/44000 | Loss: 3.0164 | LR: 3.63e-04 | 1381.5ms/step | 14.5 samples/s
307
+ Step 15940/44000 | Loss: 3.0122 | LR: 3.63e-04 | 1382.6ms/step | 14.5 samples/s
308
+ Step 15950/44000 | Loss: 3.0095 | LR: 3.62e-04 | 1381.7ms/step | 14.5 samples/s
309
+ Step 15960/44000 | Loss: 3.0093 | LR: 3.62e-04 | 1380.0ms/step | 14.5 samples/s
310
+ Step 15970/44000 | Loss: 3.0030 | LR: 3.62e-04 | 1381.9ms/step | 14.5 samples/s
311
+ Step 15980/44000 | Loss: 3.0254 | LR: 3.62e-04 | 1382.2ms/step | 14.5 samples/s
312
+ Step 15990/44000 | Loss: 3.0162 | LR: 3.62e-04 | 1380.0ms/step | 14.5 samples/s
313
+ Step 16000/44000 | Loss: 3.0118 | LR: 3.62e-04 | 1380.2ms/step | 14.5 samples/s
314
+ Validation at step 16000: Loss = 3.0173, Perplexity = 20.44
315
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-16000
316
+ Step 16010/44000 | Loss: 3.0096 | LR: 3.62e-04 | 3884.3ms/step | 5.1 samples/s
317
+ Step 16020/44000 | Loss: 3.0058 | LR: 3.61e-04 | 1390.9ms/step | 14.4 samples/s
318
+ Step 16030/44000 | Loss: 3.0324 | LR: 3.61e-04 | 1390.2ms/step | 14.4 samples/s
319
+ Step 16040/44000 | Loss: 2.9985 | LR: 3.61e-04 | 1387.1ms/step | 14.4 samples/s
320
+ Step 16050/44000 | Loss: 3.0099 | LR: 3.61e-04 | 1390.9ms/step | 14.4 samples/s
321
+ Step 16060/44000 | Loss: 2.9980 | LR: 3.61e-04 | 1387.1ms/step | 14.4 samples/s
322
+ Step 16070/44000 | Loss: 3.0020 | LR: 3.61e-04 | 1389.6ms/step | 14.4 samples/s
323
+ Step 16080/44000 | Loss: 3.0256 | LR: 3.60e-04 | 1390.1ms/step | 14.4 samples/s
324
+ Step 16090/44000 | Loss: 3.0010 | LR: 3.60e-04 | 1393.2ms/step | 14.4 samples/s
325
+ Step 16100/44000 | Loss: 3.0039 | LR: 3.60e-04 | 1393.9ms/step | 14.3 samples/s
326
+ Step 16110/44000 | Loss: 3.0026 | LR: 3.60e-04 | 1394.5ms/step | 14.3 samples/s
327
+ Step 16120/44000 | Loss: 3.0100 | LR: 3.60e-04 | 1395.4ms/step | 14.3 samples/s
328
+ Step 16130/44000 | Loss: 3.0298 | LR: 3.60e-04 | 1394.4ms/step | 14.3 samples/s
329
+ Step 16140/44000 | Loss: 3.0292 | LR: 3.59e-04 | 1398.0ms/step | 14.3 samples/s
330
+ Step 16150/44000 | Loss: 3.0112 | LR: 3.59e-04 | 1393.8ms/step | 14.3 samples/s
331
+ Step 16160/44000 | Loss: 3.0259 | LR: 3.59e-04 | 1398.1ms/step | 14.3 samples/s
332
+ Step 16170/44000 | Loss: 3.0327 | LR: 3.59e-04 | 1397.1ms/step | 14.3 samples/s
333
+ Step 16180/44000 | Loss: 3.0044 | LR: 3.59e-04 | 1396.9ms/step | 14.3 samples/s
334
+ Step 16190/44000 | Loss: 2.9992 | LR: 3.59e-04 | 1388.0ms/step | 14.4 samples/s
335
+ Step 16200/44000 | Loss: 2.9916 | LR: 3.58e-04 | 1380.0ms/step | 14.5 samples/s
336
+ Step 16210/44000 | Loss: 3.0107 | LR: 3.58e-04 | 1380.0ms/step | 14.5 samples/s
337
+ Step 16220/44000 | Loss: 3.0111 | LR: 3.58e-04 | 1381.8ms/step | 14.5 samples/s
338
+ Step 16230/44000 | Loss: 3.0175 | LR: 3.58e-04 | 1382.3ms/step | 14.5 samples/s
339
+ Step 16240/44000 | Loss: 3.0278 | LR: 3.58e-04 | 1382.0ms/step | 14.5 samples/s
340
+ Step 16250/44000 | Loss: 2.9954 | LR: 3.58e-04 | 1382.4ms/step | 14.5 samples/s
341
+ Step 16260/44000 | Loss: 2.9768 | LR: 3.57e-04 | 1381.6ms/step | 14.5 samples/s
342
+ Step 16270/44000 | Loss: 2.9961 | LR: 3.57e-04 | 1381.9ms/step | 14.5 samples/s
343
+ Step 16280/44000 | Loss: 3.0008 | LR: 3.57e-04 | 1381.8ms/step | 14.5 samples/s
344
+ Step 16290/44000 | Loss: 2.9926 | LR: 3.57e-04 | 1382.8ms/step | 14.5 samples/s
345
+ Step 16300/44000 | Loss: 2.9982 | LR: 3.57e-04 | 1381.9ms/step | 14.5 samples/s
346
+ Step 16310/44000 | Loss: 3.0197 | LR: 3.57e-04 | 1383.5ms/step | 14.5 samples/s
347
+ Step 16320/44000 | Loss: 3.0188 | LR: 3.56e-04 | 1380.8ms/step | 14.5 samples/s
348
+ Step 16330/44000 | Loss: 3.0053 | LR: 3.56e-04 | 1383.5ms/step | 14.5 samples/s
349
+ Step 16340/44000 | Loss: 3.0005 | LR: 3.56e-04 | 1384.1ms/step | 14.5 samples/s
350
+ Step 16350/44000 | Loss: 3.0162 | LR: 3.56e-04 | 1381.5ms/step | 14.5 samples/s
351
+ Step 16360/44000 | Loss: 3.0381 | LR: 3.56e-04 | 1382.6ms/step | 14.5 samples/s
352
+ Step 16370/44000 | Loss: 2.9824 | LR: 3.56e-04 | 1381.8ms/step | 14.5 samples/s
353
+ Step 16380/44000 | Loss: 2.9896 | LR: 3.55e-04 | 1383.5ms/step | 14.5 samples/s
354
+ Step 16390/44000 | Loss: 2.9913 | LR: 3.55e-04 | 1382.0ms/step | 14.5 samples/s
355
+ Step 16400/44000 | Loss: 3.0306 | LR: 3.55e-04 | 1381.5ms/step | 14.5 samples/s
356
+ Step 16410/44000 | Loss: 3.0181 | LR: 3.55e-04 | 1382.6ms/step | 14.5 samples/s
357
+ Step 16420/44000 | Loss: 3.0110 | LR: 3.55e-04 | 1380.8ms/step | 14.5 samples/s
358
+ Step 16430/44000 | Loss: 2.9916 | LR: 3.55e-04 | 1382.0ms/step | 14.5 samples/s
359
+ Step 16440/44000 | Loss: 3.0059 | LR: 3.54e-04 | 1379.7ms/step | 14.5 samples/s
360
+ Step 16450/44000 | Loss: 3.0211 | LR: 3.54e-04 | 1379.7ms/step | 14.5 samples/s
361
+ Step 16460/44000 | Loss: 3.0120 | LR: 3.54e-04 | 1381.1ms/step | 14.5 samples/s
362
+ Step 16470/44000 | Loss: 2.9991 | LR: 3.54e-04 | 1380.6ms/step | 14.5 samples/s
363
+ Step 16480/44000 | Loss: 2.9927 | LR: 3.54e-04 | 1380.7ms/step | 14.5 samples/s
364
+ Step 16490/44000 | Loss: 3.0051 | LR: 3.54e-04 | 1382.2ms/step | 14.5 samples/s
365
+ Step 16500/44000 | Loss: 2.9845 | LR: 3.53e-04 | 1385.3ms/step | 14.4 samples/s
366
+ Step 16510/44000 | Loss: 3.0184 | LR: 3.53e-04 | 1382.0ms/step | 14.5 samples/s
367
+ Step 16520/44000 | Loss: 3.0196 | LR: 3.53e-04 | 1382.2ms/step | 14.5 samples/s
368
+ Step 16530/44000 | Loss: 3.0493 | LR: 3.53e-04 | 1381.9ms/step | 14.5 samples/s
369
+ Step 16540/44000 | Loss: 3.0069 | LR: 3.53e-04 | 1380.6ms/step | 14.5 samples/s
370
+ Step 16550/44000 | Loss: 3.0061 | LR: 3.53e-04 | 1382.3ms/step | 14.5 samples/s
371
+ Step 16560/44000 | Loss: 3.0258 | LR: 3.52e-04 | 1384.2ms/step | 14.4 samples/s
372
+ Step 16570/44000 | Loss: 3.0314 | LR: 3.52e-04 | 1381.7ms/step | 14.5 samples/s
373
+ Step 16580/44000 | Loss: 3.0007 | LR: 3.52e-04 | 1381.4ms/step | 14.5 samples/s
374
+ Step 16590/44000 | Loss: 2.9939 | LR: 3.52e-04 | 1381.0ms/step | 14.5 samples/s
375
+ Step 16600/44000 | Loss: 2.9923 | LR: 3.52e-04 | 1380.6ms/step | 14.5 samples/s
376
+ Step 16610/44000 | Loss: 3.0024 | LR: 3.52e-04 | 1380.1ms/step | 14.5 samples/s
377
+ Step 16620/44000 | Loss: 3.0315 | LR: 3.51e-04 | 1382.2ms/step | 14.5 samples/s
378
+ Step 16630/44000 | Loss: 2.9957 | LR: 3.51e-04 | 1383.6ms/step | 14.5 samples/s
379
+ Step 16640/44000 | Loss: 2.9889 | LR: 3.51e-04 | 1381.2ms/step | 14.5 samples/s
380
+ Step 16650/44000 | Loss: 2.9832 | LR: 3.51e-04 | 1380.9ms/step | 14.5 samples/s
381
+ Step 16660/44000 | Loss: 3.0235 | LR: 3.51e-04 | 1383.8ms/step | 14.5 samples/s
382
+ Step 16670/44000 | Loss: 3.0150 | LR: 3.51e-04 | 1382.9ms/step | 14.5 samples/s
383
+ Step 16680/44000 | Loss: 2.9943 | LR: 3.50e-04 | 1383.2ms/step | 14.5 samples/s
384
+ Step 16690/44000 | Loss: 3.0117 | LR: 3.50e-04 | 1382.0ms/step | 14.5 samples/s
385
+ Step 16700/44000 | Loss: 2.9965 | LR: 3.50e-04 | 1382.4ms/step | 14.5 samples/s
386
+ Step 16710/44000 | Loss: 3.0252 | LR: 3.50e-04 | 1381.9ms/step | 14.5 samples/s
387
+ Step 16720/44000 | Loss: 3.0122 | LR: 3.50e-04 | 1382.0ms/step | 14.5 samples/s
388
+ Step 16730/44000 | Loss: 3.0275 | LR: 3.50e-04 | 1380.4ms/step | 14.5 samples/s
389
+ Step 16740/44000 | Loss: 2.9936 | LR: 3.49e-04 | 1379.6ms/step | 14.5 samples/s
390
+ Step 16750/44000 | Loss: 3.0155 | LR: 3.49e-04 | 1381.8ms/step | 14.5 samples/s
391
+ Step 16760/44000 | Loss: 2.9755 | LR: 3.49e-04 | 1381.9ms/step | 14.5 samples/s
392
+ Step 16770/44000 | Loss: 2.9897 | LR: 3.49e-04 | 1384.0ms/step | 14.5 samples/s
393
+ Step 16780/44000 | Loss: 3.0522 | LR: 3.49e-04 | 1383.5ms/step | 14.5 samples/s
394
+ Step 16790/44000 | Loss: 3.0116 | LR: 3.49e-04 | 1385.6ms/step | 14.4 samples/s
395
+ Step 16800/44000 | Loss: 2.9814 | LR: 3.48e-04 | 1383.1ms/step | 14.5 samples/s
396
+ Step 16810/44000 | Loss: 3.0145 | LR: 3.48e-04 | 1383.1ms/step | 14.5 samples/s
397
+ Step 16820/44000 | Loss: 3.0329 | LR: 3.48e-04 | 1382.8ms/step | 14.5 samples/s
398
+ Step 16830/44000 | Loss: 2.9916 | LR: 3.48e-04 | 1383.1ms/step | 14.5 samples/s
399
+ Step 16840/44000 | Loss: 2.9916 | LR: 3.48e-04 | 1382.9ms/step | 14.5 samples/s
400
+ Step 16850/44000 | Loss: 2.9933 | LR: 3.48e-04 | 1384.2ms/step | 14.4 samples/s
401
+ Step 16860/44000 | Loss: 2.9971 | LR: 3.47e-04 | 1383.2ms/step | 14.5 samples/s
402
+ Step 16870/44000 | Loss: 3.0203 | LR: 3.47e-04 | 1381.0ms/step | 14.5 samples/s
403
+ Step 16880/44000 | Loss: 3.0168 | LR: 3.47e-04 | 1381.3ms/step | 14.5 samples/s
404
+ Step 16890/44000 | Loss: 3.0029 | LR: 3.47e-04 | 1381.6ms/step | 14.5 samples/s
405
+ Step 16900/44000 | Loss: 2.9893 | LR: 3.47e-04 | 1380.8ms/step | 14.5 samples/s
406
+ Step 16910/44000 | Loss: 3.0246 | LR: 3.47e-04 | 1381.8ms/step | 14.5 samples/s
407
+ Step 16920/44000 | Loss: 3.0200 | LR: 3.46e-04 | 1380.5ms/step | 14.5 samples/s
408
+ Step 16930/44000 | Loss: 2.9938 | LR: 3.46e-04 | 1379.7ms/step | 14.5 samples/s
409
+ Step 16940/44000 | Loss: 3.0054 | LR: 3.46e-04 | 1385.0ms/step | 14.4 samples/s
410
+ Step 16950/44000 | Loss: 2.9910 | LR: 3.46e-04 | 1382.5ms/step | 14.5 samples/s
411
+ Step 16960/44000 | Loss: 2.9779 | LR: 3.46e-04 | 1381.2ms/step | 14.5 samples/s
412
+ Step 16970/44000 | Loss: 3.0061 | LR: 3.46e-04 | 1378.7ms/step | 14.5 samples/s
413
+ Step 16980/44000 | Loss: 3.0286 | LR: 3.45e-04 | 1381.2ms/step | 14.5 samples/s
414
+ Step 16990/44000 | Loss: 3.0181 | LR: 3.45e-04 | 1383.1ms/step | 14.5 samples/s
415
+ Step 17000/44000 | Loss: 3.0214 | LR: 3.45e-04 | 1383.6ms/step | 14.5 samples/s
416
+ Validation at step 17000: Loss = 3.0033, Perplexity = 20.15
417
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-17000
418
+ Step 17010/44000 | Loss: 3.0014 | LR: 3.45e-04 | 3960.9ms/step | 5.0 samples/s
419
+ Step 17020/44000 | Loss: 3.0023 | LR: 3.45e-04 | 1387.7ms/step | 14.4 samples/s
420
+ Step 17030/44000 | Loss: 2.9887 | LR: 3.45e-04 | 1391.8ms/step | 14.4 samples/s
421
+ Step 17040/44000 | Loss: 3.0081 | LR: 3.44e-04 | 1385.9ms/step | 14.4 samples/s
422
+ Step 17050/44000 | Loss: 3.0109 | LR: 3.44e-04 | 1378.2ms/step | 14.5 samples/s
423
+ Step 17060/44000 | Loss: 2.9994 | LR: 3.44e-04 | 1380.6ms/step | 14.5 samples/s
424
+ Step 17070/44000 | Loss: 2.9990 | LR: 3.44e-04 | 1381.2ms/step | 14.5 samples/s
425
+ Step 17080/44000 | Loss: 3.0163 | LR: 3.44e-04 | 1381.8ms/step | 14.5 samples/s
426
+ Step 17090/44000 | Loss: 3.0045 | LR: 3.44e-04 | 1382.5ms/step | 14.5 samples/s
427
+ Step 17100/44000 | Loss: 3.0048 | LR: 3.43e-04 | 1381.2ms/step | 14.5 samples/s
428
+ Step 17110/44000 | Loss: 3.0118 | LR: 3.43e-04 | 1380.7ms/step | 14.5 samples/s
429
+ Step 17120/44000 | Loss: 2.9760 | LR: 3.43e-04 | 1380.1ms/step | 14.5 samples/s
430
+ Step 17130/44000 | Loss: 3.0005 | LR: 3.43e-04 | 1382.0ms/step | 14.5 samples/s
431
+ Step 17140/44000 | Loss: 2.9831 | LR: 3.43e-04 | 1381.9ms/step | 14.5 samples/s
432
+ Step 17150/44000 | Loss: 3.0132 | LR: 3.43e-04 | 1382.4ms/step | 14.5 samples/s
433
+ Step 17160/44000 | Loss: 2.9875 | LR: 3.42e-04 | 1379.4ms/step | 14.5 samples/s
434
+ Step 17170/44000 | Loss: 3.0163 | LR: 3.42e-04 | 1379.9ms/step | 14.5 samples/s
435
+ Step 17180/44000 | Loss: 3.0253 | LR: 3.42e-04 | 1382.0ms/step | 14.5 samples/s
436
+ Step 17190/44000 | Loss: 2.9860 | LR: 3.42e-04 | 1383.0ms/step | 14.5 samples/s
437
+ Step 17200/44000 | Loss: 3.0115 | LR: 3.42e-04 | 1382.0ms/step | 14.5 samples/s
438
+ Step 17210/44000 | Loss: 3.0037 | LR: 3.42e-04 | 1380.8ms/step | 14.5 samples/s
439
+ Step 17220/44000 | Loss: 2.9925 | LR: 3.41e-04 | 1384.0ms/step | 14.5 samples/s
440
+ Step 17230/44000 | Loss: 3.0244 | LR: 3.41e-04 | 1384.4ms/step | 14.4 samples/s
441
+ Step 17240/44000 | Loss: 2.9996 | LR: 3.41e-04 | 1379.9ms/step | 14.5 samples/s
442
+ Step 17250/44000 | Loss: 3.0261 | LR: 3.41e-04 | 1381.3ms/step | 14.5 samples/s
443
+ Step 17260/44000 | Loss: 2.9982 | LR: 3.41e-04 | 1381.6ms/step | 14.5 samples/s
444
+ Step 17270/44000 | Loss: 2.9750 | LR: 3.41e-04 | 1381.2ms/step | 14.5 samples/s
445
+ Step 17280/44000 | Loss: 3.0168 | LR: 3.40e-04 | 1381.6ms/step | 14.5 samples/s
446
+ Step 17290/44000 | Loss: 2.9897 | LR: 3.40e-04 | 1383.7ms/step | 14.5 samples/s
447
+ Step 17300/44000 | Loss: 2.9955 | LR: 3.40e-04 | 1382.7ms/step | 14.5 samples/s
448
+ Step 17310/44000 | Loss: 2.9770 | LR: 3.40e-04 | 1382.5ms/step | 14.5 samples/s
449
+ Step 17320/44000 | Loss: 2.9880 | LR: 3.40e-04 | 1381.8ms/step | 14.5 samples/s
450
+ Step 17330/44000 | Loss: 2.9879 | LR: 3.40e-04 | 1382.9ms/step | 14.5 samples/s
451
+ Step 17340/44000 | Loss: 3.0105 | LR: 3.39e-04 | 1381.6ms/step | 14.5 samples/s
452
+ Step 17350/44000 | Loss: 2.9734 | LR: 3.39e-04 | 1381.4ms/step | 14.5 samples/s
453
+ Step 17360/44000 | Loss: 3.0121 | LR: 3.39e-04 | 1384.1ms/step | 14.4 samples/s
454
+ Step 17370/44000 | Loss: 2.9960 | LR: 3.39e-04 | 1384.7ms/step | 14.4 samples/s
455
+ Step 17380/44000 | Loss: 2.9806 | LR: 3.39e-04 | 1382.8ms/step | 14.5 samples/s
456
+ Step 17390/44000 | Loss: 2.9971 | LR: 3.39e-04 | 1383.6ms/step | 14.5 samples/s
457
+ Step 17400/44000 | Loss: 3.0013 | LR: 3.38e-04 | 1382.5ms/step | 14.5 samples/s
458
+ Step 17410/44000 | Loss: 2.9969 | LR: 3.38e-04 | 1380.7ms/step | 14.5 samples/s
459
+ Step 17420/44000 | Loss: 2.9929 | LR: 3.38e-04 | 1381.3ms/step | 14.5 samples/s
460
+ Step 17430/44000 | Loss: 3.0156 | LR: 3.38e-04 | 1380.6ms/step | 14.5 samples/s
461
+ Step 17440/44000 | Loss: 3.0278 | LR: 3.38e-04 | 1381.6ms/step | 14.5 samples/s
462
+ Step 17450/44000 | Loss: 2.9866 | LR: 3.38e-04 | 1380.2ms/step | 14.5 samples/s
463
+ Step 17460/44000 | Loss: 3.0027 | LR: 3.37e-04 | 1381.0ms/step | 14.5 samples/s
464
+ Step 17470/44000 | Loss: 2.9944 | LR: 3.37e-04 | 1382.1ms/step | 14.5 samples/s
465
+ Step 17480/44000 | Loss: 3.0006 | LR: 3.37e-04 | 1383.9ms/step | 14.5 samples/s
466
+ Step 17490/44000 | Loss: 2.9942 | LR: 3.37e-04 | 1382.7ms/step | 14.5 samples/s
467
+ Step 17500/44000 | Loss: 2.9898 | LR: 3.37e-04 | 1383.7ms/step | 14.5 samples/s
468
+ Step 17510/44000 | Loss: 2.9653 | LR: 3.37e-04 | 1382.1ms/step | 14.5 samples/s
469
+ Step 17520/44000 | Loss: 2.9832 | LR: 3.36e-04 | 1380.3ms/step | 14.5 samples/s
epfl_final/SparseAtt/wandb/run-20260820_041559-zovy63wq/files/requirements.txt ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pip==26.0.1
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.6.3
8
+ typing_extensions==4.15.0
9
+ triton==3.6.0
10
+ tqdm==4.67.3
11
+ torchinfo==1.8.0
12
+ sympy==1.14.0
13
+ smmap==5.0.3
14
+ six==1.17.0
15
+ shellingham==1.5.4
16
+ setuptools==81.0.0
17
+ safetensors==0.7.0
18
+ regex==2026.4.4
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.4.1
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.4.4
38
+ networkx==3.6.1
39
+ multidict==6.7.1
40
+ mdurl==0.1.2
41
+ MarkupSafe==3.0.3
42
+ idna==3.13
43
+ hf-xet==1.4.3
44
+ h11==0.16.0
45
+ fsspec==2026.2.0
46
+ frozenlist==1.8.0
47
+ filelock==3.29.0
48
+ dill==0.4.1
49
+ cuda-pathfinder==1.5.4
50
+ click==8.3.3
51
+ charset-normalizer==3.4.7
52
+ certifi==2026.4.22
53
+ attrs==26.1.0
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ yarl==1.23.0
58
+ typing-inspection==0.4.2
59
+ sentry-sdk==2.58.0
60
+ requests==2.33.1
61
+ python-dateutil==2.9.0.post0
62
+ pydantic_core==2.46.3
63
+ nvidia-cusparse==12.6.3.3
64
+ nvidia-cufft==12.0.0.61
65
+ nvidia-cudnn-cu13==9.19.0.56
66
+ multiprocess==0.70.19
67
+ markdown-it-py==4.0.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ cuda-bindings==13.2.0
72
+ anyio==4.13.0
73
+ aiosignal==1.4.0
74
+ rich==15.0.0
75
+ pydantic==2.13.3
76
+ pandas==3.0.2
77
+ nvidia-cusolver==12.0.4.66
78
+ httpx==0.28.1
79
+ GitPython==3.1.49
80
+ aiohttp==3.13.5
81
+ wandb==0.26.1
82
+ typer==0.25.1
83
+ torch==2.11.0
84
+ huggingface_hub==1.13.0
85
+ tokenizers==0.22.2
86
+ datasets==4.8.5
87
+ accelerate==1.13.0
88
+ transformers==5.7.0
89
+ word2number==1.1
90
+ sqlitedict==2.1.0
91
+ pytz==2026.2
92
+ threadpoolctl==3.6.0
93
+ tcolorpy==0.1.7
94
+ tabulate==0.10.0
95
+ scipy==1.17.1
96
+ portalocker==3.2.0
97
+ pathvalidate==3.3.1
98
+ narwhals==2.22.1
99
+ more-itertools==11.1.0
100
+ lxml==6.1.1
101
+ joblib==1.5.3
102
+ colorama==0.4.6
103
+ chardet==6.0.0.post1
104
+ absl-py==2.4.0
105
+ scikit-learn==1.9.0
106
+ sacrebleu==2.6.0
107
+ nltk==3.9.4
108
+ mbstrdecoder==1.1.5
109
+ typepy==1.3.5
110
+ rouge_score==0.1.2
111
+ DataProperty==1.1.1
112
+ tabledata==1.3.5
113
+ evaluate==0.4.6
114
+ pytablewriter==1.2.1
115
+ lm_eval==0.4.12
116
+ pluggy==1.6.0
117
+ iniconfig==2.3.0
118
+ pytest==9.1.0
119
+ py-spy==0.4.2
120
+ pillow==12.2.0
121
+ tifffile==2026.6.1
122
+ imagecodecs==2026.6.26
123
+ zipp==4.1.0
124
+ importlib_metadata==9.0.0
125
+ derivative==0.6.3
126
+ pysindy==2.1.0
127
+ pyparsing==3.3.2
128
+ kiwisolver==1.5.0
129
+ fonttools==4.63.0
130
+ cycler==0.12.1
131
+ contourpy==1.3.3
132
+ matplotlib==3.11.0
133
+ sortedcontainers==2.4.0
134
+ wcwidth==0.8.2
135
+ sigfig==1.3.19
136
+ importlib_resources==7.1.0
137
+ enum-tools==0.13.0
138
+ cramjam==2.11.0
139
+ python-snappy==0.7.3
140
+ compact-json==1.8.2
141
+ numbers-parser==4.18.5
142
+ entmax==1.3
epfl_final/SparseAtt/wandb/run-20260820_041559-zovy63wq/files/wandb-metadata.json ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.13.13",
4
+ "startedAt": "2026-08-20T04:15:59.140772Z",
5
+ "args": [
6
+ "--config",
7
+ "/mloscratch/homes/navasard/sparse/configs_ropectl/resume_a1p0_wandb/seed46.json"
8
+ ],
9
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
10
+ "codePath": "train.py",
11
+ "codePathLocal": "train.py",
12
+ "git": {
13
+ "remote": "https://github.com/rubentium/TokenPrune.git",
14
+ "commit": "5637886f0cc760009989b14e68c6ae06d4827418"
15
+ },
16
+ "email": "ruben.navasardyan@mail.utoronto.ca",
17
+ "root": "/mloscratch/homes/navasard/sparse/SparseAtt",
18
+ "host": "h100-workspace-5-gpu-0-2",
19
+ "executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
20
+ "cpu_count": 96,
21
+ "cpu_count_logical": 192,
22
+ "gpu": "NVIDIA H100 80GB HBM3",
23
+ "gpu_count": 5,
24
+ "disk": {
25
+ "/": {
26
+ "total": "9374931091456",
27
+ "used": "5285341044736"
28
+ }
29
+ },
30
+ "memory": {
31
+ "total": "1623005626368"
32
+ },
33
+ "gpu_nvidia": [
34
+ {
35
+ "name": "NVIDIA H100 80GB HBM3",
36
+ "memoryTotal": "85520809984",
37
+ "cudaCores": 16896,
38
+ "architecture": "Hopper",
39
+ "uuid": "GPU-6fb12770-6b51-9595-91c9-0d20801e3896"
40
+ },
41
+ {
42
+ "name": "NVIDIA H100 80GB HBM3",
43
+ "memoryTotal": "85520809984",
44
+ "cudaCores": 16896,
45
+ "architecture": "Hopper",
46
+ "uuid": "GPU-5cb9f8b7-4a13-4f85-d73e-6bcbebebefb8"
47
+ },
48
+ {
49
+ "name": "NVIDIA H100 80GB HBM3",
50
+ "memoryTotal": "85520809984",
51
+ "cudaCores": 16896,
52
+ "architecture": "Hopper",
53
+ "uuid": "GPU-3512802a-d501-0a3c-9812-4ab9deb3a3a9"
54
+ },
55
+ {
56
+ "name": "NVIDIA H100 80GB HBM3",
57
+ "memoryTotal": "85520809984",
58
+ "cudaCores": 16896,
59
+ "architecture": "Hopper",
60
+ "uuid": "GPU-24436def-beee-c9f1-bd05-c97df556de08"
61
+ },
62
+ {
63
+ "name": "NVIDIA H100 80GB HBM3",
64
+ "memoryTotal": "85520809984",
65
+ "cudaCores": 16896,
66
+ "architecture": "Hopper",
67
+ "uuid": "GPU-7b827c86-854a-af51-b5d3-6aaa43307d4a"
68
+ }
69
+ ],
70
+ "cudaVersion": "13.0",
71
+ "writerId": "coszg0msvzctygm9xdydyfdbivurb1ok"
72
+ }
epfl_final/SparseAtt/wandb/run-20260820_041559-zovy63wq/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/SparseAtt/wandb/run-20260820_041559-zovy63wq/logs/debug.log ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-08-20 04:15:59,194 INFO MainThread:373 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-08-20 04:15:59,194 INFO MainThread:373 [wandb_setup.py:_flush():81] Configure stats pid to 373
3
+ 2026-08-20 04:15:59,194 INFO MainThread:373 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-08-20 04:15:59,194 INFO MainThread:373 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_041559-zovy63wq/logs/debug.log
5
+ 2026-08-20 04:15:59,194 INFO MainThread:373 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260820_041559-zovy63wq/logs/debug-internal.log
6
+ 2026-08-20 04:15:59,195 INFO MainThread:373 [wandb_init.py:init():850] calling init triggers
7
+ 2026-08-20 04:15:59,195 INFO MainThread:373 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46/run_20260819_153735/checkpoint-13000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 44000, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 1000, 'eval_batches': 50, 'logging_steps': 10, 'seed': 46, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'l8k365m_dense_ropectl_a1p0_seed46', 'wandb_tags': ['qwen3', '365m', '8k', 'dense', 'rope-control'], 'hidden_size': 768, 'intermediate_size': 3072, 'num_attention_heads': 6, 'num_key_value_heads': 2, 'num_hidden_layers': 28, 'head_dim': None, 'attention_dropout': 0.0, 'rope_train_position_scale_set': [1.0, 0.9896339488176222, 0.9692159692159692, 0.9420289855072463, 0.8730158730158729], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'sparse_mixed_nrb_set': None, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
9
+ 2026-08-20 04:15:59,195 INFO MainThread:373 [wandb_init.py:init():898] starting backend
10
+ 2026-08-20 04:15:59,637 INFO MainThread:373 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-08-20 04:16:00,159 INFO MainThread:373 [wandb_init.py:init():918] backend started and connected
12
+ 2026-08-20 04:16:00,163 INFO MainThread:373 [wandb_init.py:init():988] updated telemetry
13
+ 2026-08-20 04:16:00,187 INFO MainThread:373 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-08-20 04:16:00,701 INFO MainThread:373 [wandb_init.py:init():1051] run resumed
15
+ 2026-08-20 04:16:00,703 INFO MainThread:373 [wandb_init.py:init():1056] starting run threads in backend
16
+ 2026-08-20 04:16:00,863 INFO MainThread:373 [wandb_run.py:_console_start():2554] atexit reg
17
+ 2026-08-20 04:16:00,863 INFO MainThread:373 [wandb_run.py:_redirect():2403] redirect: wrap_raw
18
+ 2026-08-20 04:16:00,863 INFO MainThread:373 [wandb_run.py:_redirect():2472] Wrapping output streams.
19
+ 2026-08-20 04:16:00,863 INFO MainThread:373 [wandb_run.py:_redirect():2495] Redirects installed.
20
+ 2026-08-20 04:16:00,865 INFO MainThread:373 [wandb_init.py:init():1094] run started, returning control to user process