EPFL final checkpoints and W&B history 501-550 of 1463

.gitattributes

@@ -62,3 +62,8 @@ epfl_final/SparseAtt/wandb/run-20260709_222958-hn868f80/run-hn868f80.wandb filte
 epfl_final/SparseAtt/wandb/run-20260709_223121-9blizq3m/run-9blizq3m.wandb filter=lfs diff=lfs merge=lfs -text
 epfl_final/SparseAtt/wandb/run-20260709_223859-cd66e13p/run-cd66e13p.wandb filter=lfs diff=lfs merge=lfs -text
 epfl_final/SparseAtt/wandb/run-20260710_000026-hn868f80/run-hn868f80.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260710_214547-cd66e13p/run-cd66e13p.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260711_014922-hn868f80/run-hn868f80.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260711_075132-cd66e13p/run-cd66e13p.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260711_080025-9blizq3m/run-9blizq3m.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/SparseAtt/wandb/run-20260713_031902-9blizq3m/run-9blizq3m.wandb filter=lfs diff=lfs merge=lfs -text

epfl_final/SparseAtt/wandb/run-20260710_214547-cd66e13p/files/output.log

@@ -0,0 +1,378 @@
+Starting training...
+Validation at step 13500: Loss = 2.1827, Perplexity = 8.87
+wandb: WARNING Tried to log to step 13500 that is less than the current step 13591. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 13501: dense (dense step interval)
+Step 13510/37465 | Loss: 2.1730 | LR: 3.65e-04 | 32607.2ms/step | 0.6 samples/s
+wandb: WARNING Tried to log to step 13510 that is less than the current step 13591. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 13520/37465 | Loss: 2.3480 | LR: 3.65e-04 | 6337.0ms/step | 3.2 samples/s
+wandb: WARNING Tried to log to step 13520 that is less than the current step 13591. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 13530/37465 | Loss: 2.3052 | LR: 3.65e-04 | 6358.5ms/step | 3.1 samples/s
+wandb: WARNING Tried to log to step 13530 that is less than the current step 13591. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 13540/37465 | Loss: 2.0984 | LR: 3.65e-04 | 6380.4ms/step | 3.1 samples/s
+wandb: WARNING Tried to log to step 13540 that is less than the current step 13591. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 13550/37465 | Loss: 2.2042 | LR: 3.64e-04 | 6407.0ms/step | 3.1 samples/s
+wandb: WARNING Tried to log to step 13550 that is less than the current step 13591. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 13560/37465 | Loss: 2.0372 | LR: 3.64e-04 | 6387.9ms/step | 3.1 samples/s
+wandb: WARNING Tried to log to step 13560 that is less than the current step 13591. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 13570/37465 | Loss: 2.0305 | LR: 3.64e-04 | 6387.0ms/step | 3.1 samples/s
+wandb: WARNING Tried to log to step 13570 that is less than the current step 13591. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 13580/37465 | Loss: 2.1331 | LR: 3.64e-04 | 6398.7ms/step | 3.1 samples/s
+wandb: WARNING Tried to log to step 13580 that is less than the current step 13591. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 13590/37465 | Loss: 2.2168 | LR: 3.64e-04 | 6387.7ms/step | 3.1 samples/s
+wandb: WARNING Tried to log to step 13590 that is less than the current step 13591. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 13600/37465 | Loss: 2.0527 | LR: 3.64e-04 | 6401.6ms/step | 3.1 samples/s
+Step 13610/37465 | Loss: 2.2849 | LR: 3.63e-04 | 6416.2ms/step | 3.1 samples/s
+Step 13620/37465 | Loss: 2.2750 | LR: 3.63e-04 | 6413.3ms/step | 3.1 samples/s
+Step 13630/37465 | Loss: 2.2619 | LR: 3.63e-04 | 6394.1ms/step | 3.1 samples/s
+Step 13640/37465 | Loss: 2.1195 | LR: 3.63e-04 | 6410.8ms/step | 3.1 samples/s
+Step 13650/37465 | Loss: 1.9432 | LR: 3.63e-04 | 6395.6ms/step | 3.1 samples/s
+Step 13660/37465 | Loss: 2.1889 | LR: 3.62e-04 | 6398.2ms/step | 3.1 samples/s
+Step 13670/37465 | Loss: 2.3743 | LR: 3.62e-04 | 6398.2ms/step | 3.1 samples/s
+Step 13680/37465 | Loss: 2.1759 | LR: 3.62e-04 | 6390.3ms/step | 3.1 samples/s
+Step 13690/37465 | Loss: 2.3683 | LR: 3.62e-04 | 6385.7ms/step | 3.1 samples/s
+Step 13700/37465 | Loss: 2.2851 | LR: 3.62e-04 | 6418.6ms/step | 3.1 samples/s
+Step 13710/37465 | Loss: 2.1569 | LR: 3.61e-04 | 6389.6ms/step | 3.1 samples/s
+Step 13720/37465 | Loss: 2.2308 | LR: 3.61e-04 | 6390.4ms/step | 3.1 samples/s
+Step 13730/37465 | Loss: 2.2380 | LR: 3.61e-04 | 6396.1ms/step | 3.1 samples/s
+Step 13740/37465 | Loss: 2.1942 | LR: 3.61e-04 | 6389.5ms/step | 3.1 samples/s
+Step 13750/37465 | Loss: 2.1337 | LR: 3.61e-04 | 6408.5ms/step | 3.1 samples/s
+Step 13760/37465 | Loss: 2.1211 | LR: 3.60e-04 | 6394.9ms/step | 3.1 samples/s
+Step 13770/37465 | Loss: 2.1965 | LR: 3.60e-04 | 6390.7ms/step | 3.1 samples/s
+Step 13780/37465 | Loss: 2.1241 | LR: 3.60e-04 | 6403.3ms/step | 3.1 samples/s
+Step 13790/37465 | Loss: 2.4519 | LR: 3.60e-04 | 6400.6ms/step | 3.1 samples/s
+Step 13800/37465 | Loss: 2.1514 | LR: 3.60e-04 | 6392.8ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-13800
+Step 13810/37465 | Loss: 1.9849 | LR: 3.59e-04 | 7295.9ms/step | 2.7 samples/s
+Step 13820/37465 | Loss: 2.3685 | LR: 3.59e-04 | 6367.5ms/step | 3.1 samples/s
+Step 13830/37465 | Loss: 2.0261 | LR: 3.59e-04 | 6369.7ms/step | 3.1 samples/s
+Step 13840/37465 | Loss: 2.1250 | LR: 3.59e-04 | 6371.8ms/step | 3.1 samples/s
+Step 13850/37465 | Loss: 2.1507 | LR: 3.59e-04 | 6366.4ms/step | 3.1 samples/s
+Step 13860/37465 | Loss: 2.3489 | LR: 3.59e-04 | 6369.3ms/step | 3.1 samples/s
+Step 13870/37465 | Loss: 2.2081 | LR: 3.58e-04 | 6372.2ms/step | 3.1 samples/s
+Step 13880/37465 | Loss: 1.9191 | LR: 3.58e-04 | 6371.1ms/step | 3.1 samples/s
+Step 13890/37465 | Loss: 2.1213 | LR: 3.58e-04 | 6371.9ms/step | 3.1 samples/s
+Step 13900/37465 | Loss: 2.0575 | LR: 3.58e-04 | 6371.1ms/step | 3.1 samples/s
+Step 13910/37465 | Loss: 2.0749 | LR: 3.58e-04 | 6369.0ms/step | 3.1 samples/s
+Step 13920/37465 | Loss: 2.4667 | LR: 3.57e-04 | 6368.6ms/step | 3.1 samples/s
+Step 13930/37465 | Loss: 1.9530 | LR: 3.57e-04 | 6370.5ms/step | 3.1 samples/s
+Step 13940/37465 | Loss: 2.1053 | LR: 3.57e-04 | 6368.9ms/step | 3.1 samples/s
+Step 13950/37465 | Loss: 2.2413 | LR: 3.57e-04 | 6370.3ms/step | 3.1 samples/s
+Step 13960/37465 | Loss: 2.2849 | LR: 3.57e-04 | 6368.5ms/step | 3.1 samples/s
+Step 13970/37465 | Loss: 1.9871 | LR: 3.56e-04 | 6369.2ms/step | 3.1 samples/s
+Step 13980/37465 | Loss: 2.0952 | LR: 3.56e-04 | 6371.1ms/step | 3.1 samples/s
+Step 13990/37465 | Loss: 2.0158 | LR: 3.56e-04 | 6369.4ms/step | 3.1 samples/s
+Step 14000/37465 | Loss: 2.0204 | LR: 3.56e-04 | 6369.3ms/step | 3.1 samples/s
+Validation at step 14000: Loss = 2.1755, Perplexity = 8.81
+Step 14010/37465 | Loss: 2.1946 | LR: 3.56e-04 | 7139.8ms/step | 2.8 samples/s
+Step 14020/37465 | Loss: 2.2801 | LR: 3.55e-04 | 6383.3ms/step | 3.1 samples/s
+Step 14030/37465 | Loss: 2.0314 | LR: 3.55e-04 | 6381.8ms/step | 3.1 samples/s
+Step 14040/37465 | Loss: 2.1638 | LR: 3.55e-04 | 6390.7ms/step | 3.1 samples/s
+Step 14050/37465 | Loss: 2.1134 | LR: 3.55e-04 | 6400.1ms/step | 3.1 samples/s
+Step 14060/37465 | Loss: 2.1680 | LR: 3.55e-04 | 6393.4ms/step | 3.1 samples/s
+Step 14070/37465 | Loss: 2.2838 | LR: 3.54e-04 | 6392.0ms/step | 3.1 samples/s
+Step 14080/37465 | Loss: 2.0191 | LR: 3.54e-04 | 6396.2ms/step | 3.1 samples/s
+Step 14090/37465 | Loss: 1.9681 | LR: 3.54e-04 | 6388.8ms/step | 3.1 samples/s
+Step 14100/37465 | Loss: 2.1645 | LR: 3.54e-04 | 6383.4ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-14100
+Step 14110/37465 | Loss: 2.3167 | LR: 3.54e-04 | 7217.2ms/step | 2.8 samples/s
+Step 14120/37465 | Loss: 2.1068 | LR: 3.53e-04 | 6363.0ms/step | 3.1 samples/s
+Step 14130/37465 | Loss: 2.1149 | LR: 3.53e-04 | 6369.7ms/step | 3.1 samples/s
+Step 14140/37465 | Loss: 2.3787 | LR: 3.53e-04 | 6367.3ms/step | 3.1 samples/s
+Step 14150/37465 | Loss: 2.2132 | LR: 3.53e-04 | 6367.5ms/step | 3.1 samples/s
+Step 14160/37465 | Loss: 1.8507 | LR: 3.53e-04 | 6366.5ms/step | 3.1 samples/s
+Step 14170/37465 | Loss: 2.3047 | LR: 3.53e-04 | 6368.6ms/step | 3.1 samples/s
+Step 14180/37465 | Loss: 2.0906 | LR: 3.52e-04 | 6369.7ms/step | 3.1 samples/s
+Step 14190/37465 | Loss: 2.0775 | LR: 3.52e-04 | 6371.0ms/step | 3.1 samples/s
+Step 14200/37465 | Loss: 2.2080 | LR: 3.52e-04 | 6364.2ms/step | 3.1 samples/s
+Step 14210/37465 | Loss: 2.1505 | LR: 3.52e-04 | 6367.7ms/step | 3.1 samples/s
+Step 14220/37465 | Loss: 2.2481 | LR: 3.52e-04 | 6374.0ms/step | 3.1 samples/s
+Step 14230/37465 | Loss: 2.2816 | LR: 3.51e-04 | 6366.2ms/step | 3.1 samples/s
+Step 14240/37465 | Loss: 1.9671 | LR: 3.51e-04 | 6365.9ms/step | 3.1 samples/s
+Step 14250/37465 | Loss: 2.2365 | LR: 3.51e-04 | 6370.1ms/step | 3.1 samples/s
+Step 14260/37465 | Loss: 2.2084 | LR: 3.51e-04 | 6370.8ms/step | 3.1 samples/s
+Step 14270/37465 | Loss: 2.2190 | LR: 3.51e-04 | 6366.6ms/step | 3.1 samples/s
+Step 14280/37465 | Loss: 2.2877 | LR: 3.50e-04 | 6366.0ms/step | 3.1 samples/s
+Step 14290/37465 | Loss: 1.9823 | LR: 3.50e-04 | 6368.4ms/step | 3.1 samples/s
+Step 14300/37465 | Loss: 2.5094 | LR: 3.50e-04 | 6369.1ms/step | 3.1 samples/s
+Step 14310/37465 | Loss: 2.1061 | LR: 3.50e-04 | 6367.3ms/step | 3.1 samples/s
+Step 14320/37465 | Loss: 2.2721 | LR: 3.50e-04 | 6367.2ms/step | 3.1 samples/s
+Step 14330/37465 | Loss: 2.1312 | LR: 3.49e-04 | 6366.6ms/step | 3.1 samples/s
+Step 14340/37465 | Loss: 2.1073 | LR: 3.49e-04 | 6368.3ms/step | 3.1 samples/s
+Step 14350/37465 | Loss: 2.0273 | LR: 3.49e-04 | 6367.2ms/step | 3.1 samples/s
+Step 14360/37465 | Loss: 2.2089 | LR: 3.49e-04 | 6367.3ms/step | 3.1 samples/s
+Step 14370/37465 | Loss: 2.1985 | LR: 3.49e-04 | 6369.6ms/step | 3.1 samples/s
+Step 14380/37465 | Loss: 2.1767 | LR: 3.48e-04 | 6368.6ms/step | 3.1 samples/s
+Step 14390/37465 | Loss: 2.1296 | LR: 3.48e-04 | 6363.6ms/step | 3.1 samples/s
+Step 14400/37465 | Loss: 2.1224 | LR: 3.48e-04 | 6366.7ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-14400
+Step 14410/37465 | Loss: 2.1755 | LR: 3.48e-04 | 7254.4ms/step | 2.8 samples/s
+Step 14420/37465 | Loss: 2.3244 | LR: 3.48e-04 | 6360.1ms/step | 3.1 samples/s
+Step 14430/37465 | Loss: 1.9281 | LR: 3.47e-04 | 6367.7ms/step | 3.1 samples/s
+Step 14440/37465 | Loss: 2.1976 | LR: 3.47e-04 | 6385.8ms/step | 3.1 samples/s
+Step 14450/37465 | Loss: 2.3460 | LR: 3.47e-04 | 6378.5ms/step | 3.1 samples/s
+Step 14460/37465 | Loss: 2.1171 | LR: 3.47e-04 | 6389.2ms/step | 3.1 samples/s
+Step 14470/37465 | Loss: 2.0977 | LR: 3.47e-04 | 6383.2ms/step | 3.1 samples/s
+Step 14480/37465 | Loss: 2.1791 | LR: 3.46e-04 | 6370.1ms/step | 3.1 samples/s
+Step 14490/37465 | Loss: 2.0231 | LR: 3.46e-04 | 6390.1ms/step | 3.1 samples/s
+Step 14500/37465 | Loss: 2.1271 | LR: 3.46e-04 | 6389.5ms/step | 3.1 samples/s
+Validation at step 14500: Loss = 2.1650, Perplexity = 8.71
+Step 14510/37465 | Loss: 2.1169 | LR: 3.46e-04 | 7114.3ms/step | 2.8 samples/s
+Step 14520/37465 | Loss: 2.3124 | LR: 3.46e-04 | 6369.9ms/step | 3.1 samples/s
+Step 14530/37465 | Loss: 2.1476 | LR: 3.45e-04 | 6373.6ms/step | 3.1 samples/s
+Step 14540/37465 | Loss: 2.0613 | LR: 3.45e-04 | 6370.3ms/step | 3.1 samples/s
+Step 14550/37465 | Loss: 2.0775 | LR: 3.45e-04 | 6378.5ms/step | 3.1 samples/s
+Step 14560/37465 | Loss: 2.0322 | LR: 3.45e-04 | 6391.0ms/step | 3.1 samples/s
+Step 14570/37465 | Loss: 2.2577 | LR: 3.45e-04 | 6383.7ms/step | 3.1 samples/s
+Step 14580/37465 | Loss: 2.3854 | LR: 3.44e-04 | 6387.5ms/step | 3.1 samples/s
+Step 14590/37465 | Loss: 2.0735 | LR: 3.44e-04 | 6387.0ms/step | 3.1 samples/s
+Step 14600/37465 | Loss: 2.2532 | LR: 3.44e-04 | 6395.8ms/step | 3.1 samples/s
+Step 14610/37465 | Loss: 2.1828 | LR: 3.44e-04 | 6390.9ms/step | 3.1 samples/s
+Step 14620/37465 | Loss: 2.3182 | LR: 3.44e-04 | 6390.2ms/step | 3.1 samples/s
+Step 14630/37465 | Loss: 2.0965 | LR: 3.43e-04 | 6382.5ms/step | 3.1 samples/s
+Step 14640/37465 | Loss: 2.2808 | LR: 3.43e-04 | 6389.8ms/step | 3.1 samples/s
+Step 14650/37465 | Loss: 2.0155 | LR: 3.43e-04 | 6387.0ms/step | 3.1 samples/s
+Step 14660/37465 | Loss: 1.9774 | LR: 3.43e-04 | 6396.4ms/step | 3.1 samples/s
+Step 14670/37465 | Loss: 2.1642 | LR: 3.43e-04 | 6375.4ms/step | 3.1 samples/s
+Step 14680/37465 | Loss: 2.0299 | LR: 3.42e-04 | 6396.3ms/step | 3.1 samples/s
+Step 14690/37465 | Loss: 2.1358 | LR: 3.42e-04 | 6372.7ms/step | 3.1 samples/s
+Step 14700/37465 | Loss: 2.3069 | LR: 3.42e-04 | 6385.0ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-14700
+Step 14710/37465 | Loss: 2.1485 | LR: 3.42e-04 | 7207.6ms/step | 2.8 samples/s
+Step 14720/37465 | Loss: 2.2051 | LR: 3.42e-04 | 6349.6ms/step | 3.1 samples/s
+Step 14730/37465 | Loss: 2.1022 | LR: 3.41e-04 | 6364.9ms/step | 3.1 samples/s
+Step 14740/37465 | Loss: 2.1708 | LR: 3.41e-04 | 6375.6ms/step | 3.1 samples/s
+Step 14750/37465 | Loss: 2.0972 | LR: 3.41e-04 | 6389.1ms/step | 3.1 samples/s
+Step 14760/37465 | Loss: 2.1225 | LR: 3.41e-04 | 6400.2ms/step | 3.1 samples/s
+Step 14770/37465 | Loss: 2.2417 | LR: 3.41e-04 | 6403.9ms/step | 3.1 samples/s
+Step 14780/37465 | Loss: 2.2120 | LR: 3.40e-04 | 6397.2ms/step | 3.1 samples/s
+Step 14790/37465 | Loss: 1.9596 | LR: 3.40e-04 | 6400.4ms/step | 3.1 samples/s
+Step 14800/37465 | Loss: 2.1538 | LR: 3.40e-04 | 6405.0ms/step | 3.1 samples/s
+Step 14810/37465 | Loss: 2.1852 | LR: 3.40e-04 | 6407.6ms/step | 3.1 samples/s
+Step 14820/37465 | Loss: 2.1338 | LR: 3.40e-04 | 6396.9ms/step | 3.1 samples/s
+Step 14830/37465 | Loss: 2.0693 | LR: 3.39e-04 | 6384.8ms/step | 3.1 samples/s
+Step 14840/37465 | Loss: 2.1841 | LR: 3.39e-04 | 6395.3ms/step | 3.1 samples/s
+Step 14850/37465 | Loss: 2.3232 | LR: 3.39e-04 | 6406.9ms/step | 3.1 samples/s
+Step 14860/37465 | Loss: 2.0659 | LR: 3.39e-04 | 6404.8ms/step | 3.1 samples/s
+Step 14870/37465 | Loss: 2.3366 | LR: 3.39e-04 | 6396.5ms/step | 3.1 samples/s
+Step 14880/37465 | Loss: 2.2500 | LR: 3.38e-04 | 6396.2ms/step | 3.1 samples/s
+Step 14890/37465 | Loss: 2.1977 | LR: 3.38e-04 | 6394.3ms/step | 3.1 samples/s
+Step 14900/37465 | Loss: 1.9957 | LR: 3.38e-04 | 6403.1ms/step | 3.1 samples/s
+Step 14910/37465 | Loss: 2.0481 | LR: 3.38e-04 | 6382.3ms/step | 3.1 samples/s
+Step 14920/37465 | Loss: 2.2255 | LR: 3.38e-04 | 6401.6ms/step | 3.1 samples/s
+Step 14930/37465 | Loss: 2.0322 | LR: 3.37e-04 | 6386.2ms/step | 3.1 samples/s
+Step 14940/37465 | Loss: 2.2190 | LR: 3.37e-04 | 6395.6ms/step | 3.1 samples/s
+Step 14950/37465 | Loss: 2.1823 | LR: 3.37e-04 | 6420.9ms/step | 3.1 samples/s
+Step 14960/37465 | Loss: 2.3548 | LR: 3.37e-04 | 6389.8ms/step | 3.1 samples/s
+Step 14970/37465 | Loss: 2.2362 | LR: 3.37e-04 | 6382.5ms/step | 3.1 samples/s
+Step 14980/37465 | Loss: 1.9586 | LR: 3.36e-04 | 6393.5ms/step | 3.1 samples/s
+Step 14990/37465 | Loss: 2.0592 | LR: 3.36e-04 | 6390.4ms/step | 3.1 samples/s
+Step 15000/37465 | Loss: 2.1358 | LR: 3.36e-04 | 6381.1ms/step | 3.1 samples/s
+Validation at step 15000: Loss = 2.1561, Perplexity = 8.64
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-15000
+Step 15010/37465 | Loss: 2.0846 | LR: 3.36e-04 | 7978.9ms/step | 2.5 samples/s
+Step 15020/37465 | Loss: 2.3434 | LR: 3.36e-04 | 6386.0ms/step | 3.1 samples/s
+Step 15030/37465 | Loss: 2.4108 | LR: 3.35e-04 | 6381.3ms/step | 3.1 samples/s
+Step 15040/37465 | Loss: 2.2740 | LR: 3.35e-04 | 6385.0ms/step | 3.1 samples/s
+Step 15050/37465 | Loss: 2.1656 | LR: 3.35e-04 | 6373.9ms/step | 3.1 samples/s
+Step 15060/37465 | Loss: 2.1107 | LR: 3.35e-04 | 6380.1ms/step | 3.1 samples/s
+Step 15070/37465 | Loss: 2.0331 | LR: 3.35e-04 | 6391.6ms/step | 3.1 samples/s
+Step 15080/37465 | Loss: 2.2044 | LR: 3.34e-04 | 6386.2ms/step | 3.1 samples/s
+Step 15090/37465 | Loss: 1.9704 | LR: 3.34e-04 | 6385.7ms/step | 3.1 samples/s
+Step 15100/37465 | Loss: 2.3616 | LR: 3.34e-04 | 6373.8ms/step | 3.1 samples/s
+Step 15110/37465 | Loss: 2.1920 | LR: 3.34e-04 | 6389.0ms/step | 3.1 samples/s
+Step 15120/37465 | Loss: 2.2021 | LR: 3.34e-04 | 6389.8ms/step | 3.1 samples/s
+Step 15130/37465 | Loss: 2.1482 | LR: 3.33e-04 | 6390.6ms/step | 3.1 samples/s
+Step 15140/37465 | Loss: 2.2383 | LR: 3.33e-04 | 6381.3ms/step | 3.1 samples/s
+Step 15150/37465 | Loss: 2.1096 | LR: 3.33e-04 | 6403.3ms/step | 3.1 samples/s
+Step 15160/37465 | Loss: 2.1027 | LR: 3.33e-04 | 6389.5ms/step | 3.1 samples/s
+Step 15170/37465 | Loss: 2.0981 | LR: 3.33e-04 | 6381.7ms/step | 3.1 samples/s
+Step 15180/37465 | Loss: 2.1780 | LR: 3.32e-04 | 6396.7ms/step | 3.1 samples/s
+Step 15190/37465 | Loss: 2.2559 | LR: 3.32e-04 | 6386.3ms/step | 3.1 samples/s
+Step 15200/37465 | Loss: 2.0781 | LR: 3.32e-04 | 6379.1ms/step | 3.1 samples/s
+Step 15210/37465 | Loss: 2.3151 | LR: 3.32e-04 | 6385.6ms/step | 3.1 samples/s
+Step 15220/37465 | Loss: 2.3866 | LR: 3.32e-04 | 6379.2ms/step | 3.1 samples/s
+Step 15230/37465 | Loss: 2.1246 | LR: 3.31e-04 | 6383.0ms/step | 3.1 samples/s
+Step 15240/37465 | Loss: 2.2300 | LR: 3.31e-04 | 6375.4ms/step | 3.1 samples/s
+Step 15250/37465 | Loss: 2.1732 | LR: 3.31e-04 | 6387.0ms/step | 3.1 samples/s
+Step 15260/37465 | Loss: 2.1922 | LR: 3.31e-04 | 6376.0ms/step | 3.1 samples/s
+Step 15270/37465 | Loss: 2.2474 | LR: 3.31e-04 | 6375.8ms/step | 3.1 samples/s
+Step 15280/37465 | Loss: 2.1362 | LR: 3.30e-04 | 6381.2ms/step | 3.1 samples/s
+Step 15290/37465 | Loss: 2.2197 | LR: 3.30e-04 | 6384.8ms/step | 3.1 samples/s
+Step 15300/37465 | Loss: 2.1255 | LR: 3.30e-04 | 6385.0ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-15300
+Step 15310/37465 | Loss: 2.1754 | LR: 3.30e-04 | 7222.4ms/step | 2.8 samples/s
+Step 15320/37465 | Loss: 2.4196 | LR: 3.30e-04 | 6362.3ms/step | 3.1 samples/s
+Step 15330/37465 | Loss: 2.0001 | LR: 3.29e-04 | 6369.6ms/step | 3.1 samples/s
+Step 15340/37465 | Loss: 2.1723 | LR: 3.29e-04 | 6368.4ms/step | 3.1 samples/s
+Step 15350/37465 | Loss: 2.2263 | LR: 3.29e-04 | 6370.1ms/step | 3.1 samples/s
+Step 15360/37465 | Loss: 2.3249 | LR: 3.29e-04 | 6367.8ms/step | 3.1 samples/s
+Step 15370/37465 | Loss: 2.2169 | LR: 3.29e-04 | 6370.4ms/step | 3.1 samples/s
+Step 15380/37465 | Loss: 2.2066 | LR: 3.28e-04 | 6368.3ms/step | 3.1 samples/s
+Step 15390/37465 | Loss: 2.1188 | LR: 3.28e-04 | 6370.1ms/step | 3.1 samples/s
+Step 15400/37465 | Loss: 2.3455 | LR: 3.28e-04 | 6371.2ms/step | 3.1 samples/s
+Step 15410/37465 | Loss: 2.1736 | LR: 3.28e-04 | 6370.1ms/step | 3.1 samples/s
+Step 15420/37465 | Loss: 1.9758 | LR: 3.28e-04 | 6367.6ms/step | 3.1 samples/s
+Step 15430/37465 | Loss: 2.0108 | LR: 3.27e-04 | 6369.9ms/step | 3.1 samples/s
+Step 15440/37465 | Loss: 2.3506 | LR: 3.27e-04 | 6369.1ms/step | 3.1 samples/s
+Step 15450/37465 | Loss: 2.1285 | LR: 3.27e-04 | 6369.8ms/step | 3.1 samples/s
+Step 15460/37465 | Loss: 2.1943 | LR: 3.27e-04 | 6370.3ms/step | 3.1 samples/s
+Step 15470/37465 | Loss: 2.2823 | LR: 3.27e-04 | 6369.6ms/step | 3.1 samples/s
+Step 15480/37465 | Loss: 2.0418 | LR: 3.26e-04 | 6369.4ms/step | 3.1 samples/s
+Step 15490/37465 | Loss: 2.3550 | LR: 3.26e-04 | 6371.0ms/step | 3.1 samples/s
+Step 15500/37465 | Loss: 2.4904 | LR: 3.26e-04 | 6367.0ms/step | 3.1 samples/s
+Validation at step 15500: Loss = 2.1511, Perplexity = 8.59
+Step 15510/37465 | Loss: 2.1592 | LR: 3.26e-04 | 7117.9ms/step | 2.8 samples/s
+Step 15520/37465 | Loss: 2.1246 | LR: 3.26e-04 | 6369.5ms/step | 3.1 samples/s
+Step 15530/37465 | Loss: 2.0678 | LR: 3.25e-04 | 6372.0ms/step | 3.1 samples/s
+Step 15540/37465 | Loss: 2.2972 | LR: 3.25e-04 | 6366.1ms/step | 3.1 samples/s
+Step 15550/37465 | Loss: 2.0049 | LR: 3.25e-04 | 6369.4ms/step | 3.1 samples/s
+Step 15560/37465 | Loss: 1.9591 | LR: 3.25e-04 | 6369.0ms/step | 3.1 samples/s
+Step 15570/37465 | Loss: 2.1559 | LR: 3.25e-04 | 6363.6ms/step | 3.1 samples/s
+Step 15580/37465 | Loss: 2.0345 | LR: 3.24e-04 | 6365.1ms/step | 3.1 samples/s
+Step 15590/37465 | Loss: 2.1562 | LR: 3.24e-04 | 6369.2ms/step | 3.1 samples/s
+Step 15600/37465 | Loss: 2.0848 | LR: 3.24e-04 | 6367.9ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-15600
+Step 15610/37465 | Loss: 2.2405 | LR: 3.24e-04 | 7192.2ms/step | 2.8 samples/s
+Step 15620/37465 | Loss: 2.0629 | LR: 3.24e-04 | 6375.6ms/step | 3.1 samples/s
+Step 15630/37465 | Loss: 2.3102 | LR: 3.23e-04 | 6379.5ms/step | 3.1 samples/s
+Step 15640/37465 | Loss: 2.0751 | LR: 3.23e-04 | 6382.3ms/step | 3.1 samples/s
+Step 15650/37465 | Loss: 2.2569 | LR: 3.23e-04 | 6388.6ms/step | 3.1 samples/s
+Step 15660/37465 | Loss: 2.1427 | LR: 3.23e-04 | 6383.2ms/step | 3.1 samples/s
+Step 15670/37465 | Loss: 2.3874 | LR: 3.22e-04 | 6380.1ms/step | 3.1 samples/s
+Step 15680/37465 | Loss: 2.0656 | LR: 3.22e-04 | 6382.6ms/step | 3.1 samples/s
+Step 15690/37465 | Loss: 2.3272 | LR: 3.22e-04 | 6396.0ms/step | 3.1 samples/s
+Step 15700/37465 | Loss: 2.3459 | LR: 3.22e-04 | 6385.9ms/step | 3.1 samples/s
+Step 15710/37465 | Loss: 2.0873 | LR: 3.22e-04 | 6386.3ms/step | 3.1 samples/s
+Step 15720/37465 | Loss: 2.1562 | LR: 3.21e-04 | 6400.2ms/step | 3.1 samples/s
+Step 15730/37465 | Loss: 2.2470 | LR: 3.21e-04 | 6386.2ms/step | 3.1 samples/s
+Step 15740/37465 | Loss: 2.2241 | LR: 3.21e-04 | 6382.2ms/step | 3.1 samples/s
+Step 15750/37465 | Loss: 2.2091 | LR: 3.21e-04 | 6391.8ms/step | 3.1 samples/s
+Step 15760/37465 | Loss: 2.4460 | LR: 3.21e-04 | 6386.2ms/step | 3.1 samples/s
+Step 15770/37465 | Loss: 2.0713 | LR: 3.20e-04 | 6384.7ms/step | 3.1 samples/s
+Step 15780/37465 | Loss: 2.0548 | LR: 3.20e-04 | 6384.1ms/step | 3.1 samples/s
+Step 15790/37465 | Loss: 2.2096 | LR: 3.20e-04 | 6380.8ms/step | 3.1 samples/s
+Step 15800/37465 | Loss: 2.0127 | LR: 3.20e-04 | 6390.2ms/step | 3.1 samples/s
+Step 15810/37465 | Loss: 2.2154 | LR: 3.20e-04 | 6388.2ms/step | 3.1 samples/s
+Step 15820/37465 | Loss: 2.2781 | LR: 3.19e-04 | 6383.2ms/step | 3.1 samples/s
+Step 15830/37465 | Loss: 2.4152 | LR: 3.19e-04 | 6380.4ms/step | 3.1 samples/s
+Step 15840/37465 | Loss: 2.0720 | LR: 3.19e-04 | 6394.5ms/step | 3.1 samples/s
+Step 15850/37465 | Loss: 2.1361 | LR: 3.19e-04 | 6394.7ms/step | 3.1 samples/s
+Step 15860/37465 | Loss: 1.9575 | LR: 3.19e-04 | 6398.7ms/step | 3.1 samples/s
+Step 15870/37465 | Loss: 2.0221 | LR: 3.18e-04 | 6389.0ms/step | 3.1 samples/s
+Step 15880/37465 | Loss: 1.9096 | LR: 3.18e-04 | 6380.7ms/step | 3.1 samples/s
+Step 15890/37465 | Loss: 2.1362 | LR: 3.18e-04 | 6391.9ms/step | 3.1 samples/s
+Step 15900/37465 | Loss: 2.1514 | LR: 3.18e-04 | 6391.8ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-15900
+Step 15910/37465 | Loss: 2.0833 | LR: 3.18e-04 | 7258.5ms/step | 2.8 samples/s
+Step 15920/37465 | Loss: 2.2520 | LR: 3.17e-04 | 6370.7ms/step | 3.1 samples/s
+Step 15930/37465 | Loss: 2.1908 | LR: 3.17e-04 | 6382.3ms/step | 3.1 samples/s
+Step 15940/37465 | Loss: 2.0790 | LR: 3.17e-04 | 6391.1ms/step | 3.1 samples/s
+Step 15950/37465 | Loss: 2.2482 | LR: 3.17e-04 | 6394.8ms/step | 3.1 samples/s
+Step 15960/37465 | Loss: 2.2337 | LR: 3.17e-04 | 6397.3ms/step | 3.1 samples/s
+Step 15970/37465 | Loss: 2.1050 | LR: 3.16e-04 | 6390.0ms/step | 3.1 samples/s
+Step 15980/37465 | Loss: 2.1370 | LR: 3.16e-04 | 6387.0ms/step | 3.1 samples/s
+Step 15990/37465 | Loss: 2.1387 | LR: 3.16e-04 | 6398.6ms/step | 3.1 samples/s
+Step 16000/37465 | Loss: 2.1974 | LR: 3.16e-04 | 6385.1ms/step | 3.1 samples/s
+Validation at step 16000: Loss = 2.1419, Perplexity = 8.52
+Step 16010/37465 | Loss: 2.0627 | LR: 3.15e-04 | 7119.8ms/step | 2.8 samples/s
+Step 16020/37465 | Loss: 2.1877 | LR: 3.15e-04 | 6367.5ms/step | 3.1 samples/s
+Step 16030/37465 | Loss: 2.1707 | LR: 3.15e-04 | 6367.0ms/step | 3.1 samples/s
+Step 16040/37465 | Loss: 2.1585 | LR: 3.15e-04 | 6367.6ms/step | 3.1 samples/s
+Step 16050/37465 | Loss: 1.9886 | LR: 3.15e-04 | 6365.0ms/step | 3.1 samples/s
+Step 16060/37465 | Loss: 1.9092 | LR: 3.14e-04 | 6367.6ms/step | 3.1 samples/s
+Step 16070/37465 | Loss: 2.1091 | LR: 3.14e-04 | 6365.1ms/step | 3.1 samples/s
+Step 16080/37465 | Loss: 2.0553 | LR: 3.14e-04 | 6365.5ms/step | 3.1 samples/s
+Step 16090/37465 | Loss: 2.2302 | LR: 3.14e-04 | 6366.3ms/step | 3.1 samples/s
+Step 16100/37465 | Loss: 2.0317 | LR: 3.14e-04 | 6369.1ms/step | 3.1 samples/s
+Step 16110/37465 | Loss: 2.2083 | LR: 3.13e-04 | 6367.3ms/step | 3.1 samples/s
+Step 16120/37465 | Loss: 2.0373 | LR: 3.13e-04 | 6365.5ms/step | 3.1 samples/s
+Step 16130/37465 | Loss: 1.9851 | LR: 3.13e-04 | 6366.0ms/step | 3.1 samples/s
+Step 16140/37465 | Loss: 2.3132 | LR: 3.13e-04 | 6365.7ms/step | 3.1 samples/s
+Step 16150/37465 | Loss: 2.1328 | LR: 3.13e-04 | 6367.2ms/step | 3.1 samples/s
+Step 16160/37465 | Loss: 1.8949 | LR: 3.12e-04 | 6365.5ms/step | 3.1 samples/s
+Step 16170/37465 | Loss: 2.2175 | LR: 3.12e-04 | 6365.7ms/step | 3.1 samples/s
+Step 16180/37465 | Loss: 2.3401 | LR: 3.12e-04 | 6368.7ms/step | 3.1 samples/s
+Step 16190/37465 | Loss: 2.3139 | LR: 3.12e-04 | 6365.5ms/step | 3.1 samples/s
+Step 16200/37465 | Loss: 2.0547 | LR: 3.12e-04 | 6364.4ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-16200
+Step 16210/37465 | Loss: 2.0493 | LR: 3.11e-04 | 7198.6ms/step | 2.8 samples/s
+Step 16220/37465 | Loss: 2.0250 | LR: 3.11e-04 | 6359.0ms/step | 3.1 samples/s
+Step 16230/37465 | Loss: 2.0703 | LR: 3.11e-04 | 6375.3ms/step | 3.1 samples/s
+Step 16240/37465 | Loss: 2.2373 | LR: 3.11e-04 | 6393.3ms/step | 3.1 samples/s
+Step 16250/37465 | Loss: 2.0281 | LR: 3.11e-04 | 6369.1ms/step | 3.1 samples/s
+Step 16260/37465 | Loss: 2.1965 | LR: 3.10e-04 | 6383.1ms/step | 3.1 samples/s
+Step 16270/37465 | Loss: 2.2070 | LR: 3.10e-04 | 6382.9ms/step | 3.1 samples/s
+Step 16280/37465 | Loss: 2.2367 | LR: 3.10e-04 | 6391.6ms/step | 3.1 samples/s
+Step 16290/37465 | Loss: 2.2437 | LR: 3.10e-04 | 6385.5ms/step | 3.1 samples/s
+Step 16300/37465 | Loss: 2.2265 | LR: 3.09e-04 | 6388.4ms/step | 3.1 samples/s
+Step 16310/37465 | Loss: 1.9594 | LR: 3.09e-04 | 6392.7ms/step | 3.1 samples/s
+Step 16320/37465 | Loss: 2.1062 | LR: 3.09e-04 | 6383.2ms/step | 3.1 samples/s
+Step 16330/37465 | Loss: 2.1146 | LR: 3.09e-04 | 6392.0ms/step | 3.1 samples/s
+Step 16340/37465 | Loss: 2.1178 | LR: 3.09e-04 | 6394.8ms/step | 3.1 samples/s
+Step 16350/37465 | Loss: 2.2422 | LR: 3.08e-04 | 6381.1ms/step | 3.1 samples/s
+Step 16360/37465 | Loss: 2.0249 | LR: 3.08e-04 | 6385.7ms/step | 3.1 samples/s
+Step 16370/37465 | Loss: 2.2969 | LR: 3.08e-04 | 6385.4ms/step | 3.1 samples/s
+Step 16380/37465 | Loss: 2.1431 | LR: 3.08e-04 | 6379.8ms/step | 3.1 samples/s
+Step 16390/37465 | Loss: 1.9768 | LR: 3.08e-04 | 6390.3ms/step | 3.1 samples/s
+Step 16400/37465 | Loss: 1.8444 | LR: 3.07e-04 | 6398.2ms/step | 3.1 samples/s
+Step 16410/37465 | Loss: 2.0089 | LR: 3.07e-04 | 6386.9ms/step | 3.1 samples/s
+Step 16420/37465 | Loss: 2.3197 | LR: 3.07e-04 | 6382.6ms/step | 3.1 samples/s
+Step 16430/37465 | Loss: 2.1429 | LR: 3.07e-04 | 6389.8ms/step | 3.1 samples/s
+Step 16440/37465 | Loss: 2.1560 | LR: 3.07e-04 | 6376.0ms/step | 3.1 samples/s
+Step 16450/37465 | Loss: 2.1730 | LR: 3.06e-04 | 6397.7ms/step | 3.1 samples/s
+Step 16460/37465 | Loss: 2.2393 | LR: 3.06e-04 | 6398.8ms/step | 3.1 samples/s
+Step 16470/37465 | Loss: 2.2210 | LR: 3.06e-04 | 6389.8ms/step | 3.1 samples/s
+Step 16480/37465 | Loss: 2.0427 | LR: 3.06e-04 | 6401.2ms/step | 3.1 samples/s
+Step 16490/37465 | Loss: 2.1295 | LR: 3.06e-04 | 6386.0ms/step | 3.1 samples/s
+Step 16500/37465 | Loss: 2.0503 | LR: 3.05e-04 | 6377.3ms/step | 3.1 samples/s
+Validation at step 16500: Loss = 2.1341, Perplexity = 8.45
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-16500
+Step 16510/37465 | Loss: 2.1940 | LR: 3.05e-04 | 7983.2ms/step | 2.5 samples/s
+Step 16520/37465 | Loss: 2.2603 | LR: 3.05e-04 | 6361.2ms/step | 3.1 samples/s
+Step 16530/37465 | Loss: 1.9918 | LR: 3.05e-04 | 6381.9ms/step | 3.1 samples/s
+Step 16540/37465 | Loss: 2.1904 | LR: 3.04e-04 | 6381.4ms/step | 3.1 samples/s
+Step 16550/37465 | Loss: 2.3764 | LR: 3.04e-04 | 6385.2ms/step | 3.1 samples/s
+Step 16560/37465 | Loss: 2.1837 | LR: 3.04e-04 | 6382.3ms/step | 3.1 samples/s
+Step 16570/37465 | Loss: 2.0634 | LR: 3.04e-04 | 6386.5ms/step | 3.1 samples/s
+Step 16580/37465 | Loss: 2.1037 | LR: 3.04e-04 | 6398.1ms/step | 3.1 samples/s
+Step 16590/37465 | Loss: 2.1111 | LR: 3.03e-04 | 6378.3ms/step | 3.1 samples/s
+Step 16600/37465 | Loss: 2.1911 | LR: 3.03e-04 | 6373.5ms/step | 3.1 samples/s
+Step 16610/37465 | Loss: 2.1009 | LR: 3.03e-04 | 6382.0ms/step | 3.1 samples/s
+Step 16620/37465 | Loss: 2.1424 | LR: 3.03e-04 | 6381.6ms/step | 3.1 samples/s
+Step 16630/37465 | Loss: 2.1943 | LR: 3.03e-04 | 6392.4ms/step | 3.1 samples/s
+Step 16640/37465 | Loss: 2.0890 | LR: 3.02e-04 | 6385.4ms/step | 3.1 samples/s
+Step 16650/37465 | Loss: 2.0347 | LR: 3.02e-04 | 6394.5ms/step | 3.1 samples/s
+Step 16660/37465 | Loss: 2.1890 | LR: 3.02e-04 | 6380.1ms/step | 3.1 samples/s
+Step 16670/37465 | Loss: 2.0895 | LR: 3.02e-04 | 6393.6ms/step | 3.1 samples/s
+Step 16680/37465 | Loss: 2.2781 | LR: 3.02e-04 | 6392.6ms/step | 3.1 samples/s
+Step 16690/37465 | Loss: 2.3166 | LR: 3.01e-04 | 6378.0ms/step | 3.1 samples/s
+Step 16700/37465 | Loss: 1.9661 | LR: 3.01e-04 | 6390.0ms/step | 3.1 samples/s
+Step 16710/37465 | Loss: 2.0900 | LR: 3.01e-04 | 6377.9ms/step | 3.1 samples/s
+Step 16720/37465 | Loss: 2.1978 | LR: 3.01e-04 | 6368.1ms/step | 3.1 samples/s
+Step 16730/37465 | Loss: 2.1452 | LR: 3.01e-04 | 6375.5ms/step | 3.1 samples/s
+Step 16740/37465 | Loss: 1.9208 | LR: 3.00e-04 | 6386.6ms/step | 3.1 samples/s
+Step 16750/37465 | Loss: 2.1976 | LR: 3.00e-04 | 6373.1ms/step | 3.1 samples/s
+Step 16760/37465 | Loss: 2.3627 | LR: 3.00e-04 | 6379.2ms/step | 3.1 samples/s
+Step 16770/37465 | Loss: 1.9211 | LR: 3.00e-04 | 6377.7ms/step | 3.1 samples/s
+Step 16780/37465 | Loss: 1.9545 | LR: 2.99e-04 | 6374.1ms/step | 3.1 samples/s
+Step 16790/37465 | Loss: 2.1300 | LR: 2.99e-04 | 6383.0ms/step | 3.1 samples/s
+Step 16800/37465 | Loss: 2.0184 | LR: 2.99e-04 | 6385.9ms/step | 3.1 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-16800
+Step 16810/37465 | Loss: 2.1033 | LR: 2.99e-04 | 7232.8ms/step | 2.8 samples/s
+Step 16820/37465 | Loss: 1.9405 | LR: 2.99e-04 | 6369.6ms/step | 3.1 samples/s
+Step 16830/37465 | Loss: 2.0058 | LR: 2.98e-04 | 6380.2ms/step | 3.1 samples/s
+Step 16840/37465 | Loss: 2.1582 | LR: 2.98e-04 | 6371.5ms/step | 3.1 samples/s
+Step 16850/37465 | Loss: 2.3082 | LR: 2.98e-04 | 6389.5ms/step | 3.1 samples/s
+Step 16860/37465 | Loss: 2.1048 | LR: 2.98e-04 | 6400.8ms/step | 3.1 samples/s
+Step 16870/37465 | Loss: 1.8812 | LR: 2.98e-04 | 6385.1ms/step | 3.1 samples/s
+Step 16880/37465 | Loss: 2.1725 | LR: 2.97e-04 | 6394.5ms/step | 3.1 samples/s
+Step 16890/37465 | Loss: 2.1955 | LR: 2.97e-04 | 6385.9ms/step | 3.1 samples/s
+Step 16900/37465 | Loss: 2.0537 | LR: 2.97e-04 | 6381.5ms/step | 3.1 samples/s
+Step 16910/37465 | Loss: 2.2637 | LR: 2.97e-04 | 6384.0ms/step | 3.1 samples/s
+Step 16920/37465 | Loss: 2.5258 | LR: 2.97e-04 | 6388.4ms/step | 3.1 samples/s
+Step 16930/37465 | Loss: 2.2026 | LR: 2.96e-04 | 6404.8ms/step | 3.1 samples/s
+Step 16940/37465 | Loss: 2.3028 | LR: 2.96e-04 | 6397.3ms/step | 3.1 samples/s
+Step 16950/37465 | Loss: 2.3473 | LR: 2.96e-04 | 6383.4ms/step | 3.1 samples/s
+Step 16960/37465 | Loss: 2.3319 | LR: 2.96e-04 | 6385.2ms/step | 3.1 samples/s
+Step 16970/37465 | Loss: 2.1164 | LR: 2.95e-04 | 6403.0ms/step | 3.1 samples/s
+Step 16980/37465 | Loss: 1.9927 | LR: 2.95e-04 | 6383.8ms/step | 3.1 samples/s

epfl_final/SparseAtt/wandb/run-20260710_214547-cd66e13p/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260710_214547-cd66e13p/run-cd66e13p.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efbe4f94466ff9e8de7cefeb77f29b340c9d5272032572a0c439a9b6d4831a83
+size 6750208

epfl_final/SparseAtt/wandb/run-20260711_014922-hn868f80/files/config.yaml

@@ -0,0 +1,190 @@
+_wandb:
+    value:
+        cli_version: 0.26.1
+        e:
+            pspecsuhmt7dtlbip70bxzvlz0boe7t3:
+                args:
+                    - --config
+                    - configs/train_config_30k_resampled_b128.json
+                codePath: train.py
+                codePathLocal: train.py
+                cpu_count: 96
+                cpu_count_logical: 192
+                cudaVersion: "13.0"
+                disk:
+                    /:
+                        total: "9374931091456"
+                        used: "6273430933504"
+                email: ruben.navasardyan@mail.utoronto.ca
+                executable: /mloscratch/homes/navasard/conda/bin/python3.13
+                git:
+                    commit: 88298c7a418622530ca5426b22b1e5ebd10ca0cb
+                    remote: https://github.com/rubentium/TokenPrune.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 4
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-cd5c7d97-0f62-2f41-36c8-233c8f57b814
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-b97c2df9-b38b-c4e9-4d9d-780e91b62380
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-337db0d9-bb90-c768-bb02-7b5dfde9a5d1
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-129f6620-c2e8-b738-a48f-5b9473847237
+                host: h100-train-4-gpu-0-3
+                memory:
+                    total: "1623005732864"
+                os: Linux-6.8.0-124-generic-x86_64-with-glibc2.35
+                program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
+                python: CPython 3.13.13
+                root: /mloscratch/homes/navasard/sparse/SparseAtt
+                startedAt: "2026-07-11T01:49:22.406280Z"
+                writerId: pspecsuhmt7dtlbip70bxzvlz0boe7t3
+        m: []
+        python_version: 3.13.13
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 5
+                - 13
+                - 14
+                - 15
+                - 16
+                - 61
+                - 62
+            "4": 3.13.13
+            "5": 0.26.1
+            "6": 5.7.0
+            "12": 0.26.1
+            "13": linux-x86_64
+attention_type:
+    value: sparse_stochastic
+checkpoint_activations:
+    value: true
+dense_mode:
+    value: false
+dense_step_intervals:
+    value: null
+eval_batches:
+    value: 20
+eval_steps:
+    value: 500
+force_new_run:
+    value: false
+gradient_accumulation_steps:
+    value: 5
+head_dim:
+    value: null
+hidden_size:
+    value: 1536
+init_from_scratch:
+    value: true
+intermediate_size:
+    value: 6144
+learning_rate:
+    value: 0.0005
+logging_steps:
+    value: 10
+lr_scheduler:
+    value: cosine
+max_grad_norm:
+    value: 1
+max_seq_length:
+    value: 30720
+max_steps:
+    value: null
+mixed_precision:
+    value: bf16
+model_dtype:
+    value: bfloat16
+model_path:
+    value: /mloscratch/homes/navasard/qwen3_06b_base
+num_attention_heads:
+    value: 12
+num_epochs:
+    value: 1
+num_hidden_layers:
+    value: null
+num_key_value_heads:
+    value: 4
+num_workers:
+    value: 4
+output_dir:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209
+per_device_batch_size:
+    value: 1
+resume_from_checkpoint:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-8700
+save_steps:
+    value: 300
+seed:
+    value: 42
+sparse_block_size:
+    value: 128
+sparse_layer_indices:
+    value: null
+sparse_mask_cache_dir:
+    value: /mloscratch/homes/navasard/sparse/mask_cache_30k_b128_nrb120
+sparse_mask_pool_size:
+    value: 1000
+sparse_mask_seed:
+    value: 0
+sparse_num_random_blocks:
+    value: 120
+sparse_val_dense_compare:
+    value: false
+strategy:
+    value: ddp
+train_path:
+    value: /mloscratch/homes/navasard/data/longctx30k/tokenized
+val_path:
+    value: null
+verbose:
+    value: true
+wandb_enabled:
+    value: true
+wandb_name:
+    value: 1.2b resampled 30k longctx
+wandb_project:
+    value: stochastic_sparsity
+wandb_tags:
+    value:
+        - qwen3
+        - 1.2b
+        - resampled
+        - 30k
+        - longctx
+        - b128
+        - scaleladder
+warmup_steps:
+    value: 750
+weight_decay:
+    value: 0.01

epfl_final/SparseAtt/wandb/run-20260711_014922-hn868f80/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260711_014922-hn868f80/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-07-11T01:49:22.406280Z",
+  "args":  [
+    "--config",
+    "configs/train_config_30k_resampled_b128.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "88298c7a418622530ca5426b22b1e5ebd10ca0cb"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-train-4-gpu-0-3",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "6273430933504"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-cd5c7d97-0f62-2f41-36c8-233c8f57b814"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-b97c2df9-b38b-c4e9-4d9d-780e91b62380"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-337db0d9-bb90-c768-bb02-7b5dfde9a5d1"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-129f6620-c2e8-b738-a48f-5b9473847237"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "pspecsuhmt7dtlbip70bxzvlz0boe7t3"
+}

epfl_final/SparseAtt/wandb/run-20260711_014922-hn868f80/files/wandb-summary.json

@@ -0,0 +1 @@
+{"speed/ms_per_step":8129.328346252441,"eval/uses_sparse_attention":1,"train/epoch":0.9998665421059656,"eval/loss":2.070084353983402,"speed/samples_per_sec":2.4602278500929105,"speed/steps_per_sec":0.12301139250464553,"eval/dense_interval_active":0,"_step":37460,"eval/dense_interval_start":-1,"train/learning_rate":2.2880351668863596e-11,"train/global_step":37460,"eval/perplexity":7.925491636442943,"eval/dense_interval_end":-1,"eval/uses_dense_attention":0,"_runtime":382723.979437986,"train/loss":1.9458291056379675,"_timestamp":1.7839699260158184e+09,"_wandb":{"runtime":382723}}

epfl_final/SparseAtt/wandb/run-20260711_014922-hn868f80/logs/debug.log

@@ -0,0 +1,25 @@
+2026-07-11 01:49:22,428 INFO    MainThread:3626 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-07-11 01:49:22,428 INFO    MainThread:3626 [wandb_setup.py:_flush():81] Configure stats pid to 3626
+2026-07-11 01:49:22,428 INFO    MainThread:3626 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-11 01:49:22,428 INFO    MainThread:3626 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260711_014922-hn868f80/logs/debug.log
+2026-07-11 01:49:22,428 INFO    MainThread:3626 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260711_014922-hn868f80/logs/debug-internal.log
+2026-07-11 01:49:22,428 INFO    MainThread:3626 [wandb_init.py:init():850] calling init triggers
+2026-07-11 01:49:22,429 INFO    MainThread:3626 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/longctx30k/tokenized', 'val_path': None, 'max_seq_length': 30720, 'num_workers': 4, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_resampled/run_20260708_112209/checkpoint-8700', 'force_new_run': False, 'num_epochs': 1, 'max_steps': None, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 5, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 300, 'eval_steps': 500, 'eval_batches': 20, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b resampled 30k longctx', 'wandb_tags': ['qwen3', '1.2b', 'resampled', '30k', 'longctx', 'b128', 'scaleladder'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 120, 'sparse_mask_pool_size': 1000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_30k_b128_nrb120', 'sparse_mask_seed': 0, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-07-11 01:49:22,429 INFO    MainThread:3626 [wandb_init.py:init():898] starting backend
+2026-07-11 01:49:22,693 INFO    MainThread:3626 [wandb_init.py:init():913] sending inform_init request
+2026-07-11 01:49:23,023 INFO    MainThread:3626 [wandb_init.py:init():918] backend started and connected
+2026-07-11 01:49:23,027 INFO    MainThread:3626 [wandb_init.py:init():988] updated telemetry
+2026-07-11 01:49:23,060 INFO    MainThread:3626 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-07-11 01:49:23,434 INFO    MainThread:3626 [wandb_init.py:init():1051] run resumed
+2026-07-11 01:49:23,436 INFO    MainThread:3626 [wandb_init.py:init():1056] starting run threads in backend
+2026-07-11 01:49:23,687 INFO    MainThread:3626 [wandb_run.py:_console_start():2554] atexit reg
+2026-07-11 01:49:23,687 INFO    MainThread:3626 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-07-11 01:49:23,687 INFO    MainThread:3626 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-07-11 01:49:23,687 INFO    MainThread:3626 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-07-11 01:49:23,693 INFO    MainThread:3626 [wandb_init.py:init():1094] run started, returning control to user process
+2026-07-13 19:12:55,392 INFO    MainThread:3626 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/hn868f80
+2026-07-13 19:12:55,393 INFO    MainThread:3626 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
+2026-07-13 19:12:55,394 INFO    MainThread:3626 [wandb_run.py:_restore():2502] restore
+2026-07-13 19:12:55,394 INFO    MainThread:3626 [wandb_run.py:_restore():2508] restore done
+2026-07-13 19:12:57,406 INFO    MainThread:3626 [wandb_run.py:_footer_sync_info():3889] logging synced files

epfl_final/SparseAtt/wandb/run-20260711_014922-hn868f80/run-hn868f80.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbd51c453c01fab62703a85a941be67c0007898f745fc89f38d82641484847fe
+size 76708694

epfl_final/SparseAtt/wandb/run-20260711_074750-cd66e13p/files/output.log

@@ -0,0 +1,3 @@
+Starting training...
+Validation at step 16800: Loss = 2.1319, Perplexity = 8.43
+wandb: WARNING Tried to log to step 16800 that is less than the current step 16971. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.

epfl_final/SparseAtt/wandb/run-20260711_074750-cd66e13p/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260711_074750-cd66e13p/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-07-11T07:47:50.263885Z",
+  "args":  [
+    "--config",
+    "configs/train_config_30k_dense_b128.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "88298c7a418622530ca5426b22b1e5ebd10ca0cb"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h200-train-4-gpu-0-4",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H200",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "7679361548288",
+      "used":  "3627044163584"
+    }
+  },
+  "memory":  {
+    "total":  "1623010586624"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H200",
+      "memoryTotal":  "150754820096",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-6d391a7d-303b-75a5-985c-7980ae7165a0"
+    },
+    {
+      "name":  "NVIDIA H200",
+      "memoryTotal":  "150754820096",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-5e2907f1-1a74-acc8-3916-4a4242799fbe"
+    },
+    {
+      "name":  "NVIDIA H200",
+      "memoryTotal":  "150754820096",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bb9585a1-d507-7096-5a0e-a9667db36e4a"
+    },
+    {
+      "name":  "NVIDIA H200",
+      "memoryTotal":  "150754820096",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-112ee834-613d-54f0-1940-9e0f764fc8e9"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "qzb75kngnssa6tjvtd49ace5r5ez9gcy"
+}

epfl_final/SparseAtt/wandb/run-20260711_074750-cd66e13p/logs/debug-internal.log

@@ -0,0 +1,20 @@
+{"time":"2026-07-11T07:47:51.054449975Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-07-11T07:47:51.059928755Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-07-11T07:47:51.455046102Z","level":"INFO","msg":"stream: created new stream","id":"cd66e13p"}
+{"time":"2026-07-11T07:47:51.455169807Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-07-11T07:47:51.457348338Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-07-11T07:47:51.458510998Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-07-11T07:47:51.458515585Z","level":"INFO","msg":"writer: started","stream_id":"cd66e13p"}
+{"time":"2026-07-11T07:47:51.897827795Z","level":"INFO","msg":"filestream: sending request","total_files":1}
+{"time":"2026-07-11T07:47:52.170358494Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-11T07:48:06.897807756Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1,"uploaded_len":2}
+{"time":"2026-07-11T07:48:07.097596817Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-11T07:48:21.898686782Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":0,"events_lines":2}
+{"time":"2026-07-11T07:48:22.085388586Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-11T07:48:36.897865586Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
+{"time":"2026-07-11T07:48:37.122934981Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-11T07:48:44.12675121Z","level":"WARN","msg":"handler: ignoring partial history record","step":16800,"current":16971}
+{"time":"2026-07-11T07:48:51.898387601Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":4,"events_lines":2,"console_offset":1,"console_lines":1}
+{"time":"2026-07-11T07:48:52.092742244Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-11T07:49:06.898713604Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":6,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-07-11T07:49:07.106833639Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

epfl_final/SparseAtt/wandb/run-20260711_074750-cd66e13p/logs/debug.log

@@ -0,0 +1,20 @@
+2026-07-11 07:47:50,283 INFO    MainThread:199 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-07-11 07:47:50,284 INFO    MainThread:199 [wandb_setup.py:_flush():81] Configure stats pid to 199
+2026-07-11 07:47:50,284 INFO    MainThread:199 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-11 07:47:50,284 INFO    MainThread:199 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260711_074750-cd66e13p/logs/debug.log
+2026-07-11 07:47:50,284 INFO    MainThread:199 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260711_074750-cd66e13p/logs/debug-internal.log
+2026-07-11 07:47:50,284 INFO    MainThread:199 [wandb_init.py:init():850] calling init triggers
+2026-07-11 07:47:50,284 INFO    MainThread:199 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/longctx30k/tokenized', 'val_path': None, 'max_seq_length': 30720, 'num_workers': 4, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-16800', 'force_new_run': False, 'num_epochs': 1, 'max_steps': None, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 5, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 300, 'eval_steps': 500, 'eval_batches': 20, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b dense 30k longctx', 'wandb_tags': ['qwen3', '1.2b', 'dense', '30k', 'longctx', 'b128', 'scaleladder'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 120, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
+2026-07-11 07:47:50,284 INFO    MainThread:199 [wandb_init.py:init():898] starting backend
+2026-07-11 07:47:51,035 INFO    MainThread:199 [wandb_init.py:init():913] sending inform_init request
+2026-07-11 07:47:51,458 INFO    MainThread:199 [wandb_init.py:init():918] backend started and connected
+2026-07-11 07:47:51,469 INFO    MainThread:199 [wandb_init.py:init():988] updated telemetry
+2026-07-11 07:47:51,499 INFO    MainThread:199 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-07-11 07:47:51,892 INFO    MainThread:199 [wandb_init.py:init():1051] run resumed
+2026-07-11 07:47:51,896 INFO    MainThread:199 [wandb_init.py:init():1056] starting run threads in backend
+2026-07-11 07:47:52,079 INFO    MainThread:199 [wandb_run.py:_console_start():2554] atexit reg
+2026-07-11 07:47:52,079 INFO    MainThread:199 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-07-11 07:47:52,079 INFO    MainThread:199 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-07-11 07:47:52,080 INFO    MainThread:199 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-07-11 07:47:52,091 INFO    MainThread:199 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/run-20260711_075132-cd66e13p/files/config.yaml

@@ -0,0 +1,190 @@
+_wandb:
+    value:
+        cli_version: 0.26.1
+        e:
+            co4vsxbsrtv0f5d9wuv9fdcdauh1rl9y:
+                args:
+                    - --config
+                    - configs/train_config_30k_dense_b128.json
+                codePath: train.py
+                codePathLocal: train.py
+                cpu_count: 96
+                cpu_count_logical: 192
+                cudaVersion: "13.0"
+                disk:
+                    /:
+                        total: "9374931091456"
+                        used: "6243339026432"
+                email: ruben.navasardyan@mail.utoronto.ca
+                executable: /mloscratch/homes/navasard/conda/bin/python3.13
+                git:
+                    commit: 88298c7a418622530ca5426b22b1e5ebd10ca0cb
+                    remote: https://github.com/rubentium/TokenPrune.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 4
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-e8ce0e9a-801b-03d2-c898-c0cc7d746d53
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-b817d241-acce-e5dd-a749-3447239c5207
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-47444fc7-ea8c-cef6-aac8-ef408dc8047b
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-37dc97ad-46ef-50d9-3619-3a6eaea1b6aa
+                host: h100-train-4-gpu-2-0-4
+                memory:
+                    total: "1623005732864"
+                os: Linux-6.8.0-124-generic-x86_64-with-glibc2.35
+                program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
+                python: CPython 3.13.13
+                root: /mloscratch/homes/navasard/sparse/SparseAtt
+                startedAt: "2026-07-11T07:51:32.432252Z"
+                writerId: co4vsxbsrtv0f5d9wuv9fdcdauh1rl9y
+        m: []
+        python_version: 3.13.13
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 5
+                - 13
+                - 14
+                - 15
+                - 16
+                - 61
+                - 62
+            "4": 3.13.13
+            "5": 0.26.1
+            "6": 5.7.0
+            "12": 0.26.1
+            "13": linux-x86_64
+attention_type:
+    value: sparse_stochastic
+checkpoint_activations:
+    value: true
+dense_mode:
+    value: true
+dense_step_intervals:
+    value: null
+eval_batches:
+    value: 20
+eval_steps:
+    value: 500
+force_new_run:
+    value: false
+gradient_accumulation_steps:
+    value: 5
+head_dim:
+    value: null
+hidden_size:
+    value: 1536
+init_from_scratch:
+    value: true
+intermediate_size:
+    value: 6144
+learning_rate:
+    value: 0.0005
+logging_steps:
+    value: 10
+lr_scheduler:
+    value: cosine
+max_grad_norm:
+    value: 1
+max_seq_length:
+    value: 30720
+max_steps:
+    value: null
+mixed_precision:
+    value: bf16
+model_dtype:
+    value: bfloat16
+model_path:
+    value: /mloscratch/homes/navasard/qwen3_06b_base
+num_attention_heads:
+    value: 12
+num_epochs:
+    value: 1
+num_hidden_layers:
+    value: null
+num_key_value_heads:
+    value: 4
+num_workers:
+    value: 4
+output_dir:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119
+per_device_batch_size:
+    value: 1
+resume_from_checkpoint:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-16800
+save_steps:
+    value: 300
+seed:
+    value: 42
+sparse_block_size:
+    value: 128
+sparse_layer_indices:
+    value: null
+sparse_mask_cache_dir:
+    value: null
+sparse_mask_pool_size:
+    value: 1
+sparse_mask_seed:
+    value: 0
+sparse_num_random_blocks:
+    value: 120
+sparse_val_dense_compare:
+    value: false
+strategy:
+    value: ddp
+train_path:
+    value: /mloscratch/homes/navasard/data/longctx30k/tokenized
+val_path:
+    value: null
+verbose:
+    value: true
+wandb_enabled:
+    value: true
+wandb_name:
+    value: 1.2b dense 30k longctx
+wandb_project:
+    value: stochastic_sparsity
+wandb_tags:
+    value:
+        - qwen3
+        - 1.2b
+        - dense
+        - 30k
+        - longctx
+        - b128
+        - scaleladder
+warmup_steps:
+    value: 750
+weight_decay:
+    value: 0.01

epfl_final/SparseAtt/wandb/run-20260711_075132-cd66e13p/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260711_075132-cd66e13p/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-07-11T07:51:32.432252Z",
+  "args":  [
+    "--config",
+    "configs/train_config_30k_dense_b128.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "88298c7a418622530ca5426b22b1e5ebd10ca0cb"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-train-4-gpu-2-0-4",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "6243339026432"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-e8ce0e9a-801b-03d2-c898-c0cc7d746d53"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-b817d241-acce-e5dd-a749-3447239c5207"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-47444fc7-ea8c-cef6-aac8-ef408dc8047b"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-37dc97ad-46ef-50d9-3619-3a6eaea1b6aa"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "co4vsxbsrtv0f5d9wuv9fdcdauh1rl9y"
+}

epfl_final/SparseAtt/wandb/run-20260711_075132-cd66e13p/files/wandb-summary.json

@@ -0,0 +1 @@
+{"eval/dense_interval_end":-1,"eval/dense_interval_start":-1,"eval/uses_dense_attention":1,"train/global_step":37460,"speed/steps_per_sec":0.15911030026630077,"eval/dense_interval_active":0,"train/learning_rate":2.2880351668863596e-11,"train/epoch":0.9998665421059656,"_wandb":{"runtime":247914},"_step":37460,"eval/uses_sparse_attention":0,"_timestamp":1.7838876839270074e+09,"train/loss":1.917395345494151,"eval/loss":2.0424968034029005,"_runtime":247914.72359931,"speed/ms_per_step":6284.948229789734,"speed/samples_per_sec":3.182206005326015,"eval/perplexity":7.70983512979409}

epfl_final/SparseAtt/wandb/run-20260711_075132-cd66e13p/logs/debug.log

@@ -0,0 +1,25 @@
+2026-07-11 07:51:32,450 INFO    MainThread:199 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-07-11 07:51:32,450 INFO    MainThread:199 [wandb_setup.py:_flush():81] Configure stats pid to 199
+2026-07-11 07:51:32,450 INFO    MainThread:199 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-11 07:51:32,451 INFO    MainThread:199 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260711_075132-cd66e13p/logs/debug.log
+2026-07-11 07:51:32,451 INFO    MainThread:199 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260711_075132-cd66e13p/logs/debug-internal.log
+2026-07-11 07:51:32,451 INFO    MainThread:199 [wandb_init.py:init():850] calling init triggers
+2026-07-11 07:51:32,451 INFO    MainThread:199 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/longctx30k/tokenized', 'val_path': None, 'max_seq_length': 30720, 'num_workers': 4, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_dense/run_20260708_065119/checkpoint-16800', 'force_new_run': False, 'num_epochs': 1, 'max_steps': None, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 5, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 300, 'eval_steps': 500, 'eval_batches': 20, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b dense 30k longctx', 'wandb_tags': ['qwen3', '1.2b', 'dense', '30k', 'longctx', 'b128', 'scaleladder'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 120, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'dense_mode': True, 'dense_step_intervals': None, '_wandb': {}}
+2026-07-11 07:51:32,451 INFO    MainThread:199 [wandb_init.py:init():898] starting backend
+2026-07-11 07:51:32,978 INFO    MainThread:199 [wandb_init.py:init():913] sending inform_init request
+2026-07-11 07:51:33,400 INFO    MainThread:199 [wandb_init.py:init():918] backend started and connected
+2026-07-11 07:51:33,406 INFO    MainThread:199 [wandb_init.py:init():988] updated telemetry
+2026-07-11 07:51:33,429 INFO    MainThread:199 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-07-11 07:51:33,862 INFO    MainThread:199 [wandb_init.py:init():1051] run resumed
+2026-07-11 07:51:33,864 INFO    MainThread:199 [wandb_init.py:init():1056] starting run threads in backend
+2026-07-11 07:51:34,062 INFO    MainThread:199 [wandb_run.py:_console_start():2554] atexit reg
+2026-07-11 07:51:34,062 INFO    MainThread:199 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-07-11 07:51:34,062 INFO    MainThread:199 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-07-11 07:51:34,062 INFO    MainThread:199 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-07-11 07:51:34,068 INFO    MainThread:199 [wandb_init.py:init():1094] run started, returning control to user process
+2026-07-12 20:22:05,584 INFO    MainThread:199 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/cd66e13p
+2026-07-12 20:22:05,585 INFO    MainThread:199 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
+2026-07-12 20:22:05,586 INFO    MainThread:199 [wandb_run.py:_restore():2502] restore
+2026-07-12 20:22:05,586 INFO    MainThread:199 [wandb_run.py:_restore():2508] restore done
+2026-07-12 20:22:07,140 INFO    MainThread:199 [wandb_run.py:_footer_sync_info():3889] logging synced files

epfl_final/SparseAtt/wandb/run-20260711_075132-cd66e13p/run-cd66e13p.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a9992167643df61ada1a24698d387a099c7140ef606038d56257c87f630de49
+size 42788185

epfl_final/SparseAtt/wandb/run-20260711_080025-9blizq3m/files/output.log

@@ -0,0 +1,287 @@
+Starting training...
+Validation at step 5700: Loss = 2.4255, Perplexity = 11.31
+wandb: WARNING Tried to log to step 5700 that is less than the current step 5731. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Switching attention mode for step 5701: sparse_stochastic (configured sparse attention)
+Step 5710/37465 | Loss: 2.4233 | LR: 4.78e-04 | 25615.7ms/step | 0.8 samples/s
+wandb: WARNING Tried to log to step 5710 that is less than the current step 5731. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 5720/37465 | Loss: 2.3971 | LR: 4.78e-04 | 8106.0ms/step | 2.5 samples/s
+wandb: WARNING Tried to log to step 5720 that is less than the current step 5731. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 5730/37465 | Loss: 2.4188 | LR: 4.78e-04 | 8148.0ms/step | 2.5 samples/s
+wandb: WARNING Tried to log to step 5730 that is less than the current step 5731. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Step 5740/37465 | Loss: 2.3991 | LR: 4.78e-04 | 8130.2ms/step | 2.5 samples/s
+Step 5750/37465 | Loss: 2.4403 | LR: 4.77e-04 | 8157.3ms/step | 2.5 samples/s
+Step 5760/37465 | Loss: 2.1459 | LR: 4.77e-04 | 8150.0ms/step | 2.5 samples/s
+Step 5770/37465 | Loss: 2.4807 | LR: 4.77e-04 | 8176.4ms/step | 2.4 samples/s
+Step 5780/37465 | Loss: 2.4041 | LR: 4.77e-04 | 8155.3ms/step | 2.5 samples/s
+Step 5790/37465 | Loss: 2.2887 | LR: 4.77e-04 | 8185.4ms/step | 2.4 samples/s
+Step 5800/37465 | Loss: 2.4940 | LR: 4.77e-04 | 8195.5ms/step | 2.4 samples/s
+Step 5810/37465 | Loss: 2.4236 | LR: 4.77e-04 | 8181.2ms/step | 2.4 samples/s
+Step 5820/37465 | Loss: 2.1601 | LR: 4.77e-04 | 8181.0ms/step | 2.4 samples/s
+Step 5830/37465 | Loss: 2.3791 | LR: 4.77e-04 | 8183.4ms/step | 2.4 samples/s
+Step 5840/37465 | Loss: 2.4035 | LR: 4.77e-04 | 8200.5ms/step | 2.4 samples/s
+Step 5850/37465 | Loss: 2.2279 | LR: 4.77e-04 | 8146.1ms/step | 2.5 samples/s
+Step 5860/37465 | Loss: 2.1539 | LR: 4.76e-04 | 8175.4ms/step | 2.4 samples/s
+Step 5870/37465 | Loss: 2.4310 | LR: 4.76e-04 | 8168.1ms/step | 2.4 samples/s
+Step 5880/37465 | Loss: 2.4335 | LR: 4.76e-04 | 8099.7ms/step | 2.5 samples/s
+Step 5890/37465 | Loss: 2.5099 | LR: 4.76e-04 | 8151.9ms/step | 2.5 samples/s
+Step 5900/37465 | Loss: 2.5700 | LR: 4.76e-04 | 8134.2ms/step | 2.5 samples/s
+Step 5910/37465 | Loss: 2.2665 | LR: 4.76e-04 | 8145.1ms/step | 2.5 samples/s
+Step 5920/37465 | Loss: 2.4116 | LR: 4.76e-04 | 8143.3ms/step | 2.5 samples/s
+Step 5930/37465 | Loss: 2.3634 | LR: 4.76e-04 | 8128.2ms/step | 2.5 samples/s
+Step 5940/37465 | Loss: 2.6368 | LR: 4.76e-04 | 8131.6ms/step | 2.5 samples/s
+Step 5950/37465 | Loss: 2.5229 | LR: 4.76e-04 | 8116.3ms/step | 2.5 samples/s
+Step 5960/37465 | Loss: 2.2806 | LR: 4.76e-04 | 8141.0ms/step | 2.5 samples/s
+Step 5970/37465 | Loss: 2.2798 | LR: 4.75e-04 | 8138.6ms/step | 2.5 samples/s
+Step 5980/37465 | Loss: 2.3335 | LR: 4.75e-04 | 8157.1ms/step | 2.5 samples/s
+Step 5990/37465 | Loss: 2.3756 | LR: 4.75e-04 | 8141.2ms/step | 2.5 samples/s
+Step 6000/37465 | Loss: 2.2740 | LR: 4.75e-04 | 8117.4ms/step | 2.5 samples/s
+Validation at step 6000: Loss = 2.4133, Perplexity = 11.17
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_fixed/run_20260709_000503/checkpoint-6000
+Step 6010/37465 | Loss: 2.2714 | LR: 4.75e-04 | 9634.0ms/step | 2.1 samples/s
+Step 6020/37465 | Loss: 2.5321 | LR: 4.75e-04 | 8083.3ms/step | 2.5 samples/s
+Step 6030/37465 | Loss: 2.4918 | LR: 4.75e-04 | 8106.1ms/step | 2.5 samples/s
+Step 6040/37465 | Loss: 2.3314 | LR: 4.75e-04 | 8101.6ms/step | 2.5 samples/s
+Step 6050/37465 | Loss: 2.2412 | LR: 4.75e-04 | 8168.9ms/step | 2.4 samples/s
+Step 6060/37465 | Loss: 2.5855 | LR: 4.75e-04 | 8174.5ms/step | 2.4 samples/s
+Step 6070/37465 | Loss: 2.3413 | LR: 4.75e-04 | 8172.7ms/step | 2.4 samples/s
+Step 6080/37465 | Loss: 2.5651 | LR: 4.74e-04 | 8196.3ms/step | 2.4 samples/s
+Step 6090/37465 | Loss: 2.3406 | LR: 4.74e-04 | 8193.4ms/step | 2.4 samples/s
+Step 6100/37465 | Loss: 2.1750 | LR: 4.74e-04 | 8184.3ms/step | 2.4 samples/s
+Step 6110/37465 | Loss: 2.5999 | LR: 4.74e-04 | 8157.3ms/step | 2.5 samples/s
+Step 6120/37465 | Loss: 2.5244 | LR: 4.74e-04 | 8183.5ms/step | 2.4 samples/s
+Step 6130/37465 | Loss: 2.4220 | LR: 4.74e-04 | 8200.1ms/step | 2.4 samples/s
+Step 6140/37465 | Loss: 2.3484 | LR: 4.74e-04 | 8198.3ms/step | 2.4 samples/s
+Step 6150/37465 | Loss: 2.5200 | LR: 4.74e-04 | 8141.7ms/step | 2.5 samples/s
+Step 6160/37465 | Loss: 2.4814 | LR: 4.74e-04 | 8120.6ms/step | 2.5 samples/s
+Step 6170/37465 | Loss: 2.3418 | LR: 4.74e-04 | 8141.2ms/step | 2.5 samples/s
+Step 6180/37465 | Loss: 2.4501 | LR: 4.73e-04 | 8049.7ms/step | 2.5 samples/s
+Step 6190/37465 | Loss: 2.4962 | LR: 4.73e-04 | 8232.9ms/step | 2.4 samples/s
+Step 6200/37465 | Loss: 2.3876 | LR: 4.73e-04 | 8211.8ms/step | 2.4 samples/s
+Step 6210/37465 | Loss: 2.2009 | LR: 4.73e-04 | 8231.0ms/step | 2.4 samples/s
+Step 6220/37465 | Loss: 2.4581 | LR: 4.73e-04 | 8234.9ms/step | 2.4 samples/s
+Step 6230/37465 | Loss: 2.3382 | LR: 4.73e-04 | 8233.5ms/step | 2.4 samples/s
+Step 6240/37465 | Loss: 2.4628 | LR: 4.73e-04 | 8188.2ms/step | 2.4 samples/s
+Step 6250/37465 | Loss: 2.3168 | LR: 4.73e-04 | 8166.0ms/step | 2.4 samples/s
+Step 6260/37465 | Loss: 2.2641 | LR: 4.73e-04 | 8188.7ms/step | 2.4 samples/s
+Step 6270/37465 | Loss: 1.9939 | LR: 4.73e-04 | 8173.5ms/step | 2.4 samples/s
+Step 6280/37465 | Loss: 2.2060 | LR: 4.73e-04 | 8177.8ms/step | 2.4 samples/s
+Step 6290/37465 | Loss: 2.3970 | LR: 4.72e-04 | 8196.8ms/step | 2.4 samples/s
+Step 6300/37465 | Loss: 2.4667 | LR: 4.72e-04 | 8114.9ms/step | 2.5 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_fixed/run_20260709_000503/checkpoint-6300
+Step 6310/37465 | Loss: 2.2527 | LR: 4.72e-04 | 9896.9ms/step | 2.0 samples/s
+Step 6320/37465 | Loss: 2.3347 | LR: 4.72e-04 | 8162.6ms/step | 2.5 samples/s
+Step 6330/37465 | Loss: 2.4564 | LR: 4.72e-04 | 8161.8ms/step | 2.5 samples/s
+Step 6340/37465 | Loss: 2.2622 | LR: 4.72e-04 | 8169.7ms/step | 2.4 samples/s
+Step 6350/37465 | Loss: 2.6875 | LR: 4.72e-04 | 8166.6ms/step | 2.4 samples/s
+Step 6360/37465 | Loss: 2.2354 | LR: 4.72e-04 | 8183.3ms/step | 2.4 samples/s
+Step 6370/37465 | Loss: 2.1165 | LR: 4.72e-04 | 8210.6ms/step | 2.4 samples/s
+Step 6380/37465 | Loss: 2.5141 | LR: 4.72e-04 | 8191.8ms/step | 2.4 samples/s
+Step 6390/37465 | Loss: 2.5378 | LR: 4.71e-04 | 8218.6ms/step | 2.4 samples/s
+Step 6400/37465 | Loss: 2.3583 | LR: 4.71e-04 | 8170.3ms/step | 2.4 samples/s
+Step 6410/37465 | Loss: 2.4088 | LR: 4.71e-04 | 8177.7ms/step | 2.4 samples/s
+Step 6420/37465 | Loss: 2.2487 | LR: 4.71e-04 | 8226.4ms/step | 2.4 samples/s
+Step 6430/37465 | Loss: 2.2890 | LR: 4.71e-04 | 8239.9ms/step | 2.4 samples/s
+Step 6440/37465 | Loss: 2.2322 | LR: 4.71e-04 | 8236.4ms/step | 2.4 samples/s
+Step 6450/37465 | Loss: 2.3595 | LR: 4.71e-04 | 8172.2ms/step | 2.4 samples/s
+Step 6460/37465 | Loss: 2.3734 | LR: 4.71e-04 | 8105.1ms/step | 2.5 samples/s
+Step 6470/37465 | Loss: 2.3530 | LR: 4.71e-04 | 8126.2ms/step | 2.5 samples/s
+Step 6480/37465 | Loss: 2.3063 | LR: 4.71e-04 | 8204.0ms/step | 2.4 samples/s
+Step 6490/37465 | Loss: 2.3493 | LR: 4.70e-04 | 8162.5ms/step | 2.5 samples/s
+Step 6500/37465 | Loss: 2.5273 | LR: 4.70e-04 | 8223.7ms/step | 2.4 samples/s
+Validation at step 6500: Loss = 2.3911, Perplexity = 10.93
+Step 6510/37465 | Loss: 2.3985 | LR: 4.70e-04 | 8884.6ms/step | 2.3 samples/s
+Step 6520/37465 | Loss: 2.4548 | LR: 4.70e-04 | 8207.0ms/step | 2.4 samples/s
+Step 6530/37465 | Loss: 2.2737 | LR: 4.70e-04 | 8177.7ms/step | 2.4 samples/s
+Step 6540/37465 | Loss: 2.4096 | LR: 4.70e-04 | 8219.9ms/step | 2.4 samples/s
+Step 6550/37465 | Loss: 2.3655 | LR: 4.70e-04 | 8177.0ms/step | 2.4 samples/s
+Step 6560/37465 | Loss: 2.4906 | LR: 4.70e-04 | 8139.6ms/step | 2.5 samples/s
+Step 6570/37465 | Loss: 2.5839 | LR: 4.70e-04 | 8224.5ms/step | 2.4 samples/s
+Step 6580/37465 | Loss: 2.2572 | LR: 4.70e-04 | 7998.1ms/step | 2.5 samples/s
+Step 6590/37465 | Loss: 2.2486 | LR: 4.69e-04 | 8212.1ms/step | 2.4 samples/s
+Step 6600/37465 | Loss: 2.2569 | LR: 4.69e-04 | 8185.3ms/step | 2.4 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_fixed/run_20260709_000503/checkpoint-6600
+Step 6610/37465 | Loss: 2.3613 | LR: 4.69e-04 | 9094.6ms/step | 2.2 samples/s
+Step 6620/37465 | Loss: 2.5718 | LR: 4.69e-04 | 8179.7ms/step | 2.4 samples/s
+Step 6630/37465 | Loss: 2.3047 | LR: 4.69e-04 | 8214.3ms/step | 2.4 samples/s
+Step 6640/37465 | Loss: 2.3896 | LR: 4.69e-04 | 8201.9ms/step | 2.4 samples/s
+Step 6650/37465 | Loss: 2.5540 | LR: 4.69e-04 | 8178.4ms/step | 2.4 samples/s
+Step 6660/37465 | Loss: 2.4475 | LR: 4.69e-04 | 8090.6ms/step | 2.5 samples/s
+Step 6670/37465 | Loss: 2.2657 | LR: 4.69e-04 | 8114.4ms/step | 2.5 samples/s
+Step 6680/37465 | Loss: 2.4151 | LR: 4.69e-04 | 8184.5ms/step | 2.4 samples/s
+Step 6690/37465 | Loss: 2.6053 | LR: 4.68e-04 | 8201.4ms/step | 2.4 samples/s
+Step 6700/37465 | Loss: 2.1858 | LR: 4.68e-04 | 8195.0ms/step | 2.4 samples/s
+Step 6710/37465 | Loss: 2.5396 | LR: 4.68e-04 | 8219.8ms/step | 2.4 samples/s
+Step 6720/37465 | Loss: 2.2380 | LR: 4.68e-04 | 8210.9ms/step | 2.4 samples/s
+Step 6730/37465 | Loss: 2.4316 | LR: 4.68e-04 | 8185.2ms/step | 2.4 samples/s
+Step 6740/37465 | Loss: 2.2572 | LR: 4.68e-04 | 8208.6ms/step | 2.4 samples/s
+Step 6750/37465 | Loss: 2.2984 | LR: 4.68e-04 | 8211.7ms/step | 2.4 samples/s
+Step 6760/37465 | Loss: 2.4685 | LR: 4.68e-04 | 8202.9ms/step | 2.4 samples/s
+Step 6770/37465 | Loss: 2.2939 | LR: 4.68e-04 | 8199.0ms/step | 2.4 samples/s
+Step 6780/37465 | Loss: 2.5630 | LR: 4.67e-04 | 8211.3ms/step | 2.4 samples/s
+Step 6790/37465 | Loss: 2.4770 | LR: 4.67e-04 | 8196.7ms/step | 2.4 samples/s
+Step 6800/37465 | Loss: 2.3050 | LR: 4.67e-04 | 8169.9ms/step | 2.4 samples/s
+Step 6810/37465 | Loss: 2.5238 | LR: 4.67e-04 | 8170.5ms/step | 2.4 samples/s
+Step 6820/37465 | Loss: 2.5106 | LR: 4.67e-04 | 8176.7ms/step | 2.4 samples/s
+Step 6830/37465 | Loss: 2.6382 | LR: 4.67e-04 | 8194.0ms/step | 2.4 samples/s
+Step 6840/37465 | Loss: 2.4227 | LR: 4.67e-04 | 8168.3ms/step | 2.4 samples/s
+Step 6850/37465 | Loss: 2.3855 | LR: 4.67e-04 | 8183.9ms/step | 2.4 samples/s
+Step 6860/37465 | Loss: 2.3495 | LR: 4.67e-04 | 8175.4ms/step | 2.4 samples/s
+Step 6870/37465 | Loss: 2.4752 | LR: 4.66e-04 | 8130.8ms/step | 2.5 samples/s
+Step 6880/37465 | Loss: 2.3842 | LR: 4.66e-04 | 8118.0ms/step | 2.5 samples/s
+Step 6890/37465 | Loss: 2.5300 | LR: 4.66e-04 | 8087.4ms/step | 2.5 samples/s
+Step 6900/37465 | Loss: 2.3989 | LR: 4.66e-04 | 8137.5ms/step | 2.5 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_fixed/run_20260709_000503/checkpoint-6900
+Step 6910/37465 | Loss: 2.4433 | LR: 4.66e-04 | 8961.7ms/step | 2.2 samples/s
+Step 6920/37465 | Loss: 2.4076 | LR: 4.66e-04 | 8075.6ms/step | 2.5 samples/s
+Step 6930/37465 | Loss: 2.2864 | LR: 4.66e-04 | 8103.1ms/step | 2.5 samples/s
+Step 6940/37465 | Loss: 2.1097 | LR: 4.66e-04 | 8107.7ms/step | 2.5 samples/s
+Step 6950/37465 | Loss: 2.5191 | LR: 4.66e-04 | 8043.5ms/step | 2.5 samples/s
+Step 6960/37465 | Loss: 2.4665 | LR: 4.66e-04 | 8100.3ms/step | 2.5 samples/s
+Step 6970/37465 | Loss: 2.4891 | LR: 4.65e-04 | 8025.7ms/step | 2.5 samples/s
+Step 6980/37465 | Loss: 2.4306 | LR: 4.65e-04 | 8115.8ms/step | 2.5 samples/s
+Step 6990/37465 | Loss: 2.4929 | LR: 4.65e-04 | 8124.1ms/step | 2.5 samples/s
+Step 7000/37465 | Loss: 2.1510 | LR: 4.65e-04 | 8104.7ms/step | 2.5 samples/s
+Validation at step 7000: Loss = 2.3711, Perplexity = 10.71
+Step 7010/37465 | Loss: 2.2900 | LR: 4.65e-04 | 8746.1ms/step | 2.3 samples/s
+Step 7020/37465 | Loss: 2.2995 | LR: 4.65e-04 | 8105.8ms/step | 2.5 samples/s
+Step 7030/37465 | Loss: 2.3253 | LR: 4.65e-04 | 8143.7ms/step | 2.5 samples/s
+Step 7040/37465 | Loss: 2.1986 | LR: 4.65e-04 | 8143.9ms/step | 2.5 samples/s
+Step 7050/37465 | Loss: 2.3172 | LR: 4.65e-04 | 8174.8ms/step | 2.4 samples/s
+Step 7060/37465 | Loss: 2.4077 | LR: 4.64e-04 | 8164.4ms/step | 2.4 samples/s
+Step 7070/37465 | Loss: 2.2571 | LR: 4.64e-04 | 8165.3ms/step | 2.4 samples/s
+Step 7080/37465 | Loss: 2.4516 | LR: 4.64e-04 | 8167.4ms/step | 2.4 samples/s
+Step 7090/37465 | Loss: 2.2556 | LR: 4.64e-04 | 8026.7ms/step | 2.5 samples/s
+Step 7100/37465 | Loss: 2.3510 | LR: 4.64e-04 | 8176.6ms/step | 2.4 samples/s
+Step 7110/37465 | Loss: 2.1612 | LR: 4.64e-04 | 8077.1ms/step | 2.5 samples/s
+Step 7120/37465 | Loss: 1.9000 | LR: 4.64e-04 | 8154.8ms/step | 2.5 samples/s
+Step 7130/37465 | Loss: 2.3495 | LR: 4.64e-04 | 8238.0ms/step | 2.4 samples/s
+Step 7140/37465 | Loss: 2.4536 | LR: 4.64e-04 | 8211.4ms/step | 2.4 samples/s
+Step 7150/37465 | Loss: 2.5172 | LR: 4.63e-04 | 8118.8ms/step | 2.5 samples/s
+Step 7160/37465 | Loss: 2.4395 | LR: 4.63e-04 | 8237.3ms/step | 2.4 samples/s
+Step 7170/37465 | Loss: 2.3546 | LR: 4.63e-04 | 8177.9ms/step | 2.4 samples/s
+Step 7180/37465 | Loss: 2.5471 | LR: 4.63e-04 | 8185.2ms/step | 2.4 samples/s
+Step 7190/37465 | Loss: 2.6000 | LR: 4.63e-04 | 8225.6ms/step | 2.4 samples/s
+Step 7200/37465 | Loss: 2.3475 | LR: 4.63e-04 | 8132.2ms/step | 2.5 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_fixed/run_20260709_000503/checkpoint-7200
+Step 7210/37465 | Loss: 2.6240 | LR: 4.63e-04 | 9976.2ms/step | 2.0 samples/s
+Step 7220/37465 | Loss: 2.3894 | LR: 4.63e-04 | 8216.2ms/step | 2.4 samples/s
+Step 7230/37465 | Loss: 2.4416 | LR: 4.63e-04 | 8234.5ms/step | 2.4 samples/s
+Step 7240/37465 | Loss: 2.3953 | LR: 4.62e-04 | 8155.0ms/step | 2.5 samples/s
+Step 7250/37465 | Loss: 2.3746 | LR: 4.62e-04 | 8174.5ms/step | 2.4 samples/s
+Step 7260/37465 | Loss: 2.3994 | LR: 4.62e-04 | 8180.1ms/step | 2.4 samples/s
+Step 7270/37465 | Loss: 2.2348 | LR: 4.62e-04 | 8056.5ms/step | 2.5 samples/s
+Step 7280/37465 | Loss: 2.4378 | LR: 4.62e-04 | 8159.2ms/step | 2.5 samples/s
+Step 7290/37465 | Loss: 2.6233 | LR: 4.62e-04 | 8155.0ms/step | 2.5 samples/s
+Step 7300/37465 | Loss: 2.3456 | LR: 4.62e-04 | 8197.2ms/step | 2.4 samples/s
+Step 7310/37465 | Loss: 2.3139 | LR: 4.62e-04 | 8240.0ms/step | 2.4 samples/s
+Step 7320/37465 | Loss: 2.2889 | LR: 4.62e-04 | 8202.1ms/step | 2.4 samples/s
+Step 7330/37465 | Loss: 2.4655 | LR: 4.61e-04 | 8245.5ms/step | 2.4 samples/s
+Step 7340/37465 | Loss: 2.3131 | LR: 4.61e-04 | 8219.7ms/step | 2.4 samples/s
+Step 7350/37465 | Loss: 2.4320 | LR: 4.61e-04 | 8105.2ms/step | 2.5 samples/s
+Step 7360/37465 | Loss: 2.4561 | LR: 4.61e-04 | 8164.5ms/step | 2.4 samples/s
+Step 7370/37465 | Loss: 2.1081 | LR: 4.61e-04 | 8134.0ms/step | 2.5 samples/s
+Step 7380/37465 | Loss: 2.4054 | LR: 4.61e-04 | 8181.1ms/step | 2.4 samples/s
+Step 7390/37465 | Loss: 2.3375 | LR: 4.61e-04 | 8172.4ms/step | 2.4 samples/s
+Step 7400/37465 | Loss: 2.3454 | LR: 4.61e-04 | 8221.0ms/step | 2.4 samples/s
+Step 7410/37465 | Loss: 2.2249 | LR: 4.60e-04 | 8192.7ms/step | 2.4 samples/s
+Step 7420/37465 | Loss: 2.4995 | LR: 4.60e-04 | 8208.0ms/step | 2.4 samples/s
+Step 7430/37465 | Loss: 2.4542 | LR: 4.60e-04 | 8215.9ms/step | 2.4 samples/s
+Step 7440/37465 | Loss: 2.1935 | LR: 4.60e-04 | 8215.2ms/step | 2.4 samples/s
+Step 7450/37465 | Loss: 2.2351 | LR: 4.60e-04 | 8232.4ms/step | 2.4 samples/s
+Step 7460/37465 | Loss: 2.3569 | LR: 4.60e-04 | 8171.1ms/step | 2.4 samples/s
+Step 7470/37465 | Loss: 2.1997 | LR: 4.60e-04 | 8132.9ms/step | 2.5 samples/s
+Step 7480/37465 | Loss: 2.4366 | LR: 4.60e-04 | 8132.6ms/step | 2.5 samples/s
+Step 7490/37465 | Loss: 2.4346 | LR: 4.60e-04 | 8136.7ms/step | 2.5 samples/s
+Step 7500/37465 | Loss: 2.2755 | LR: 4.59e-04 | 8204.4ms/step | 2.4 samples/s
+Validation at step 7500: Loss = 2.3556, Perplexity = 10.54
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_fixed/run_20260709_000503/checkpoint-7500
+Step 7510/37465 | Loss: 2.3687 | LR: 4.59e-04 | 9753.3ms/step | 2.1 samples/s
+Step 7520/37465 | Loss: 2.4069 | LR: 4.59e-04 | 8148.1ms/step | 2.5 samples/s
+Step 7530/37465 | Loss: 2.4684 | LR: 4.59e-04 | 8205.8ms/step | 2.4 samples/s
+Step 7540/37465 | Loss: 2.2359 | LR: 4.59e-04 | 8167.7ms/step | 2.4 samples/s
+Step 7550/37465 | Loss: 2.4971 | LR: 4.59e-04 | 8180.7ms/step | 2.4 samples/s
+Step 7560/37465 | Loss: 2.3584 | LR: 4.59e-04 | 8203.0ms/step | 2.4 samples/s
+Step 7570/37465 | Loss: 2.3746 | LR: 4.59e-04 | 8166.4ms/step | 2.4 samples/s
+Step 7580/37465 | Loss: 2.5086 | LR: 4.59e-04 | 8169.6ms/step | 2.4 samples/s
+Step 7590/37465 | Loss: 2.3596 | LR: 4.58e-04 | 8188.5ms/step | 2.4 samples/s
+Step 7600/37465 | Loss: 2.5025 | LR: 4.58e-04 | 8177.6ms/step | 2.4 samples/s
+Step 7610/37465 | Loss: 2.4599 | LR: 4.58e-04 | 8200.8ms/step | 2.4 samples/s
+Step 7620/37465 | Loss: 2.1612 | LR: 4.58e-04 | 8218.8ms/step | 2.4 samples/s
+Step 7630/37465 | Loss: 2.3423 | LR: 4.58e-04 | 8224.4ms/step | 2.4 samples/s
+Step 7640/37465 | Loss: 2.2358 | LR: 4.58e-04 | 8214.2ms/step | 2.4 samples/s
+Step 7650/37465 | Loss: 2.2991 | LR: 4.58e-04 | 8186.7ms/step | 2.4 samples/s
+Step 7660/37465 | Loss: 2.3484 | LR: 4.58e-04 | 8181.6ms/step | 2.4 samples/s
+Step 7670/37465 | Loss: 2.2338 | LR: 4.57e-04 | 8178.5ms/step | 2.4 samples/s
+Step 7680/37465 | Loss: 2.3069 | LR: 4.57e-04 | 8180.8ms/step | 2.4 samples/s
+Step 7690/37465 | Loss: 2.1779 | LR: 4.57e-04 | 8201.0ms/step | 2.4 samples/s
+Step 7700/37465 | Loss: 2.2300 | LR: 4.57e-04 | 8185.0ms/step | 2.4 samples/s
+Step 7710/37465 | Loss: 2.6032 | LR: 4.57e-04 | 8193.9ms/step | 2.4 samples/s
+Step 7720/37465 | Loss: 2.1712 | LR: 4.57e-04 | 8201.8ms/step | 2.4 samples/s
+Step 7730/37465 | Loss: 2.3737 | LR: 4.57e-04 | 8178.0ms/step | 2.4 samples/s
+Step 7740/37465 | Loss: 2.4715 | LR: 4.57e-04 | 8200.8ms/step | 2.4 samples/s
+Step 7750/37465 | Loss: 2.4677 | LR: 4.56e-04 | 8207.4ms/step | 2.4 samples/s
+Step 7760/37465 | Loss: 2.3958 | LR: 4.56e-04 | 8205.5ms/step | 2.4 samples/s
+Step 7770/37465 | Loss: 2.1620 | LR: 4.56e-04 | 8206.5ms/step | 2.4 samples/s
+Step 7780/37465 | Loss: 2.1917 | LR: 4.56e-04 | 8202.3ms/step | 2.4 samples/s
+Step 7790/37465 | Loss: 2.1820 | LR: 4.56e-04 | 8177.7ms/step | 2.4 samples/s
+Step 7800/37465 | Loss: 2.1496 | LR: 4.56e-04 | 8176.2ms/step | 2.4 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_fixed/run_20260709_000503/checkpoint-7800
+Step 7810/37465 | Loss: 2.3445 | LR: 4.56e-04 | 9051.5ms/step | 2.2 samples/s
+Step 7820/37465 | Loss: 2.3243 | LR: 4.56e-04 | 8191.0ms/step | 2.4 samples/s
+Step 7830/37465 | Loss: 2.2762 | LR: 4.56e-04 | 8182.2ms/step | 2.4 samples/s
+Step 7840/37465 | Loss: 2.4698 | LR: 4.55e-04 | 8157.2ms/step | 2.5 samples/s
+Step 7850/37465 | Loss: 2.2608 | LR: 4.55e-04 | 8187.7ms/step | 2.4 samples/s
+Step 7860/37465 | Loss: 2.2268 | LR: 4.55e-04 | 8196.4ms/step | 2.4 samples/s
+Step 7870/37465 | Loss: 2.3566 | LR: 4.55e-04 | 8210.0ms/step | 2.4 samples/s
+Step 7880/37465 | Loss: 2.2290 | LR: 4.55e-04 | 8207.9ms/step | 2.4 samples/s
+Step 7890/37465 | Loss: 2.4396 | LR: 4.55e-04 | 8201.9ms/step | 2.4 samples/s
+Step 7900/37465 | Loss: 2.2428 | LR: 4.55e-04 | 8206.0ms/step | 2.4 samples/s
+Step 7910/37465 | Loss: 2.2175 | LR: 4.55e-04 | 8156.6ms/step | 2.5 samples/s
+Step 7920/37465 | Loss: 2.1634 | LR: 4.54e-04 | 8014.5ms/step | 2.5 samples/s
+Step 7930/37465 | Loss: 2.2800 | LR: 4.54e-04 | 8081.7ms/step | 2.5 samples/s
+Step 7940/37465 | Loss: 2.1433 | LR: 4.54e-04 | 8093.1ms/step | 2.5 samples/s
+Step 7950/37465 | Loss: 2.3007 | LR: 4.54e-04 | 8103.2ms/step | 2.5 samples/s
+Step 7960/37465 | Loss: 2.4438 | LR: 4.54e-04 | 8096.1ms/step | 2.5 samples/s
+Step 7970/37465 | Loss: 2.3276 | LR: 4.54e-04 | 8083.8ms/step | 2.5 samples/s
+Step 7980/37465 | Loss: 2.3070 | LR: 4.54e-04 | 8108.2ms/step | 2.5 samples/s
+Step 7990/37465 | Loss: 2.4770 | LR: 4.54e-04 | 8144.9ms/step | 2.5 samples/s
+Step 8000/37465 | Loss: 2.3272 | LR: 4.53e-04 | 8162.2ms/step | 2.5 samples/s
+Validation at step 8000: Loss = 2.3388, Perplexity = 10.37
+Step 8010/37465 | Loss: 2.0893 | LR: 4.53e-04 | 8718.1ms/step | 2.3 samples/s
+Step 8020/37465 | Loss: 2.3537 | LR: 4.53e-04 | 8087.2ms/step | 2.5 samples/s
+Step 8030/37465 | Loss: 2.3397 | LR: 4.53e-04 | 8111.2ms/step | 2.5 samples/s
+Step 8040/37465 | Loss: 2.3065 | LR: 4.53e-04 | 8110.2ms/step | 2.5 samples/s
+Step 8050/37465 | Loss: 2.1418 | LR: 4.53e-04 | 8139.4ms/step | 2.5 samples/s
+Step 8060/37465 | Loss: 2.4284 | LR: 4.53e-04 | 8095.5ms/step | 2.5 samples/s
+Step 8070/37465 | Loss: 2.4237 | LR: 4.53e-04 | 8058.4ms/step | 2.5 samples/s
+Step 8080/37465 | Loss: 2.3563 | LR: 4.52e-04 | 8163.7ms/step | 2.4 samples/s
+Step 8090/37465 | Loss: 2.2095 | LR: 4.52e-04 | 8144.3ms/step | 2.5 samples/s
+Step 8100/37465 | Loss: 2.4239 | LR: 4.52e-04 | 8176.8ms/step | 2.4 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_fixed/run_20260709_000503/checkpoint-8100
+Step 8110/37465 | Loss: 2.4102 | LR: 4.52e-04 | 9041.5ms/step | 2.2 samples/s
+Step 8120/37465 | Loss: 2.4006 | LR: 4.52e-04 | 8162.7ms/step | 2.5 samples/s
+Step 8130/37465 | Loss: 2.3514 | LR: 4.52e-04 | 8043.4ms/step | 2.5 samples/s
+Step 8140/37465 | Loss: 2.2464 | LR: 4.52e-04 | 8121.2ms/step | 2.5 samples/s
+Step 8150/37465 | Loss: 2.4491 | LR: 4.52e-04 | 8135.2ms/step | 2.5 samples/s
+Step 8160/37465 | Loss: 2.2588 | LR: 4.51e-04 | 8196.7ms/step | 2.4 samples/s
+Step 8170/37465 | Loss: 2.3349 | LR: 4.51e-04 | 8081.3ms/step | 2.5 samples/s
+Step 8180/37465 | Loss: 2.4152 | LR: 4.51e-04 | 7996.8ms/step | 2.5 samples/s
+Step 8190/37465 | Loss: 2.2468 | LR: 4.51e-04 | 8094.2ms/step | 2.5 samples/s
+Step 8200/37465 | Loss: 2.3353 | LR: 4.51e-04 | 8123.1ms/step | 2.5 samples/s
+Step 8210/37465 | Loss: 2.4674 | LR: 4.51e-04 | 8035.2ms/step | 2.5 samples/s
+Step 8220/37465 | Loss: 2.3570 | LR: 4.51e-04 | 8156.5ms/step | 2.5 samples/s
+Step 8230/37465 | Loss: 2.2833 | LR: 4.51e-04 | 8231.9ms/step | 2.4 samples/s
+Step 8240/37465 | Loss: 2.2904 | LR: 4.50e-04 | 8042.1ms/step | 2.5 samples/s
+Step 8250/37465 | Loss: 2.2846 | LR: 4.50e-04 | 8132.7ms/step | 2.5 samples/s
+Step 8260/37465 | Loss: 2.5686 | LR: 4.50e-04 | 8096.2ms/step | 2.5 samples/s
+Step 8270/37465 | Loss: 2.4240 | LR: 4.50e-04 | 8086.1ms/step | 2.5 samples/s
+Step 8280/37465 | Loss: 2.1655 | LR: 4.50e-04 | 8169.1ms/step | 2.4 samples/s
+Step 8290/37465 | Loss: 2.4233 | LR: 4.50e-04 | 8231.7ms/step | 2.4 samples/s
+Step 8300/37465 | Loss: 2.2389 | LR: 4.50e-04 | 8191.6ms/step | 2.4 samples/s
+Step 8310/37465 | Loss: 2.4158 | LR: 4.49e-04 | 8181.8ms/step | 2.4 samples/s
+Step 8320/37465 | Loss: 2.4746 | LR: 4.49e-04 | 8235.6ms/step | 2.4 samples/s
+Step 8330/37465 | Loss: 2.1171 | LR: 4.49e-04 | 8195.8ms/step | 2.4 samples/s
+Step 8340/37465 | Loss: 2.4831 | LR: 4.49e-04 | 8188.3ms/step | 2.4 samples/s
+Step 8350/37465 | Loss: 2.4076 | LR: 4.49e-04 | 8185.2ms/step | 2.4 samples/s
+Step 8360/37465 | Loss: 2.0461 | LR: 4.49e-04 | 8230.2ms/step | 2.4 samples/s
+Step 8370/37465 | Loss: 2.3678 | LR: 4.49e-04 | 8169.6ms/step | 2.4 samples/s

epfl_final/SparseAtt/wandb/run-20260711_080025-9blizq3m/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260711_080025-9blizq3m/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-07-11T08:00:25.742753Z",
+  "args":  [
+    "--config",
+    "configs/train_config_30k_fixed_b128.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "88298c7a418622530ca5426b22b1e5ebd10ca0cb"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h200-train-4-gpu-0-4",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H200",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "7679361548288",
+      "used":  "3627275882496"
+    }
+  },
+  "memory":  {
+    "total":  "1623010586624"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H200",
+      "memoryTotal":  "150754820096",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-6d391a7d-303b-75a5-985c-7980ae7165a0"
+    },
+    {
+      "name":  "NVIDIA H200",
+      "memoryTotal":  "150754820096",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-5e2907f1-1a74-acc8-3916-4a4242799fbe"
+    },
+    {
+      "name":  "NVIDIA H200",
+      "memoryTotal":  "150754820096",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bb9585a1-d507-7096-5a0e-a9667db36e4a"
+    },
+    {
+      "name":  "NVIDIA H200",
+      "memoryTotal":  "150754820096",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-112ee834-613d-54f0-1940-9e0f764fc8e9"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "pk6ppm41s40gx5kde0d4ks5xjyp6u3xo"
+}

epfl_final/SparseAtt/wandb/run-20260711_080025-9blizq3m/logs/debug.log

@@ -0,0 +1,20 @@
+2026-07-11 08:00:25,765 INFO    MainThread:2259 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-07-11 08:00:25,765 INFO    MainThread:2259 [wandb_setup.py:_flush():81] Configure stats pid to 2259
+2026-07-11 08:00:25,765 INFO    MainThread:2259 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-11 08:00:25,765 INFO    MainThread:2259 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260711_080025-9blizq3m/logs/debug.log
+2026-07-11 08:00:25,765 INFO    MainThread:2259 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260711_080025-9blizq3m/logs/debug-internal.log
+2026-07-11 08:00:25,766 INFO    MainThread:2259 [wandb_init.py:init():850] calling init triggers
+2026-07-11 08:00:25,766 INFO    MainThread:2259 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/longctx30k/tokenized', 'val_path': None, 'max_seq_length': 30720, 'num_workers': 4, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_fixed/run_20260709_000503', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_fixed/run_20260709_000503/checkpoint-5700', 'force_new_run': False, 'num_epochs': 1, 'max_steps': None, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 5, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 300, 'eval_steps': 500, 'eval_batches': 20, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b fixed 30k longctx', 'wandb_tags': ['qwen3', '1.2b', 'fixed', '30k', 'longctx', 'b128', 'scaleladder'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 120, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_30k_b128_nrb120', 'sparse_mask_seed': 1, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-07-11 08:00:25,766 INFO    MainThread:2259 [wandb_init.py:init():898] starting backend
+2026-07-11 08:00:26,013 INFO    MainThread:2259 [wandb_init.py:init():913] sending inform_init request
+2026-07-11 08:00:26,316 INFO    MainThread:2259 [wandb_init.py:init():918] backend started and connected
+2026-07-11 08:00:26,320 INFO    MainThread:2259 [wandb_init.py:init():988] updated telemetry
+2026-07-11 08:00:26,337 INFO    MainThread:2259 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-07-11 08:00:26,745 INFO    MainThread:2259 [wandb_init.py:init():1051] run resumed
+2026-07-11 08:00:26,747 INFO    MainThread:2259 [wandb_init.py:init():1056] starting run threads in backend
+2026-07-11 08:00:26,960 INFO    MainThread:2259 [wandb_run.py:_console_start():2554] atexit reg
+2026-07-11 08:00:26,960 INFO    MainThread:2259 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-07-11 08:00:26,960 INFO    MainThread:2259 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-07-11 08:00:26,961 INFO    MainThread:2259 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-07-11 08:00:26,967 INFO    MainThread:2259 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/run-20260711_080025-9blizq3m/run-9blizq3m.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3027c287c7f56607a7e9d67cc572b5ca4bff0159c45ab803d3b71c553f900b65
+size 6553600

epfl_final/SparseAtt/wandb/run-20260713_031902-9blizq3m/files/config.yaml

@@ -0,0 +1,190 @@
+_wandb:
+    value:
+        cli_version: 0.26.1
+        e:
+            vfbd6u6afm7i15wtvccnshegb2rm9vr5:
+                args:
+                    - --config
+                    - configs/train_config_30k_fixed_b128.json
+                codePath: train.py
+                codePathLocal: train.py
+                cpu_count: 96
+                cpu_count_logical: 192
+                cudaVersion: "13.0"
+                disk:
+                    /:
+                        total: "9374931091456"
+                        used: "4251919761408"
+                email: ruben.navasardyan@mail.utoronto.ca
+                executable: /mloscratch/homes/navasard/conda/bin/python3.13
+                git:
+                    commit: 88298c7a418622530ca5426b22b1e5ebd10ca0cb
+                    remote: https://github.com/rubentium/TokenPrune.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 4
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-99f99f92-e501-0b15-5782-6beb2c34fa1a
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-7004b604-21a2-890e-0312-3eee1eb93c67
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-a2cf418a-a552-c05b-f5dc-8b4f120a914a
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-3f5bace4-0066-71eb-863d-73a3a6d79e3b
+                host: h100-train-4-gpu-2-0-5
+                memory:
+                    total: "1623005732864"
+                os: Linux-6.8.0-124-generic-x86_64-with-glibc2.35
+                program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
+                python: CPython 3.13.13
+                root: /mloscratch/homes/navasard/sparse/SparseAtt
+                startedAt: "2026-07-13T03:19:02.573325Z"
+                writerId: vfbd6u6afm7i15wtvccnshegb2rm9vr5
+        m: []
+        python_version: 3.13.13
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 5
+                - 13
+                - 14
+                - 15
+                - 16
+                - 61
+                - 62
+            "4": 3.13.13
+            "5": 0.26.1
+            "6": 5.7.0
+            "12": 0.26.1
+            "13": linux-x86_64
+attention_type:
+    value: sparse_stochastic
+checkpoint_activations:
+    value: true
+dense_mode:
+    value: false
+dense_step_intervals:
+    value: null
+eval_batches:
+    value: 20
+eval_steps:
+    value: 500
+force_new_run:
+    value: false
+gradient_accumulation_steps:
+    value: 5
+head_dim:
+    value: null
+hidden_size:
+    value: 1536
+init_from_scratch:
+    value: true
+intermediate_size:
+    value: 6144
+learning_rate:
+    value: 0.0005
+logging_steps:
+    value: 10
+lr_scheduler:
+    value: cosine
+max_grad_norm:
+    value: 1
+max_seq_length:
+    value: 30720
+max_steps:
+    value: null
+mixed_precision:
+    value: bf16
+model_dtype:
+    value: bfloat16
+model_path:
+    value: /mloscratch/homes/navasard/qwen3_06b_base
+num_attention_heads:
+    value: 12
+num_epochs:
+    value: 1
+num_hidden_layers:
+    value: null
+num_key_value_heads:
+    value: 4
+num_workers:
+    value: 4
+output_dir:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_fixed/run_20260709_000503
+per_device_batch_size:
+    value: 1
+resume_from_checkpoint:
+    value: /mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_fixed/run_20260709_000503/checkpoint-8100
+save_steps:
+    value: 300
+seed:
+    value: 42
+sparse_block_size:
+    value: 128
+sparse_layer_indices:
+    value: null
+sparse_mask_cache_dir:
+    value: /mloscratch/homes/navasard/sparse/mask_cache_30k_b128_nrb120
+sparse_mask_pool_size:
+    value: 1
+sparse_mask_seed:
+    value: 1
+sparse_num_random_blocks:
+    value: 120
+sparse_val_dense_compare:
+    value: false
+strategy:
+    value: ddp
+train_path:
+    value: /mloscratch/homes/navasard/data/longctx30k/tokenized
+val_path:
+    value: null
+verbose:
+    value: true
+wandb_enabled:
+    value: true
+wandb_name:
+    value: 1.2b fixed 30k longctx
+wandb_project:
+    value: stochastic_sparsity
+wandb_tags:
+    value:
+        - qwen3
+        - 1.2b
+        - fixed
+        - 30k
+        - longctx
+        - b128
+        - scaleladder
+warmup_steps:
+    value: 750
+weight_decay:
+    value: 0.01

epfl_final/SparseAtt/wandb/run-20260713_031902-9blizq3m/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260713_031902-9blizq3m/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-07-13T03:19:02.573325Z",
+  "args":  [
+    "--config",
+    "configs/train_config_30k_fixed_b128.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "train.py",
+  "codePathLocal":  "train.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "88298c7a418622530ca5426b22b1e5ebd10ca0cb"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "h100-train-4-gpu-2-0-5",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "9374931091456",
+      "used":  "4251919761408"
+    }
+  },
+  "memory":  {
+    "total":  "1623005732864"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-99f99f92-e501-0b15-5782-6beb2c34fa1a"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-7004b604-21a2-890e-0312-3eee1eb93c67"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-a2cf418a-a552-c05b-f5dc-8b4f120a914a"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-3f5bace4-0066-71eb-863d-73a3a6d79e3b"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "vfbd6u6afm7i15wtvccnshegb2rm9vr5"
+}

epfl_final/SparseAtt/wandb/run-20260713_031902-9blizq3m/files/wandb-summary.json

@@ -0,0 +1 @@
+{"speed/steps_per_sec":0.12368149206180723,"_runtime":394203.596395206,"eval/loss":2.0703051459789275,"speed/ms_per_step":8085.284090042114,"eval/perplexity":7.927241714751403,"eval/dense_interval_start":-1,"speed/samples_per_sec":2.4736298412361446,"train/loss":1.9446697549521923,"train/learning_rate":2.2880351668863596e-11,"train/epoch":0.9998665421059656,"eval/uses_sparse_attention":1,"_wandb":{"runtime":394203},"eval/dense_interval_end":-1,"eval/dense_interval_active":0,"train/global_step":37460,"_step":37460,"eval/uses_dense_attention":0,"_timestamp":1.784151810859256e+09}

epfl_final/SparseAtt/wandb/run-20260713_031902-9blizq3m/logs/debug.log

@@ -0,0 +1,25 @@
+2026-07-13 03:19:02,594 INFO    MainThread:217 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-07-13 03:19:02,594 INFO    MainThread:217 [wandb_setup.py:_flush():81] Configure stats pid to 217
+2026-07-13 03:19:02,594 INFO    MainThread:217 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-13 03:19:02,594 INFO    MainThread:217 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260713_031902-9blizq3m/logs/debug.log
+2026-07-13 03:19:02,594 INFO    MainThread:217 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260713_031902-9blizq3m/logs/debug-internal.log
+2026-07-13 03:19:02,594 INFO    MainThread:217 [wandb_init.py:init():850] calling init triggers
+2026-07-13 03:19:02,594 INFO    MainThread:217 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/longctx30k/tokenized', 'val_path': None, 'max_seq_length': 30720, 'num_workers': 4, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_fixed/run_20260709_000503', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l30k1_2b_fixed/run_20260709_000503/checkpoint-8100', 'force_new_run': False, 'num_epochs': 1, 'max_steps': None, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 5, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 300, 'eval_steps': 500, 'eval_batches': 20, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b fixed 30k longctx', 'wandb_tags': ['qwen3', '1.2b', 'fixed', '30k', 'longctx', 'b128', 'scaleladder'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 120, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_30k_b128_nrb120', 'sparse_mask_seed': 1, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-07-13 03:19:02,594 INFO    MainThread:217 [wandb_init.py:init():898] starting backend
+2026-07-13 03:19:03,132 INFO    MainThread:217 [wandb_init.py:init():913] sending inform_init request
+2026-07-13 03:19:03,583 INFO    MainThread:217 [wandb_init.py:init():918] backend started and connected
+2026-07-13 03:19:03,586 INFO    MainThread:217 [wandb_init.py:init():988] updated telemetry
+2026-07-13 03:19:03,619 INFO    MainThread:217 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-07-13 03:19:04,114 INFO    MainThread:217 [wandb_init.py:init():1051] run resumed
+2026-07-13 03:19:04,117 INFO    MainThread:217 [wandb_init.py:init():1056] starting run threads in backend
+2026-07-13 03:19:04,359 INFO    MainThread:217 [wandb_run.py:_console_start():2554] atexit reg
+2026-07-13 03:19:04,359 INFO    MainThread:217 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-07-13 03:19:04,359 INFO    MainThread:217 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-07-13 03:19:04,359 INFO    MainThread:217 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-07-13 03:19:04,367 INFO    MainThread:217 [wandb_init.py:init():1094] run started, returning control to user process
+2026-07-15 21:44:20,711 INFO    MainThread:217 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/9blizq3m
+2026-07-15 21:44:20,711 INFO    MainThread:217 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
+2026-07-15 21:44:20,713 INFO    MainThread:217 [wandb_run.py:_restore():2502] restore
+2026-07-15 21:44:20,713 INFO    MainThread:217 [wandb_run.py:_restore():2508] restore done
+2026-07-15 21:45:28,863 INFO    MainThread:217 [wandb_run.py:_footer_sync_info():3889] logging synced files

epfl_final/SparseAtt/wandb/run-20260713_031902-9blizq3m/run-9blizq3m.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:acec3222b6f3576b8e3e4c28d59c9392df3b095352379599fdf4074a734c87f9
+size 77668423

epfl_final/SparseAtt/wandb/run-20260726_072540-33yw8345/files/output.log

@@ -0,0 +1,2 @@
+Starting training...
+Validation at step 47000: Loss = 2.5335, Perplexity = 12.60

epfl_final/SparseAtt/wandb/run-20260726_072540-33yw8345/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260726_072540-33yw8345/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-07-26T07:25:40.449889Z",
+  "args":  [
+    "--config",
+    "configs/train_config_1mask_perlayerfix_resume_3gpu.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train_fixed_resume.py",
+  "codePath":  "train_fixed_resume.py",
+  "codePathLocal":  "train_fixed_resume.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "88298c7a418622530ca5426b22b1e5ebd10ca0cb"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "a100-workspace-4-gpu-0-1",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  64,
+  "cpu_count_logical":  128,
+  "gpu":  "NVIDIA A100-SXM4-80GB",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "7679361679360",
+      "used":  "5455846473728"
+    }
+  },
+  "memory":  {
+    "total":  "1081884934144"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-943525a1-9808-8c31-17d9-c4180bb03955"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-b4caa2bd-09d3-3702-9fd1-293525d8e08c"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-46161407-c19c-cdae-c140-0963b35a3047"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-f1d949cd-9bad-cd83-2a2b-ed36b22f13a6"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "31bh9xxac323znkkwgqxbo3evgdi2yt3"
+}

epfl_final/SparseAtt/wandb/run-20260726_072540-33yw8345/logs/debug-internal.log

@@ -0,0 +1,93 @@
+{"time":"2026-07-26T07:25:41.039304707Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-07-26T07:25:41.0436407Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
+{"time":"2026-07-26T07:25:41.426218774Z","level":"INFO","msg":"stream: created new stream","id":"33yw8345"}
+{"time":"2026-07-26T07:25:41.426289265Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-07-26T07:25:41.428294604Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-07-26T07:25:41.428329098Z","level":"INFO","msg":"writer: started","stream_id":"33yw8345"}
+{"time":"2026-07-26T07:25:41.428358663Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-07-26T07:25:41.953219196Z","level":"INFO","msg":"filestream: sending request","total_files":1}
+{"time":"2026-07-26T07:25:42.294511179Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:25:56.953410277Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1,"uploaded_len":2}
+{"time":"2026-07-26T07:25:57.146278793Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:26:11.953569381Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":0,"events_lines":2}
+{"time":"2026-07-26T07:26:12.190245242Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:26:26.95389428Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
+{"time":"2026-07-26T07:26:27.254169792Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:26:41.953848978Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":4,"events_lines":2}
+{"time":"2026-07-26T07:26:42.265001345Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:26:56.954131177Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":6,"events_lines":2}
+{"time":"2026-07-26T07:26:57.24948496Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:27:11.95411566Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":8,"events_lines":2}
+{"time":"2026-07-26T07:27:12.326079959Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:27:26.953902625Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":10,"events_lines":2,"console_offset":1,"console_lines":1}
+{"time":"2026-07-26T07:27:27.35562387Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:27:41.953502852Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":12,"events_lines":2}
+{"time":"2026-07-26T07:27:42.202676611Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:27:56.953491172Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":14,"events_lines":2}
+{"time":"2026-07-26T07:27:57.178337121Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:28:11.95396338Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":16,"events_lines":2}
+{"time":"2026-07-26T07:28:12.206595315Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:28:26.953773241Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":18,"events_lines":2}
+{"time":"2026-07-26T07:28:27.340498951Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:28:41.953323928Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":20,"events_lines":2}
+{"time":"2026-07-26T07:28:42.233487041Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:28:56.953423959Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":22,"events_lines":2}
+{"time":"2026-07-26T07:28:57.224619717Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:29:11.953965734Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":24,"events_lines":2}
+{"time":"2026-07-26T07:29:12.275139172Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:29:26.953998399Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":26,"events_lines":2}
+{"time":"2026-07-26T07:29:27.247916312Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:29:41.953222457Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":28,"events_lines":2}
+{"time":"2026-07-26T07:29:42.250941305Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:29:56.954255467Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":30,"events_lines":2}
+{"time":"2026-07-26T07:29:57.261442895Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:30:11.95344445Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":32,"events_lines":2}
+{"time":"2026-07-26T07:30:12.174528939Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:30:26.953621125Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":34,"events_lines":2}
+{"time":"2026-07-26T07:30:27.312095471Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:30:41.953589164Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":36,"events_lines":2}
+{"time":"2026-07-26T07:30:42.206318833Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:30:56.953398296Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":38,"events_lines":2}
+{"time":"2026-07-26T07:30:57.297019153Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:31:11.953440052Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":40,"events_lines":2}
+{"time":"2026-07-26T07:31:12.217339382Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:31:26.953184623Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":42,"events_lines":2}
+{"time":"2026-07-26T07:31:27.14420975Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:31:41.953490982Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":44,"events_lines":2}
+{"time":"2026-07-26T07:31:42.251672247Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:31:56.953430397Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":46,"events_lines":2}
+{"time":"2026-07-26T07:31:57.363535818Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:32:11.953788162Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":48,"events_lines":2}
+{"time":"2026-07-26T07:32:12.210171632Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:32:26.953393495Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":50,"events_lines":2}
+{"time":"2026-07-26T07:32:27.249414943Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:32:41.954126725Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":52,"events_lines":2}
+{"time":"2026-07-26T07:32:42.21049248Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:32:56.953820854Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":54,"events_lines":2}
+{"time":"2026-07-26T07:32:57.284792797Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:33:11.953357799Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":56,"events_lines":2}
+{"time":"2026-07-26T07:33:12.281955837Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:33:26.954127102Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":58,"events_lines":2}
+{"time":"2026-07-26T07:33:27.298310041Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:33:41.954222609Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":60,"events_lines":2}
+{"time":"2026-07-26T07:33:42.301933354Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:33:56.953920805Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":62,"events_lines":2}
+{"time":"2026-07-26T07:33:57.212599461Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:34:11.954222747Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":64,"events_lines":2}
+{"time":"2026-07-26T07:34:12.271572969Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:34:26.954070272Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":66,"events_lines":2}
+{"time":"2026-07-26T07:34:27.294954271Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:34:41.953868596Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":68,"events_lines":2}
+{"time":"2026-07-26T07:34:42.224200869Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:34:56.953541814Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":70,"events_lines":2}
+{"time":"2026-07-26T07:34:57.310265898Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:35:11.953670452Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":72,"events_lines":2}
+{"time":"2026-07-26T07:35:12.307525257Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:35:26.95323702Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":74,"events_lines":2}
+{"time":"2026-07-26T07:35:27.276680589Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:35:41.95337307Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":76,"events_lines":2}
+{"time":"2026-07-26T07:35:42.27517548Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:35:56.953667316Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":78,"events_lines":2}
+{"time":"2026-07-26T07:35:57.178478934Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-07-26T07:36:11.954083939Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":80,"events_lines":2}
+{"time":"2026-07-26T07:36:12.223191068Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

epfl_final/SparseAtt/wandb/run-20260726_072540-33yw8345/logs/debug.log

@@ -0,0 +1,20 @@
+2026-07-26 07:25:40,474 INFO    MainThread:1749 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-07-26 07:25:40,474 INFO    MainThread:1749 [wandb_setup.py:_flush():81] Configure stats pid to 1749
+2026-07-26 07:25:40,474 INFO    MainThread:1749 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-07-26 07:25:40,475 INFO    MainThread:1749 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260726_072540-33yw8345/logs/debug.log
+2026-07-26 07:25:40,475 INFO    MainThread:1749 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260726_072540-33yw8345/logs/debug-internal.log
+2026-07-26 07:25:40,475 INFO    MainThread:1749 [wandb_init.py:init():850] calling init triggers
+2026-07-26 07:25:40,475 INFO    MainThread:1749 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-47000', 'resume_micro_step_override': None, 'preserve_checkpoint_lr': True, 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'sparse 1mask perlayer-fix', 'wandb_tags': ['qwen3', '1.2b', 'sparse_stochastic', '1mask', 'perlayer-fix'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': 28, 'head_dim': 128, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'sparse_mask_seed': 0, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-07-26 07:25:40,475 INFO    MainThread:1749 [wandb_init.py:init():898] starting backend
+2026-07-26 07:25:41,025 INFO    MainThread:1749 [wandb_init.py:init():913] sending inform_init request
+2026-07-26 07:25:41,428 INFO    MainThread:1749 [wandb_init.py:init():918] backend started and connected
+2026-07-26 07:25:41,432 INFO    MainThread:1749 [wandb_init.py:init():988] updated telemetry
+2026-07-26 07:25:41,454 INFO    MainThread:1749 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-07-26 07:25:41,951 INFO    MainThread:1749 [wandb_init.py:init():1051] run resumed
+2026-07-26 07:25:41,952 INFO    MainThread:1749 [wandb_init.py:init():1056] starting run threads in backend
+2026-07-26 07:25:42,165 INFO    MainThread:1749 [wandb_run.py:_console_start():2554] atexit reg
+2026-07-26 07:25:42,165 INFO    MainThread:1749 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-07-26 07:25:42,165 INFO    MainThread:1749 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-07-26 07:25:42,165 INFO    MainThread:1749 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-07-26 07:25:42,172 INFO    MainThread:1749 [wandb_init.py:init():1094] run started, returning control to user process

epfl_final/SparseAtt/wandb/run-20260726_073805-33yw8345/files/output.log

@@ -0,0 +1,366 @@
+Starting training...
+Validation at step 47000: Loss = 2.5335, Perplexity = 12.60
+Switching attention mode for step 47001: sparse_stochastic (configured sparse attention)
+Step 47010/50462 | Loss: 2.5459 | LR: 5.93e-06 | 22806.7ms/step | 2.6 samples/s
+Step 47020/50462 | Loss: 2.5222 | LR: 5.89e-06 | 8265.6ms/step | 7.3 samples/s
+Step 47030/50462 | Loss: 2.5425 | LR: 5.86e-06 | 8287.9ms/step | 7.2 samples/s
+Step 47040/50462 | Loss: 2.4869 | LR: 5.82e-06 | 8300.5ms/step | 7.2 samples/s
+Step 47050/50462 | Loss: 2.5257 | LR: 5.79e-06 | 8302.8ms/step | 7.2 samples/s
+Step 47060/50462 | Loss: 2.5221 | LR: 5.76e-06 | 8301.5ms/step | 7.2 samples/s
+Step 47070/50462 | Loss: 2.5138 | LR: 5.72e-06 | 8300.5ms/step | 7.2 samples/s
+Step 47080/50462 | Loss: 2.5226 | LR: 5.69e-06 | 8300.0ms/step | 7.2 samples/s
+Step 47090/50462 | Loss: 2.5188 | LR: 5.65e-06 | 8301.1ms/step | 7.2 samples/s
+Step 47100/50462 | Loss: 2.5219 | LR: 5.62e-06 | 8301.2ms/step | 7.2 samples/s
+Step 47110/50462 | Loss: 2.5261 | LR: 5.59e-06 | 8302.8ms/step | 7.2 samples/s
+Step 47120/50462 | Loss: 2.4759 | LR: 5.56e-06 | 8303.9ms/step | 7.2 samples/s
+Step 47130/50462 | Loss: 2.5224 | LR: 5.52e-06 | 8301.2ms/step | 7.2 samples/s
+Step 47140/50462 | Loss: 2.5344 | LR: 5.49e-06 | 8304.0ms/step | 7.2 samples/s
+Step 47150/50462 | Loss: 2.4921 | LR: 5.46e-06 | 8303.3ms/step | 7.2 samples/s
+Step 47160/50462 | Loss: 2.5337 | LR: 5.42e-06 | 8301.3ms/step | 7.2 samples/s
+Step 47170/50462 | Loss: 2.5162 | LR: 5.39e-06 | 8301.1ms/step | 7.2 samples/s
+Step 47180/50462 | Loss: 2.5263 | LR: 5.36e-06 | 8301.5ms/step | 7.2 samples/s
+Step 47190/50462 | Loss: 2.5194 | LR: 5.33e-06 | 8301.6ms/step | 7.2 samples/s
+Step 47200/50462 | Loss: 2.5175 | LR: 5.29e-06 | 8304.1ms/step | 7.2 samples/s
+Step 47210/50462 | Loss: 2.4912 | LR: 5.26e-06 | 8303.6ms/step | 7.2 samples/s
+Step 47220/50462 | Loss: 2.5329 | LR: 5.23e-06 | 8304.0ms/step | 7.2 samples/s
+Step 47230/50462 | Loss: 2.5304 | LR: 5.20e-06 | 8303.6ms/step | 7.2 samples/s
+Step 47240/50462 | Loss: 2.4988 | LR: 5.16e-06 | 8303.8ms/step | 7.2 samples/s
+Step 47250/50462 | Loss: 2.5299 | LR: 5.13e-06 | 8303.2ms/step | 7.2 samples/s
+Validation at step 47250: Loss = 2.5335, Perplexity = 12.60
+Step 47260/50462 | Loss: 2.5234 | LR: 5.10e-06 | 12599.4ms/step | 4.8 samples/s
+Step 47270/50462 | Loss: 2.5359 | LR: 5.07e-06 | 8301.7ms/step | 7.2 samples/s
+Step 47280/50462 | Loss: 2.5108 | LR: 5.04e-06 | 8303.0ms/step | 7.2 samples/s
+Step 47290/50462 | Loss: 2.5375 | LR: 5.01e-06 | 8305.3ms/step | 7.2 samples/s
+Step 47300/50462 | Loss: 2.5374 | LR: 4.97e-06 | 8305.8ms/step | 7.2 samples/s
+Step 47310/50462 | Loss: 2.4979 | LR: 4.94e-06 | 8304.9ms/step | 7.2 samples/s
+Step 47320/50462 | Loss: 2.5143 | LR: 4.91e-06 | 8301.7ms/step | 7.2 samples/s
+Step 47330/50462 | Loss: 2.5177 | LR: 4.88e-06 | 8304.6ms/step | 7.2 samples/s
+Step 47340/50462 | Loss: 2.4832 | LR: 4.85e-06 | 8307.4ms/step | 7.2 samples/s
+Step 47350/50462 | Loss: 2.5310 | LR: 4.82e-06 | 8313.0ms/step | 7.2 samples/s
+Step 47360/50462 | Loss: 2.5016 | LR: 4.79e-06 | 8311.6ms/step | 7.2 samples/s
+Step 47370/50462 | Loss: 2.5399 | LR: 4.76e-06 | 8314.5ms/step | 7.2 samples/s
+Step 47380/50462 | Loss: 2.5329 | LR: 4.73e-06 | 8310.7ms/step | 7.2 samples/s
+Step 47390/50462 | Loss: 2.5150 | LR: 4.70e-06 | 8310.5ms/step | 7.2 samples/s
+Step 47400/50462 | Loss: 2.5178 | LR: 4.67e-06 | 8312.5ms/step | 7.2 samples/s
+Step 47410/50462 | Loss: 2.5297 | LR: 4.64e-06 | 8311.8ms/step | 7.2 samples/s
+Step 47420/50462 | Loss: 2.4808 | LR: 4.61e-06 | 8313.6ms/step | 7.2 samples/s
+Step 47430/50462 | Loss: 2.5087 | LR: 4.58e-06 | 8311.7ms/step | 7.2 samples/s
+Step 47440/50462 | Loss: 2.5168 | LR: 4.55e-06 | 8313.8ms/step | 7.2 samples/s
+Step 47450/50462 | Loss: 2.4910 | LR: 4.52e-06 | 8314.4ms/step | 7.2 samples/s
+Step 47460/50462 | Loss: 2.5360 | LR: 4.49e-06 | 8309.8ms/step | 7.2 samples/s
+Step 47470/50462 | Loss: 2.5346 | LR: 4.46e-06 | 8312.2ms/step | 7.2 samples/s
+Step 47480/50462 | Loss: 2.5321 | LR: 4.43e-06 | 8318.5ms/step | 7.2 samples/s
+Step 47490/50462 | Loss: 2.5256 | LR: 4.40e-06 | 8316.7ms/step | 7.2 samples/s
+Step 47500/50462 | Loss: 2.5176 | LR: 4.37e-06 | 8311.8ms/step | 7.2 samples/s
+Validation at step 47500: Loss = 2.5335, Perplexity = 12.60
+Step 47510/50462 | Loss: 2.4883 | LR: 4.34e-06 | 12619.5ms/step | 4.8 samples/s
+Step 47520/50462 | Loss: 2.5365 | LR: 4.31e-06 | 8312.0ms/step | 7.2 samples/s
+Step 47530/50462 | Loss: 2.5318 | LR: 4.28e-06 | 8315.7ms/step | 7.2 samples/s
+Step 47540/50462 | Loss: 2.5003 | LR: 4.25e-06 | 8319.1ms/step | 7.2 samples/s
+Step 47550/50462 | Loss: 2.5290 | LR: 4.22e-06 | 8318.5ms/step | 7.2 samples/s
+Step 47560/50462 | Loss: 2.5153 | LR: 4.19e-06 | 8322.5ms/step | 7.2 samples/s
+Step 47570/50462 | Loss: 2.5249 | LR: 4.16e-06 | 8323.7ms/step | 7.2 samples/s
+Step 47580/50462 | Loss: 2.5613 | LR: 4.13e-06 | 8334.2ms/step | 7.2 samples/s
+Step 47590/50462 | Loss: 2.5304 | LR: 4.11e-06 | 8326.2ms/step | 7.2 samples/s
+Step 47600/50462 | Loss: 2.5215 | LR: 4.08e-06 | 8318.7ms/step | 7.2 samples/s
+Step 47610/50462 | Loss: 2.5013 | LR: 4.05e-06 | 8315.5ms/step | 7.2 samples/s
+Step 47620/50462 | Loss: 2.5141 | LR: 4.02e-06 | 8321.4ms/step | 7.2 samples/s
+Step 47630/50462 | Loss: 2.5422 | LR: 3.99e-06 | 8318.7ms/step | 7.2 samples/s
+Step 47640/50462 | Loss: 2.4982 | LR: 3.97e-06 | 8317.9ms/step | 7.2 samples/s
+Step 47650/50462 | Loss: 2.5175 | LR: 3.94e-06 | 8320.7ms/step | 7.2 samples/s
+Step 47660/50462 | Loss: 2.5164 | LR: 3.91e-06 | 8318.2ms/step | 7.2 samples/s
+Step 47670/50462 | Loss: 2.5088 | LR: 3.88e-06 | 8321.0ms/step | 7.2 samples/s
+Step 47680/50462 | Loss: 2.5261 | LR: 3.85e-06 | 8313.1ms/step | 7.2 samples/s
+Step 47690/50462 | Loss: 2.5224 | LR: 3.83e-06 | 8311.6ms/step | 7.2 samples/s
+Step 47700/50462 | Loss: 2.5305 | LR: 3.80e-06 | 8311.4ms/step | 7.2 samples/s
+Step 47710/50462 | Loss: 2.4971 | LR: 3.77e-06 | 8311.3ms/step | 7.2 samples/s
+Step 47720/50462 | Loss: 2.5235 | LR: 3.74e-06 | 8309.2ms/step | 7.2 samples/s
+Step 47730/50462 | Loss: 2.5239 | LR: 3.72e-06 | 8310.0ms/step | 7.2 samples/s
+Step 47740/50462 | Loss: 2.5085 | LR: 3.69e-06 | 8315.3ms/step | 7.2 samples/s
+Step 47750/50462 | Loss: 2.5353 | LR: 3.66e-06 | 8305.3ms/step | 7.2 samples/s
+Validation at step 47750: Loss = 2.5335, Perplexity = 12.60
+Step 47760/50462 | Loss: 2.4930 | LR: 3.64e-06 | 12610.0ms/step | 4.8 samples/s
+Step 47770/50462 | Loss: 2.5552 | LR: 3.61e-06 | 8309.1ms/step | 7.2 samples/s
+Step 47780/50462 | Loss: 2.5137 | LR: 3.58e-06 | 8311.2ms/step | 7.2 samples/s
+Step 47790/50462 | Loss: 2.5084 | LR: 3.56e-06 | 8311.6ms/step | 7.2 samples/s
+Step 47800/50462 | Loss: 2.4999 | LR: 3.53e-06 | 8308.8ms/step | 7.2 samples/s
+Step 47810/50462 | Loss: 2.5327 | LR: 3.50e-06 | 8308.0ms/step | 7.2 samples/s
+Step 47820/50462 | Loss: 2.5265 | LR: 3.48e-06 | 8312.4ms/step | 7.2 samples/s
+Step 47830/50462 | Loss: 2.5188 | LR: 3.45e-06 | 8310.1ms/step | 7.2 samples/s
+Step 47840/50462 | Loss: 2.5409 | LR: 3.42e-06 | 8311.2ms/step | 7.2 samples/s
+Step 47850/50462 | Loss: 2.5313 | LR: 3.40e-06 | 8310.1ms/step | 7.2 samples/s
+Step 47860/50462 | Loss: 2.5337 | LR: 3.37e-06 | 8306.8ms/step | 7.2 samples/s
+Step 47870/50462 | Loss: 2.5285 | LR: 3.35e-06 | 8308.1ms/step | 7.2 samples/s
+Step 47880/50462 | Loss: 2.4817 | LR: 3.32e-06 | 8313.9ms/step | 7.2 samples/s
+Step 47890/50462 | Loss: 2.5415 | LR: 3.30e-06 | 8307.6ms/step | 7.2 samples/s
+Step 47900/50462 | Loss: 2.5291 | LR: 3.27e-06 | 8309.2ms/step | 7.2 samples/s
+Step 47910/50462 | Loss: 2.5277 | LR: 3.24e-06 | 8306.6ms/step | 7.2 samples/s
+Step 47920/50462 | Loss: 2.5026 | LR: 3.22e-06 | 8305.8ms/step | 7.2 samples/s
+Step 47930/50462 | Loss: 2.5405 | LR: 3.19e-06 | 8308.2ms/step | 7.2 samples/s
+Step 47940/50462 | Loss: 2.5155 | LR: 3.17e-06 | 8304.3ms/step | 7.2 samples/s
+Step 47950/50462 | Loss: 2.5226 | LR: 3.14e-06 | 8306.7ms/step | 7.2 samples/s
+Step 47960/50462 | Loss: 2.4898 | LR: 3.12e-06 | 8306.9ms/step | 7.2 samples/s
+Step 47970/50462 | Loss: 2.5259 | LR: 3.09e-06 | 8310.4ms/step | 7.2 samples/s
+Step 47980/50462 | Loss: 2.5237 | LR: 3.07e-06 | 8307.9ms/step | 7.2 samples/s
+Step 47990/50462 | Loss: 2.5060 | LR: 3.04e-06 | 8307.2ms/step | 7.2 samples/s
+Step 48000/50462 | Loss: 2.5163 | LR: 3.02e-06 | 8309.1ms/step | 7.2 samples/s
+Validation at step 48000: Loss = 2.5335, Perplexity = 12.60
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-48000
+Step 48010/50462 | Loss: 2.5290 | LR: 3.00e-06 | 13601.3ms/step | 4.4 samples/s
+Step 48020/50462 | Loss: 2.5583 | LR: 2.97e-06 | 8300.8ms/step | 7.2 samples/s
+Step 48030/50462 | Loss: 2.4996 | LR: 2.95e-06 | 8315.9ms/step | 7.2 samples/s
+Step 48040/50462 | Loss: 2.5126 | LR: 2.92e-06 | 8313.4ms/step | 7.2 samples/s
+Step 48050/50462 | Loss: 2.5062 | LR: 2.90e-06 | 8317.0ms/step | 7.2 samples/s
+Step 48060/50462 | Loss: 2.5220 | LR: 2.87e-06 | 8315.4ms/step | 7.2 samples/s
+Step 48070/50462 | Loss: 2.5258 | LR: 2.85e-06 | 8317.1ms/step | 7.2 samples/s
+Step 48080/50462 | Loss: 2.5499 | LR: 2.83e-06 | 8320.2ms/step | 7.2 samples/s
+Step 48090/50462 | Loss: 2.5252 | LR: 2.80e-06 | 8320.8ms/step | 7.2 samples/s
+Step 48100/50462 | Loss: 2.5286 | LR: 2.78e-06 | 8321.8ms/step | 7.2 samples/s
+Step 48110/50462 | Loss: 2.5176 | LR: 2.76e-06 | 8314.1ms/step | 7.2 samples/s
+Step 48120/50462 | Loss: 2.5165 | LR: 2.73e-06 | 8332.9ms/step | 7.2 samples/s
+Step 48130/50462 | Loss: 2.5169 | LR: 2.71e-06 | 8307.5ms/step | 7.2 samples/s
+Step 48140/50462 | Loss: 2.4988 | LR: 2.69e-06 | 8300.1ms/step | 7.2 samples/s
+Step 48150/50462 | Loss: 2.5116 | LR: 2.66e-06 | 8314.2ms/step | 7.2 samples/s
+Step 48160/50462 | Loss: 2.5313 | LR: 2.64e-06 | 8316.9ms/step | 7.2 samples/s
+Step 48170/50462 | Loss: 2.5122 | LR: 2.62e-06 | 8315.9ms/step | 7.2 samples/s
+Step 48180/50462 | Loss: 2.5238 | LR: 2.60e-06 | 8295.5ms/step | 7.2 samples/s
+Step 48190/50462 | Loss: 2.5189 | LR: 2.57e-06 | 8295.8ms/step | 7.2 samples/s
+Step 48200/50462 | Loss: 2.5296 | LR: 2.55e-06 | 8307.3ms/step | 7.2 samples/s
+Step 48210/50462 | Loss: 2.5054 | LR: 2.53e-06 | 8312.2ms/step | 7.2 samples/s
+Step 48220/50462 | Loss: 2.5126 | LR: 2.51e-06 | 8309.4ms/step | 7.2 samples/s
+Step 48230/50462 | Loss: 2.5187 | LR: 2.48e-06 | 8311.0ms/step | 7.2 samples/s
+Step 48240/50462 | Loss: 2.5174 | LR: 2.46e-06 | 8312.0ms/step | 7.2 samples/s
+Step 48250/50462 | Loss: 2.5548 | LR: 2.44e-06 | 8310.4ms/step | 7.2 samples/s
+Validation at step 48250: Loss = 2.5335, Perplexity = 12.60
+Step 48260/50462 | Loss: 2.4928 | LR: 2.42e-06 | 12613.1ms/step | 4.8 samples/s
+Step 48270/50462 | Loss: 2.5189 | LR: 2.39e-06 | 8309.9ms/step | 7.2 samples/s
+Step 48280/50462 | Loss: 2.5276 | LR: 2.37e-06 | 8309.1ms/step | 7.2 samples/s
+Step 48290/50462 | Loss: 2.5148 | LR: 2.35e-06 | 8311.2ms/step | 7.2 samples/s
+Step 48300/50462 | Loss: 2.5072 | LR: 2.33e-06 | 8311.6ms/step | 7.2 samples/s
+Step 48310/50462 | Loss: 2.5283 | LR: 2.31e-06 | 8313.2ms/step | 7.2 samples/s
+Step 48320/50462 | Loss: 2.5262 | LR: 2.29e-06 | 8313.0ms/step | 7.2 samples/s
+Step 48330/50462 | Loss: 2.5108 | LR: 2.27e-06 | 8308.4ms/step | 7.2 samples/s
+Step 48340/50462 | Loss: 2.5324 | LR: 2.24e-06 | 8307.0ms/step | 7.2 samples/s
+Step 48350/50462 | Loss: 2.5471 | LR: 2.22e-06 | 8302.7ms/step | 7.2 samples/s
+Step 48360/50462 | Loss: 2.5045 | LR: 2.20e-06 | 8307.6ms/step | 7.2 samples/s
+Step 48370/50462 | Loss: 2.5406 | LR: 2.18e-06 | 8306.6ms/step | 7.2 samples/s
+Step 48380/50462 | Loss: 2.5068 | LR: 2.16e-06 | 8304.8ms/step | 7.2 samples/s
+Step 48390/50462 | Loss: 2.5545 | LR: 2.14e-06 | 8311.2ms/step | 7.2 samples/s
+Step 48400/50462 | Loss: 2.5099 | LR: 2.12e-06 | 8309.1ms/step | 7.2 samples/s
+Step 48410/50462 | Loss: 2.5393 | LR: 2.10e-06 | 8307.0ms/step | 7.2 samples/s
+Step 48420/50462 | Loss: 2.5151 | LR: 2.08e-06 | 8315.2ms/step | 7.2 samples/s
+Step 48430/50462 | Loss: 2.5124 | LR: 2.06e-06 | 8317.9ms/step | 7.2 samples/s
+Step 48440/50462 | Loss: 2.5282 | LR: 2.04e-06 | 8311.5ms/step | 7.2 samples/s
+Step 48450/50462 | Loss: 2.4945 | LR: 2.02e-06 | 8311.7ms/step | 7.2 samples/s
+Step 48460/50462 | Loss: 2.5426 | LR: 2.00e-06 | 8310.4ms/step | 7.2 samples/s
+Step 48470/50462 | Loss: 2.5294 | LR: 1.98e-06 | 8308.9ms/step | 7.2 samples/s
+Step 48480/50462 | Loss: 2.5130 | LR: 1.96e-06 | 8308.1ms/step | 7.2 samples/s
+Step 48490/50462 | Loss: 2.5209 | LR: 1.94e-06 | 8312.5ms/step | 7.2 samples/s
+Step 48500/50462 | Loss: 2.4894 | LR: 1.92e-06 | 8308.1ms/step | 7.2 samples/s
+Validation at step 48500: Loss = 2.5335, Perplexity = 12.60
+Step 48510/50462 | Loss: 2.5232 | LR: 1.90e-06 | 12607.5ms/step | 4.8 samples/s
+Step 48520/50462 | Loss: 2.5231 | LR: 1.88e-06 | 8307.3ms/step | 7.2 samples/s
+Step 48530/50462 | Loss: 2.5403 | LR: 1.86e-06 | 8307.4ms/step | 7.2 samples/s
+Step 48540/50462 | Loss: 2.4961 | LR: 1.84e-06 | 8305.6ms/step | 7.2 samples/s
+Step 48550/50462 | Loss: 2.5063 | LR: 1.82e-06 | 8305.2ms/step | 7.2 samples/s
+Step 48560/50462 | Loss: 2.5160 | LR: 1.80e-06 | 8304.5ms/step | 7.2 samples/s
+Step 48570/50462 | Loss: 2.5190 | LR: 1.78e-06 | 8305.3ms/step | 7.2 samples/s
+Step 48580/50462 | Loss: 2.5112 | LR: 1.77e-06 | 8304.5ms/step | 7.2 samples/s
+Step 48590/50462 | Loss: 2.5361 | LR: 1.75e-06 | 8304.3ms/step | 7.2 samples/s
+Step 48600/50462 | Loss: 2.5268 | LR: 1.73e-06 | 8305.3ms/step | 7.2 samples/s
+Step 48610/50462 | Loss: 2.5125 | LR: 1.71e-06 | 8303.1ms/step | 7.2 samples/s
+Step 48620/50462 | Loss: 2.5178 | LR: 1.69e-06 | 8302.0ms/step | 7.2 samples/s
+Step 48630/50462 | Loss: 2.5415 | LR: 1.67e-06 | 8301.3ms/step | 7.2 samples/s
+Step 48640/50462 | Loss: 2.5219 | LR: 1.66e-06 | 8301.4ms/step | 7.2 samples/s
+Step 48650/50462 | Loss: 2.5388 | LR: 1.64e-06 | 8303.4ms/step | 7.2 samples/s
+Step 48660/50462 | Loss: 2.5340 | LR: 1.62e-06 | 8304.2ms/step | 7.2 samples/s
+Step 48670/50462 | Loss: 2.5358 | LR: 1.60e-06 | 8304.0ms/step | 7.2 samples/s
+Step 48680/50462 | Loss: 2.5181 | LR: 1.58e-06 | 8300.6ms/step | 7.2 samples/s
+Step 48690/50462 | Loss: 2.5325 | LR: 1.57e-06 | 8302.3ms/step | 7.2 samples/s
+Step 48700/50462 | Loss: 2.4926 | LR: 1.55e-06 | 8301.5ms/step | 7.2 samples/s
+Step 48710/50462 | Loss: 2.5158 | LR: 1.53e-06 | 8303.5ms/step | 7.2 samples/s
+Step 48720/50462 | Loss: 2.5367 | LR: 1.51e-06 | 8304.8ms/step | 7.2 samples/s
+Step 48730/50462 | Loss: 2.5213 | LR: 1.50e-06 | 8306.3ms/step | 7.2 samples/s
+Step 48740/50462 | Loss: 2.4993 | LR: 1.48e-06 | 8307.2ms/step | 7.2 samples/s
+Step 48750/50462 | Loss: 2.5413 | LR: 1.46e-06 | 8302.0ms/step | 7.2 samples/s
+Validation at step 48750: Loss = 2.5335, Perplexity = 12.60
+Step 48760/50462 | Loss: 2.5534 | LR: 1.44e-06 | 12599.8ms/step | 4.8 samples/s
+Step 48770/50462 | Loss: 2.4933 | LR: 1.43e-06 | 8305.9ms/step | 7.2 samples/s
+Step 48780/50462 | Loss: 2.5156 | LR: 1.41e-06 | 8303.6ms/step | 7.2 samples/s
+Step 48790/50462 | Loss: 2.5237 | LR: 1.39e-06 | 8304.5ms/step | 7.2 samples/s
+Step 48800/50462 | Loss: 2.5043 | LR: 1.38e-06 | 8304.5ms/step | 7.2 samples/s
+Step 48810/50462 | Loss: 2.5133 | LR: 1.36e-06 | 8303.1ms/step | 7.2 samples/s
+Step 48820/50462 | Loss: 2.5177 | LR: 1.34e-06 | 8303.5ms/step | 7.2 samples/s
+Step 48830/50462 | Loss: 2.5341 | LR: 1.33e-06 | 8304.0ms/step | 7.2 samples/s
+Step 48840/50462 | Loss: 2.5479 | LR: 1.31e-06 | 8302.1ms/step | 7.2 samples/s
+Step 48850/50462 | Loss: 2.5190 | LR: 1.30e-06 | 8303.6ms/step | 7.2 samples/s
+Step 48860/50462 | Loss: 2.4863 | LR: 1.28e-06 | 8302.8ms/step | 7.2 samples/s
+Step 48870/50462 | Loss: 2.5281 | LR: 1.26e-06 | 8304.2ms/step | 7.2 samples/s
+Step 48880/50462 | Loss: 2.5103 | LR: 1.25e-06 | 8303.5ms/step | 7.2 samples/s
+Step 48890/50462 | Loss: 2.5414 | LR: 1.23e-06 | 8303.2ms/step | 7.2 samples/s
+Step 48900/50462 | Loss: 2.5190 | LR: 1.22e-06 | 8302.0ms/step | 7.2 samples/s
+Step 48910/50462 | Loss: 2.5328 | LR: 1.20e-06 | 8303.5ms/step | 7.2 samples/s
+Step 48920/50462 | Loss: 2.5194 | LR: 1.19e-06 | 8303.9ms/step | 7.2 samples/s
+Step 48930/50462 | Loss: 2.5159 | LR: 1.17e-06 | 8308.8ms/step | 7.2 samples/s
+Step 48940/50462 | Loss: 2.4965 | LR: 1.16e-06 | 8305.4ms/step | 7.2 samples/s
+Step 48950/50462 | Loss: 2.4975 | LR: 1.14e-06 | 8307.8ms/step | 7.2 samples/s
+Step 48960/50462 | Loss: 2.5283 | LR: 1.13e-06 | 8303.9ms/step | 7.2 samples/s
+Step 48970/50462 | Loss: 2.4958 | LR: 1.11e-06 | 8303.8ms/step | 7.2 samples/s
+Step 48980/50462 | Loss: 2.5632 | LR: 1.10e-06 | 8305.3ms/step | 7.2 samples/s
+Step 48990/50462 | Loss: 2.5476 | LR: 1.08e-06 | 8303.3ms/step | 7.2 samples/s
+Step 49000/50462 | Loss: 2.5296 | LR: 1.07e-06 | 8302.8ms/step | 7.2 samples/s
+Validation at step 49000: Loss = 2.5335, Perplexity = 12.60
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-49000
+Step 49010/50462 | Loss: 2.5185 | LR: 1.05e-06 | 13591.3ms/step | 4.4 samples/s
+Step 49020/50462 | Loss: 2.5272 | LR: 1.04e-06 | 8292.2ms/step | 7.2 samples/s
+Step 49030/50462 | Loss: 2.5273 | LR: 1.02e-06 | 8298.0ms/step | 7.2 samples/s
+Step 49040/50462 | Loss: 2.5241 | LR: 1.01e-06 | 8296.5ms/step | 7.2 samples/s
+Step 49050/50462 | Loss: 2.5247 | LR: 9.95e-07 | 8296.2ms/step | 7.2 samples/s
+Step 49060/50462 | Loss: 2.5449 | LR: 9.81e-07 | 8296.7ms/step | 7.2 samples/s
+Step 49070/50462 | Loss: 2.5413 | LR: 9.67e-07 | 8300.0ms/step | 7.2 samples/s
+Step 49080/50462 | Loss: 2.5440 | LR: 9.53e-07 | 8299.2ms/step | 7.2 samples/s
+Step 49090/50462 | Loss: 2.4979 | LR: 9.39e-07 | 8300.2ms/step | 7.2 samples/s
+Step 49100/50462 | Loss: 2.5458 | LR: 9.25e-07 | 8296.5ms/step | 7.2 samples/s
+Step 49110/50462 | Loss: 2.5313 | LR: 9.12e-07 | 8295.5ms/step | 7.2 samples/s
+Step 49120/50462 | Loss: 2.5012 | LR: 8.99e-07 | 8295.2ms/step | 7.2 samples/s
+Step 49130/50462 | Loss: 2.5311 | LR: 8.85e-07 | 8298.9ms/step | 7.2 samples/s
+Step 49140/50462 | Loss: 2.5022 | LR: 8.72e-07 | 8300.6ms/step | 7.2 samples/s
+Step 49150/50462 | Loss: 2.5322 | LR: 8.59e-07 | 8301.3ms/step | 7.2 samples/s
+Step 49160/50462 | Loss: 2.5229 | LR: 8.46e-07 | 8298.8ms/step | 7.2 samples/s
+Step 49170/50462 | Loss: 2.5215 | LR: 8.33e-07 | 8298.3ms/step | 7.2 samples/s
+Step 49180/50462 | Loss: 2.5109 | LR: 8.20e-07 | 8299.1ms/step | 7.2 samples/s
+Step 49190/50462 | Loss: 2.4997 | LR: 8.07e-07 | 8302.0ms/step | 7.2 samples/s
+Step 49200/50462 | Loss: 2.5524 | LR: 7.95e-07 | 8299.7ms/step | 7.2 samples/s
+Step 49210/50462 | Loss: 2.5242 | LR: 7.82e-07 | 8300.9ms/step | 7.2 samples/s
+Step 49220/50462 | Loss: 2.5342 | LR: 7.70e-07 | 8301.4ms/step | 7.2 samples/s
+Step 49230/50462 | Loss: 2.5362 | LR: 7.57e-07 | 8299.7ms/step | 7.2 samples/s
+Step 49240/50462 | Loss: 2.5259 | LR: 7.45e-07 | 8299.7ms/step | 7.2 samples/s
+Step 49250/50462 | Loss: 2.5209 | LR: 7.33e-07 | 8301.0ms/step | 7.2 samples/s
+Validation at step 49250: Loss = 2.5335, Perplexity = 12.60
+Step 49260/50462 | Loss: 2.4930 | LR: 7.21e-07 | 12598.9ms/step | 4.8 samples/s
+Step 49270/50462 | Loss: 2.5122 | LR: 7.09e-07 | 8300.6ms/step | 7.2 samples/s
+Step 49280/50462 | Loss: 2.4956 | LR: 6.97e-07 | 8301.9ms/step | 7.2 samples/s
+Step 49290/50462 | Loss: 2.5164 | LR: 6.85e-07 | 8302.5ms/step | 7.2 samples/s
+Step 49300/50462 | Loss: 2.5157 | LR: 6.74e-07 | 8300.1ms/step | 7.2 samples/s
+Step 49310/50462 | Loss: 2.5206 | LR: 6.62e-07 | 8300.8ms/step | 7.2 samples/s
+Step 49320/50462 | Loss: 2.5092 | LR: 6.51e-07 | 8300.1ms/step | 7.2 samples/s
+Step 49330/50462 | Loss: 2.5182 | LR: 6.39e-07 | 8301.4ms/step | 7.2 samples/s
+Step 49340/50462 | Loss: 2.5277 | LR: 6.28e-07 | 8300.6ms/step | 7.2 samples/s
+Step 49350/50462 | Loss: 2.5310 | LR: 6.17e-07 | 8300.9ms/step | 7.2 samples/s
+Step 49360/50462 | Loss: 2.5419 | LR: 6.06e-07 | 8301.2ms/step | 7.2 samples/s
+Step 49370/50462 | Loss: 2.5235 | LR: 5.95e-07 | 8301.6ms/step | 7.2 samples/s
+Step 49380/50462 | Loss: 2.5364 | LR: 5.84e-07 | 8302.3ms/step | 7.2 samples/s
+Step 49390/50462 | Loss: 2.5353 | LR: 5.73e-07 | 8303.3ms/step | 7.2 samples/s
+Step 49400/50462 | Loss: 2.5336 | LR: 5.63e-07 | 8301.9ms/step | 7.2 samples/s
+Step 49410/50462 | Loss: 2.5249 | LR: 5.52e-07 | 8302.4ms/step | 7.2 samples/s
+Step 49420/50462 | Loss: 2.5232 | LR: 5.42e-07 | 8302.3ms/step | 7.2 samples/s
+Step 49430/50462 | Loss: 2.5027 | LR: 5.31e-07 | 8301.8ms/step | 7.2 samples/s
+Step 49440/50462 | Loss: 2.5115 | LR: 5.21e-07 | 8299.5ms/step | 7.2 samples/s
+Step 49450/50462 | Loss: 2.4985 | LR: 5.11e-07 | 8300.2ms/step | 7.2 samples/s
+Step 49460/50462 | Loss: 2.5319 | LR: 5.01e-07 | 8303.0ms/step | 7.2 samples/s
+Step 49470/50462 | Loss: 2.5332 | LR: 4.91e-07 | 8302.3ms/step | 7.2 samples/s
+Step 49480/50462 | Loss: 2.5268 | LR: 4.81e-07 | 8304.3ms/step | 7.2 samples/s
+Step 49490/50462 | Loss: 2.5080 | LR: 4.72e-07 | 8304.6ms/step | 7.2 samples/s
+Step 49500/50462 | Loss: 2.5294 | LR: 4.62e-07 | 8304.1ms/step | 7.2 samples/s
+Validation at step 49500: Loss = 2.5335, Perplexity = 12.60
+Step 49510/50462 | Loss: 2.5378 | LR: 4.52e-07 | 12603.5ms/step | 4.8 samples/s
+Step 49520/50462 | Loss: 2.5177 | LR: 4.43e-07 | 8303.6ms/step | 7.2 samples/s
+Step 49530/50462 | Loss: 2.5214 | LR: 4.34e-07 | 8303.7ms/step | 7.2 samples/s
+Step 49540/50462 | Loss: 2.5133 | LR: 4.24e-07 | 8299.3ms/step | 7.2 samples/s
+Step 49550/50462 | Loss: 2.5205 | LR: 4.15e-07 | 8299.9ms/step | 7.2 samples/s
+Step 49560/50462 | Loss: 2.5360 | LR: 4.06e-07 | 8301.6ms/step | 7.2 samples/s
+Step 49570/50462 | Loss: 2.5380 | LR: 3.97e-07 | 8301.2ms/step | 7.2 samples/s
+Step 49580/50462 | Loss: 2.5368 | LR: 3.88e-07 | 8300.1ms/step | 7.2 samples/s
+Step 49590/50462 | Loss: 2.5114 | LR: 3.79e-07 | 8301.6ms/step | 7.2 samples/s
+Step 49600/50462 | Loss: 2.5420 | LR: 3.71e-07 | 8300.4ms/step | 7.2 samples/s
+Step 49610/50462 | Loss: 2.5213 | LR: 3.62e-07 | 8298.5ms/step | 7.2 samples/s
+Step 49620/50462 | Loss: 2.5215 | LR: 3.54e-07 | 8299.9ms/step | 7.2 samples/s
+Step 49630/50462 | Loss: 2.5218 | LR: 3.45e-07 | 8301.6ms/step | 7.2 samples/s
+Step 49640/50462 | Loss: 2.5239 | LR: 3.37e-07 | 8304.3ms/step | 7.2 samples/s
+Step 49650/50462 | Loss: 2.5271 | LR: 3.29e-07 | 8301.0ms/step | 7.2 samples/s
+Step 49660/50462 | Loss: 2.5097 | LR: 3.21e-07 | 8303.2ms/step | 7.2 samples/s
+Step 49670/50462 | Loss: 2.4974 | LR: 3.13e-07 | 8303.2ms/step | 7.2 samples/s
+Step 49680/50462 | Loss: 2.5413 | LR: 3.05e-07 | 8303.1ms/step | 7.2 samples/s
+Step 49690/50462 | Loss: 2.5591 | LR: 2.97e-07 | 8307.7ms/step | 7.2 samples/s
+Step 49700/50462 | Loss: 2.5311 | LR: 2.90e-07 | 8302.2ms/step | 7.2 samples/s
+Step 49710/50462 | Loss: 2.5247 | LR: 2.82e-07 | 8301.6ms/step | 7.2 samples/s
+Step 49720/50462 | Loss: 2.5162 | LR: 2.75e-07 | 8301.9ms/step | 7.2 samples/s
+Step 49730/50462 | Loss: 2.5314 | LR: 2.67e-07 | 8302.9ms/step | 7.2 samples/s
+Step 49740/50462 | Loss: 2.5521 | LR: 2.60e-07 | 8301.8ms/step | 7.2 samples/s
+Step 49750/50462 | Loss: 2.5234 | LR: 2.53e-07 | 8301.5ms/step | 7.2 samples/s
+Validation at step 49750: Loss = 2.5335, Perplexity = 12.60
+Step 49760/50462 | Loss: 2.5410 | LR: 2.46e-07 | 12609.1ms/step | 4.8 samples/s
+Step 49770/50462 | Loss: 2.5297 | LR: 2.39e-07 | 8302.3ms/step | 7.2 samples/s
+Step 49780/50462 | Loss: 2.5150 | LR: 2.32e-07 | 8300.3ms/step | 7.2 samples/s
+Step 49790/50462 | Loss: 2.5073 | LR: 2.25e-07 | 8302.9ms/step | 7.2 samples/s
+Step 49800/50462 | Loss: 2.5506 | LR: 2.19e-07 | 8302.4ms/step | 7.2 samples/s
+Step 49810/50462 | Loss: 2.5370 | LR: 2.12e-07 | 8303.0ms/step | 7.2 samples/s
+Step 49820/50462 | Loss: 2.5353 | LR: 2.06e-07 | 8303.1ms/step | 7.2 samples/s
+Step 49830/50462 | Loss: 2.5247 | LR: 1.99e-07 | 8302.7ms/step | 7.2 samples/s
+Step 49840/50462 | Loss: 2.5392 | LR: 1.93e-07 | 8302.5ms/step | 7.2 samples/s
+Step 49850/50462 | Loss: 2.5159 | LR: 1.87e-07 | 8303.9ms/step | 7.2 samples/s
+Step 49860/50462 | Loss: 2.5150 | LR: 1.81e-07 | 8300.6ms/step | 7.2 samples/s
+Step 49870/50462 | Loss: 2.5295 | LR: 1.75e-07 | 8302.3ms/step | 7.2 samples/s
+Step 49880/50462 | Loss: 2.5155 | LR: 1.69e-07 | 8302.5ms/step | 7.2 samples/s
+Step 49890/50462 | Loss: 2.5236 | LR: 1.63e-07 | 8303.6ms/step | 7.2 samples/s
+Step 49900/50462 | Loss: 2.5372 | LR: 1.58e-07 | 8301.2ms/step | 7.2 samples/s
+Step 49910/50462 | Loss: 2.5234 | LR: 1.52e-07 | 8299.2ms/step | 7.2 samples/s
+Step 49920/50462 | Loss: 2.5371 | LR: 1.47e-07 | 8298.1ms/step | 7.2 samples/s
+Step 49930/50462 | Loss: 2.5078 | LR: 1.41e-07 | 8302.4ms/step | 7.2 samples/s
+Step 49940/50462 | Loss: 2.5202 | LR: 1.36e-07 | 8300.5ms/step | 7.2 samples/s
+Step 49950/50462 | Loss: 2.4798 | LR: 1.31e-07 | 8300.8ms/step | 7.2 samples/s
+Step 49960/50462 | Loss: 2.5108 | LR: 1.26e-07 | 8303.2ms/step | 7.2 samples/s
+Step 49970/50462 | Loss: 2.5031 | LR: 1.21e-07 | 8306.8ms/step | 7.2 samples/s
+Step 49980/50462 | Loss: 2.5172 | LR: 1.16e-07 | 8307.4ms/step | 7.2 samples/s
+Step 49990/50462 | Loss: 2.4949 | LR: 1.11e-07 | 8303.7ms/step | 7.2 samples/s
+Step 50000/50462 | Loss: 2.5055 | LR: 1.07e-07 | 8304.4ms/step | 7.2 samples/s
+Validation at step 50000: Loss = 2.5335, Perplexity = 12.60
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-50000
+Step 50010/50462 | Loss: 2.5019 | LR: 1.02e-07 | 13609.2ms/step | 4.4 samples/s
+Step 50020/50462 | Loss: 2.5096 | LR: 9.75e-08 | 8301.5ms/step | 7.2 samples/s
+Step 50030/50462 | Loss: 2.5396 | LR: 9.32e-08 | 8302.4ms/step | 7.2 samples/s
+Step 50040/50462 | Loss: 2.5697 | LR: 8.89e-08 | 8300.5ms/step | 7.2 samples/s
+Step 50050/50462 | Loss: 2.5236 | LR: 8.47e-08 | 8301.3ms/step | 7.2 samples/s
+Step 50060/50462 | Loss: 2.5167 | LR: 8.07e-08 | 8301.7ms/step | 7.2 samples/s
+Step 50070/50462 | Loss: 2.5362 | LR: 7.67e-08 | 8304.8ms/step | 7.2 samples/s
+Step 50080/50462 | Loss: 2.5090 | LR: 7.28e-08 | 8306.7ms/step | 7.2 samples/s
+Step 50090/50462 | Loss: 2.5006 | LR: 6.91e-08 | 8306.6ms/step | 7.2 samples/s
+Step 50100/50462 | Loss: 2.5067 | LR: 6.54e-08 | 8303.9ms/step | 7.2 samples/s
+Step 50110/50462 | Loss: 2.5461 | LR: 6.19e-08 | 8305.7ms/step | 7.2 samples/s
+Step 50120/50462 | Loss: 2.5285 | LR: 5.84e-08 | 8305.3ms/step | 7.2 samples/s
+Step 50130/50462 | Loss: 2.5281 | LR: 5.50e-08 | 8303.8ms/step | 7.2 samples/s
+Step 50140/50462 | Loss: 2.5460 | LR: 5.18e-08 | 8304.7ms/step | 7.2 samples/s
+Step 50150/50462 | Loss: 2.5023 | LR: 4.86e-08 | 8305.7ms/step | 7.2 samples/s
+Step 50160/50462 | Loss: 2.5168 | LR: 4.55e-08 | 8308.6ms/step | 7.2 samples/s
+Step 50170/50462 | Loss: 2.4855 | LR: 4.26e-08 | 8306.8ms/step | 7.2 samples/s
+Step 50180/50462 | Loss: 2.4896 | LR: 3.97e-08 | 8303.6ms/step | 7.2 samples/s
+Step 50190/50462 | Loss: 2.5365 | LR: 3.69e-08 | 8306.1ms/step | 7.2 samples/s
+Step 50200/50462 | Loss: 2.5073 | LR: 3.43e-08 | 8306.1ms/step | 7.2 samples/s
+Step 50210/50462 | Loss: 2.5316 | LR: 3.17e-08 | 8305.9ms/step | 7.2 samples/s
+Step 50220/50462 | Loss: 2.5271 | LR: 2.92e-08 | 8303.5ms/step | 7.2 samples/s
+Step 50230/50462 | Loss: 2.5451 | LR: 2.69e-08 | 8307.0ms/step | 7.2 samples/s
+Step 50240/50462 | Loss: 2.5096 | LR: 2.46e-08 | 8310.3ms/step | 7.2 samples/s
+Step 50250/50462 | Loss: 2.5043 | LR: 2.24e-08 | 8313.0ms/step | 7.2 samples/s
+Validation at step 50250: Loss = 2.5335, Perplexity = 12.60
+Step 50260/50462 | Loss: 2.5517 | LR: 2.04e-08 | 12611.1ms/step | 4.8 samples/s
+Step 50270/50462 | Loss: 2.5620 | LR: 1.84e-08 | 8306.2ms/step | 7.2 samples/s
+Step 50280/50462 | Loss: 2.5334 | LR: 1.65e-08 | 8313.6ms/step | 7.2 samples/s
+Step 50290/50462 | Loss: 2.5465 | LR: 1.48e-08 | 8314.4ms/step | 7.2 samples/s
+Step 50300/50462 | Loss: 2.5021 | LR: 1.31e-08 | 8311.1ms/step | 7.2 samples/s
+Step 50310/50462 | Loss: 2.4976 | LR: 1.15e-08 | 8312.0ms/step | 7.2 samples/s
+Step 50320/50462 | Loss: 2.5183 | LR: 1.01e-08 | 8307.1ms/step | 7.2 samples/s
+Step 50330/50462 | Loss: 2.5457 | LR: 8.70e-09 | 8311.5ms/step | 7.2 samples/s
+Step 50340/50462 | Loss: 2.5261 | LR: 7.43e-09 | 8307.2ms/step | 7.2 samples/s
+Step 50350/50462 | Loss: 2.5079 | LR: 6.26e-09 | 8307.2ms/step | 7.2 samples/s
+Step 50360/50462 | Loss: 2.5305 | LR: 5.19e-09 | 8312.3ms/step | 7.2 samples/s
+Step 50370/50462 | Loss: 2.5062 | LR: 4.23e-09 | 8311.0ms/step | 7.2 samples/s
+Step 50380/50462 | Loss: 2.4987 | LR: 3.36e-09 | 8310.9ms/step | 7.2 samples/s
+Step 50390/50462 | Loss: 2.5374 | LR: 2.59e-09 | 8312.0ms/step | 7.2 samples/s
+Step 50400/50462 | Loss: 2.5212 | LR: 1.92e-09 | 8310.2ms/step | 7.2 samples/s
+Step 50410/50462 | Loss: 2.5112 | LR: 1.35e-09 | 8309.0ms/step | 7.2 samples/s
+Step 50420/50462 | Loss: 2.5384 | LR: 8.81e-10 | 8307.2ms/step | 7.2 samples/s
+Step 50430/50462 | Loss: 2.5279 | LR: 5.11e-10 | 8311.9ms/step | 7.2 samples/s
+Step 50440/50462 | Loss: 2.4989 | LR: 2.42e-10 | 8314.6ms/step | 7.2 samples/s
+Step 50450/50462 | Loss: 2.5403 | LR: 7.19e-11 | 8313.5ms/step | 7.2 samples/s
+Step 50460/50462 | Loss: 2.5120 | LR: 2.00e-12 | 8313.8ms/step | 7.2 samples/s
+Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928/checkpoint-50462

epfl_final/SparseAtt/wandb/run-20260726_073805-33yw8345/files/requirements.txt

@@ -0,0 +1,142 @@
+pip==26.0.1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+setuptools==81.0.0
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.4.4
+networkx==3.6.1
+multidict==6.7.1
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+yarl==1.23.0
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+cuda-bindings==13.2.0
+anyio==4.13.0
+aiosignal==1.4.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==3.0.2
+nvidia-cusolver==12.0.4.66
+httpx==0.28.1
+GitPython==3.1.49
+aiohttp==3.13.5
+wandb==0.26.1
+typer==0.25.1
+torch==2.11.0
+huggingface_hub==1.13.0
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+word2number==1.1
+sqlitedict==2.1.0
+pytz==2026.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.10.0
+scipy==1.17.1
+portalocker==3.2.0
+pathvalidate==3.3.1
+narwhals==2.22.1
+more-itertools==11.1.0
+lxml==6.1.1
+joblib==1.5.3
+colorama==0.4.6
+chardet==6.0.0.post1
+absl-py==2.4.0
+scikit-learn==1.9.0
+sacrebleu==2.6.0
+nltk==3.9.4
+mbstrdecoder==1.1.5
+typepy==1.3.5
+rouge_score==0.1.2
+DataProperty==1.1.1
+tabledata==1.3.5
+evaluate==0.4.6
+pytablewriter==1.2.1
+lm_eval==0.4.12
+pluggy==1.6.0
+iniconfig==2.3.0
+pytest==9.1.0
+py-spy==0.4.2
+pillow==12.2.0
+tifffile==2026.6.1
+imagecodecs==2026.6.26
+zipp==4.1.0
+importlib_metadata==9.0.0
+derivative==0.6.3
+pysindy==2.1.0
+pyparsing==3.3.2
+kiwisolver==1.5.0
+fonttools==4.63.0
+cycler==0.12.1
+contourpy==1.3.3
+matplotlib==3.11.0
+sortedcontainers==2.4.0
+wcwidth==0.8.2
+sigfig==1.3.19
+importlib_resources==7.1.0
+enum-tools==0.13.0
+cramjam==2.11.0
+python-snappy==0.7.3
+compact-json==1.8.2
+numbers-parser==4.18.5
+entmax==1.3

epfl_final/SparseAtt/wandb/run-20260726_073805-33yw8345/files/wandb-metadata.json

@@ -0,0 +1,65 @@
+{
+  "os":  "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.13.13",
+  "startedAt":  "2026-07-26T07:38:05.450183Z",
+  "args":  [
+    "--config",
+    "configs/train_config_1mask_perlayerfix_resume_3gpu.json"
+  ],
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train_fixed_resume.py",
+  "codePath":  "train_fixed_resume.py",
+  "codePathLocal":  "train_fixed_resume.py",
+  "git":  {
+    "remote":  "https://github.com/rubentium/TokenPrune.git",
+    "commit":  "88298c7a418622530ca5426b22b1e5ebd10ca0cb"
+  },
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse/SparseAtt",
+  "host":  "a100-workspace-4-gpu-0-1",
+  "executable":  "/mloscratch/homes/navasard/conda/bin/python3.13",
+  "cpu_count":  64,
+  "cpu_count_logical":  128,
+  "gpu":  "NVIDIA A100-SXM4-80GB",
+  "gpu_count":  4,
+  "disk":  {
+    "/":  {
+      "total":  "7679361679360",
+      "used":  "5455843303424"
+    }
+  },
+  "memory":  {
+    "total":  "1081884934144"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-943525a1-9808-8c31-17d9-c4180bb03955"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-b4caa2bd-09d3-3702-9fd1-293525d8e08c"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-46161407-c19c-cdae-c140-0963b35a3047"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-f1d949cd-9bad-cd83-2a2b-ed36b22f13a6"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "azps7d1jv0uw4lrmtl92nlgaedih8j0b"
+}