Hugging Face's logo

rubentium
/
sparse-checkpoints

Model card Files
xet
 Community
rubentium commited on
Commit
cad088e
·
verified ·
1 Parent(s): f2f2768

EPFL ALiBi and training provenance 1001-1050 of 1184

Browse files
Files changed (50) hide show
  1. epfl_final/train_logs/.space_probe +0 -0
  2. epfl_final/train_logs/bounded_temp_validation_a_20260907.log +98 -0
  3. epfl_final/train_logs/bounded_temp_validation_b_20260907.log +106 -0
  4. epfl_final/train_logs/dense_central_20260906_long.log +32 -0
  5. epfl_final/train_logs/dense_central_20260906_short_a.log +22 -0
  6. epfl_final/train_logs/dense_central_20260906_short_b.log +22 -0
  7. epfl_final/train_logs/dense_diag_20260906_a10040_a.log +116 -0
  8. epfl_final/train_logs/dense_diag_20260906_a10040_a_phase2.log +0 -0
  9. epfl_final/train_logs/dense_diag_20260906_a10040_b.log +88 -0
  10. epfl_final/train_logs/dense_diag_20260906_a10040_b_phase2.log +0 -0
  11. epfl_final/train_logs/dense_diag_20260906_a10080.log +42 -0
  12. epfl_final/train_logs/dense_diag_20260906_a10080_phase2.log +0 -0
  13. epfl_final/train_logs/dense_diag_20260906_controls_012.log +114 -0
  14. epfl_final/train_logs/dense_diag_20260906_native30_3.log +18 -0
  15. epfl_final/train_logs/generation_sanity_cachefixed_20260907.log +8 -0
  16. epfl_final/train_logs/p2_1p2b_dense_seed72_4gpu_chained.log +1021 -0
  17. epfl_final/train_logs/p2_1p2b_dense_seed72_4gpu_resume2750_fixed.log +588 -0
  18. epfl_final/train_logs/p2_1p2b_dense_seed72_4gpu_resume2750_live.log +186 -0
  19. epfl_final/train_logs/p2_1p2b_dense_seed72_6gpu_repair_from9250.log +1136 -0
  20. epfl_final/train_logs/p2_1p2b_dense_seed72_6gpu_resume18000.log +496 -0
  21. epfl_final/train_logs/p2_1p2b_dense_seed72_6gpu_resume18000_retry.log +789 -0
  22. epfl_final/train_logs/p2_1p2b_dense_seed72_6gpu_resume23500.log +0 -0
  23. epfl_final/train_logs/p2_1p2b_dense_seed72_6gpu_resume23500.pid +1 -0
  24. epfl_final/train_logs/p2_1p2b_dense_seed72_6gpu_resume31000.log +0 -0
  25. epfl_final/train_logs/p2_1p2b_dense_seed72_6gpu_resume44250.log +606 -0
  26. epfl_final/train_logs/p2_1p2b_dense_seed72_6gpu_resume44250.pid +1 -0
  27. epfl_final/train_logs/p2_1p2b_dense_seed72_6gpu_resume9250.log +623 -0
  28. epfl_final/train_logs/p2_1p2b_dense_seed72_6gpu_resume9250_retry.log +0 -0
  29. epfl_final/train_logs/p2_1p2b_dense_seed72_resume6000.log +515 -0
  30. epfl_final/train_logs/p2_1p2b_dense_seed72_resume8750.log +270 -0
  31. epfl_final/train_logs/p2_1p2b_mixed_seed72_4gpu_resume45250_live.log +762 -0
  32. epfl_final/train_logs/p2_1p2b_mixed_seed72_h100_resume35000.log +0 -0
  33. epfl_final/train_logs/p4_1p2b_retrieval_40gb.log +2 -0
  34. epfl_final/train_logs/p4_1p2b_retrieval_40gb_retry.log +2 -0
  35. epfl_final/train_logs/p4_3b_mixed_resume_partial.log +10 -0
  36. epfl_final/train_logs/p4_40gb_safe_remaining.log +2 -0
  37. epfl_final/train_logs/p4_40gb_safe_remaining_retry.log +14 -0
  38. epfl_final/train_logs/p4_a10080_recovery_latest.log +6 -0
  39. epfl_final/train_logs/p4_a10080_resume_latest.log +13 -0
  40. epfl_final/train_logs/p4_large_resume_mixed_after_restart.log +2 -0
  41. epfl_final/train_logs/p4_large_retry_after_health.log +13 -0
  42. epfl_final/train_logs/p4_safe16_b.log +14 -0
  43. epfl_final/train_logs/queue_seed70_mixed_repaired_after_seed72.log +1 -0
  44. epfl_final/train_logs/queue_seed70_mixed_repaired_after_seed72.pid +1 -0
  45. epfl_final/train_logs/retry_crashed_40gb.log +18 -0
  46. epfl_final/train_logs/temp_dense_downstream_a_20260906.log +22 -0
  47. epfl_final/train_logs/temp_dense_downstream_b_20260906.log +30 -0
  48. epfl_final/train_logs/temp_protocol_40a_20260906.log +74 -0
  49. epfl_final/train_logs/temp_protocol_40b_20260906.log +50 -0
  50. epfl_final/train_logs/temp_protocol_80_20260906.log +120 -0
epfl_final/train_logs/.space_probe ADDED
Binary file (1 Bytes). View file
 
epfl_final/train_logs/bounded_temp_validation_a_20260907.log ADDED
@@ -0,0 +1,98 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ SHARD=0 JOBS=48 DONE=0
2
+ START 0 retr30_1p2b_dense_s71_passkey_tau0.8
3
+ START 1 retr30_1p2b_dense_s71_vt_tau0.8
4
+ START 2 retr30_1p2b_dense_s71_niah_multikey_tau0.85
5
+ START 3 retr30_1p2b_dense_s71_niah_multivalue_tau0.85
6
+ START 4 retr30_1p2b_dense_s71_niah_multikey_tau1
7
+ DONE 0 retr30_1p2b_dense_s71_passkey_tau0.8 0 True
8
+ DONE 3 retr30_1p2b_dense_s71_niah_multivalue_tau0.85 0 True
9
+ START 0 retr30_1p2b_dense_s71_niah_multivalue_tau1
10
+ START 3 retr30_1p2b_mixed_s71_passkey_tau0.8
11
+ DONE 1 retr30_1p2b_dense_s71_vt_tau0.8 0 True
12
+ DONE 4 retr30_1p2b_dense_s71_niah_multikey_tau1 0 True
13
+ START 1 retr30_1p2b_mixed_s71_vt_tau0.8
14
+ START 4 retr30_1p2b_mixed_s71_niah_multikey_tau0.85
15
+ DONE 2 retr30_1p2b_dense_s71_niah_multikey_tau0.85 0 True
16
+ START 2 retr30_1p2b_mixed_s71_niah_multivalue_tau0.85
17
+ DONE 0 retr30_1p2b_dense_s71_niah_multivalue_tau1 0 True
18
+ START 0 retr30_1p2b_mixed_s71_niah_multikey_tau1
19
+ DONE 3 retr30_1p2b_mixed_s71_passkey_tau0.8 0 True
20
+ DONE 1 retr30_1p2b_mixed_s71_vt_tau0.8 0 True
21
+ DONE 4 retr30_1p2b_mixed_s71_niah_multikey_tau0.85 0 True
22
+ START 1 retr30_1p2b_mixed_s71_niah_multivalue_tau1
23
+ START 3 retr30_3b_dense_passkey_tau0.8
24
+ START 4 retr30_3b_dense_vt_tau0.8
25
+ DONE 2 retr30_1p2b_mixed_s71_niah_multivalue_tau0.85 0 True
26
+ START 2 retr30_3b_dense_niah_multikey_tau0.85
27
+ DONE 0 retr30_1p2b_mixed_s71_niah_multikey_tau1 0 True
28
+ START 0 retr30_3b_dense_niah_multivalue_tau0.85
29
+ DONE 1 retr30_1p2b_mixed_s71_niah_multivalue_tau1 0 True
30
+ START 1 retr30_3b_dense_niah_multikey_tau1
31
+ DONE 3 retr30_3b_dense_passkey_tau0.8 0 True
32
+ DONE 4 retr30_3b_dense_vt_tau0.8 0 True
33
+ START 3 retr30_3b_dense_niah_multivalue_tau1
34
+ START 4 retr30_3b_mixed_passkey_tau0.8
35
+ DONE 2 retr30_3b_dense_niah_multikey_tau0.85 0 True
36
+ START 2 retr30_3b_mixed_vt_tau0.8
37
+ DONE 0 retr30_3b_dense_niah_multivalue_tau0.85 0 True
38
+ START 0 retr30_3b_mixed_niah_multikey_tau0.85
39
+ DONE 1 retr30_3b_dense_niah_multikey_tau1 0 True
40
+ START 1 retr30_3b_mixed_niah_multivalue_tau0.85
41
+ DONE 3 retr30_3b_dense_niah_multivalue_tau1 0 True
42
+ START 3 retr30_3b_mixed_niah_multikey_tau1
43
+ DONE 4 retr30_3b_mixed_passkey_tau0.8 0 True
44
+ START 4 retr30_3b_mixed_niah_multivalue_tau1
45
+ DONE 2 retr30_3b_mixed_vt_tau0.8 0 True
46
+ START 2 retr8_1p2b_dense_s71_passkey_tau1
47
+ DONE 2 retr8_1p2b_dense_s71_passkey_tau1 0 True
48
+ START 2 retr8_1p2b_dense_s71_passkey_selected
49
+ DONE 2 retr8_1p2b_dense_s71_passkey_selected 0 True
50
+ START 2 retr8_1p2b_dense_s71_vt_tau1
51
+ DONE 2 retr8_1p2b_dense_s71_vt_tau1 0 True
52
+ START 2 retr8_1p2b_dense_s71_vt_selected
53
+ DONE 0 retr30_3b_mixed_niah_multikey_tau0.85 0 True
54
+ START 0 retr8_1p2b_mixed_s71_passkey_tau1
55
+ DONE 1 retr30_3b_mixed_niah_multivalue_tau0.85 0 True
56
+ START 1 retr8_1p2b_mixed_s71_passkey_selected
57
+ DONE 2 retr8_1p2b_dense_s71_vt_selected 0 True
58
+ START 2 retr8_1p2b_mixed_s71_vt_tau1
59
+ DONE 0 retr8_1p2b_mixed_s71_passkey_tau1 0 True
60
+ START 0 retr8_1p2b_mixed_s71_vt_selected
61
+ DONE 1 retr8_1p2b_mixed_s71_passkey_selected 0 True
62
+ START 1 retr8_3b_dense_passkey_tau1
63
+ DONE 2 retr8_1p2b_mixed_s71_vt_tau1 0 True
64
+ START 2 retr8_3b_dense_passkey_selected
65
+ DONE 0 retr8_1p2b_mixed_s71_vt_selected 0 True
66
+ START 0 retr8_3b_dense_vt_tau1
67
+ DONE 1 retr8_3b_dense_passkey_tau1 0 True
68
+ START 1 retr8_3b_dense_vt_selected
69
+ DONE 2 retr8_3b_dense_passkey_selected 0 True
70
+ START 2 retr8_3b_mixed_passkey_tau1
71
+ DONE 0 retr8_3b_dense_vt_tau1 0 True
72
+ START 0 retr8_3b_mixed_passkey_selected
73
+ DONE 1 retr8_3b_dense_vt_selected 0 True
74
+ START 1 retr8_3b_mixed_vt_tau1
75
+ DONE 3 retr30_3b_mixed_niah_multikey_tau1 0 True
76
+ START 3 retr8_3b_mixed_vt_selected
77
+ DONE 4 retr30_3b_mixed_niah_multivalue_tau1 0 True
78
+ START 4 gen30_1p2b_dense_s71_passkey_tau1
79
+ DONE 2 retr8_3b_mixed_passkey_tau1 0 True
80
+ START 2 gen30_1p2b_dense_s71_passkey_selected
81
+ DONE 0 retr8_3b_mixed_passkey_selected 0 True
82
+ START 0 gen30_1p2b_mixed_s71_passkey_tau1
83
+ DONE 4 gen30_1p2b_dense_s71_passkey_tau1 0 True
84
+ START 4 gen30_1p2b_mixed_s71_passkey_selected
85
+ DONE 2 gen30_1p2b_dense_s71_passkey_selected 0 True
86
+ START 2 gen30_3b_dense_passkey_tau1
87
+ DONE 0 gen30_1p2b_mixed_s71_passkey_tau1 0 True
88
+ START 0 gen30_3b_dense_passkey_selected
89
+ DONE 4 gen30_1p2b_mixed_s71_passkey_selected 0 True
90
+ START 4 gen30_3b_mixed_passkey_tau1
91
+ DONE 2 gen30_3b_dense_passkey_tau1 0 True
92
+ DONE 0 gen30_3b_dense_passkey_selected 0 True
93
+ START 0 gen30_3b_mixed_passkey_selected
94
+ DONE 1 retr8_3b_mixed_vt_tau1 0 True
95
+ DONE 3 retr8_3b_mixed_vt_selected 0 True
96
+ DONE 4 gen30_3b_mixed_passkey_tau1 0 True
97
+ DONE 0 gen30_3b_mixed_passkey_selected 0 True
98
+ COMPLETE=48/48
epfl_final/train_logs/bounded_temp_validation_b_20260907.log ADDED
@@ -0,0 +1,106 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ SHARD=1 JOBS=52 DONE=0
2
+ START 0 cal_1p2b_dense_s71_L8192
3
+ START 1 cal_1p2b_dense_s71_L30720
4
+ START 2 cal_1p2b_mixed_s71_L8192
5
+ START 3 cal_1p2b_mixed_s71_L30720
6
+ START 4 retr30_1p2b_dense_s71_niah_multikey_tau0.8
7
+ DONE 0 cal_1p2b_dense_s71_L8192 0 True
8
+ START 0 retr30_1p2b_dense_s71_niah_multivalue_tau0.8
9
+ DONE 2 cal_1p2b_mixed_s71_L8192 0 True
10
+ START 2 retr30_1p2b_dense_s71_passkey_tau0.85
11
+ DONE 1 cal_1p2b_dense_s71_L30720 0 True
12
+ DONE 3 cal_1p2b_mixed_s71_L30720 0 True
13
+ START 1 retr30_1p2b_dense_s71_vt_tau0.85
14
+ START 3 retr30_1p2b_dense_s71_passkey_tau1
15
+ DONE 4 retr30_1p2b_dense_s71_niah_multikey_tau0.8 0 True
16
+ START 4 retr30_1p2b_dense_s71_vt_tau1
17
+ DONE 0 retr30_1p2b_dense_s71_niah_multivalue_tau0.8 0 True
18
+ START 0 retr30_1p2b_mixed_s71_niah_multikey_tau0.8
19
+ DONE 2 retr30_1p2b_dense_s71_passkey_tau0.85 0 True
20
+ START 2 retr30_1p2b_mixed_s71_niah_multivalue_tau0.8
21
+ DONE 1 retr30_1p2b_dense_s71_vt_tau0.85 0 True
22
+ DONE 3 retr30_1p2b_dense_s71_passkey_tau1 0 True
23
+ START 1 retr30_1p2b_mixed_s71_passkey_tau0.85
24
+ START 3 retr30_1p2b_mixed_s71_vt_tau0.85
25
+ DONE 4 retr30_1p2b_dense_s71_vt_tau1 0 True
26
+ START 4 retr30_1p2b_mixed_s71_passkey_tau1
27
+ DONE 0 retr30_1p2b_mixed_s71_niah_multikey_tau0.8 0 True
28
+ START 0 retr30_1p2b_mixed_s71_vt_tau1
29
+ DONE 2 retr30_1p2b_mixed_s71_niah_multivalue_tau0.8 0 True
30
+ START 2 retr30_3b_dense_niah_multikey_tau0.8
31
+ DONE 3 retr30_1p2b_mixed_s71_vt_tau0.85 0 True
32
+ START 3 retr30_3b_dense_niah_multivalue_tau0.8
33
+ DONE 1 retr30_1p2b_mixed_s71_passkey_tau0.85 0 True
34
+ START 1 retr30_3b_dense_passkey_tau0.85
35
+ DONE 4 retr30_1p2b_mixed_s71_passkey_tau1 0 True
36
+ START 4 retr30_3b_dense_vt_tau0.85
37
+ DONE 0 retr30_1p2b_mixed_s71_vt_tau1 0 True
38
+ START 0 retr30_3b_dense_passkey_tau1
39
+ DONE 2 retr30_3b_dense_niah_multikey_tau0.8 0 True
40
+ DONE 3 retr30_3b_dense_niah_multivalue_tau0.8 0 True
41
+ DONE 1 retr30_3b_dense_passkey_tau0.85 0 True
42
+ START 1 retr30_3b_dense_vt_tau1
43
+ START 2 retr30_3b_mixed_niah_multikey_tau0.8
44
+ START 3 retr30_3b_mixed_niah_multivalue_tau0.8
45
+ DONE 4 retr30_3b_dense_vt_tau0.85 0 True
46
+ START 4 retr30_3b_mixed_passkey_tau0.85
47
+ DONE 0 retr30_3b_dense_passkey_tau1 0 True
48
+ START 0 retr30_3b_mixed_vt_tau0.85
49
+ DONE 1 retr30_3b_dense_vt_tau1 0 True
50
+ START 1 retr30_3b_mixed_passkey_tau1
51
+ DONE 3 retr30_3b_mixed_niah_multivalue_tau0.8 0 True
52
+ START 3 retr30_3b_mixed_vt_tau1
53
+ DONE 2 retr30_3b_mixed_niah_multikey_tau0.8 0 True
54
+ START 2 retr8_1p2b_dense_s71_niah_multikey_tau1
55
+ DONE 2 retr8_1p2b_dense_s71_niah_multikey_tau1 0 True
56
+ START 2 retr8_1p2b_dense_s71_niah_multikey_selected
57
+ DONE 2 retr8_1p2b_dense_s71_niah_multikey_selected 0 True
58
+ START 2 retr8_1p2b_dense_s71_niah_multivalue_tau1
59
+ DONE 4 retr30_3b_mixed_passkey_tau0.85 0 True
60
+ START 4 retr8_1p2b_dense_s71_niah_multivalue_selected
61
+ DONE 2 retr8_1p2b_dense_s71_niah_multivalue_tau1 0 True
62
+ START 2 retr8_1p2b_mixed_s71_niah_multikey_tau1
63
+ DONE 0 retr30_3b_mixed_vt_tau0.85 0 True
64
+ START 0 retr8_1p2b_mixed_s71_niah_multikey_selected
65
+ DONE 4 retr8_1p2b_dense_s71_niah_multivalue_selected 0 True
66
+ DONE 2 retr8_1p2b_mixed_s71_niah_multikey_tau1 0 True
67
+ START 2 retr8_1p2b_mixed_s71_niah_multivalue_tau1
68
+ START 4 retr8_1p2b_mixed_s71_niah_multivalue_selected
69
+ DONE 0 retr8_1p2b_mixed_s71_niah_multikey_selected 0 True
70
+ START 0 retr8_3b_dense_niah_multikey_tau1
71
+ DONE 2 retr8_1p2b_mixed_s71_niah_multivalue_tau1 0 True
72
+ DONE 4 retr8_1p2b_mixed_s71_niah_multivalue_selected 0 True
73
+ START 2 retr8_3b_dense_niah_multikey_selected
74
+ START 4 retr8_3b_dense_niah_multivalue_tau1
75
+ DONE 0 retr8_3b_dense_niah_multikey_tau1 0 True
76
+ START 0 retr8_3b_dense_niah_multivalue_selected
77
+ DONE 2 retr8_3b_dense_niah_multikey_selected 0 True
78
+ DONE 4 retr8_3b_dense_niah_multivalue_tau1 0 True
79
+ START 2 retr8_3b_mixed_niah_multikey_tau1
80
+ START 4 retr8_3b_mixed_niah_multikey_selected
81
+ DONE 0 retr8_3b_dense_niah_multivalue_selected 0 True
82
+ START 0 retr8_3b_mixed_niah_multivalue_tau1
83
+ DONE 2 retr8_3b_mixed_niah_multikey_tau1 0 True
84
+ START 2 retr8_3b_mixed_niah_multivalue_selected
85
+ DONE 4 retr8_3b_mixed_niah_multikey_selected 0 True
86
+ START 4 gen30_1p2b_dense_s71_niah_multikey_tau1
87
+ DONE 1 retr30_3b_mixed_passkey_tau1 0 True
88
+ START 1 gen30_1p2b_dense_s71_niah_multikey_selected
89
+ DONE 3 retr30_3b_mixed_vt_tau1 0 True
90
+ START 3 gen30_1p2b_mixed_s71_niah_multikey_tau1
91
+ DONE 4 gen30_1p2b_dense_s71_niah_multikey_tau1 0 True
92
+ DONE 1 gen30_1p2b_dense_s71_niah_multikey_selected 0 True
93
+ START 1 gen30_1p2b_mixed_s71_niah_multikey_selected
94
+ START 4 gen30_3b_dense_niah_multikey_tau1
95
+ DONE 3 gen30_1p2b_mixed_s71_niah_multikey_tau1 0 True
96
+ START 3 gen30_3b_dense_niah_multikey_selected
97
+ DONE 1 gen30_1p2b_mixed_s71_niah_multikey_selected 0 True
98
+ START 1 gen30_3b_mixed_niah_multikey_tau1
99
+ DONE 0 retr8_3b_mixed_niah_multivalue_tau1 0 True
100
+ START 0 gen30_3b_mixed_niah_multikey_selected
101
+ DONE 4 gen30_3b_dense_niah_multikey_tau1 0 True
102
+ DONE 3 gen30_3b_dense_niah_multikey_selected 0 True
103
+ DONE 1 gen30_3b_mixed_niah_multikey_tau1 0 True
104
+ DONE 2 retr8_3b_mixed_niah_multivalue_selected 0 True
105
+ DONE 0 gen30_3b_mixed_niah_multikey_selected 0 True
106
+ COMPLETE=52/52
epfl_final/train_logs/dense_central_20260906_long.log ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ PROFILE=long JOBS=15 ALREADY=0
2
+ START 0 ppl_mixed_s72_30720_fineweb
3
+ START 1 ppl_mixed_s72_30720_pg19
4
+ START 2 ppl_mixed_s72_30720_arxiv
5
+ START 3 retr_dense_s70_30720_passkey
6
+ DONE 0 ppl_mixed_s72_30720_fineweb 0 True
7
+ DONE 1 ppl_mixed_s72_30720_pg19 0 True
8
+ DONE 2 ppl_mixed_s72_30720_arxiv 0 True
9
+ START 0 retr_dense_s70_30720_niah_multikey
10
+ START 1 retr_dense_s70_30720_niah_multivalue
11
+ START 2 retr_dense_s70_30720_vt
12
+ DONE 3 retr_dense_s70_30720_passkey 0 True
13
+ START 3 retr_dense_s71_30720_passkey
14
+ DONE 0 retr_dense_s70_30720_niah_multikey 0 True
15
+ START 0 retr_dense_s71_30720_niah_multikey
16
+ DONE 1 retr_dense_s70_30720_niah_multivalue 0 True
17
+ START 1 retr_dense_s71_30720_niah_multivalue
18
+ DONE 2 retr_dense_s70_30720_vt 0 True
19
+ START 2 retr_dense_s71_30720_vt
20
+ DONE 3 retr_dense_s71_30720_passkey 0 True
21
+ START 3 retr_mixed_s72_30720_passkey
22
+ DONE 0 retr_dense_s71_30720_niah_multikey 0 True
23
+ START 0 retr_mixed_s72_30720_niah_multikey
24
+ DONE 1 retr_dense_s71_30720_niah_multivalue 0 True
25
+ START 1 retr_mixed_s72_30720_niah_multivalue
26
+ DONE 2 retr_dense_s71_30720_vt 0 True
27
+ START 2 retr_mixed_s72_30720_vt
28
+ DONE 3 retr_mixed_s72_30720_passkey 0 True
29
+ DONE 0 retr_mixed_s72_30720_niah_multikey 0 True
30
+ DONE 1 retr_mixed_s72_30720_niah_multivalue 0 True
31
+ DONE 2 retr_mixed_s72_30720_vt 0 True
32
+ COMPLETE=15/15 FAILED=[]
epfl_final/train_logs/dense_central_20260906_short_a.log ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ PROFILE=short JOBS=20 ALREADY=0
2
+ START 0 ppl_mixed_s72_8192_pg19
3
+ START 1 ppl_mixed_s72_16384_fineweb
4
+ START 2 ppl_mixed_s72_16384_arxiv
5
+ START 3 ppl_mixed_s72_24576_pg19
6
+ START 4 retr_dense_s70_8192_passkey
7
+ DONE 1 ppl_mixed_s72_16384_fineweb 0 True
8
+ DONE 2 ppl_mixed_s72_16384_arxiv 0 True
9
+ DONE 3 ppl_mixed_s72_24576_pg19 0 True
10
+ START 1 retr_dense_s70_8192_niah_multikey
11
+ START 2 retr_dense_s70_8192_niah_multivalue
12
+ START 3 retr_dense_s70_8192_vt
13
+ DONE 0 ppl_mixed_s72_8192_pg19 0 True
14
+ START 0 retr_dense_s71_8192_niah_multikey
15
+ DONE 4 retr_dense_s70_8192_passkey 0 True
16
+ START 4 retr_mixed_s72_8192_vt
17
+ DONE 1 retr_dense_s70_8192_niah_multikey 0 True
18
+ DONE 2 retr_dense_s70_8192_niah_multivalue 0 True
19
+ DONE 3 retr_dense_s70_8192_vt 0 True
20
+ DONE 0 retr_dense_s71_8192_niah_multikey 0 True
21
+ DONE 4 retr_mixed_s72_8192_vt 0 True
22
+ COMPLETE=20/20 FAILED=[]
epfl_final/train_logs/dense_central_20260906_short_b.log ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ PROFILE=short JOBS=20 ALREADY=0
2
+ START 0 ppl_mixed_s72_8192_fineweb
3
+ START 1 ppl_mixed_s72_8192_arxiv
4
+ START 2 ppl_mixed_s72_16384_pg19
5
+ START 3 ppl_mixed_s72_24576_fineweb
6
+ START 4 ppl_mixed_s72_24576_arxiv
7
+ DONE 0 ppl_mixed_s72_8192_fineweb 0 True
8
+ DONE 1 ppl_mixed_s72_8192_arxiv 0 True
9
+ DONE 2 ppl_mixed_s72_16384_pg19 0 True
10
+ DONE 3 ppl_mixed_s72_24576_fineweb 0 True
11
+ DONE 4 ppl_mixed_s72_24576_arxiv 0 True
12
+ START 0 retr_dense_s71_8192_niah_multivalue
13
+ START 1 retr_dense_s71_8192_vt
14
+ START 2 retr_mixed_s72_8192_passkey
15
+ START 3 retr_mixed_s72_8192_niah_multikey
16
+ START 4 retr_mixed_s72_8192_niah_multivalue
17
+ DONE 0 retr_dense_s71_8192_niah_multivalue 0 True
18
+ DONE 1 retr_dense_s71_8192_vt 0 True
19
+ DONE 2 retr_mixed_s72_8192_passkey 0 True
20
+ DONE 3 retr_mixed_s72_8192_niah_multikey 0 True
21
+ DONE 4 retr_mixed_s72_8192_niah_multivalue 0 True
22
+ COMPLETE=19/20 FAILED=['retr_mixed_s72_8192_vt']
epfl_final/train_logs/dense_diag_20260906_a10040_a.log ADDED
@@ -0,0 +1,116 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ PROFILE=a10040 JOBS=100 ALREADY=0
2
+ START 0 365m_dense_s42_8192
3
+ START 1 365m_dense_s42_16384
4
+ START 2 365m_dense_s42_24576
5
+ START 3 365m_dense_s42_30720
6
+ START 4 365m_mixed_s42_8192
7
+ DONE 1 365m_dense_s42_16384 0 True
8
+ DONE 2 365m_dense_s42_24576 0 True
9
+ START 1 365m_dense_s43_24576
10
+ START 2 365m_dense_s43_30720
11
+ DONE 0 365m_dense_s42_8192 0 True
12
+ DONE 4 365m_mixed_s42_8192 0 True
13
+ START 0 365m_mixed_s43_8192
14
+ START 4 365m_mixed_s43_16384
15
+ DONE 3 365m_dense_s42_30720 0 True
16
+ START 3 365m_mixed_s43_24576
17
+ DONE 1 365m_dense_s43_24576 0 True
18
+ START 1 365m_mixed_s44_8192
19
+ DONE 4 365m_mixed_s43_16384 0 True
20
+ DONE 3 365m_mixed_s43_24576 0 True
21
+ START 3 365m_mixed_s44_16384
22
+ START 4 365m_mixed_s44_24576
23
+ DONE 2 365m_dense_s43_30720 0 True
24
+ DONE 0 365m_mixed_s43_8192 0 True
25
+ START 0 365m_mixed_s44_30720
26
+ START 2 365m_dense_s45_8192
27
+ DONE 3 365m_mixed_s44_16384 0 True
28
+ DONE 4 365m_mixed_s44_24576 0 True
29
+ START 3 365m_mixed_s45_8192
30
+ START 4 365m_mixed_s45_16384
31
+ DONE 1 365m_mixed_s44_8192 0 True
32
+ START 1 365m_mixed_s45_30720
33
+ DONE 2 365m_dense_s45_8192 0 True
34
+ START 2 365m_dense_s46_8192
35
+ DONE 0 365m_mixed_s44_30720 0 True
36
+ START 0 365m_dense_s46_16384
37
+ DONE 4 365m_mixed_s45_16384 0 True
38
+ DONE 3 365m_mixed_s45_8192 0 True
39
+ START 3 365m_mixed_s46_8192
40
+ START 4 365m_mixed_s46_16384
41
+ DONE 2 365m_dense_s46_8192 0 True
42
+ START 2 365m_dense_s47_8192
43
+ DONE 0 365m_dense_s46_16384 0 True
44
+ START 0 365m_dense_s47_16384
45
+ DONE 1 365m_mixed_s45_30720 0 True
46
+ START 1 365m_dense_s47_24576
47
+ DONE 4 365m_mixed_s46_16384 0 True
48
+ START 4 365m_dense_s47_30720
49
+ DONE 3 365m_mixed_s46_8192 0 True
50
+ START 3 365m_mixed_s47_8192
51
+ DONE 0 365m_dense_s47_16384 0 True
52
+ DONE 2 365m_dense_s47_8192 0 True
53
+ START 0 365m_mixed_s47_24576
54
+ START 2 365m_mixed_s47_30720
55
+ DONE 1 365m_dense_s47_24576 0 True
56
+ START 1 365m_dropout_p28_s42_8192
57
+ DONE 1 365m_dropout_p28_s42_8192 0 True
58
+ START 1 365m_dropout_p28_s42_30720
59
+ DONE 3 365m_mixed_s47_8192 0 True
60
+ START 3 365m_ropectl_s42_16384
61
+ DONE 0 365m_mixed_s47_24576 0 True
62
+ START 0 365m_dropout_p28_s43_8192
63
+ DONE 4 365m_dense_s47_30720 0 True
64
+ DONE 1 365m_dropout_p28_s42_30720 0 True
65
+ START 1 365m_dropout_p28_s43_16384
66
+ START 4 365m_dropout_p28_s43_24576
67
+ DONE 3 365m_ropectl_s42_16384 0 True
68
+ START 3 365m_dropout_p28_s43_30720
69
+ DONE 2 365m_mixed_s47_30720 0 True
70
+ START 2 365m_ropectl_s43_8192
71
+ DONE 0 365m_dropout_p28_s43_8192 0 True
72
+ DONE 1 365m_dropout_p28_s43_16384 0 True
73
+ START 1 365m_dropout_p28_s44_8192
74
+ DONE 4 365m_dropout_p28_s43_24576 0 True
75
+ START 0 365m_dropout_p28_s44_16384
76
+ START 4 365m_dropout_p28_s44_24576
77
+ DONE 2 365m_ropectl_s43_8192 0 True
78
+ START 2 365m_ropectl_s44_16384
79
+ DONE 3 365m_dropout_p28_s43_30720 0 True
80
+ START 3 365m_ropectl_s44_24576
81
+ DONE 1 365m_dropout_p28_s44_8192 0 True
82
+ START 1 365m_ropectl_s44_30720
83
+ DONE 0 365m_dropout_p28_s44_16384 0 True
84
+ DONE 4 365m_dropout_p28_s44_24576 0 True
85
+ START 0 365m_dropout_p28_s45_24576
86
+ START 4 365m_dropout_p28_s45_30720
87
+ DONE 2 365m_ropectl_s44_16384 0 True
88
+ DONE 3 365m_ropectl_s44_24576 0 True
89
+ START 3 365m_ropectl_s45_24576
90
+ START 2 365m_ropectl_s45_30720
91
+ DONE 1 365m_ropectl_s44_30720 0 True
92
+ START 1 365m_dropout_p28_s46_8192
93
+ DONE 0 365m_dropout_p28_s45_24576 0 True
94
+ DONE 4 365m_dropout_p28_s45_30720 0 True
95
+ START 0 365m_ropectl_s46_8192
96
+ START 4 365m_ropectl_s46_16384
97
+ DONE 3 365m_ropectl_s45_24576 0 True
98
+ DONE 1 365m_dropout_p28_s46_8192 0 True
99
+ START 3 1p2b_legacy_dense_s42_8192
100
+ START 1 1p2b_legacy_dense_s42_16384
101
+ DONE 2 365m_ropectl_s45_30720 0 True
102
+ START 2 1p2b_legacy_mixed_s42_8192
103
+ DONE 0 365m_ropectl_s46_8192 0 True
104
+ START 0 1p2b_legacy_mixed_s42_16384
105
+ DONE 4 365m_ropectl_s46_16384 0 True
106
+ START 4 1p2b_dense_s71_16384
107
+ DONE 1 1p2b_legacy_dense_s42_16384 0 True
108
+ START 1 1p2b_mixed_s72_8192
109
+ DONE 3 1p2b_legacy_dense_s42_8192 0 True
110
+ DONE 0 1p2b_legacy_mixed_s42_16384 0 True
111
+ START 0 1p2b_mixed_s72_16384
112
+ DONE 2 1p2b_legacy_mixed_s42_8192 0 True
113
+ DONE 4 1p2b_dense_s71_16384 0 True
114
+ DONE 1 1p2b_mixed_s72_8192 0 True
115
+ DONE 0 1p2b_mixed_s72_16384 0 True
116
+ COMPLETE=98/100 FAILED=['1p2b_mixed_s71_8192', '1p2b_mixed_s71_16384']
epfl_final/train_logs/dense_diag_20260906_a10040_a_phase2.log ADDED
File without changes
epfl_final/train_logs/dense_diag_20260906_a10040_b.log ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ PROFILE=a10040 JOBS=100 ALREADY=0
2
+ START 0 365m_mixed_s42_16384
3
+ START 1 365m_mixed_s42_24576
4
+ START 2 365m_mixed_s42_30720
5
+ START 3 365m_dense_s43_8192
6
+ START 4 365m_dense_s43_16384
7
+ DONE 1 365m_mixed_s42_24576 0 True
8
+ START 1 365m_mixed_s43_30720
9
+ DONE 2 365m_mixed_s42_30720 0 True
10
+ DONE 4 365m_dense_s43_16384 0 True
11
+ START 2 365m_dense_s44_8192
12
+ START 4 365m_dense_s44_16384
13
+ DONE 0 365m_mixed_s42_16384 0 True
14
+ START 0 365m_dense_s44_24576
15
+ DONE 3 365m_dense_s43_8192 0 True
16
+ START 3 365m_dense_s44_30720
17
+ DONE 1 365m_mixed_s43_30720 0 True
18
+ START 1 365m_dense_s45_16384
19
+ DONE 4 365m_dense_s44_16384 0 True
20
+ DONE 0 365m_dense_s44_24576 0 True
21
+ START 0 365m_dense_s45_24576
22
+ START 4 365m_dense_s45_30720
23
+ DONE 2 365m_dense_s44_8192 0 True
24
+ START 2 365m_mixed_s45_24576
25
+ DONE 3 365m_dense_s44_30720 0 True
26
+ DONE 0 365m_dense_s45_24576 0 True
27
+ START 0 365m_dense_s46_24576
28
+ START 3 365m_dense_s46_30720
29
+ DONE 1 365m_dense_s45_16384 0 True
30
+ DONE 4 365m_dense_s45_30720 0 True
31
+ START 4 365m_mixed_s46_24576
32
+ START 1 365m_mixed_s46_30720
33
+ DONE 2 365m_mixed_s45_24576 0 True
34
+ START 2 365m_mixed_s47_16384
35
+ DONE 0 365m_dense_s46_24576 0 True
36
+ START 0 365m_dropout_p28_s42_16384
37
+ DONE 3 365m_dense_s46_30720 0 True
38
+ START 3 365m_dropout_p28_s42_24576
39
+ DONE 4 365m_mixed_s46_24576 0 True
40
+ START 4 365m_ropectl_s42_8192
41
+ DONE 0 365m_dropout_p28_s42_16384 0 True
42
+ START 0 365m_ropectl_s42_24576
43
+ DONE 3 365m_dropout_p28_s42_24576 0 True
44
+ START 3 365m_ropectl_s42_30720
45
+ DONE 1 365m_mixed_s46_30720 0 True
46
+ DONE 4 365m_ropectl_s42_8192 0 True
47
+ START 1 365m_ropectl_s43_16384
48
+ START 4 365m_ropectl_s43_24576
49
+ DONE 0 365m_ropectl_s42_24576 0 True
50
+ START 0 365m_ropectl_s43_30720
51
+ DONE 2 365m_mixed_s47_16384 0 True
52
+ DONE 3 365m_ropectl_s42_30720 0 True
53
+ START 2 365m_dropout_p28_s44_30720
54
+ START 3 365m_ropectl_s44_8192
55
+ DONE 1 365m_ropectl_s43_16384 0 True
56
+ DONE 4 365m_ropectl_s43_24576 0 True
57
+ START 4 365m_dropout_p28_s45_8192
58
+ START 1 365m_dropout_p28_s45_16384
59
+ DONE 0 365m_ropectl_s43_30720 0 True
60
+ START 0 365m_ropectl_s45_8192
61
+ DONE 3 365m_ropectl_s44_8192 0 True
62
+ START 3 365m_ropectl_s45_16384
63
+ DONE 2 365m_dropout_p28_s44_30720 0 True
64
+ DONE 4 365m_dropout_p28_s45_8192 0 True
65
+ START 4 365m_dropout_p28_s46_16384
66
+ START 2 365m_dropout_p28_s46_24576
67
+ DONE 1 365m_dropout_p28_s45_16384 0 True
68
+ START 1 365m_dropout_p28_s46_30720
69
+ DONE 0 365m_ropectl_s45_8192 0 True
70
+ DONE 3 365m_ropectl_s45_16384 0 True
71
+ START 0 365m_ropectl_s46_24576
72
+ START 3 365m_ropectl_s46_30720
73
+ DONE 4 365m_dropout_p28_s46_16384 0 True
74
+ START 4 1p2b_dense_s70_8192
75
+ DONE 2 365m_dropout_p28_s46_24576 0 True
76
+ START 2 1p2b_dense_s70_16384
77
+ DONE 1 365m_dropout_p28_s46_30720 0 True
78
+ START 1 1p2b_dense_s71_8192
79
+ DONE 0 365m_ropectl_s46_24576 0 True
80
+ DONE 3 365m_ropectl_s46_30720 0 True
81
+ START 0 1p2b_mixed_s71_8192
82
+ START 3 1p2b_mixed_s71_16384
83
+ DONE 2 1p2b_dense_s70_16384 0 True
84
+ DONE 4 1p2b_dense_s70_8192 0 True
85
+ DONE 1 1p2b_dense_s71_8192 0 True
86
+ DONE 3 1p2b_mixed_s71_16384 0 True
87
+ DONE 0 1p2b_mixed_s71_8192 0 True
88
+ COMPLETE=100/100 FAILED=[]
epfl_final/train_logs/dense_diag_20260906_a10040_b_phase2.log ADDED
File without changes
epfl_final/train_logs/dense_diag_20260906_a10080.log ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ PROFILE=a10080 JOBS=20 ALREADY=0
2
+ START 0 1p2b_legacy_dense_s42_24576
3
+ START 1 1p2b_legacy_dense_s42_30720
4
+ START 2 1p2b_legacy_mixed_s42_24576
5
+ START 3 1p2b_legacy_mixed_s42_30720
6
+ DONE 2 1p2b_legacy_mixed_s42_24576 0 True
7
+ START 2 1p2b_dense_s70_24576
8
+ DONE 0 1p2b_legacy_dense_s42_24576 0 True
9
+ START 0 1p2b_dense_s70_30720
10
+ DONE 3 1p2b_legacy_mixed_s42_30720 0 True
11
+ START 3 1p2b_dense_s71_24576
12
+ DONE 1 1p2b_legacy_dense_s42_30720 0 True
13
+ START 1 1p2b_dense_s71_30720
14
+ DONE 2 1p2b_dense_s70_24576 0 True
15
+ START 2 1p2b_mixed_s71_24576
16
+ DONE 3 1p2b_dense_s71_24576 0 True
17
+ START 3 1p2b_mixed_s71_30720
18
+ DONE 0 1p2b_dense_s70_30720 0 True
19
+ START 0 1p2b_mixed_s72_24576
20
+ DONE 1 1p2b_dense_s71_30720 0 True
21
+ START 1 1p2b_mixed_s72_30720
22
+ DONE 2 1p2b_mixed_s71_24576 0 True
23
+ START 2 3b_dense_8192
24
+ DONE 0 1p2b_mixed_s72_24576 0 True
25
+ START 0 3b_dense_16384
26
+ DONE 3 1p2b_mixed_s71_30720 0 True
27
+ START 3 3b_dense_24576
28
+ DONE 1 1p2b_mixed_s72_30720 0 True
29
+ START 1 3b_dense_30720
30
+ DONE 2 3b_dense_8192 0 True
31
+ START 2 3b_mixed_8192
32
+ DONE 0 3b_dense_16384 0 True
33
+ START 0 3b_mixed_16384
34
+ DONE 3 3b_dense_24576 0 True
35
+ START 3 3b_mixed_24576
36
+ DONE 2 3b_mixed_8192 0 True
37
+ START 2 3b_mixed_30720
38
+ DONE 1 3b_dense_30720 0 True
39
+ DONE 0 3b_mixed_16384 0 True
40
+ DONE 3 3b_mixed_24576 0 True
41
+ DONE 2 3b_mixed_30720 0 True
42
+ COMPLETE=20/20 FAILED=[]
epfl_final/train_logs/dense_diag_20260906_a10080_phase2.log ADDED
File without changes
epfl_final/train_logs/dense_diag_20260906_controls_012.log ADDED
@@ -0,0 +1,114 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ PROFILE=a10040 JOBS=156 ALREADY=100
2
+ START 0 365m_static_nrb35_s42_8192
3
+ START 1 365m_static_nrb35_s42_16384
4
+ START 2 365m_static_nrb35_s42_24576
5
+ DONE 0 365m_static_nrb35_s42_8192 0 True
6
+ START 0 365m_static_nrb35_s42_30720
7
+ DONE 1 365m_static_nrb35_s42_16384 0 True
8
+ START 1 365m_resampled_nrb35_s42_8192
9
+ DONE 2 365m_static_nrb35_s42_24576 0 True
10
+ START 2 365m_resampled_nrb35_s42_16384
11
+ DONE 1 365m_resampled_nrb35_s42_8192 0 True
12
+ START 1 365m_resampled_nrb35_s42_24576
13
+ DONE 0 365m_static_nrb35_s42_30720 0 True
14
+ START 0 365m_resampled_nrb35_s42_30720
15
+ DONE 2 365m_resampled_nrb35_s42_16384 0 True
16
+ START 2 365m_static_nrb35_s43_8192
17
+ DONE 1 365m_resampled_nrb35_s42_24576 0 True
18
+ START 1 365m_static_nrb35_s43_16384
19
+ DONE 2 365m_static_nrb35_s43_8192 0 True
20
+ START 2 365m_static_nrb35_s43_24576
21
+ DONE 0 365m_resampled_nrb35_s42_30720 0 True
22
+ START 0 365m_static_nrb35_s43_30720
23
+ DONE 1 365m_static_nrb35_s43_16384 0 True
24
+ START 1 365m_resampled_nrb35_s43_8192
25
+ DONE 2 365m_static_nrb35_s43_24576 0 True
26
+ START 2 365m_resampled_nrb35_s43_16384
27
+ DONE 0 365m_static_nrb35_s43_30720 0 True
28
+ START 0 365m_resampled_nrb35_s43_24576
29
+ DONE 1 365m_resampled_nrb35_s43_8192 0 True
30
+ START 1 365m_resampled_nrb35_s43_30720
31
+ DONE 2 365m_resampled_nrb35_s43_16384 0 True
32
+ START 2 365m_static_nrb35_s44_8192
33
+ DONE 0 365m_resampled_nrb35_s43_24576 0 True
34
+ START 0 365m_static_nrb35_s44_16384
35
+ DONE 2 365m_static_nrb35_s44_8192 0 True
36
+ START 2 365m_static_nrb35_s44_24576
37
+ DONE 1 365m_resampled_nrb35_s43_30720 0 True
38
+ START 1 365m_static_nrb35_s44_30720
39
+ DONE 0 365m_static_nrb35_s44_16384 0 True
40
+ START 0 365m_resampled_nrb35_s44_8192
41
+ DONE 2 365m_static_nrb35_s44_24576 0 True
42
+ START 2 365m_resampled_nrb35_s44_16384
43
+ DONE 1 365m_static_nrb35_s44_30720 0 True
44
+ START 1 365m_resampled_nrb35_s44_24576
45
+ DONE 0 365m_resampled_nrb35_s44_8192 0 True
46
+ START 0 365m_resampled_nrb35_s44_30720
47
+ DONE 2 365m_resampled_nrb35_s44_16384 0 True
48
+ START 2 365m_static_nrb35_s45_8192
49
+ DONE 1 365m_resampled_nrb35_s44_24576 0 True
50
+ START 1 365m_static_nrb35_s45_16384
51
+ DONE 0 365m_resampled_nrb35_s44_30720 0 True
52
+ START 0 365m_static_nrb35_s45_24576
53
+ DONE 2 365m_static_nrb35_s45_8192 0 True
54
+ START 2 365m_static_nrb35_s45_30720
55
+ DONE 1 365m_static_nrb35_s45_16384 0 True
56
+ START 1 365m_resampled_nrb35_s45_8192
57
+ DONE 0 365m_static_nrb35_s45_24576 0 True
58
+ START 0 365m_resampled_nrb35_s45_16384
59
+ DONE 2 365m_static_nrb35_s45_30720 0 True
60
+ DONE 1 365m_resampled_nrb35_s45_8192 0 True
61
+ START 1 365m_resampled_nrb35_s45_24576
62
+ START 2 365m_resampled_nrb35_s45_30720
63
+ DONE 0 365m_resampled_nrb35_s45_16384 0 True
64
+ START 0 365m_static_nrb35_s46_8192
65
+ DONE 1 365m_resampled_nrb35_s45_24576 0 True
66
+ START 1 365m_static_nrb35_s46_16384
67
+ DONE 2 365m_resampled_nrb35_s45_30720 0 True
68
+ START 2 365m_static_nrb35_s46_24576
69
+ DONE 0 365m_static_nrb35_s46_8192 0 True
70
+ START 0 365m_static_nrb35_s46_30720
71
+ DONE 1 365m_static_nrb35_s46_16384 0 True
72
+ START 1 365m_resampled_nrb35_s46_8192
73
+ DONE 2 365m_static_nrb35_s46_24576 0 True
74
+ START 2 365m_resampled_nrb35_s46_16384
75
+ DONE 0 365m_static_nrb35_s46_30720 0 True
76
+ START 0 365m_resampled_nrb35_s46_24576
77
+ DONE 1 365m_resampled_nrb35_s46_8192 0 True
78
+ START 1 365m_resampled_nrb35_s46_30720
79
+ DONE 2 365m_resampled_nrb35_s46_16384 0 True
80
+ START 2 365m_static_nrb35_s47_8192
81
+ DONE 0 365m_resampled_nrb35_s46_24576 0 True
82
+ START 0 365m_static_nrb35_s47_16384
83
+ DONE 2 365m_static_nrb35_s47_8192 0 True
84
+ START 2 365m_static_nrb35_s47_24576
85
+ DONE 1 365m_resampled_nrb35_s46_30720 0 True
86
+ START 1 365m_static_nrb35_s47_30720
87
+ DONE 0 365m_static_nrb35_s47_16384 0 True
88
+ START 0 365m_resampled_nrb35_s47_8192
89
+ DONE 2 365m_static_nrb35_s47_24576 0 True
90
+ START 2 365m_resampled_nrb35_s47_16384
91
+ DONE 1 365m_static_nrb35_s47_30720 0 True
92
+ START 1 365m_resampled_nrb35_s47_24576
93
+ DONE 0 365m_resampled_nrb35_s47_8192 0 True
94
+ START 0 365m_resampled_nrb35_s47_30720
95
+ DONE 2 365m_resampled_nrb35_s47_16384 0 True
96
+ START 2 1p2b_native30_dense_8192
97
+ DONE 1 365m_resampled_nrb35_s47_24576 0 True
98
+ START 1 1p2b_native30_dense_16384
99
+ DONE 0 365m_resampled_nrb35_s47_30720 0 True
100
+ START 0 1p2b_native30_mixed_8192
101
+ DONE 2 1p2b_native30_dense_8192 0 True
102
+ START 2 1p2b_native30_mixed_16384
103
+ DONE 1 1p2b_native30_dense_16384 0 True
104
+ DONE 0 1p2b_native30_mixed_8192 0 True
105
+ START 0 1p2b_native30_fixed_8192
106
+ START 1 1p2b_native30_fixed_16384
107
+ DONE 2 1p2b_native30_mixed_16384 0 True
108
+ START 2 1p2b_native30_resampled_8192
109
+ DONE 0 1p2b_native30_fixed_8192 0 True
110
+ START 0 1p2b_native30_resampled_16384
111
+ DONE 1 1p2b_native30_fixed_16384 0 True
112
+ DONE 2 1p2b_native30_resampled_8192 0 True
113
+ DONE 0 1p2b_native30_resampled_16384 0 True
114
+ COMPLETE=156/156 FAILED=[]
epfl_final/train_logs/dense_diag_20260906_native30_3.log ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ PROFILE=a10080 JOBS=28 ALREADY=20
2
+ START 3 1p2b_native30_dense_24576
3
+ DONE 3 1p2b_native30_dense_24576 0 True
4
+ START 3 1p2b_native30_dense_30720
5
+ DONE 3 1p2b_native30_dense_30720 0 True
6
+ START 3 1p2b_native30_mixed_24576
7
+ DONE 3 1p2b_native30_mixed_24576 0 True
8
+ START 3 1p2b_native30_mixed_30720
9
+ DONE 3 1p2b_native30_mixed_30720 0 True
10
+ START 3 1p2b_native30_fixed_24576
11
+ DONE 3 1p2b_native30_fixed_24576 0 True
12
+ START 3 1p2b_native30_fixed_30720
13
+ DONE 3 1p2b_native30_fixed_30720 0 True
14
+ START 3 1p2b_native30_resampled_24576
15
+ DONE 3 1p2b_native30_resampled_24576 0 True
16
+ START 3 1p2b_native30_resampled_30720
17
+ DONE 3 1p2b_native30_resampled_30720 0 True
18
+ COMPLETE=28/28 FAILED=[]
epfl_final/train_logs/generation_sanity_cachefixed_20260907.log ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ DONE 1p2b_dense_s71/passkey rc=0
2
+ DONE 1p2b_dense_s71/niah_multikey rc=0
3
+ DONE 1p2b_mixed_s71/passkey rc=0
4
+ DONE 1p2b_mixed_s71/niah_multikey rc=0
5
+ DONE 3b_dense/passkey rc=0
6
+ DONE 3b_dense/niah_multikey rc=0
7
+ DONE 3b_mixed/passkey rc=0
8
+ DONE 3b_mixed/niah_multikey rc=0
epfl_final/train_logs/p2_1p2b_dense_seed72_4gpu_chained.log ADDED
@@ -0,0 +1,1021 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Thu Sep 3 05:24:02 AM UTC 2026 waiting for mixed seed72 endpoint
2
+ Thu Sep 3 05:24:32 AM UTC 2026 waiting for mixed seed72 endpoint
3
+ Thu Sep 3 05:25:02 AM UTC 2026 waiting for mixed seed72 endpoint
4
+ Thu Sep 3 05:25:32 AM UTC 2026 waiting for mixed seed72 endpoint
5
+ Thu Sep 3 05:26:02 AM UTC 2026 waiting for mixed seed72 endpoint
6
+ Thu Sep 3 05:26:32 AM UTC 2026 waiting for mixed seed72 endpoint
7
+ Thu Sep 3 05:27:02 AM UTC 2026 waiting for mixed seed72 endpoint
8
+ Thu Sep 3 05:27:32 AM UTC 2026 waiting for mixed seed72 endpoint
9
+ Thu Sep 3 05:28:02 AM UTC 2026 waiting for mixed seed72 endpoint
10
+ Thu Sep 3 05:28:32 AM UTC 2026 waiting for mixed seed72 endpoint
11
+ Thu Sep 3 05:29:02 AM UTC 2026 waiting for mixed seed72 endpoint
12
+ Thu Sep 3 05:29:32 AM UTC 2026 waiting for mixed seed72 endpoint
13
+ Thu Sep 3 05:30:02 AM UTC 2026 waiting for mixed seed72 endpoint
14
+ Thu Sep 3 05:30:32 AM UTC 2026 waiting for mixed seed72 endpoint
15
+ Thu Sep 3 05:31:02 AM UTC 2026 waiting for mixed seed72 endpoint
16
+ Thu Sep 3 05:31:32 AM UTC 2026 waiting for mixed seed72 endpoint
17
+ Thu Sep 3 05:32:02 AM UTC 2026 waiting for mixed seed72 endpoint
18
+ Thu Sep 3 05:32:32 AM UTC 2026 waiting for mixed seed72 endpoint
19
+ Thu Sep 3 05:33:02 AM UTC 2026 waiting for mixed seed72 endpoint
20
+ Thu Sep 3 05:33:32 AM UTC 2026 waiting for mixed seed72 endpoint
21
+ Thu Sep 3 05:34:02 AM UTC 2026 waiting for mixed seed72 endpoint
22
+ Thu Sep 3 05:34:32 AM UTC 2026 waiting for mixed seed72 endpoint
23
+ Thu Sep 3 05:35:02 AM UTC 2026 waiting for mixed seed72 endpoint
24
+ Thu Sep 3 05:35:32 AM UTC 2026 waiting for mixed seed72 endpoint
25
+ Thu Sep 3 05:36:02 AM UTC 2026 waiting for mixed seed72 endpoint
26
+ Thu Sep 3 05:36:32 AM UTC 2026 waiting for mixed seed72 endpoint
27
+ Thu Sep 3 05:37:02 AM UTC 2026 waiting for mixed seed72 endpoint
28
+ Thu Sep 3 05:37:32 AM UTC 2026 waiting for mixed seed72 endpoint
29
+ Thu Sep 3 05:38:02 AM UTC 2026 waiting for mixed seed72 endpoint
30
+ Thu Sep 3 05:38:32 AM UTC 2026 waiting for mixed seed72 endpoint
31
+ Thu Sep 3 05:39:02 AM UTC 2026 waiting for mixed seed72 endpoint
32
+ Thu Sep 3 05:39:32 AM UTC 2026 waiting for mixed seed72 endpoint
33
+ Thu Sep 3 05:40:02 AM UTC 2026 waiting for mixed seed72 endpoint
34
+ Thu Sep 3 05:40:32 AM UTC 2026 waiting for mixed seed72 endpoint
35
+ Thu Sep 3 05:41:02 AM UTC 2026 waiting for mixed seed72 endpoint
36
+ Thu Sep 3 05:41:32 AM UTC 2026 waiting for mixed seed72 endpoint
37
+ Thu Sep 3 05:42:02 AM UTC 2026 waiting for mixed seed72 endpoint
38
+ Thu Sep 3 05:42:32 AM UTC 2026 waiting for mixed seed72 endpoint
39
+ Thu Sep 3 05:43:02 AM UTC 2026 waiting for mixed seed72 endpoint
40
+ Thu Sep 3 05:43:32 AM UTC 2026 waiting for mixed seed72 endpoint
41
+ Thu Sep 3 05:44:02 AM UTC 2026 waiting for mixed seed72 endpoint
42
+ Thu Sep 3 05:44:32 AM UTC 2026 waiting for mixed seed72 endpoint
43
+ Thu Sep 3 05:45:02 AM UTC 2026 waiting for mixed seed72 endpoint
44
+ Thu Sep 3 05:45:32 AM UTC 2026 waiting for mixed seed72 endpoint
45
+ Thu Sep 3 05:46:02 AM UTC 2026 waiting for mixed seed72 endpoint
46
+ Thu Sep 3 05:46:32 AM UTC 2026 waiting for mixed seed72 endpoint
47
+ Thu Sep 3 05:47:02 AM UTC 2026 waiting for mixed seed72 endpoint
48
+ Thu Sep 3 05:47:32 AM UTC 2026 waiting for mixed seed72 endpoint
49
+ Thu Sep 3 05:48:02 AM UTC 2026 waiting for mixed seed72 endpoint
50
+ Thu Sep 3 05:48:32 AM UTC 2026 waiting for mixed seed72 endpoint
51
+ Thu Sep 3 05:49:02 AM UTC 2026 waiting for mixed seed72 endpoint
52
+ Thu Sep 3 05:49:32 AM UTC 2026 waiting for mixed seed72 endpoint
53
+ Thu Sep 3 05:50:02 AM UTC 2026 waiting for mixed seed72 endpoint
54
+ Thu Sep 3 05:50:32 AM UTC 2026 waiting for mixed seed72 endpoint
55
+ Thu Sep 3 05:51:02 AM UTC 2026 waiting for mixed seed72 endpoint
56
+ Thu Sep 3 05:51:32 AM UTC 2026 waiting for mixed seed72 endpoint
57
+ Thu Sep 3 05:52:02 AM UTC 2026 waiting for mixed seed72 endpoint
58
+ Thu Sep 3 05:52:32 AM UTC 2026 waiting for mixed seed72 endpoint
59
+ Thu Sep 3 05:53:02 AM UTC 2026 waiting for mixed seed72 endpoint
60
+ Thu Sep 3 05:53:32 AM UTC 2026 waiting for mixed seed72 endpoint
61
+ Thu Sep 3 05:54:02 AM UTC 2026 waiting for mixed seed72 endpoint
62
+ Thu Sep 3 05:54:32 AM UTC 2026 waiting for mixed seed72 endpoint
63
+ Thu Sep 3 05:55:02 AM UTC 2026 waiting for mixed seed72 endpoint
64
+ Thu Sep 3 05:55:32 AM UTC 2026 waiting for mixed seed72 endpoint
65
+ Thu Sep 3 05:56:02 AM UTC 2026 waiting for mixed seed72 endpoint
66
+ Thu Sep 3 05:56:32 AM UTC 2026 waiting for mixed seed72 endpoint
67
+ Thu Sep 3 05:57:02 AM UTC 2026 waiting for mixed seed72 endpoint
68
+ Thu Sep 3 05:57:32 AM UTC 2026 waiting for mixed seed72 endpoint
69
+ Thu Sep 3 05:58:02 AM UTC 2026 waiting for mixed seed72 endpoint
70
+ Thu Sep 3 05:58:32 AM UTC 2026 waiting for mixed seed72 endpoint
71
+ Thu Sep 3 05:59:02 AM UTC 2026 waiting for mixed seed72 endpoint
72
+ Thu Sep 3 05:59:32 AM UTC 2026 waiting for mixed seed72 endpoint
73
+ Thu Sep 3 06:00:02 AM UTC 2026 waiting for mixed seed72 endpoint
74
+ Thu Sep 3 06:00:32 AM UTC 2026 waiting for mixed seed72 endpoint
75
+ Thu Sep 3 06:01:02 AM UTC 2026 waiting for mixed seed72 endpoint
76
+ Thu Sep 3 06:01:32 AM UTC 2026 waiting for mixed seed72 endpoint
77
+ Thu Sep 3 06:02:02 AM UTC 2026 waiting for mixed seed72 endpoint
78
+ Thu Sep 3 06:02:32 AM UTC 2026 waiting for mixed seed72 endpoint
79
+ Thu Sep 3 06:03:02 AM UTC 2026 waiting for mixed seed72 endpoint
80
+ Thu Sep 3 06:03:32 AM UTC 2026 waiting for mixed seed72 endpoint
81
+ Thu Sep 3 06:04:02 AM UTC 2026 waiting for mixed seed72 endpoint
82
+ Thu Sep 3 06:04:32 AM UTC 2026 waiting for mixed seed72 endpoint
83
+ Thu Sep 3 06:05:02 AM UTC 2026 waiting for mixed seed72 endpoint
84
+ Thu Sep 3 06:05:32 AM UTC 2026 waiting for mixed seed72 endpoint
85
+ Thu Sep 3 06:06:02 AM UTC 2026 waiting for mixed seed72 endpoint
86
+ Thu Sep 3 06:06:32 AM UTC 2026 waiting for mixed seed72 endpoint
87
+ Thu Sep 3 06:07:02 AM UTC 2026 waiting for mixed seed72 endpoint
88
+ Thu Sep 3 06:07:32 AM UTC 2026 waiting for mixed seed72 endpoint
89
+ Thu Sep 3 06:08:02 AM UTC 2026 waiting for mixed seed72 endpoint
90
+ Thu Sep 3 06:08:32 AM UTC 2026 waiting for mixed seed72 endpoint
91
+ Thu Sep 3 06:09:02 AM UTC 2026 waiting for mixed seed72 endpoint
92
+ Thu Sep 3 06:09:32 AM UTC 2026 waiting for mixed seed72 endpoint
93
+ Thu Sep 3 06:10:02 AM UTC 2026 waiting for mixed seed72 endpoint
94
+ Thu Sep 3 06:10:32 AM UTC 2026 waiting for mixed seed72 endpoint
95
+ Thu Sep 3 06:11:02 AM UTC 2026 waiting for mixed seed72 endpoint
96
+ Thu Sep 3 06:11:32 AM UTC 2026 waiting for mixed seed72 endpoint
97
+ Thu Sep 3 06:12:02 AM UTC 2026 waiting for mixed seed72 endpoint
98
+ Thu Sep 3 06:12:33 AM UTC 2026 waiting for mixed seed72 endpoint
99
+ Thu Sep 3 06:13:03 AM UTC 2026 waiting for mixed seed72 endpoint
100
+ Thu Sep 3 06:13:33 AM UTC 2026 waiting for mixed seed72 endpoint
101
+ Thu Sep 3 06:14:03 AM UTC 2026 waiting for mixed seed72 endpoint
102
+ Thu Sep 3 06:14:33 AM UTC 2026 waiting for mixed seed72 endpoint
103
+ Thu Sep 3 06:15:03 AM UTC 2026 waiting for mixed seed72 endpoint
104
+ Thu Sep 3 06:15:33 AM UTC 2026 waiting for mixed seed72 endpoint
105
+ Thu Sep 3 06:16:03 AM UTC 2026 waiting for mixed seed72 endpoint
106
+ Thu Sep 3 06:16:33 AM UTC 2026 waiting for mixed seed72 endpoint
107
+ Thu Sep 3 06:17:03 AM UTC 2026 waiting for mixed seed72 endpoint
108
+ Thu Sep 3 06:17:33 AM UTC 2026 waiting for mixed seed72 endpoint
109
+ Thu Sep 3 06:18:03 AM UTC 2026 waiting for mixed seed72 endpoint
110
+ Thu Sep 3 06:18:33 AM UTC 2026 waiting for mixed seed72 endpoint
111
+ Thu Sep 3 06:19:03 AM UTC 2026 waiting for mixed seed72 endpoint
112
+ Thu Sep 3 06:19:33 AM UTC 2026 waiting for mixed seed72 endpoint
113
+ Thu Sep 3 06:20:03 AM UTC 2026 waiting for mixed seed72 endpoint
114
+ Thu Sep 3 06:20:33 AM UTC 2026 waiting for mixed seed72 endpoint
115
+ Thu Sep 3 06:21:03 AM UTC 2026 waiting for mixed seed72 endpoint
116
+ Thu Sep 3 06:21:33 AM UTC 2026 waiting for mixed seed72 endpoint
117
+ Thu Sep 3 06:22:03 AM UTC 2026 waiting for mixed seed72 endpoint
118
+ Thu Sep 3 06:22:33 AM UTC 2026 waiting for mixed seed72 endpoint
119
+ Thu Sep 3 06:23:03 AM UTC 2026 waiting for mixed seed72 endpoint
120
+ Thu Sep 3 06:23:33 AM UTC 2026 waiting for mixed seed72 endpoint
121
+ Thu Sep 3 06:24:03 AM UTC 2026 waiting for mixed seed72 endpoint
122
+ Thu Sep 3 06:24:33 AM UTC 2026 waiting for mixed seed72 endpoint
123
+ Thu Sep 3 06:25:03 AM UTC 2026 waiting for mixed seed72 endpoint
124
+ Thu Sep 3 06:25:33 AM UTC 2026 waiting for mixed seed72 endpoint
125
+ Thu Sep 3 06:26:03 AM UTC 2026 waiting for mixed seed72 endpoint
126
+ Thu Sep 3 06:26:33 AM UTC 2026 waiting for mixed seed72 endpoint
127
+ Thu Sep 3 06:27:03 AM UTC 2026 waiting for mixed seed72 endpoint
128
+ Thu Sep 3 06:27:33 AM UTC 2026 waiting for mixed seed72 endpoint
129
+ Thu Sep 3 06:28:03 AM UTC 2026 waiting for mixed seed72 endpoint
130
+ Thu Sep 3 06:28:33 AM UTC 2026 waiting for mixed seed72 endpoint
131
+ Thu Sep 3 06:29:03 AM UTC 2026 waiting for mixed seed72 endpoint
132
+ Thu Sep 3 06:29:33 AM UTC 2026 waiting for mixed seed72 endpoint
133
+ Thu Sep 3 06:30:03 AM UTC 2026 waiting for mixed seed72 endpoint
134
+ Thu Sep 3 06:30:33 AM UTC 2026 waiting for mixed seed72 endpoint
135
+ Thu Sep 3 06:31:03 AM UTC 2026 waiting for mixed seed72 endpoint
136
+ Thu Sep 3 06:31:33 AM UTC 2026 waiting for mixed seed72 endpoint
137
+ Thu Sep 3 06:32:03 AM UTC 2026 waiting for mixed seed72 endpoint
138
+ Thu Sep 3 06:32:33 AM UTC 2026 waiting for mixed seed72 endpoint
139
+ Thu Sep 3 06:33:03 AM UTC 2026 waiting for mixed seed72 endpoint
140
+ Thu Sep 3 06:33:33 AM UTC 2026 waiting for mixed seed72 endpoint
141
+ Thu Sep 3 06:34:03 AM UTC 2026 waiting for mixed seed72 endpoint
142
+ Thu Sep 3 06:34:33 AM UTC 2026 waiting for mixed seed72 endpoint
143
+ Thu Sep 3 06:35:03 AM UTC 2026 waiting for mixed seed72 endpoint
144
+ Thu Sep 3 06:35:33 AM UTC 2026 waiting for mixed seed72 endpoint
145
+ Thu Sep 3 06:36:03 AM UTC 2026 waiting for mixed seed72 endpoint
146
+ Thu Sep 3 06:36:33 AM UTC 2026 waiting for mixed seed72 endpoint
147
+ Thu Sep 3 06:37:03 AM UTC 2026 waiting for mixed seed72 endpoint
148
+ Thu Sep 3 06:37:33 AM UTC 2026 waiting for mixed seed72 endpoint
149
+ Thu Sep 3 06:38:03 AM UTC 2026 waiting for mixed seed72 endpoint
150
+ Thu Sep 3 06:38:33 AM UTC 2026 waiting for mixed seed72 endpoint
151
+ Thu Sep 3 06:39:03 AM UTC 2026 waiting for mixed seed72 endpoint
152
+ Thu Sep 3 06:39:33 AM UTC 2026 waiting for mixed seed72 endpoint
153
+ Thu Sep 3 06:40:03 AM UTC 2026 waiting for mixed seed72 endpoint
154
+ Thu Sep 3 06:40:33 AM UTC 2026 waiting for mixed seed72 endpoint
155
+ Thu Sep 3 06:41:03 AM UTC 2026 waiting for mixed seed72 endpoint
156
+ Thu Sep 3 06:41:33 AM UTC 2026 waiting for mixed seed72 endpoint
157
+ Thu Sep 3 06:42:03 AM UTC 2026 waiting for mixed seed72 endpoint
158
+ Thu Sep 3 06:42:33 AM UTC 2026 waiting for mixed seed72 endpoint
159
+ Thu Sep 3 06:43:03 AM UTC 2026 waiting for mixed seed72 endpoint
160
+ Thu Sep 3 06:43:33 AM UTC 2026 waiting for mixed seed72 endpoint
161
+ Thu Sep 3 06:44:03 AM UTC 2026 waiting for mixed seed72 endpoint
162
+ Thu Sep 3 06:44:33 AM UTC 2026 waiting for mixed seed72 endpoint
163
+ Thu Sep 3 06:45:03 AM UTC 2026 waiting for mixed seed72 endpoint
164
+ Thu Sep 3 06:45:33 AM UTC 2026 waiting for mixed seed72 endpoint
165
+ Thu Sep 3 06:46:03 AM UTC 2026 waiting for mixed seed72 endpoint
166
+ Thu Sep 3 06:46:33 AM UTC 2026 waiting for mixed seed72 endpoint
167
+ Thu Sep 3 06:47:03 AM UTC 2026 waiting for mixed seed72 endpoint
168
+ Thu Sep 3 06:47:33 AM UTC 2026 waiting for mixed seed72 endpoint
169
+ Thu Sep 3 06:48:03 AM UTC 2026 waiting for mixed seed72 endpoint
170
+ Thu Sep 3 06:48:33 AM UTC 2026 waiting for mixed seed72 endpoint
171
+ Thu Sep 3 06:49:03 AM UTC 2026 waiting for mixed seed72 endpoint
172
+ Thu Sep 3 06:49:33 AM UTC 2026 waiting for mixed seed72 endpoint
173
+ Thu Sep 3 06:50:03 AM UTC 2026 waiting for mixed seed72 endpoint
174
+ Thu Sep 3 06:50:33 AM UTC 2026 waiting for mixed seed72 endpoint
175
+ Thu Sep 3 06:51:03 AM UTC 2026 waiting for mixed seed72 endpoint
176
+ Thu Sep 3 06:51:33 AM UTC 2026 waiting for mixed seed72 endpoint
177
+ Thu Sep 3 06:52:03 AM UTC 2026 waiting for mixed seed72 endpoint
178
+ Thu Sep 3 06:52:33 AM UTC 2026 waiting for mixed seed72 endpoint
179
+ Thu Sep 3 06:53:03 AM UTC 2026 waiting for mixed seed72 endpoint
180
+ Thu Sep 3 06:53:33 AM UTC 2026 waiting for mixed seed72 endpoint
181
+ Thu Sep 3 06:54:03 AM UTC 2026 waiting for mixed seed72 endpoint
182
+ Thu Sep 3 06:54:33 AM UTC 2026 waiting for mixed seed72 endpoint
183
+ Thu Sep 3 06:55:03 AM UTC 2026 waiting for mixed seed72 endpoint
184
+ Thu Sep 3 06:55:33 AM UTC 2026 waiting for mixed seed72 endpoint
185
+ Thu Sep 3 06:56:03 AM UTC 2026 waiting for mixed seed72 endpoint
186
+ Thu Sep 3 06:56:33 AM UTC 2026 waiting for mixed seed72 endpoint
187
+ Thu Sep 3 06:57:03 AM UTC 2026 waiting for mixed seed72 endpoint
188
+ Thu Sep 3 06:57:33 AM UTC 2026 waiting for mixed seed72 endpoint
189
+ Thu Sep 3 06:58:03 AM UTC 2026 waiting for mixed seed72 endpoint
190
+ Thu Sep 3 06:58:33 AM UTC 2026 waiting for mixed seed72 endpoint
191
+ Thu Sep 3 06:59:03 AM UTC 2026 waiting for mixed seed72 endpoint
192
+ Thu Sep 3 06:59:33 AM UTC 2026 waiting for mixed seed72 endpoint
193
+ Thu Sep 3 07:00:03 AM UTC 2026 waiting for mixed seed72 endpoint
194
+ Thu Sep 3 07:00:33 AM UTC 2026 waiting for mixed seed72 endpoint
195
+ Thu Sep 3 07:01:03 AM UTC 2026 waiting for mixed seed72 endpoint
196
+ Thu Sep 3 07:01:33 AM UTC 2026 waiting for mixed seed72 endpoint
197
+ Thu Sep 3 07:02:03 AM UTC 2026 waiting for mixed seed72 endpoint
198
+ Thu Sep 3 07:02:33 AM UTC 2026 waiting for mixed seed72 endpoint
199
+ Thu Sep 3 07:03:03 AM UTC 2026 waiting for mixed seed72 endpoint
200
+ Thu Sep 3 07:03:33 AM UTC 2026 waiting for mixed seed72 endpoint
201
+ Thu Sep 3 07:04:03 AM UTC 2026 waiting for mixed seed72 endpoint
202
+ Thu Sep 3 07:04:33 AM UTC 2026 waiting for mixed seed72 endpoint
203
+ Thu Sep 3 07:05:03 AM UTC 2026 waiting for mixed seed72 endpoint
204
+ Thu Sep 3 07:05:33 AM UTC 2026 waiting for mixed seed72 endpoint
205
+ Thu Sep 3 07:06:03 AM UTC 2026 waiting for mixed seed72 endpoint
206
+ Thu Sep 3 07:06:34 AM UTC 2026 waiting for mixed seed72 endpoint
207
+ Thu Sep 3 07:07:04 AM UTC 2026 waiting for mixed seed72 endpoint
208
+ Thu Sep 3 07:07:34 AM UTC 2026 waiting for mixed seed72 endpoint
209
+ Thu Sep 3 07:08:04 AM UTC 2026 waiting for mixed seed72 endpoint
210
+ Thu Sep 3 07:08:34 AM UTC 2026 waiting for mixed seed72 endpoint
211
+ Thu Sep 3 07:09:04 AM UTC 2026 waiting for mixed seed72 endpoint
212
+ Thu Sep 3 07:09:34 AM UTC 2026 waiting for mixed seed72 endpoint
213
+ Thu Sep 3 07:10:04 AM UTC 2026 waiting for mixed seed72 endpoint
214
+ Thu Sep 3 07:10:34 AM UTC 2026 waiting for mixed seed72 endpoint
215
+ Thu Sep 3 07:11:04 AM UTC 2026 waiting for mixed seed72 endpoint
216
+ Thu Sep 3 07:11:34 AM UTC 2026 waiting for mixed seed72 endpoint
217
+ Thu Sep 3 07:12:04 AM UTC 2026 waiting for mixed seed72 endpoint
218
+ Thu Sep 3 07:12:34 AM UTC 2026 waiting for mixed seed72 endpoint
219
+ Thu Sep 3 07:13:04 AM UTC 2026 waiting for mixed seed72 endpoint
220
+ Thu Sep 3 07:13:34 AM UTC 2026 waiting for mixed seed72 endpoint
221
+ Thu Sep 3 07:14:04 AM UTC 2026 waiting for mixed seed72 endpoint
222
+ Thu Sep 3 07:14:34 AM UTC 2026 waiting for mixed seed72 endpoint
223
+ Thu Sep 3 07:15:04 AM UTC 2026 waiting for mixed seed72 endpoint
224
+ Thu Sep 3 07:15:34 AM UTC 2026 waiting for mixed seed72 endpoint
225
+ Thu Sep 3 07:16:04 AM UTC 2026 waiting for mixed seed72 endpoint
226
+ Thu Sep 3 07:16:34 AM UTC 2026 waiting for mixed seed72 endpoint
227
+ Thu Sep 3 07:17:04 AM UTC 2026 waiting for mixed seed72 endpoint
228
+ Thu Sep 3 07:17:34 AM UTC 2026 waiting for mixed seed72 endpoint
229
+ Thu Sep 3 07:18:04 AM UTC 2026 waiting for mixed seed72 endpoint
230
+ Thu Sep 3 07:18:34 AM UTC 2026 waiting for mixed seed72 endpoint
231
+ Thu Sep 3 07:19:04 AM UTC 2026 waiting for mixed seed72 endpoint
232
+ Thu Sep 3 07:19:34 AM UTC 2026 waiting for mixed seed72 endpoint
233
+ Thu Sep 3 07:20:04 AM UTC 2026 waiting for mixed seed72 endpoint
234
+ Thu Sep 3 07:20:34 AM UTC 2026 waiting for mixed seed72 endpoint
235
+ Thu Sep 3 07:21:04 AM UTC 2026 waiting for mixed seed72 endpoint
236
+ Thu Sep 3 07:21:34 AM UTC 2026 waiting for mixed seed72 endpoint
237
+ Thu Sep 3 07:22:04 AM UTC 2026 waiting for mixed seed72 endpoint
238
+ Thu Sep 3 07:22:34 AM UTC 2026 waiting for mixed seed72 endpoint
239
+ Thu Sep 3 07:23:04 AM UTC 2026 waiting for mixed seed72 endpoint
240
+ Thu Sep 3 07:23:34 AM UTC 2026 waiting for mixed seed72 endpoint
241
+ Thu Sep 3 07:24:04 AM UTC 2026 waiting for mixed seed72 endpoint
242
+ Thu Sep 3 07:24:34 AM UTC 2026 waiting for mixed seed72 endpoint
243
+ Thu Sep 3 07:25:04 AM UTC 2026 waiting for mixed seed72 endpoint
244
+ Thu Sep 3 07:25:34 AM UTC 2026 waiting for mixed seed72 endpoint
245
+ Thu Sep 3 07:26:04 AM UTC 2026 waiting for mixed seed72 endpoint
246
+ Thu Sep 3 07:26:34 AM UTC 2026 waiting for mixed seed72 endpoint
247
+ Thu Sep 3 07:27:04 AM UTC 2026 waiting for mixed seed72 endpoint
248
+ Thu Sep 3 07:27:34 AM UTC 2026 waiting for mixed seed72 endpoint
249
+ Thu Sep 3 07:28:04 AM UTC 2026 waiting for mixed seed72 endpoint
250
+ Thu Sep 3 07:28:34 AM UTC 2026 waiting for mixed seed72 endpoint
251
+ Thu Sep 3 07:29:04 AM UTC 2026 waiting for mixed seed72 endpoint
252
+ Thu Sep 3 07:29:34 AM UTC 2026 waiting for mixed seed72 endpoint
253
+ Thu Sep 3 07:30:04 AM UTC 2026 waiting for mixed seed72 endpoint
254
+ Thu Sep 3 07:30:34 AM UTC 2026 waiting for mixed seed72 endpoint
255
+ Thu Sep 3 07:31:04 AM UTC 2026 waiting for mixed seed72 endpoint
256
+ Thu Sep 3 07:31:34 AM UTC 2026 waiting for mixed seed72 endpoint
257
+ Thu Sep 3 07:32:04 AM UTC 2026 waiting for mixed seed72 endpoint
258
+ Thu Sep 3 07:32:34 AM UTC 2026 waiting for mixed seed72 endpoint
259
+ Thu Sep 3 07:33:04 AM UTC 2026 waiting for mixed seed72 endpoint
260
+ Thu Sep 3 07:33:34 AM UTC 2026 waiting for mixed seed72 endpoint
261
+ Thu Sep 3 07:34:04 AM UTC 2026 waiting for mixed seed72 endpoint
262
+ Thu Sep 3 07:34:34 AM UTC 2026 waiting for mixed seed72 endpoint
263
+ Thu Sep 3 07:35:04 AM UTC 2026 waiting for mixed seed72 endpoint
264
+ Thu Sep 3 07:35:34 AM UTC 2026 waiting for mixed seed72 endpoint
265
+ Thu Sep 3 07:36:04 AM UTC 2026 waiting for mixed seed72 endpoint
266
+ Thu Sep 3 07:36:34 AM UTC 2026 waiting for mixed seed72 endpoint
267
+ Thu Sep 3 07:37:04 AM UTC 2026 waiting for mixed seed72 endpoint
268
+ Thu Sep 3 07:37:34 AM UTC 2026 waiting for mixed seed72 endpoint
269
+ Thu Sep 3 07:38:04 AM UTC 2026 waiting for mixed seed72 endpoint
270
+ Thu Sep 3 07:38:34 AM UTC 2026 waiting for mixed seed72 endpoint
271
+ Thu Sep 3 07:39:04 AM UTC 2026 waiting for mixed seed72 endpoint
272
+ Thu Sep 3 07:39:34 AM UTC 2026 waiting for mixed seed72 endpoint
273
+ Thu Sep 3 07:40:04 AM UTC 2026 waiting for mixed seed72 endpoint
274
+ Thu Sep 3 07:40:34 AM UTC 2026 waiting for mixed seed72 endpoint
275
+ Thu Sep 3 07:41:04 AM UTC 2026 waiting for mixed seed72 endpoint
276
+ Thu Sep 3 07:41:34 AM UTC 2026 waiting for mixed seed72 endpoint
277
+ Thu Sep 3 07:42:04 AM UTC 2026 waiting for mixed seed72 endpoint
278
+ Thu Sep 3 07:42:34 AM UTC 2026 waiting for mixed seed72 endpoint
279
+ Thu Sep 3 07:43:04 AM UTC 2026 waiting for mixed seed72 endpoint
280
+ Thu Sep 3 07:43:34 AM UTC 2026 waiting for mixed seed72 endpoint
281
+ Thu Sep 3 07:44:04 AM UTC 2026 waiting for mixed seed72 endpoint
282
+ Thu Sep 3 07:44:34 AM UTC 2026 waiting for mixed seed72 endpoint
283
+ Thu Sep 3 07:45:04 AM UTC 2026 waiting for mixed seed72 endpoint
284
+ Thu Sep 3 07:45:34 AM UTC 2026 waiting for mixed seed72 endpoint
285
+ Thu Sep 3 07:46:04 AM UTC 2026 waiting for mixed seed72 endpoint
286
+ Thu Sep 3 07:46:34 AM UTC 2026 waiting for mixed seed72 endpoint
287
+ Thu Sep 3 07:47:04 AM UTC 2026 waiting for mixed seed72 endpoint
288
+ Thu Sep 3 07:47:34 AM UTC 2026 waiting for mixed seed72 endpoint
289
+ Thu Sep 3 07:48:04 AM UTC 2026 waiting for mixed seed72 endpoint
290
+ Thu Sep 3 07:48:34 AM UTC 2026 waiting for mixed seed72 endpoint
291
+ Thu Sep 3 07:49:04 AM UTC 2026 waiting for mixed seed72 endpoint
292
+ Thu Sep 3 07:49:34 AM UTC 2026 waiting for mixed seed72 endpoint
293
+ Thu Sep 3 07:50:04 AM UTC 2026 waiting for mixed seed72 endpoint
294
+ Thu Sep 3 07:50:34 AM UTC 2026 waiting for mixed seed72 endpoint
295
+ Thu Sep 3 07:51:04 AM UTC 2026 waiting for mixed seed72 endpoint
296
+ Thu Sep 3 07:51:34 AM UTC 2026 waiting for mixed seed72 endpoint
297
+ Thu Sep 3 07:52:04 AM UTC 2026 waiting for mixed seed72 endpoint
298
+ Thu Sep 3 07:52:34 AM UTC 2026 waiting for mixed seed72 endpoint
299
+ Thu Sep 3 07:53:04 AM UTC 2026 waiting for mixed seed72 endpoint
300
+ Thu Sep 3 07:53:34 AM UTC 2026 waiting for mixed seed72 endpoint
301
+ Thu Sep 3 07:54:04 AM UTC 2026 waiting for mixed seed72 endpoint
302
+ Thu Sep 3 07:54:34 AM UTC 2026 waiting for mixed seed72 endpoint
303
+ Thu Sep 3 07:55:04 AM UTC 2026 waiting for mixed seed72 endpoint
304
+ Thu Sep 3 07:55:34 AM UTC 2026 waiting for mixed seed72 endpoint
305
+ Thu Sep 3 07:56:04 AM UTC 2026 waiting for mixed seed72 endpoint
306
+ Thu Sep 3 07:56:34 AM UTC 2026 waiting for mixed seed72 endpoint
307
+ Thu Sep 3 07:57:04 AM UTC 2026 waiting for mixed seed72 endpoint
308
+ Thu Sep 3 07:57:34 AM UTC 2026 waiting for mixed seed72 endpoint
309
+ Thu Sep 3 07:58:04 AM UTC 2026 waiting for mixed seed72 endpoint
310
+ Thu Sep 3 07:58:34 AM UTC 2026 waiting for mixed seed72 endpoint
311
+ Thu Sep 3 07:59:04 AM UTC 2026 waiting for mixed seed72 endpoint
312
+ Thu Sep 3 07:59:34 AM UTC 2026 waiting for mixed seed72 endpoint
313
+ Thu Sep 3 08:00:04 AM UTC 2026 waiting for mixed seed72 endpoint
314
+ Thu Sep 3 08:00:34 AM UTC 2026 waiting for mixed seed72 endpoint
315
+ Thu Sep 3 08:01:05 AM UTC 2026 waiting for mixed seed72 endpoint
316
+ Thu Sep 3 08:01:35 AM UTC 2026 waiting for mixed seed72 endpoint
317
+ Thu Sep 3 08:02:05 AM UTC 2026 waiting for mixed seed72 endpoint
318
+ Thu Sep 3 08:02:35 AM UTC 2026 waiting for mixed seed72 endpoint
319
+ Thu Sep 3 08:03:05 AM UTC 2026 waiting for mixed seed72 endpoint
320
+ Thu Sep 3 08:03:35 AM UTC 2026 waiting for mixed seed72 endpoint
321
+ Thu Sep 3 08:04:05 AM UTC 2026 waiting for mixed seed72 endpoint
322
+ Thu Sep 3 08:04:35 AM UTC 2026 waiting for mixed seed72 endpoint
323
+ Thu Sep 3 08:05:05 AM UTC 2026 waiting for mixed seed72 endpoint
324
+ Thu Sep 3 08:05:35 AM UTC 2026 waiting for mixed seed72 endpoint
325
+ Thu Sep 3 08:06:05 AM UTC 2026 waiting for mixed seed72 endpoint
326
+ Thu Sep 3 08:06:35 AM UTC 2026 waiting for mixed seed72 endpoint
327
+ Thu Sep 3 08:07:05 AM UTC 2026 waiting for mixed seed72 endpoint
328
+ Thu Sep 3 08:07:35 AM UTC 2026 waiting for mixed seed72 endpoint
329
+ Thu Sep 3 08:08:05 AM UTC 2026 waiting for mixed seed72 endpoint
330
+ Thu Sep 3 08:08:35 AM UTC 2026 waiting for mixed seed72 endpoint
331
+ Thu Sep 3 08:09:05 AM UTC 2026 waiting for mixed seed72 endpoint
332
+ Thu Sep 3 08:09:35 AM UTC 2026 waiting for mixed seed72 endpoint
333
+ Thu Sep 3 08:10:05 AM UTC 2026 waiting for mixed seed72 endpoint
334
+ Thu Sep 3 08:10:35 AM UTC 2026 waiting for mixed seed72 endpoint
335
+ Thu Sep 3 08:11:05 AM UTC 2026 waiting for mixed seed72 endpoint
336
+ Thu Sep 3 08:11:35 AM UTC 2026 waiting for mixed seed72 endpoint
337
+ Thu Sep 3 08:12:05 AM UTC 2026 waiting for mixed seed72 endpoint
338
+ Thu Sep 3 08:12:35 AM UTC 2026 waiting for mixed seed72 endpoint
339
+ Thu Sep 3 08:13:05 AM UTC 2026 waiting for mixed seed72 endpoint
340
+ Thu Sep 3 08:13:35 AM UTC 2026 waiting for mixed seed72 endpoint
341
+ Thu Sep 3 08:14:05 AM UTC 2026 waiting for mixed seed72 endpoint
342
+ Thu Sep 3 08:14:35 AM UTC 2026 waiting for mixed seed72 endpoint
343
+ Thu Sep 3 08:15:05 AM UTC 2026 waiting for mixed seed72 endpoint
344
+ Thu Sep 3 08:15:35 AM UTC 2026 waiting for mixed seed72 endpoint
345
+ Thu Sep 3 08:16:05 AM UTC 2026 waiting for mixed seed72 endpoint
346
+ Thu Sep 3 08:16:35 AM UTC 2026 waiting for mixed seed72 endpoint
347
+ Thu Sep 3 08:17:05 AM UTC 2026 waiting for mixed seed72 endpoint
348
+ Thu Sep 3 08:17:35 AM UTC 2026 waiting for mixed seed72 endpoint
349
+ Thu Sep 3 08:18:05 AM UTC 2026 waiting for mixed seed72 endpoint
350
+ Thu Sep 3 08:18:35 AM UTC 2026 waiting for mixed seed72 endpoint
351
+ Thu Sep 3 08:19:05 AM UTC 2026 waiting for mixed seed72 endpoint
352
+ Thu Sep 3 08:19:35 AM UTC 2026 waiting for mixed seed72 endpoint
353
+ Thu Sep 3 08:20:05 AM UTC 2026 waiting for mixed seed72 endpoint
354
+ Thu Sep 3 08:20:35 AM UTC 2026 waiting for mixed seed72 endpoint
355
+ Thu Sep 3 08:21:05 AM UTC 2026 waiting for mixed seed72 endpoint
356
+ Thu Sep 3 08:21:35 AM UTC 2026 waiting for mixed seed72 endpoint
357
+ Thu Sep 3 08:22:05 AM UTC 2026 waiting for mixed seed72 endpoint
358
+ Thu Sep 3 08:22:35 AM UTC 2026 waiting for mixed seed72 endpoint
359
+ Thu Sep 3 08:23:05 AM UTC 2026 waiting for mixed seed72 endpoint
360
+ Thu Sep 3 08:23:35 AM UTC 2026 waiting for mixed seed72 endpoint
361
+ Thu Sep 3 08:24:05 AM UTC 2026 waiting for mixed seed72 endpoint
362
+ Thu Sep 3 08:24:35 AM UTC 2026 waiting for mixed seed72 endpoint
363
+ Thu Sep 3 08:25:05 AM UTC 2026 waiting for mixed seed72 endpoint
364
+ Thu Sep 3 08:25:35 AM UTC 2026 waiting for mixed seed72 endpoint
365
+ Thu Sep 3 08:26:05 AM UTC 2026 waiting for mixed seed72 endpoint
366
+ Thu Sep 3 08:26:35 AM UTC 2026 waiting for mixed seed72 endpoint
367
+ Thu Sep 3 08:27:05 AM UTC 2026 waiting for mixed seed72 endpoint
368
+ Thu Sep 3 08:27:35 AM UTC 2026 waiting for mixed seed72 endpoint
369
+ Thu Sep 3 08:28:05 AM UTC 2026 waiting for mixed seed72 endpoint
370
+ Thu Sep 3 08:28:35 AM UTC 2026 waiting for mixed seed72 endpoint
371
+ Thu Sep 3 08:29:05 AM UTC 2026 waiting for mixed seed72 endpoint
372
+ Thu Sep 3 08:29:35 AM UTC 2026 waiting for mixed seed72 endpoint
373
+ Thu Sep 3 08:30:05 AM UTC 2026 waiting for mixed seed72 endpoint
374
+ Thu Sep 3 08:30:35 AM UTC 2026 waiting for mixed seed72 endpoint
375
+ Thu Sep 3 08:31:05 AM UTC 2026 waiting for mixed seed72 endpoint
376
+ Thu Sep 3 08:31:35 AM UTC 2026 waiting for mixed seed72 endpoint
377
+ Thu Sep 3 08:32:05 AM UTC 2026 waiting for mixed seed72 endpoint
378
+ Thu Sep 3 08:32:35 AM UTC 2026 waiting for mixed seed72 endpoint
379
+ Thu Sep 3 08:33:05 AM UTC 2026 waiting for mixed seed72 endpoint
380
+ Thu Sep 3 08:33:35 AM UTC 2026 waiting for mixed seed72 endpoint
381
+ Thu Sep 3 08:34:05 AM UTC 2026 waiting for mixed seed72 endpoint
382
+ Thu Sep 3 08:34:35 AM UTC 2026 waiting for mixed seed72 endpoint
383
+ Thu Sep 3 08:35:05 AM UTC 2026 waiting for mixed seed72 endpoint
384
+ Thu Sep 3 08:35:35 AM UTC 2026 waiting for mixed seed72 endpoint
385
+ Thu Sep 3 08:36:05 AM UTC 2026 waiting for mixed seed72 endpoint
386
+ Thu Sep 3 08:36:35 AM UTC 2026 waiting for mixed seed72 endpoint
387
+ Thu Sep 3 08:37:05 AM UTC 2026 waiting for mixed seed72 endpoint
388
+ Thu Sep 3 08:37:35 AM UTC 2026 waiting for mixed seed72 endpoint
389
+ Thu Sep 3 08:38:05 AM UTC 2026 waiting for mixed seed72 endpoint
390
+ Thu Sep 3 08:38:35 AM UTC 2026 waiting for mixed seed72 endpoint
391
+ Thu Sep 3 08:39:05 AM UTC 2026 waiting for mixed seed72 endpoint
392
+ Thu Sep 3 08:39:35 AM UTC 2026 waiting for mixed seed72 endpoint
393
+ Thu Sep 3 08:40:05 AM UTC 2026 waiting for mixed seed72 endpoint
394
+ Thu Sep 3 08:40:35 AM UTC 2026 waiting for mixed seed72 endpoint
395
+ Thu Sep 3 08:41:05 AM UTC 2026 waiting for mixed seed72 endpoint
396
+ Thu Sep 3 08:41:35 AM UTC 2026 waiting for mixed seed72 endpoint
397
+ Thu Sep 3 08:42:05 AM UTC 2026 waiting for mixed seed72 endpoint
398
+ Thu Sep 3 08:42:35 AM UTC 2026 waiting for mixed seed72 endpoint
399
+ Thu Sep 3 08:43:05 AM UTC 2026 waiting for mixed seed72 endpoint
400
+ Thu Sep 3 08:43:35 AM UTC 2026 waiting for mixed seed72 endpoint
401
+ Thu Sep 3 08:44:05 AM UTC 2026 waiting for mixed seed72 endpoint
402
+ Thu Sep 3 08:44:35 AM UTC 2026 waiting for mixed seed72 endpoint
403
+ Thu Sep 3 08:45:05 AM UTC 2026 waiting for mixed seed72 endpoint
404
+ Thu Sep 3 08:45:35 AM UTC 2026 waiting for mixed seed72 endpoint
405
+ Thu Sep 3 08:46:05 AM UTC 2026 waiting for mixed seed72 endpoint
406
+ Thu Sep 3 08:46:35 AM UTC 2026 waiting for mixed seed72 endpoint
407
+ Thu Sep 3 08:47:05 AM UTC 2026 waiting for mixed seed72 endpoint
408
+ Thu Sep 3 08:47:35 AM UTC 2026 waiting for mixed seed72 endpoint
409
+ Thu Sep 3 08:48:05 AM UTC 2026 waiting for mixed seed72 endpoint
410
+ Thu Sep 3 08:48:35 AM UTC 2026 waiting for mixed seed72 endpoint
411
+ Thu Sep 3 08:49:05 AM UTC 2026 waiting for mixed seed72 endpoint
412
+ Thu Sep 3 08:49:35 AM UTC 2026 waiting for mixed seed72 endpoint
413
+ Thu Sep 3 08:50:05 AM UTC 2026 waiting for mixed seed72 endpoint
414
+ Thu Sep 3 08:50:35 AM UTC 2026 waiting for mixed seed72 endpoint
415
+ Thu Sep 3 08:51:05 AM UTC 2026 waiting for mixed seed72 endpoint
416
+ Thu Sep 3 08:51:35 AM UTC 2026 waiting for mixed seed72 endpoint
417
+ Thu Sep 3 08:52:05 AM UTC 2026 waiting for mixed seed72 endpoint
418
+ Thu Sep 3 08:52:35 AM UTC 2026 waiting for mixed seed72 endpoint
419
+ Thu Sep 3 08:53:05 AM UTC 2026 waiting for mixed seed72 endpoint
420
+ Thu Sep 3 08:53:35 AM UTC 2026 waiting for mixed seed72 endpoint
421
+ Thu Sep 3 08:54:05 AM UTC 2026 waiting for mixed seed72 endpoint
422
+ Thu Sep 3 08:54:35 AM UTC 2026 waiting for mixed seed72 endpoint
423
+ Thu Sep 3 08:55:05 AM UTC 2026 waiting for mixed seed72 endpoint
424
+ Thu Sep 3 08:55:35 AM UTC 2026 waiting for mixed seed72 endpoint
425
+ Thu Sep 3 08:56:06 AM UTC 2026 waiting for mixed seed72 endpoint
426
+ Thu Sep 3 08:56:36 AM UTC 2026 waiting for mixed seed72 endpoint
427
+ Thu Sep 3 08:57:06 AM UTC 2026 waiting for mixed seed72 endpoint
428
+ Thu Sep 3 08:57:36 AM UTC 2026 waiting for mixed seed72 endpoint
429
+ Thu Sep 3 08:58:06 AM UTC 2026 waiting for mixed seed72 endpoint
430
+ Thu Sep 3 08:58:36 AM UTC 2026 waiting for mixed seed72 endpoint
431
+ Thu Sep 3 08:59:06 AM UTC 2026 waiting for mixed seed72 endpoint
432
+ Thu Sep 3 08:59:36 AM UTC 2026 waiting for mixed seed72 endpoint
433
+ Thu Sep 3 09:00:06 AM UTC 2026 waiting for mixed seed72 endpoint
434
+ Thu Sep 3 09:00:36 AM UTC 2026 waiting for mixed seed72 endpoint
435
+ Thu Sep 3 09:01:06 AM UTC 2026 waiting for mixed seed72 endpoint
436
+ Thu Sep 3 09:01:36 AM UTC 2026 waiting for mixed seed72 endpoint
437
+ Thu Sep 3 09:02:06 AM UTC 2026 waiting for mixed seed72 endpoint
438
+ Thu Sep 3 09:02:36 AM UTC 2026 waiting for mixed seed72 endpoint
439
+ Thu Sep 3 09:03:06 AM UTC 2026 waiting for mixed seed72 endpoint
440
+ Thu Sep 3 09:03:36 AM UTC 2026 waiting for mixed seed72 endpoint
441
+ Thu Sep 3 09:04:06 AM UTC 2026 waiting for mixed seed72 endpoint
442
+ Thu Sep 3 09:04:36 AM UTC 2026 waiting for mixed seed72 endpoint
443
+ Thu Sep 3 09:05:06 AM UTC 2026 waiting for mixed seed72 endpoint
444
+ Thu Sep 3 09:05:36 AM UTC 2026 waiting for mixed seed72 endpoint
445
+ Thu Sep 3 09:06:06 AM UTC 2026 waiting for mixed seed72 endpoint
446
+ Thu Sep 3 09:06:36 AM UTC 2026 waiting for mixed seed72 endpoint
447
+ Thu Sep 3 09:07:06 AM UTC 2026 waiting for mixed seed72 endpoint
448
+ Thu Sep 3 09:07:36 AM UTC 2026 waiting for mixed seed72 endpoint
449
+ Thu Sep 3 09:08:06 AM UTC 2026 waiting for mixed seed72 endpoint
450
+ Thu Sep 3 09:08:36 AM UTC 2026 waiting for mixed seed72 endpoint
451
+ Thu Sep 3 09:09:06 AM UTC 2026 waiting for mixed seed72 endpoint
452
+ Thu Sep 3 09:09:36 AM UTC 2026 waiting for mixed seed72 endpoint
453
+ Thu Sep 3 09:10:06 AM UTC 2026 waiting for mixed seed72 endpoint
454
+ Thu Sep 3 09:10:36 AM UTC 2026 waiting for mixed seed72 endpoint
455
+ Thu Sep 3 09:11:06 AM UTC 2026 waiting for mixed seed72 endpoint
456
+ Thu Sep 3 09:11:36 AM UTC 2026 waiting for mixed seed72 endpoint
457
+ Thu Sep 3 09:12:06 AM UTC 2026 waiting for mixed seed72 endpoint
458
+ Thu Sep 3 09:12:36 AM UTC 2026 waiting for mixed seed72 endpoint
459
+ Thu Sep 3 09:13:06 AM UTC 2026 waiting for mixed seed72 endpoint
460
+ Thu Sep 3 09:13:36 AM UTC 2026 waiting for mixed seed72 endpoint
461
+ Thu Sep 3 09:14:06 AM UTC 2026 waiting for mixed seed72 endpoint
462
+ Thu Sep 3 09:14:36 AM UTC 2026 waiting for mixed seed72 endpoint
463
+ Thu Sep 3 09:15:06 AM UTC 2026 waiting for mixed seed72 endpoint
464
+ Thu Sep 3 09:15:36 AM UTC 2026 waiting for mixed seed72 endpoint
465
+ Thu Sep 3 09:16:06 AM UTC 2026 waiting for mixed seed72 endpoint
466
+ Thu Sep 3 09:16:36 AM UTC 2026 waiting for mixed seed72 endpoint
467
+ Thu Sep 3 09:17:06 AM UTC 2026 waiting for mixed seed72 endpoint
468
+ Thu Sep 3 09:17:36 AM UTC 2026 waiting for mixed seed72 endpoint
469
+ Thu Sep 3 09:18:06 AM UTC 2026 waiting for mixed seed72 endpoint
470
+ Thu Sep 3 09:18:36 AM UTC 2026 waiting for mixed seed72 endpoint
471
+ Thu Sep 3 09:19:06 AM UTC 2026 waiting for mixed seed72 endpoint
472
+ Thu Sep 3 09:19:36 AM UTC 2026 waiting for mixed seed72 endpoint
473
+ Thu Sep 3 09:20:06 AM UTC 2026 waiting for mixed seed72 endpoint
474
+ Thu Sep 3 09:20:36 AM UTC 2026 waiting for mixed seed72 endpoint
475
+ Thu Sep 3 09:21:06 AM UTC 2026 waiting for mixed seed72 endpoint
476
+ Thu Sep 3 09:21:36 AM UTC 2026 waiting for mixed seed72 endpoint
477
+ Thu Sep 3 09:22:06 AM UTC 2026 waiting for mixed seed72 endpoint
478
+ Thu Sep 3 09:22:36 AM UTC 2026 waiting for mixed seed72 endpoint
479
+ Thu Sep 3 09:23:06 AM UTC 2026 waiting for mixed seed72 endpoint
480
+ Thu Sep 3 09:23:36 AM UTC 2026 waiting for mixed seed72 endpoint
481
+ Thu Sep 3 09:24:06 AM UTC 2026 waiting for mixed seed72 endpoint
482
+ Thu Sep 3 09:24:36 AM UTC 2026 waiting for mixed seed72 endpoint
483
+ Thu Sep 3 09:25:06 AM UTC 2026 waiting for mixed seed72 endpoint
484
+ Thu Sep 3 09:25:36 AM UTC 2026 waiting for mixed seed72 endpoint
485
+ Thu Sep 3 09:26:06 AM UTC 2026 waiting for mixed seed72 endpoint
486
+ Thu Sep 3 09:26:36 AM UTC 2026 waiting for mixed seed72 endpoint
487
+ Thu Sep 3 09:27:06 AM UTC 2026 waiting for mixed seed72 endpoint
488
+ Thu Sep 3 09:27:36 AM UTC 2026 waiting for mixed seed72 endpoint
489
+ Thu Sep 3 09:28:06 AM UTC 2026 waiting for mixed seed72 endpoint
490
+ Thu Sep 3 09:28:36 AM UTC 2026 waiting for mixed seed72 endpoint
491
+ Thu Sep 3 09:29:06 AM UTC 2026 waiting for mixed seed72 endpoint
492
+ Thu Sep 3 09:29:36 AM UTC 2026 waiting for mixed seed72 endpoint
493
+ Thu Sep 3 09:30:06 AM UTC 2026 waiting for mixed seed72 endpoint
494
+ Thu Sep 3 09:30:36 AM UTC 2026 waiting for mixed seed72 endpoint
495
+ Thu Sep 3 09:31:06 AM UTC 2026 waiting for mixed seed72 endpoint
496
+ Thu Sep 3 09:31:36 AM UTC 2026 waiting for mixed seed72 endpoint
497
+ Thu Sep 3 09:32:06 AM UTC 2026 waiting for mixed seed72 endpoint
498
+ Thu Sep 3 09:32:36 AM UTC 2026 waiting for mixed seed72 endpoint
499
+ Thu Sep 3 09:33:06 AM UTC 2026 waiting for mixed seed72 endpoint
500
+ Thu Sep 3 09:33:36 AM UTC 2026 waiting for mixed seed72 endpoint
501
+ Thu Sep 3 09:34:06 AM UTC 2026 waiting for mixed seed72 endpoint
502
+ Thu Sep 3 09:34:36 AM UTC 2026 waiting for mixed seed72 endpoint
503
+ Thu Sep 3 09:35:06 AM UTC 2026 waiting for mixed seed72 endpoint
504
+ Thu Sep 3 09:35:36 AM UTC 2026 waiting for mixed seed72 endpoint
505
+ Thu Sep 3 09:36:06 AM UTC 2026 waiting for mixed seed72 endpoint
506
+ Thu Sep 3 09:36:36 AM UTC 2026 waiting for mixed seed72 endpoint
507
+ Thu Sep 3 09:37:06 AM UTC 2026 waiting for mixed seed72 endpoint
508
+ Thu Sep 3 09:37:36 AM UTC 2026 waiting for mixed seed72 endpoint
509
+ Thu Sep 3 09:38:06 AM UTC 2026 waiting for mixed seed72 endpoint
510
+ Thu Sep 3 09:38:36 AM UTC 2026 waiting for mixed seed72 endpoint
511
+ Thu Sep 3 09:39:06 AM UTC 2026 waiting for mixed seed72 endpoint
512
+ Thu Sep 3 09:39:36 AM UTC 2026 waiting for mixed seed72 endpoint
513
+ Thu Sep 3 09:40:06 AM UTC 2026 waiting for mixed seed72 endpoint
514
+ Thu Sep 3 09:40:36 AM UTC 2026 waiting for mixed seed72 endpoint
515
+ Thu Sep 3 09:41:06 AM UTC 2026 waiting for mixed seed72 endpoint
516
+ Thu Sep 3 09:41:36 AM UTC 2026 waiting for mixed seed72 endpoint
517
+ Thu Sep 3 09:42:06 AM UTC 2026 mixed endpoint found; launching dense seed72
518
+ W0903 09:42:08.662000 3686 site-packages/torch/distributed/run.py:851]
519
+ W0903 09:42:08.662000 3686 site-packages/torch/distributed/run.py:851] *****************************************
520
+ W0903 09:42:08.662000 3686 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
521
+ W0903 09:42:08.662000 3686 site-packages/torch/distributed/run.py:851] *****************************************
522
+ New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216
523
+ Running config:
524
+ {
525
+ "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
526
+ "model_dtype": "bfloat16",
527
+ "init_from_scratch": true,
528
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
529
+ "val_path": null,
530
+ "max_seq_length": 8192,
531
+ "num_workers": 8,
532
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216",
533
+ "resume_from_checkpoint": null,
534
+ "force_new_run": false,
535
+ "num_epochs": 1,
536
+ "max_steps": 50462,
537
+ "per_device_batch_size": 1,
538
+ "gradient_accumulation_steps": 15,
539
+ "learning_rate": 0.0005,
540
+ "weight_decay": 0.01,
541
+ "warmup_steps": 750,
542
+ "lr_scheduler": "cosine",
543
+ "max_grad_norm": 1.0,
544
+ "save_steps": 250,
545
+ "eval_steps": 250,
546
+ "eval_batches": 50,
547
+ "logging_steps": 10,
548
+ "seed": 72,
549
+ "verbose": true,
550
+ "checkpoint_activations": false,
551
+ "resume_micro_step_override": null,
552
+ "strategy": "ddp",
553
+ "mixed_precision": "bf16",
554
+ "wandb_enabled": true,
555
+ "wandb_project": "stochastic_sparsity",
556
+ "wandb_name": "1.2b dense 8k seed72 4gpu",
557
+ "wandb_tags": [
558
+ "qwen3",
559
+ "1.2b",
560
+ "8k",
561
+ "p2",
562
+ "dense",
563
+ "seed72"
564
+ ],
565
+ "hidden_size": 1536,
566
+ "intermediate_size": 6144,
567
+ "num_attention_heads": 12,
568
+ "num_key_value_heads": 4,
569
+ "num_hidden_layers": 28,
570
+ "head_dim": null,
571
+ "attention_dropout": null,
572
+ "rope_train_position_scale_set": null,
573
+ "attention_type": "sparse_stochastic",
574
+ "sparse_block_size": 128,
575
+ "sparse_num_random_blocks": 31,
576
+ "sparse_mask_pool_size": 1,
577
+ "sparse_val_dense_compare": false,
578
+ "sparse_layer_indices": null,
579
+ "sparse_mask_cache_dir": null,
580
+ "sparse_mask_seed": 0,
581
+ "sparse_mixed_nrb_set": null,
582
+ "dense_mode": true,
583
+ "dense_step_intervals": null
584
+ }
585
+ Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
586
+ Checkpoint activations: False
587
+ Checkpoint activations: False
588
+ Initializing model from scratch (random weights)...
589
+ Checkpoint activations: False
590
+ Checkpoint activations: False
591
+ dense_mode=True: sparse attention layers will use standard causal SDPA.
592
+
593
+
594
+ ================================================================================
595
+ MODEL ARCHITECTURE
596
+ ================================================================================
597
+ Model config:
598
+ vocab_size: 151936
599
+ hidden_size: 1536
600
+ num_hidden_layers: 28
601
+ num_attention_heads: 12
602
+ num_key_value_heads: 4
603
+ head_dim: 128
604
+ intermediate_size: 6144
605
+ max_position_embeddings: 32768
606
+ rope_theta: 1000000
607
+ rope_train_position_scale_set: None
608
+ attention_type: sparse_stochastic
609
+ sparse_block_size: 128
610
+ sparse_num_random_blocks: 31
611
+ sparse_mask_pool_size: 1
612
+ sparse_max_seq_length: 8192
613
+
614
+ Parameter count:
615
+ Total parameters: 1,202,352,640
616
+ Trainable parameters: 1,202,352,640
617
+ Non-trainable parameters: 0
618
+ Model size (BF16): 2.40 GB
619
+
620
+ --------------------------------------------------------------------------------
621
+ LAYER-BY-LAYER SUMMARY
622
+ --------------------------------------------------------------------------------
623
+ ================================================================================================================================================================
624
+ Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
625
+ ================================================================================================================================================================
626
+ Qwen3ForCausalLM [1, 8192] -- -- True
627
+ ├─Qwen3Model: 1-1 -- [1, 8192, 1536] -- True
628
+ │ └─Embedding: 2-1 [1, 8192] [1, 8192, 1536] 233,373,696 True
629
+ │ └─ModuleList: 2-2 -- -- -- True
630
+ │ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
631
+ │ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
632
+ │ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
633
+ │ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
634
+ │ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
635
+ │ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
636
+ │ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
637
+ │ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
638
+ │ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
639
+ │ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
640
+ │ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
641
+ │ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
642
+ │ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
643
+ │ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
644
+ │ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
645
+ │ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
646
+ │ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
647
+ │ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
648
+ │ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
649
+ │ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
650
+ │ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
651
+ │ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
652
+ │ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
653
+ │ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
654
+ │ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
655
+ │ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
656
+ │ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
657
+ │ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
658
+ │ └─Qwen3RMSNorm: 2-3 [1, 8192, 1536] [1, 8192, 1536] 1,536 True
659
+ ├─Linear: 1-2 [1, 8192, 1536] [1, 8192, 151936] 233,373,696 True
660
+ ================================================================================================================================================================
661
+ Total params: 1,435,726,336
662
+ Trainable params: 1,435,726,336
663
+ Non-trainable params: 0
664
+ Total mult-adds (Units.GIGABYTES): 1.44
665
+ ================================================================================================================================================================
666
+ Input size (MB): 0.07
667
+ Forward/backward pass size (MB): 26218.59
668
+ Params size (MB): 2871.45
669
+ Estimated Total Size (MB): 29090.11
670
+ ================================================================================================================================================================
671
+ NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
672
+ ================================================================================
673
+
674
+ Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
675
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
676
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
677
+ Using pre-tokenized dataset (no runtime tokenization)
678
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
679
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
680
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
681
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
682
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
683
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
684
+ Dataset split: 3027764 train, 159356 validation
685
+ Training examples (packed, train split): 3027764
686
+ Validation examples (packed, val split): 159356
687
+ Per-device batch size: 1
688
+ World size (GPUs): 4
689
+ Gradient accumulation steps: 15
690
+ Effective batch size (seqs/step): 60
691
+ Sequence length: 8192
692
+ Tokens per optimizer step: 491,520
693
+ Micro-batches per epoch (per rank): 756941
694
+ Optimizer steps per epoch: 50462
695
+ Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
696
+ wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
697
+ wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
698
+ wandb: Tracking run with wandb version 0.26.1
699
+ wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260903_094250-7tfq6nwm
700
+ wandb: Run `wandb offline` to turn off syncing.
701
+ wandb: Syncing run 1.2b dense 8k seed72 4gpu
702
+ wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
703
+ wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/7tfq6nwm
704
+ Starting training...
705
+ Switching attention mode for step 1: dense (dense step interval)
706
+ Step 10/50462 | Loss: 12.2551 | LR: 6.67e-06 | 7102.4ms/step | 8.4 samples/s
707
+ Step 20/50462 | Loss: 11.9598 | LR: 1.33e-05 | 2812.8ms/step | 21.3 samples/s
708
+ Step 30/50462 | Loss: 11.0754 | LR: 2.00e-05 | 2815.7ms/step | 21.3 samples/s
709
+ Step 40/50462 | Loss: 10.1714 | LR: 2.67e-05 | 2807.9ms/step | 21.4 samples/s
710
+ Step 50/50462 | Loss: 9.7674 | LR: 3.33e-05 | 2810.0ms/step | 21.4 samples/s
711
+ Step 60/50462 | Loss: 9.3025 | LR: 4.00e-05 | 2813.9ms/step | 21.3 samples/s
712
+ Step 70/50462 | Loss: 8.7641 | LR: 4.67e-05 | 2814.0ms/step | 21.3 samples/s
713
+ Step 80/50462 | Loss: 8.3041 | LR: 5.33e-05 | 2804.3ms/step | 21.4 samples/s
714
+ Step 90/50462 | Loss: 7.9228 | LR: 6.00e-05 | 2804.0ms/step | 21.4 samples/s
715
+ Step 100/50462 | Loss: 7.5357 | LR: 6.67e-05 | 2816.2ms/step | 21.3 samples/s
716
+ Step 110/50462 | Loss: 7.2514 | LR: 7.33e-05 | 2812.8ms/step | 21.3 samples/s
717
+ Step 120/50462 | Loss: 7.0840 | LR: 8.00e-05 | 2816.8ms/step | 21.3 samples/s
718
+ Step 130/50462 | Loss: 6.9494 | LR: 8.67e-05 | 2813.0ms/step | 21.3 samples/s
719
+ Step 140/50462 | Loss: 6.8809 | LR: 9.33e-05 | 2816.8ms/step | 21.3 samples/s
720
+ Step 150/50462 | Loss: 6.7717 | LR: 1.00e-04 | 2820.0ms/step | 21.3 samples/s
721
+ Step 160/50462 | Loss: 6.6834 | LR: 1.07e-04 | 2819.2ms/step | 21.3 samples/s
722
+ Step 170/50462 | Loss: 6.5857 | LR: 1.13e-04 | 2820.4ms/step | 21.3 samples/s
723
+ Step 180/50462 | Loss: 6.4664 | LR: 1.20e-04 | 2820.4ms/step | 21.3 samples/s
724
+ Step 190/50462 | Loss: 6.3764 | LR: 1.27e-04 | 2821.6ms/step | 21.3 samples/s
725
+ Step 200/50462 | Loss: 6.3100 | LR: 1.33e-04 | 2820.9ms/step | 21.3 samples/s
726
+ Step 210/50462 | Loss: 6.2129 | LR: 1.40e-04 | 2820.0ms/step | 21.3 samples/s
727
+ Step 220/50462 | Loss: 6.1440 | LR: 1.47e-04 | 2821.8ms/step | 21.3 samples/s
728
+ Step 230/50462 | Loss: 6.1100 | LR: 1.53e-04 | 2824.6ms/step | 21.2 samples/s
729
+ Step 240/50462 | Loss: 6.0559 | LR: 1.60e-04 | 2826.9ms/step | 21.2 samples/s
730
+ Step 250/50462 | Loss: 6.0007 | LR: 1.67e-04 | 2827.9ms/step | 21.2 samples/s
731
+ Validation at step 250: Loss = 5.9550, Perplexity = 385.66
732
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-250
733
+ Step 260/50462 | Loss: 5.9754 | LR: 1.73e-04 | 7358.2ms/step | 8.2 samples/s
734
+ Step 270/50462 | Loss: 5.8872 | LR: 1.80e-04 | 2818.4ms/step | 21.3 samples/s
735
+ Step 280/50462 | Loss: 5.8567 | LR: 1.87e-04 | 2827.1ms/step | 21.2 samples/s
736
+ Step 290/50462 | Loss: 5.7731 | LR: 1.93e-04 | 2828.9ms/step | 21.2 samples/s
737
+ Step 300/50462 | Loss: 5.7089 | LR: 2.00e-04 | 2830.0ms/step | 21.2 samples/s
738
+ Step 310/50462 | Loss: 5.6774 | LR: 2.07e-04 | 2831.4ms/step | 21.2 samples/s
739
+ Step 320/50462 | Loss: 5.6239 | LR: 2.13e-04 | 2830.4ms/step | 21.2 samples/s
740
+ Step 330/50462 | Loss: 5.5787 | LR: 2.20e-04 | 2827.7ms/step | 21.2 samples/s
741
+ Step 340/50462 | Loss: 5.5174 | LR: 2.27e-04 | 2830.8ms/step | 21.2 samples/s
742
+ Step 350/50462 | Loss: 5.4471 | LR: 2.33e-04 | 2831.1ms/step | 21.2 samples/s
743
+ Step 360/50462 | Loss: 5.4239 | LR: 2.40e-04 | 2830.3ms/step | 21.2 samples/s
744
+ Step 370/50462 | Loss: 5.3810 | LR: 2.47e-04 | 2833.3ms/step | 21.2 samples/s
745
+ Step 380/50462 | Loss: 5.3305 | LR: 2.53e-04 | 2826.2ms/step | 21.2 samples/s
746
+ Step 390/50462 | Loss: 5.2998 | LR: 2.60e-04 | 2831.7ms/step | 21.2 samples/s
747
+ Step 400/50462 | Loss: 5.2540 | LR: 2.67e-04 | 2831.9ms/step | 21.2 samples/s
748
+ Step 410/50462 | Loss: 5.2052 | LR: 2.73e-04 | 2831.6ms/step | 21.2 samples/s
749
+ Step 420/50462 | Loss: 5.1640 | LR: 2.80e-04 | 2836.0ms/step | 21.2 samples/s
750
+ Step 430/50462 | Loss: 5.1449 | LR: 2.87e-04 | 2830.6ms/step | 21.2 samples/s
751
+ Step 440/50462 | Loss: 5.0836 | LR: 2.93e-04 | 2830.7ms/step | 21.2 samples/s
752
+ Step 450/50462 | Loss: 5.0509 | LR: 3.00e-04 | 2835.3ms/step | 21.2 samples/s
753
+ Step 460/50462 | Loss: 4.9936 | LR: 3.07e-04 | 2835.2ms/step | 21.2 samples/s
754
+ Step 470/50462 | Loss: 4.9375 | LR: 3.13e-04 | 2834.5ms/step | 21.2 samples/s
755
+ Step 480/50462 | Loss: 4.9336 | LR: 3.20e-04 | 2835.2ms/step | 21.2 samples/s
756
+ Step 490/50462 | Loss: 4.9644 | LR: 3.27e-04 | 2834.1ms/step | 21.2 samples/s
757
+ Step 500/50462 | Loss: 4.8714 | LR: 3.33e-04 | 2835.8ms/step | 21.2 samples/s
758
+ Validation at step 500: Loss = 4.8220, Perplexity = 124.22
759
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-500
760
+ Step 510/50462 | Loss: 4.7846 | LR: 3.40e-04 | 4784.4ms/step | 12.5 samples/s
761
+ Step 520/50462 | Loss: 4.8335 | LR: 3.47e-04 | 2823.5ms/step | 21.3 samples/s
762
+ Step 530/50462 | Loss: 4.8520 | LR: 3.53e-04 | 2830.0ms/step | 21.2 samples/s
763
+ Step 540/50462 | Loss: 4.7551 | LR: 3.60e-04 | 2829.8ms/step | 21.2 samples/s
764
+ Step 550/50462 | Loss: 4.7056 | LR: 3.67e-04 | 2828.8ms/step | 21.2 samples/s
765
+ Step 560/50462 | Loss: 4.6716 | LR: 3.73e-04 | 2831.2ms/step | 21.2 samples/s
766
+ Step 570/50462 | Loss: 4.7708 | LR: 3.80e-04 | 2835.8ms/step | 21.2 samples/s
767
+ Step 580/50462 | Loss: 4.7270 | LR: 3.87e-04 | 2829.3ms/step | 21.2 samples/s
768
+ Step 590/50462 | Loss: 4.6082 | LR: 3.93e-04 | 2830.9ms/step | 21.2 samples/s
769
+ Step 600/50462 | Loss: 4.5363 | LR: 4.00e-04 | 2828.5ms/step | 21.2 samples/s
770
+ Step 610/50462 | Loss: 4.5271 | LR: 4.07e-04 | 2828.2ms/step | 21.2 samples/s
771
+ Step 620/50462 | Loss: 4.5835 | LR: 4.13e-04 | 2829.7ms/step | 21.2 samples/s
772
+ Step 630/50462 | Loss: 4.5551 | LR: 4.20e-04 | 2826.9ms/step | 21.2 samples/s
773
+ Step 640/50462 | Loss: 4.4513 | LR: 4.27e-04 | 2829.1ms/step | 21.2 samples/s
774
+ Step 650/50462 | Loss: 4.3682 | LR: 4.33e-04 | 2828.8ms/step | 21.2 samples/s
775
+ Step 660/50462 | Loss: 4.4221 | LR: 4.40e-04 | 2830.3ms/step | 21.2 samples/s
776
+ Step 670/50462 | Loss: 4.4865 | LR: 4.47e-04 | 2826.9ms/step | 21.2 samples/s
777
+ Step 680/50462 | Loss: 4.4094 | LR: 4.53e-04 | 2817.4ms/step | 21.3 samples/s
778
+ Step 690/50462 | Loss: 4.2839 | LR: 4.60e-04 | 2828.3ms/step | 21.2 samples/s
779
+ Step 700/50462 | Loss: 4.2525 | LR: 4.67e-04 | 2829.7ms/step | 21.2 samples/s
780
+ Step 710/50462 | Loss: 4.2754 | LR: 4.73e-04 | 2830.2ms/step | 21.2 samples/s
781
+ Step 720/50462 | Loss: 4.2294 | LR: 4.80e-04 | 2833.6ms/step | 21.2 samples/s
782
+ Step 730/50462 | Loss: 4.1987 | LR: 4.87e-04 | 2832.7ms/step | 21.2 samples/s
783
+ Step 740/50462 | Loss: 4.1587 | LR: 4.93e-04 | 2832.6ms/step | 21.2 samples/s
784
+ Step 750/50462 | Loss: 4.1572 | LR: 5.00e-04 | 2832.4ms/step | 21.2 samples/s
785
+ Validation at step 750: Loss = 4.1413, Perplexity = 62.89
786
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-750
787
+ Step 760/50462 | Loss: 4.1212 | LR: 5.00e-04 | 4775.5ms/step | 12.6 samples/s
788
+ Step 770/50462 | Loss: 4.0936 | LR: 5.00e-04 | 2820.0ms/step | 21.3 samples/s
789
+ Step 780/50462 | Loss: 4.0432 | LR: 5.00e-04 | 2832.9ms/step | 21.2 samples/s
790
+ Step 790/50462 | Loss: 4.0721 | LR: 5.00e-04 | 2831.8ms/step | 21.2 samples/s
791
+ Step 800/50462 | Loss: 4.0250 | LR: 5.00e-04 | 2831.9ms/step | 21.2 samples/s
792
+ Step 810/50462 | Loss: 3.9828 | LR: 5.00e-04 | 2832.6ms/step | 21.2 samples/s
793
+ Step 820/50462 | Loss: 3.9879 | LR: 5.00e-04 | 2833.5ms/step | 21.2 samples/s
794
+ Step 830/50462 | Loss: 3.9419 | LR: 5.00e-04 | 2834.4ms/step | 21.2 samples/s
795
+ Step 840/50462 | Loss: 3.9689 | LR: 5.00e-04 | 2836.0ms/step | 21.2 samples/s
796
+ Step 850/50462 | Loss: 3.9334 | LR: 5.00e-04 | 2832.6ms/step | 21.2 samples/s
797
+ Step 860/50462 | Loss: 3.9061 | LR: 5.00e-04 | 2830.4ms/step | 21.2 samples/s
798
+ Step 870/50462 | Loss: 3.8820 | LR: 5.00e-04 | 2833.4ms/step | 21.2 samples/s
799
+ Step 880/50462 | Loss: 3.8880 | LR: 5.00e-04 | 2833.7ms/step | 21.2 samples/s
800
+ Step 890/50462 | Loss: 3.8578 | LR: 5.00e-04 | 2834.7ms/step | 21.2 samples/s
801
+ Step 900/50462 | Loss: 3.8720 | LR: 5.00e-04 | 2832.5ms/step | 21.2 samples/s
802
+ Step 910/50462 | Loss: 3.7852 | LR: 5.00e-04 | 2833.9ms/step | 21.2 samples/s
803
+ Step 920/50462 | Loss: 3.8342 | LR: 5.00e-04 | 2836.7ms/step | 21.2 samples/s
804
+ Step 930/50462 | Loss: 3.8007 | LR: 5.00e-04 | 2837.0ms/step | 21.1 samples/s
805
+ Step 940/50462 | Loss: 3.8378 | LR: 5.00e-04 | 2835.7ms/step | 21.2 samples/s
806
+ Step 950/50462 | Loss: 3.7768 | LR: 5.00e-04 | 2836.2ms/step | 21.2 samples/s
807
+ Step 960/50462 | Loss: 3.7685 | LR: 5.00e-04 | 2834.7ms/step | 21.2 samples/s
808
+ Step 970/50462 | Loss: 3.7626 | LR: 5.00e-04 | 2835.7ms/step | 21.2 samples/s
809
+ Step 980/50462 | Loss: 3.7532 | LR: 5.00e-04 | 2835.7ms/step | 21.2 samples/s
810
+ Step 990/50462 | Loss: 3.7420 | LR: 5.00e-04 | 2833.1ms/step | 21.2 samples/s
811
+ Step 1000/50462 | Loss: 3.6888 | LR: 5.00e-04 | 2835.1ms/step | 21.2 samples/s
812
+ Validation at step 1000: Loss = 3.7120, Perplexity = 40.94
813
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-1000
814
+ Step 1010/50462 | Loss: 3.7392 | LR: 5.00e-04 | 4769.8ms/step | 12.6 samples/s
815
+ Step 1020/50462 | Loss: 3.6858 | LR: 5.00e-04 | 2823.0ms/step | 21.3 samples/s
816
+ Step 1030/50462 | Loss: 3.6780 | LR: 5.00e-04 | 2821.9ms/step | 21.3 samples/s
817
+ Step 1040/50462 | Loss: 3.6940 | LR: 5.00e-04 | 2831.0ms/step | 21.2 samples/s
818
+ Step 1050/50462 | Loss: 3.6797 | LR: 5.00e-04 | 2832.5ms/step | 21.2 samples/s
819
+ Step 1060/50462 | Loss: 3.6590 | LR: 5.00e-04 | 2833.9ms/step | 21.2 samples/s
820
+ Step 1070/50462 | Loss: 3.6346 | LR: 5.00e-04 | 2834.3ms/step | 21.2 samples/s
821
+ Step 1080/50462 | Loss: 3.6330 | LR: 5.00e-04 | 2831.1ms/step | 21.2 samples/s
822
+ Step 1090/50462 | Loss: 3.6322 | LR: 5.00e-04 | 2830.9ms/step | 21.2 samples/s
823
+ Step 1100/50462 | Loss: 3.6065 | LR: 5.00e-04 | 2832.0ms/step | 21.2 samples/s
824
+ Step 1110/50462 | Loss: 3.6215 | LR: 5.00e-04 | 2835.8ms/step | 21.2 samples/s
825
+ Step 1120/50462 | Loss: 3.6081 | LR: 5.00e-04 | 2834.7ms/step | 21.2 samples/s
826
+ Step 1130/50462 | Loss: 3.5864 | LR: 5.00e-04 | 2836.9ms/step | 21.2 samples/s
827
+ Step 1140/50462 | Loss: 3.5764 | LR: 5.00e-04 | 2833.3ms/step | 21.2 samples/s
828
+ Step 1150/50462 | Loss: 3.5661 | LR: 5.00e-04 | 2834.6ms/step | 21.2 samples/s
829
+ Step 1160/50462 | Loss: 3.5526 | LR: 5.00e-04 | 2837.5ms/step | 21.1 samples/s
830
+ Step 1170/50462 | Loss: 3.5200 | LR: 5.00e-04 | 2838.2ms/step | 21.1 samples/s
831
+ Step 1180/50462 | Loss: 3.5774 | LR: 5.00e-04 | 2835.7ms/step | 21.2 samples/s
832
+ Step 1190/50462 | Loss: 3.5498 | LR: 5.00e-04 | 2832.9ms/step | 21.2 samples/s
833
+ Step 1200/50462 | Loss: 3.5842 | LR: 5.00e-04 | 2835.5ms/step | 21.2 samples/s
834
+ Step 1210/50462 | Loss: 3.5219 | LR: 5.00e-04 | 2834.7ms/step | 21.2 samples/s
835
+ Step 1220/50462 | Loss: 3.5338 | LR: 5.00e-04 | 2834.9ms/step | 21.2 samples/s
836
+ Step 1230/50462 | Loss: 3.5167 | LR: 5.00e-04 | 2830.6ms/step | 21.2 samples/s
837
+ Step 1240/50462 | Loss: 3.5601 | LR: 5.00e-04 | 2830.1ms/step | 21.2 samples/s
838
+ Step 1250/50462 | Loss: 3.5059 | LR: 5.00e-04 | 2833.6ms/step | 21.2 samples/s
839
+ Validation at step 1250: Loss = 3.5163, Perplexity = 33.66
840
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-1250
841
+ Step 1260/50462 | Loss: 3.4926 | LR: 5.00e-04 | 4778.8ms/step | 12.6 samples/s
842
+ Step 1270/50462 | Loss: 3.4753 | LR: 5.00e-04 | 2822.1ms/step | 21.3 samples/s
843
+ Step 1280/50462 | Loss: 3.5182 | LR: 5.00e-04 | 2831.1ms/step | 21.2 samples/s
844
+ Step 1290/50462 | Loss: 3.4682 | LR: 5.00e-04 | 2833.9ms/step | 21.2 samples/s
845
+ Step 1300/50462 | Loss: 3.4608 | LR: 5.00e-04 | 2831.3ms/step | 21.2 samples/s
846
+ Step 1310/50462 | Loss: 3.4649 | LR: 5.00e-04 | 2833.8ms/step | 21.2 samples/s
847
+ Step 1320/50462 | Loss: 3.4572 | LR: 5.00e-04 | 2836.6ms/step | 21.2 samples/s
848
+ Step 1330/50462 | Loss: 3.4792 | LR: 5.00e-04 | 2836.4ms/step | 21.2 samples/s
849
+ Step 1340/50462 | Loss: 3.4735 | LR: 5.00e-04 | 2833.5ms/step | 21.2 samples/s
850
+ Step 1350/50462 | Loss: 3.4699 | LR: 5.00e-04 | 2832.1ms/step | 21.2 samples/s
851
+ Step 1360/50462 | Loss: 3.4395 | LR: 5.00e-04 | 2835.7ms/step | 21.2 samples/s
852
+ Step 1370/50462 | Loss: 3.4476 | LR: 5.00e-04 | 2833.9ms/step | 21.2 samples/s
853
+ Step 1380/50462 | Loss: 3.4749 | LR: 5.00e-04 | 2836.8ms/step | 21.2 samples/s
854
+ Step 1390/50462 | Loss: 3.4820 | LR: 5.00e-04 | 2832.5ms/step | 21.2 samples/s
855
+ Step 1400/50462 | Loss: 3.4441 | LR: 5.00e-04 | 2832.5ms/step | 21.2 samples/s
856
+ Step 1410/50462 | Loss: 3.4216 | LR: 5.00e-04 | 2834.7ms/step | 21.2 samples/s
857
+ Step 1420/50462 | Loss: 3.3950 | LR: 5.00e-04 | 2833.8ms/step | 21.2 samples/s
858
+ Step 1430/50462 | Loss: 3.3444 | LR: 5.00e-04 | 2831.1ms/step | 21.2 samples/s
859
+ Step 1440/50462 | Loss: 3.4054 | LR: 5.00e-04 | 2837.5ms/step | 21.1 samples/s
860
+ Step 1450/50462 | Loss: 3.3769 | LR: 5.00e-04 | 2834.2ms/step | 21.2 samples/s
861
+ Step 1460/50462 | Loss: 3.3750 | LR: 5.00e-04 | 2834.5ms/step | 21.2 samples/s
862
+ Step 1470/50462 | Loss: 3.4102 | LR: 5.00e-04 | 2835.8ms/step | 21.2 samples/s
863
+ Step 1480/50462 | Loss: 3.3778 | LR: 5.00e-04 | 2836.2ms/step | 21.2 samples/s
864
+ Step 1490/50462 | Loss: 3.3993 | LR: 5.00e-04 | 2833.3ms/step | 21.2 samples/s
865
+ Step 1500/50462 | Loss: 3.3561 | LR: 5.00e-04 | 2832.9ms/step | 21.2 samples/s
866
+ Validation at step 1500: Loss = 3.3873, Perplexity = 29.59
867
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-1500
868
+ Step 1510/50462 | Loss: 3.3690 | LR: 5.00e-04 | 4772.0ms/step | 12.6 samples/s
869
+ Step 1520/50462 | Loss: 3.3898 | LR: 5.00e-04 | 2826.9ms/step | 21.2 samples/s
870
+ Step 1530/50462 | Loss: 3.3916 | LR: 5.00e-04 | 2831.4ms/step | 21.2 samples/s
871
+ Step 1540/50462 | Loss: 3.3581 | LR: 5.00e-04 | 2832.5ms/step | 21.2 samples/s
872
+ Step 1550/50462 | Loss: 3.3440 | LR: 5.00e-04 | 2833.3ms/step | 21.2 samples/s
873
+ Step 1560/50462 | Loss: 3.3866 | LR: 5.00e-04 | 2834.3ms/step | 21.2 samples/s
874
+ Step 1570/50462 | Loss: 3.3657 | LR: 5.00e-04 | 2834.3ms/step | 21.2 samples/s
875
+ Step 1580/50462 | Loss: 3.3420 | LR: 5.00e-04 | 2832.6ms/step | 21.2 samples/s
876
+ Step 1590/50462 | Loss: 3.3942 | LR: 5.00e-04 | 2832.5ms/step | 21.2 samples/s
877
+ Step 1600/50462 | Loss: 3.3448 | LR: 5.00e-04 | 2830.8ms/step | 21.2 samples/s
878
+ Step 1610/50462 | Loss: 3.3820 | LR: 5.00e-04 | 2831.1ms/step | 21.2 samples/s
879
+ Step 1620/50462 | Loss: 3.3357 | LR: 5.00e-04 | 2832.2ms/step | 21.2 samples/s
880
+ Step 1630/50462 | Loss: 3.3315 | LR: 5.00e-04 | 2832.8ms/step | 21.2 samples/s
881
+ Step 1640/50462 | Loss: 3.3203 | LR: 5.00e-04 | 2833.5ms/step | 21.2 samples/s
882
+ Step 1650/50462 | Loss: 3.3406 | LR: 5.00e-04 | 2835.3ms/step | 21.2 samples/s
883
+ Step 1660/50462 | Loss: 3.3265 | LR: 5.00e-04 | 2832.0ms/step | 21.2 samples/s
884
+ Step 1670/50462 | Loss: 3.3385 | LR: 5.00e-04 | 2833.9ms/step | 21.2 samples/s
885
+ Step 1680/50462 | Loss: 3.3123 | LR: 5.00e-04 | 2831.5ms/step | 21.2 samples/s
886
+ Step 1690/50462 | Loss: 3.2875 | LR: 5.00e-04 | 2833.7ms/step | 21.2 samples/s
887
+ Step 1700/50462 | Loss: 3.3084 | LR: 5.00e-04 | 2834.1ms/step | 21.2 samples/s
888
+ Step 1710/50462 | Loss: 3.2776 | LR: 5.00e-04 | 2835.5ms/step | 21.2 samples/s
889
+ Step 1720/50462 | Loss: 3.3152 | LR: 5.00e-04 | 2832.9ms/step | 21.2 samples/s
890
+ Step 1730/50462 | Loss: 3.3060 | LR: 5.00e-04 | 2832.0ms/step | 21.2 samples/s
891
+ Step 1740/50462 | Loss: 3.2981 | LR: 5.00e-04 | 2832.6ms/step | 21.2 samples/s
892
+ Step 1750/50462 | Loss: 3.3277 | LR: 5.00e-04 | 2833.0ms/step | 21.2 samples/s
893
+ Validation at step 1750: Loss = 3.3060, Perplexity = 27.27
894
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-1750
895
+ Step 1760/50462 | Loss: 3.3017 | LR: 4.99e-04 | 4767.8ms/step | 12.6 samples/s
896
+ Step 1770/50462 | Loss: 3.2843 | LR: 4.99e-04 | 2821.2ms/step | 21.3 samples/s
897
+ Step 1780/50462 | Loss: 3.2876 | LR: 4.99e-04 | 2825.9ms/step | 21.2 samples/s
898
+ Step 1790/50462 | Loss: 3.2969 | LR: 4.99e-04 | 2827.5ms/step | 21.2 samples/s
899
+ Step 1800/50462 | Loss: 3.3030 | LR: 4.99e-04 | 2826.7ms/step | 21.2 samples/s
900
+ Step 1810/50462 | Loss: 3.2772 | LR: 4.99e-04 | 2828.0ms/step | 21.2 samples/s
901
+ Step 1820/50462 | Loss: 3.2778 | LR: 4.99e-04 | 2831.3ms/step | 21.2 samples/s
902
+ Step 1830/50462 | Loss: 3.3006 | LR: 4.99e-04 | 2827.9ms/step | 21.2 samples/s
903
+ Step 1840/50462 | Loss: 3.3068 | LR: 4.99e-04 | 2831.2ms/step | 21.2 samples/s
904
+ Step 1850/50462 | Loss: 3.2800 | LR: 4.99e-04 | 2831.1ms/step | 21.2 samples/s
905
+ Step 1860/50462 | Loss: 3.2934 | LR: 4.99e-04 | 2828.0ms/step | 21.2 samples/s
906
+ Step 1870/50462 | Loss: 3.3034 | LR: 4.99e-04 | 2831.1ms/step | 21.2 samples/s
907
+ Step 1880/50462 | Loss: 3.2440 | LR: 4.99e-04 | 2836.0ms/step | 21.2 samples/s
908
+ Step 1890/50462 | Loss: 3.2497 | LR: 4.99e-04 | 2835.8ms/step | 21.2 samples/s
909
+ Step 1900/50462 | Loss: 3.2477 | LR: 4.99e-04 | 2832.5ms/step | 21.2 samples/s
910
+ Step 1910/50462 | Loss: 3.2819 | LR: 4.99e-04 | 2831.3ms/step | 21.2 samples/s
911
+ Step 1920/50462 | Loss: 3.2103 | LR: 4.99e-04 | 2831.0ms/step | 21.2 samples/s
912
+ Step 1930/50462 | Loss: 3.2519 | LR: 4.99e-04 | 2827.6ms/step | 21.2 samples/s
913
+ Step 1940/50462 | Loss: 3.2573 | LR: 4.99e-04 | 2829.5ms/step | 21.2 samples/s
914
+ Step 1950/50462 | Loss: 3.2266 | LR: 4.99e-04 | 2829.8ms/step | 21.2 samples/s
915
+ Step 1960/50462 | Loss: 3.2253 | LR: 4.99e-04 | 2827.9ms/step | 21.2 samples/s
916
+ Step 1970/50462 | Loss: 3.2139 | LR: 4.99e-04 | 2822.1ms/step | 21.3 samples/s
917
+ Step 1980/50462 | Loss: 3.2390 | LR: 4.99e-04 | 2824.2ms/step | 21.2 samples/s
918
+ Step 1990/50462 | Loss: 3.2532 | LR: 4.99e-04 | 2826.8ms/step | 21.2 samples/s
919
+ Step 2000/50462 | Loss: 3.2330 | LR: 4.99e-04 | 2827.6ms/step | 21.2 samples/s
920
+ Validation at step 2000: Loss = 3.2365, Perplexity = 25.44
921
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-2000
922
+ Step 2010/50462 | Loss: 3.2142 | LR: 4.99e-04 | 4786.8ms/step | 12.5 samples/s
923
+ Step 2020/50462 | Loss: 3.2049 | LR: 4.99e-04 | 2827.6ms/step | 21.2 samples/s
924
+ Step 2030/50462 | Loss: 3.2170 | LR: 4.99e-04 | 2829.6ms/step | 21.2 samples/s
925
+ Step 2040/50462 | Loss: 3.2108 | LR: 4.99e-04 | 2832.1ms/step | 21.2 samples/s
926
+ Step 2050/50462 | Loss: 3.2076 | LR: 4.99e-04 | 2830.7ms/step | 21.2 samples/s
927
+ Step 2060/50462 | Loss: 3.2354 | LR: 4.99e-04 | 2830.8ms/step | 21.2 samples/s
928
+ Step 2070/50462 | Loss: 3.1763 | LR: 4.99e-04 | 2830.0ms/step | 21.2 samples/s
929
+ Step 2080/50462 | Loss: 3.1992 | LR: 4.99e-04 | 2832.8ms/step | 21.2 samples/s
930
+ Step 2090/50462 | Loss: 3.1724 | LR: 4.99e-04 | 2832.2ms/step | 21.2 samples/s
931
+ Step 2100/50462 | Loss: 3.1840 | LR: 4.99e-04 | 2833.0ms/step | 21.2 samples/s
932
+ Step 2110/50462 | Loss: 3.1862 | LR: 4.99e-04 | 2829.5ms/step | 21.2 samples/s
933
+ Step 2120/50462 | Loss: 3.1750 | LR: 4.99e-04 | 2832.5ms/step | 21.2 samples/s
934
+ Step 2130/50462 | Loss: 3.2136 | LR: 4.99e-04 | 2828.8ms/step | 21.2 samples/s
935
+ Step 2140/50462 | Loss: 3.2211 | LR: 4.99e-04 | 2828.8ms/step | 21.2 samples/s
936
+ Step 2150/50462 | Loss: 3.1850 | LR: 4.99e-04 | 2830.5ms/step | 21.2 samples/s
937
+ Step 2160/50462 | Loss: 3.2123 | LR: 4.99e-04 | 2833.0ms/step | 21.2 samples/s
938
+ Step 2170/50462 | Loss: 3.1769 | LR: 4.99e-04 | 2827.5ms/step | 21.2 samples/s
939
+ Step 2180/50462 | Loss: 3.1684 | LR: 4.99e-04 | 2833.1ms/step | 21.2 samples/s
940
+ Step 2190/50462 | Loss: 3.2109 | LR: 4.99e-04 | 2832.6ms/step | 21.2 samples/s
941
+ Step 2200/50462 | Loss: 3.2057 | LR: 4.99e-04 | 2828.3ms/step | 21.2 samples/s
942
+ Step 2210/50462 | Loss: 3.1768 | LR: 4.99e-04 | 2829.8ms/step | 21.2 samples/s
943
+ Step 2220/50462 | Loss: 3.1737 | LR: 4.99e-04 | 2831.3ms/step | 21.2 samples/s
944
+ Step 2230/50462 | Loss: 3.2153 | LR: 4.99e-04 | 2829.1ms/step | 21.2 samples/s
945
+ Step 2240/50462 | Loss: 3.1607 | LR: 4.99e-04 | 2834.0ms/step | 21.2 samples/s
946
+ Step 2250/50462 | Loss: 3.1798 | LR: 4.99e-04 | 2835.3ms/step | 21.2 samples/s
947
+ Validation at step 2250: Loss = 3.1867, Perplexity = 24.21
948
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-2250
949
+ Step 2260/50462 | Loss: 3.1829 | LR: 4.99e-04 | 4783.3ms/step | 12.5 samples/s
950
+ Step 2270/50462 | Loss: 3.1597 | LR: 4.99e-04 | 2823.9ms/step | 21.2 samples/s
951
+ Step 2280/50462 | Loss: 3.1746 | LR: 4.99e-04 | 2819.4ms/step | 21.3 samples/s
952
+ Step 2290/50462 | Loss: 3.1686 | LR: 4.99e-04 | 2821.6ms/step | 21.3 samples/s
953
+ Step 2300/50462 | Loss: 3.1546 | LR: 4.99e-04 | 2827.4ms/step | 21.2 samples/s
954
+ Step 2310/50462 | Loss: 3.1692 | LR: 4.99e-04 | 2831.3ms/step | 21.2 samples/s
955
+ Step 2320/50462 | Loss: 3.1352 | LR: 4.99e-04 | 2831.3ms/step | 21.2 samples/s
956
+ Step 2330/50462 | Loss: 3.1464 | LR: 4.99e-04 | 2832.4ms/step | 21.2 samples/s
957
+ Step 2340/50462 | Loss: 3.1830 | LR: 4.99e-04 | 2832.4ms/step | 21.2 samples/s
958
+ Step 2350/50462 | Loss: 3.1474 | LR: 4.99e-04 | 2832.7ms/step | 21.2 samples/s
959
+ Step 2360/50462 | Loss: 3.1615 | LR: 4.99e-04 | 2834.3ms/step | 21.2 samples/s
960
+ Step 2370/50462 | Loss: 3.1840 | LR: 4.99e-04 | 2830.4ms/step | 21.2 samples/s
961
+ Step 2380/50462 | Loss: 3.1846 | LR: 4.99e-04 | 2832.5ms/step | 21.2 samples/s
962
+ Step 2390/50462 | Loss: 3.1568 | LR: 4.99e-04 | 2834.6ms/step | 21.2 samples/s
963
+ Step 2400/50462 | Loss: 3.1432 | LR: 4.99e-04 | 2831.5ms/step | 21.2 samples/s
964
+ Step 2410/50462 | Loss: 3.1630 | LR: 4.99e-04 | 2830.8ms/step | 21.2 samples/s
965
+ Step 2420/50462 | Loss: 3.1276 | LR: 4.99e-04 | 2835.0ms/step | 21.2 samples/s
966
+ Step 2430/50462 | Loss: 3.1093 | LR: 4.99e-04 | 2830.0ms/step | 21.2 samples/s
967
+ Step 2440/50462 | Loss: 3.1122 | LR: 4.99e-04 | 2827.3ms/step | 21.2 samples/s
968
+ Step 2450/50462 | Loss: 3.1366 | LR: 4.99e-04 | 2829.7ms/step | 21.2 samples/s
969
+ Step 2460/50462 | Loss: 3.1121 | LR: 4.99e-04 | 2829.3ms/step | 21.2 samples/s
970
+ Step 2470/50462 | Loss: 3.1566 | LR: 4.99e-04 | 2829.9ms/step | 21.2 samples/s
971
+ Step 2480/50462 | Loss: 3.1532 | LR: 4.99e-04 | 2832.4ms/step | 21.2 samples/s
972
+ Step 2490/50462 | Loss: 3.1394 | LR: 4.98e-04 | 2828.7ms/step | 21.2 samples/s
973
+ Step 2500/50462 | Loss: 3.1445 | LR: 4.98e-04 | 2832.1ms/step | 21.2 samples/s
974
+ Validation at step 2500: Loss = 3.1455, Perplexity = 23.23
975
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-2500
976
+ Step 2510/50462 | Loss: 3.1498 | LR: 4.98e-04 | 4811.5ms/step | 12.5 samples/s
977
+ Step 2520/50462 | Loss: 3.1560 | LR: 4.98e-04 | 2825.7ms/step | 21.2 samples/s
978
+ Step 2530/50462 | Loss: 3.1448 | LR: 4.98e-04 | 2828.1ms/step | 21.2 samples/s
979
+ Step 2540/50462 | Loss: 3.1235 | LR: 4.98e-04 | 2831.4ms/step | 21.2 samples/s
980
+ Step 2550/50462 | Loss: 3.1093 | LR: 4.98e-04 | 2830.7ms/step | 21.2 samples/s
981
+ Step 2560/50462 | Loss: 3.1367 | LR: 4.98e-04 | 2828.9ms/step | 21.2 samples/s
982
+ Step 2570/50462 | Loss: 3.1204 | LR: 4.98e-04 | 2830.7ms/step | 21.2 samples/s
983
+ Step 2580/50462 | Loss: 3.1045 | LR: 4.98e-04 | 2830.2ms/step | 21.2 samples/s
984
+ Step 2590/50462 | Loss: 3.0966 | LR: 4.98e-04 | 2832.2ms/step | 21.2 samples/s
985
+ Step 2600/50462 | Loss: 3.1554 | LR: 4.98e-04 | 2828.6ms/step | 21.2 samples/s
986
+ Step 2610/50462 | Loss: 3.1346 | LR: 4.98e-04 | 2823.5ms/step | 21.3 samples/s
987
+ Step 2620/50462 | Loss: 3.1211 | LR: 4.98e-04 | 2829.3ms/step | 21.2 samples/s
988
+ Step 2630/50462 | Loss: 3.1216 | LR: 4.98e-04 | 2831.2ms/step | 21.2 samples/s
989
+ Step 2640/50462 | Loss: 3.0922 | LR: 4.98e-04 | 2831.7ms/step | 21.2 samples/s
990
+ Step 2650/50462 | Loss: 3.1067 | LR: 4.98e-04 | 2834.0ms/step | 21.2 samples/s
991
+ Step 2660/50462 | Loss: 3.0699 | LR: 4.98e-04 | 2832.7ms/step | 21.2 samples/s
992
+ Step 2670/50462 | Loss: 3.0782 | LR: 4.98e-04 | 2831.8ms/step | 21.2 samples/s
993
+ Step 2680/50462 | Loss: 3.1291 | LR: 4.98e-04 | 2832.4ms/step | 21.2 samples/s
994
+ Step 2690/50462 | Loss: 3.1374 | LR: 4.98e-04 | 2824.9ms/step | 21.2 samples/s
995
+ Step 2700/50462 | Loss: 3.1127 | LR: 4.98e-04 | 2831.0ms/step | 21.2 samples/s
996
+ Step 2710/50462 | Loss: 3.0626 | LR: 4.98e-04 | 2832.2ms/step | 21.2 samples/s
997
+ Step 2720/50462 | Loss: 3.0887 | LR: 4.98e-04 | 2834.6ms/step | 21.2 samples/s
998
+ Step 2730/50462 | Loss: 3.0877 | LR: 4.98e-04 | 2832.9ms/step | 21.2 samples/s
999
+ Step 2740/50462 | Loss: 3.1018 | LR: 4.98e-04 | 2832.1ms/step | 21.2 samples/s
1000
+ Step 2750/50462 | Loss: 3.0758 | LR: 4.98e-04 | 2834.3ms/step | 21.2 samples/s
1001
+ Validation at step 2750: Loss = 3.1075, Perplexity = 22.37
1002
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-2750
1003
+ Step 2760/50462 | Loss: 3.1105 | LR: 4.98e-04 | 4764.3ms/step | 12.6 samples/s
1004
+ Step 2770/50462 | Loss: 3.0902 | LR: 4.98e-04 | 2827.1ms/step | 21.2 samples/s
1005
+ Step 2780/50462 | Loss: 3.1024 | LR: 4.98e-04 | 2831.5ms/step | 21.2 samples/s
1006
+ Step 2790/50462 | Loss: 3.1225 | LR: 4.98e-04 | 2834.2ms/step | 21.2 samples/s
1007
+ Step 2800/50462 | Loss: 3.0828 | LR: 4.98e-04 | 2834.6ms/step | 21.2 samples/s
1008
+ Step 2810/50462 | Loss: 3.0838 | LR: 4.98e-04 | 2833.9ms/step | 21.2 samples/s
1009
+ Step 2820/50462 | Loss: 3.0972 | LR: 4.98e-04 | 2831.6ms/step | 21.2 samples/s
1010
+ Step 2830/50462 | Loss: 3.0914 | LR: 4.98e-04 | 2831.6ms/step | 21.2 samples/s
1011
+ Step 2840/50462 | Loss: 3.0679 | LR: 4.98e-04 | 2835.1ms/step | 21.2 samples/s
1012
+ Step 2850/50462 | Loss: 3.0960 | LR: 4.98e-04 | 2832.3ms/step | 21.2 samples/s
1013
+ Step 2860/50462 | Loss: 3.0825 | LR: 4.98e-04 | 2830.1ms/step | 21.2 samples/s
1014
+ Step 2870/50462 | Loss: 3.0577 | LR: 4.98e-04 | 2831.8ms/step | 21.2 samples/s
1015
+ Step 2880/50462 | Loss: 3.0816 | LR: 4.98e-04 | 2830.6ms/step | 21.2 samples/s
1016
+ Step 2890/50462 | Loss: 3.0624 | LR: 4.98e-04 | 2832.9ms/step | 21.2 samples/s
1017
+ Step 2900/50462 | Loss: 3.0911 | LR: 4.98e-04 | 2828.0ms/step | 21.2 samples/s
1018
+ Step 2910/50462 | Loss: 3.0675 | LR: 4.98e-04 | 2827.4ms/step | 21.2 samples/s
1019
+ Step 2920/50462 | Loss: 3.0661 | LR: 4.98e-04 | 2825.4ms/step | 21.2 samples/s
1020
+ Step 2930/50462 | Loss: 3.0918 | LR: 4.98e-04 | 2818.3ms/step | 21.3 samples/s
1021
+ Step 2940/50462 | Loss: 3.0960 | LR: 4.98e-04 | 2825.7ms/step | 21.2 samples/s
epfl_final/train_logs/p2_1p2b_dense_seed72_4gpu_resume2750_fixed.log ADDED
@@ -0,0 +1,588 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ W0903 14:25:20.781000 373 site-packages/torch/distributed/run.py:851]
2
+ W0903 14:25:20.781000 373 site-packages/torch/distributed/run.py:851] *****************************************
3
+ W0903 14:25:20.781000 373 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
4
+ W0903 14:25:20.781000 373 site-packages/torch/distributed/run.py:851] *****************************************
5
+ Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216
6
+ Running config:
7
+ {
8
+ "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
9
+ "model_dtype": "bfloat16",
10
+ "init_from_scratch": true,
11
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
12
+ "val_path": null,
13
+ "max_seq_length": 8192,
14
+ "num_workers": 8,
15
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216",
16
+ "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-2750",
17
+ "force_new_run": false,
18
+ "num_epochs": 1,
19
+ "max_steps": 50462,
20
+ "per_device_batch_size": 1,
21
+ "gradient_accumulation_steps": 15,
22
+ "learning_rate": 0.0005,
23
+ "weight_decay": 0.01,
24
+ "warmup_steps": 750,
25
+ "lr_scheduler": "cosine",
26
+ "max_grad_norm": 1.0,
27
+ "save_steps": 250,
28
+ "eval_steps": 250,
29
+ "eval_batches": 50,
30
+ "logging_steps": 10,
31
+ "seed": 72,
32
+ "verbose": true,
33
+ "checkpoint_activations": false,
34
+ "resume_micro_step_override": null,
35
+ "strategy": "ddp",
36
+ "mixed_precision": "bf16",
37
+ "wandb_enabled": true,
38
+ "wandb_project": "stochastic_sparsity",
39
+ "wandb_name": "1.2b dense 8k seed72 4gpu",
40
+ "wandb_tags": [
41
+ "qwen3",
42
+ "1.2b",
43
+ "8k",
44
+ "p2",
45
+ "dense",
46
+ "seed72"
47
+ ],
48
+ "hidden_size": 1536,
49
+ "intermediate_size": 6144,
50
+ "num_attention_heads": 12,
51
+ "num_key_value_heads": 4,
52
+ "num_hidden_layers": 28,
53
+ "head_dim": null,
54
+ "attention_dropout": null,
55
+ "rope_train_position_scale_set": null,
56
+ "attention_type": "sparse_stochastic",
57
+ "sparse_block_size": 128,
58
+ "sparse_num_random_blocks": 31,
59
+ "sparse_mask_pool_size": 1,
60
+ "sparse_val_dense_compare": false,
61
+ "sparse_layer_indices": null,
62
+ "sparse_mask_cache_dir": null,
63
+ "sparse_mask_seed": 0,
64
+ "sparse_mixed_nrb_set": null,
65
+ "dense_mode": true,
66
+ "dense_step_intervals": null
67
+ }
68
+ Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
69
+ Checkpoint activations: False
70
+ Checkpoint activations: False
71
+ Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-2750
72
+ Checkpoint activations: False
73
+ Checkpoint activations: False
74
+ dense_mode=True: sparse attention layers will use standard causal SDPA.
75
+
76
+
77
+ ================================================================================
78
+ MODEL ARCHITECTURE
79
+ ================================================================================
80
+ Model config:
81
+ vocab_size: 151936
82
+ hidden_size: 1536
83
+ num_hidden_layers: 28
84
+ num_attention_heads: 12
85
+ num_key_value_heads: 4
86
+ head_dim: 128
87
+ intermediate_size: 6144
88
+ max_position_embeddings: 32768
89
+ rope_theta: 1000000
90
+ rope_train_position_scale_set: None
91
+ attention_type: sparse_stochastic
92
+ sparse_block_size: 128
93
+ sparse_num_random_blocks: 31
94
+ sparse_mask_pool_size: 1
95
+ sparse_max_seq_length: 8192
96
+
97
+ Parameter count:
98
+ Total parameters: 1,202,352,640
99
+ Trainable parameters: 1,202,352,640
100
+ Non-trainable parameters: 0
101
+ Model size (BF16): 2.40 GB
102
+
103
+ --------------------------------------------------------------------------------
104
+ LAYER-BY-LAYER SUMMARY
105
+ --------------------------------------------------------------------------------
106
+ ================================================================================================================================================================
107
+ Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
108
+ ================================================================================================================================================================
109
+ Qwen3ForCausalLM [1, 8192] -- -- True
110
+ ├─Qwen3Model: 1-1 -- [1, 8192, 1536] -- True
111
+ │ └─Embedding: 2-1 [1, 8192] [1, 8192, 1536] 233,373,696 True
112
+ │ └─ModuleList: 2-2 -- -- -- True
113
+ │ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
114
+ │ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
115
+ │ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
116
+ │ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
117
+ │ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
118
+ │ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
119
+ │ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
120
+ │ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
121
+ │ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
122
+ │ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
123
+ │ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
124
+ │ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
125
+ │ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
126
+ │ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
127
+ │ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
128
+ │ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
129
+ │ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
130
+ │ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
131
+ │ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
132
+ │ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
133
+ │ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
134
+ │ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
135
+ │ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
136
+ │ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
137
+ │ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
138
+ │ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
139
+ │ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
140
+ │ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
141
+ │ └─Qwen3RMSNorm: 2-3 [1, 8192, 1536] [1, 8192, 1536] 1,536 True
142
+ ├─Linear: 1-2 [1, 8192, 1536] [1, 8192, 151936] 233,373,696 True
143
+ ================================================================================================================================================================
144
+ Total params: 1,435,726,336
145
+ Trainable params: 1,435,726,336
146
+ Non-trainable params: 0
147
+ Total mult-adds (Units.GIGABYTES): 1.44
148
+ ================================================================================================================================================================
149
+ Input size (MB): 0.07
150
+ Forward/backward pass size (MB): 26218.59
151
+ Params size (MB): 2871.45
152
+ Estimated Total Size (MB): 29090.11
153
+ ================================================================================================================================================================
154
+ NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
155
+ ================================================================================
156
+
157
+ Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
158
+ Using pre-tokenized dataset (no runtime tokenization)
159
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
160
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
161
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
162
+
163
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
164
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
165
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
166
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
167
+ Dataset split: 3027764 train, 159356 validation
168
+ Training examples (packed, train split): 3027764
169
+ Validation examples (packed, val split): 159356
170
+ Per-device batch size: 1
171
+ World size (GPUs): 4
172
+ Gradient accumulation steps: 15
173
+ Effective batch size (seqs/step): 60
174
+ Sequence length: 8192
175
+ Tokens per optimizer step: 491,520
176
+ Micro-batches per epoch (per rank): 756941
177
+ Optimizer steps per epoch: 50462
178
+ Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
179
+ Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-2750...
180
+ Preserving checkpoint LR state for exact resume: current=[0.0004980057984421634], schedule_base=[0.0005]
181
+ Resumed from checkpoint at step 2750 (epoch 0, micro_step 41250)
182
+ Resuming training from step 2750
183
+ Resuming wandb run 7tfq6nwm
184
+ wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
185
+ wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
186
+ wandb: setting up run 7tfq6nwm
187
+ wandb: Tracking run with wandb version 0.26.1
188
+ wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260903_142704-7tfq6nwm
189
+ wandb: Run `wandb offline` to turn off syncing.
190
+ wandb: Resuming run 1.2b dense 8k seed72 4gpu
191
+ wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
192
+ wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/7tfq6nwm
193
+ Starting training...
194
+ Validation at step 2750: Loss = 3.1075, Perplexity = 22.37
195
+ wandb: WARNING Tried to log to step 2750 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
196
+ Switching attention mode for step 2751: dense (dense step interval)
197
+ Step 2760/50462 | Loss: 3.1106 | LR: 4.98e-04 | 19044.7ms/step | 3.2 samples/s
198
+ wandb: WARNING Tried to log to step 2760 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
199
+ Step 2770/50462 | Loss: 3.0902 | LR: 4.98e-04 | 2807.4ms/step | 21.4 samples/s
200
+ wandb: WARNING Tried to log to step 2770 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
201
+ Step 2780/50462 | Loss: 3.1024 | LR: 4.98e-04 | 2812.9ms/step | 21.3 samples/s
202
+ wandb: WARNING Tried to log to step 2780 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
203
+ Step 2790/50462 | Loss: 3.1225 | LR: 4.98e-04 | 2815.0ms/step | 21.3 samples/s
204
+ wandb: WARNING Tried to log to step 2790 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
205
+ Step 2800/50462 | Loss: 3.0828 | LR: 4.98e-04 | 2817.2ms/step | 21.3 samples/s
206
+ wandb: WARNING Tried to log to step 2800 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
207
+ Step 2810/50462 | Loss: 3.0838 | LR: 4.98e-04 | 2814.2ms/step | 21.3 samples/s
208
+ wandb: WARNING Tried to log to step 2810 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
209
+ Step 2820/50462 | Loss: 3.0972 | LR: 4.98e-04 | 2818.8ms/step | 21.3 samples/s
210
+ wandb: WARNING Tried to log to step 2820 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
211
+ Step 2830/50462 | Loss: 3.0915 | LR: 4.98e-04 | 2820.7ms/step | 21.3 samples/s
212
+ wandb: WARNING Tried to log to step 2830 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
213
+ Step 2840/50462 | Loss: 3.0679 | LR: 4.98e-04 | 2816.4ms/step | 21.3 samples/s
214
+ wandb: WARNING Tried to log to step 2840 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
215
+ Step 2850/50462 | Loss: 3.0960 | LR: 4.98e-04 | 2817.4ms/step | 21.3 samples/s
216
+ wandb: WARNING Tried to log to step 2850 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
217
+ Step 2860/50462 | Loss: 3.0825 | LR: 4.98e-04 | 2816.8ms/step | 21.3 samples/s
218
+ wandb: WARNING Tried to log to step 2860 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
219
+ Step 2870/50462 | Loss: 3.0577 | LR: 4.98e-04 | 2818.4ms/step | 21.3 samples/s
220
+ wandb: WARNING Tried to log to step 2870 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
221
+ Step 2880/50462 | Loss: 3.0816 | LR: 4.98e-04 | 2822.5ms/step | 21.3 samples/s
222
+ wandb: WARNING Tried to log to step 2880 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
223
+ Step 2890/50462 | Loss: 3.0625 | LR: 4.98e-04 | 2819.0ms/step | 21.3 samples/s
224
+ wandb: WARNING Tried to log to step 2890 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
225
+ Step 2900/50462 | Loss: 3.0911 | LR: 4.98e-04 | 2820.2ms/step | 21.3 samples/s
226
+ wandb: WARNING Tried to log to step 2900 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
227
+ Step 2910/50462 | Loss: 3.0675 | LR: 4.98e-04 | 2820.1ms/step | 21.3 samples/s
228
+ wandb: WARNING Tried to log to step 2910 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
229
+ Step 2920/50462 | Loss: 3.0661 | LR: 4.98e-04 | 2816.4ms/step | 21.3 samples/s
230
+ wandb: WARNING Tried to log to step 2920 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
231
+ Step 2930/50462 | Loss: 3.0918 | LR: 4.98e-04 | 2823.1ms/step | 21.3 samples/s
232
+ wandb: WARNING Tried to log to step 2930 that is less than the current step 2931. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
233
+ Step 2940/50462 | Loss: 3.0960 | LR: 4.98e-04 | 2820.2ms/step | 21.3 samples/s
234
+ Step 2950/50462 | Loss: 3.0752 | LR: 4.98e-04 | 2819.6ms/step | 21.3 samples/s
235
+ Step 2960/50462 | Loss: 3.0743 | LR: 4.98e-04 | 2816.4ms/step | 21.3 samples/s
236
+ Step 2970/50462 | Loss: 3.0729 | LR: 4.98e-04 | 2819.9ms/step | 21.3 samples/s
237
+ Step 2980/50462 | Loss: 3.0705 | LR: 4.98e-04 | 2817.0ms/step | 21.3 samples/s
238
+ Step 2990/50462 | Loss: 3.0481 | LR: 4.97e-04 | 2821.6ms/step | 21.3 samples/s
239
+ Step 3000/50462 | Loss: 3.0682 | LR: 4.97e-04 | 2819.9ms/step | 21.3 samples/s
240
+ Validation at step 3000: Loss = 3.0768, Perplexity = 21.69
241
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-3000
242
+ Step 3010/50462 | Loss: 3.1099 | LR: 4.97e-04 | 4716.5ms/step | 12.7 samples/s
243
+ Step 3020/50462 | Loss: 3.0968 | LR: 4.97e-04 | 2819.9ms/step | 21.3 samples/s
244
+ Step 3030/50462 | Loss: 3.0538 | LR: 4.97e-04 | 2824.1ms/step | 21.2 samples/s
245
+ Step 3040/50462 | Loss: 3.0730 | LR: 4.97e-04 | 2823.9ms/step | 21.2 samples/s
246
+ Step 3050/50462 | Loss: 3.0666 | LR: 4.97e-04 | 2827.8ms/step | 21.2 samples/s
247
+ Step 3060/50462 | Loss: 3.0805 | LR: 4.97e-04 | 2828.1ms/step | 21.2 samples/s
248
+ Step 3070/50462 | Loss: 3.0570 | LR: 4.97e-04 | 2823.8ms/step | 21.2 samples/s
249
+ Step 3080/50462 | Loss: 3.0695 | LR: 4.97e-04 | 2828.4ms/step | 21.2 samples/s
250
+ Step 3090/50462 | Loss: 3.0407 | LR: 4.97e-04 | 2821.7ms/step | 21.3 samples/s
251
+ Step 3100/50462 | Loss: 3.0628 | LR: 4.97e-04 | 2825.4ms/step | 21.2 samples/s
252
+ Step 3110/50462 | Loss: 3.0624 | LR: 4.97e-04 | 2823.5ms/step | 21.3 samples/s
253
+ Step 3120/50462 | Loss: 3.0609 | LR: 4.97e-04 | 2827.7ms/step | 21.2 samples/s
254
+ Step 3130/50462 | Loss: 3.0310 | LR: 4.97e-04 | 2824.6ms/step | 21.2 samples/s
255
+ Step 3140/50462 | Loss: 3.0491 | LR: 4.97e-04 | 2822.7ms/step | 21.3 samples/s
256
+ Step 3150/50462 | Loss: 3.0343 | LR: 4.97e-04 | 2825.1ms/step | 21.2 samples/s
257
+ Step 3160/50462 | Loss: 3.0224 | LR: 4.97e-04 | 2826.8ms/step | 21.2 samples/s
258
+ Step 3170/50462 | Loss: 3.0623 | LR: 4.97e-04 | 2828.3ms/step | 21.2 samples/s
259
+ Step 3180/50462 | Loss: 3.0616 | LR: 4.97e-04 | 2826.7ms/step | 21.2 samples/s
260
+ Step 3190/50462 | Loss: 3.0235 | LR: 4.97e-04 | 2828.1ms/step | 21.2 samples/s
261
+ Step 3200/50462 | Loss: 3.0479 | LR: 4.97e-04 | 2823.7ms/step | 21.2 samples/s
262
+ Step 3210/50462 | Loss: 3.0746 | LR: 4.97e-04 | 2829.2ms/step | 21.2 samples/s
263
+ Step 3220/50462 | Loss: 3.0593 | LR: 4.97e-04 | 2829.8ms/step | 21.2 samples/s
264
+ Step 3230/50462 | Loss: 3.0127 | LR: 4.97e-04 | 2825.7ms/step | 21.2 samples/s
265
+ Step 3240/50462 | Loss: 3.0381 | LR: 4.97e-04 | 2823.8ms/step | 21.2 samples/s
266
+ Step 3250/50462 | Loss: 3.0697 | LR: 4.97e-04 | 2818.8ms/step | 21.3 samples/s
267
+ Validation at step 3250: Loss = 3.0495, Perplexity = 21.10
268
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-3250
269
+ Step 3260/50462 | Loss: 3.0493 | LR: 4.97e-04 | 4697.0ms/step | 12.8 samples/s
270
+ Step 3270/50462 | Loss: 3.0001 | LR: 4.97e-04 | 2820.6ms/step | 21.3 samples/s
271
+ Step 3280/50462 | Loss: 3.0378 | LR: 4.97e-04 | 2829.6ms/step | 21.2 samples/s
272
+ Step 3290/50462 | Loss: 3.0070 | LR: 4.97e-04 | 2827.4ms/step | 21.2 samples/s
273
+ Step 3300/50462 | Loss: 3.0433 | LR: 4.97e-04 | 2827.4ms/step | 21.2 samples/s
274
+ Step 3310/50462 | Loss: 3.0677 | LR: 4.97e-04 | 2829.2ms/step | 21.2 samples/s
275
+ Step 3320/50462 | Loss: 3.0180 | LR: 4.97e-04 | 2831.2ms/step | 21.2 samples/s
276
+ Step 3330/50462 | Loss: 2.9940 | LR: 4.97e-04 | 2830.3ms/step | 21.2 samples/s
277
+ Step 3340/50462 | Loss: 3.0433 | LR: 4.97e-04 | 2823.6ms/step | 21.2 samples/s
278
+ Step 3350/50462 | Loss: 3.0249 | LR: 4.97e-04 | 2826.9ms/step | 21.2 samples/s
279
+ Step 3360/50462 | Loss: 3.0302 | LR: 4.97e-04 | 2829.3ms/step | 21.2 samples/s
280
+ Step 3370/50462 | Loss: 3.0396 | LR: 4.97e-04 | 2828.0ms/step | 21.2 samples/s
281
+ Step 3380/50462 | Loss: 3.0325 | LR: 4.97e-04 | 2826.0ms/step | 21.2 samples/s
282
+ Step 3390/50462 | Loss: 3.0429 | LR: 4.97e-04 | 2819.1ms/step | 21.3 samples/s
283
+ Step 3400/50462 | Loss: 3.0181 | LR: 4.97e-04 | 2826.6ms/step | 21.2 samples/s
284
+ Step 3410/50462 | Loss: 2.9776 | LR: 4.96e-04 | 2827.2ms/step | 21.2 samples/s
285
+ Step 3420/50462 | Loss: 2.9792 | LR: 4.96e-04 | 2822.3ms/step | 21.3 samples/s
286
+ Step 3430/50462 | Loss: 3.0163 | LR: 4.96e-04 | 2818.7ms/step | 21.3 samples/s
287
+ Step 3440/50462 | Loss: 2.9832 | LR: 4.96e-04 | 2825.3ms/step | 21.2 samples/s
288
+ Step 3450/50462 | Loss: 3.0095 | LR: 4.96e-04 | 2826.8ms/step | 21.2 samples/s
289
+ Step 3460/50462 | Loss: 2.9921 | LR: 4.96e-04 | 2828.3ms/step | 21.2 samples/s
290
+ Step 3470/50462 | Loss: 3.0155 | LR: 4.96e-04 | 2825.9ms/step | 21.2 samples/s
291
+ Step 3480/50462 | Loss: 3.0064 | LR: 4.96e-04 | 2829.7ms/step | 21.2 samples/s
292
+ Step 3490/50462 | Loss: 3.0077 | LR: 4.96e-04 | 2827.7ms/step | 21.2 samples/s
293
+ Step 3500/50462 | Loss: 2.9834 | LR: 4.96e-04 | 2824.3ms/step | 21.2 samples/s
294
+ Validation at step 3500: Loss = 3.0263, Perplexity = 20.62
295
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-3500
296
+ Step 3510/50462 | Loss: 3.0408 | LR: 4.96e-04 | 4701.2ms/step | 12.8 samples/s
297
+ Step 3520/50462 | Loss: 3.0308 | LR: 4.96e-04 | 2825.3ms/step | 21.2 samples/s
298
+ Step 3530/50462 | Loss: 2.9976 | LR: 4.96e-04 | 2825.0ms/step | 21.2 samples/s
299
+ Step 3540/50462 | Loss: 3.0082 | LR: 4.96e-04 | 2825.4ms/step | 21.2 samples/s
300
+ Step 3550/50462 | Loss: 2.9987 | LR: 4.96e-04 | 2824.7ms/step | 21.2 samples/s
301
+ Step 3560/50462 | Loss: 3.0430 | LR: 4.96e-04 | 2826.1ms/step | 21.2 samples/s
302
+ Step 3570/50462 | Loss: 3.0030 | LR: 4.96e-04 | 2827.4ms/step | 21.2 samples/s
303
+ Step 3580/50462 | Loss: 2.9930 | LR: 4.96e-04 | 2829.3ms/step | 21.2 samples/s
304
+ Step 3590/50462 | Loss: 2.9794 | LR: 4.96e-04 | 2825.5ms/step | 21.2 samples/s
305
+ Step 3600/50462 | Loss: 2.9900 | LR: 4.96e-04 | 2823.8ms/step | 21.2 samples/s
306
+ Step 3610/50462 | Loss: 3.0096 | LR: 4.96e-04 | 2827.5ms/step | 21.2 samples/s
307
+ Step 3620/50462 | Loss: 3.0113 | LR: 4.96e-04 | 2827.2ms/step | 21.2 samples/s
308
+ Step 3630/50462 | Loss: 2.9664 | LR: 4.96e-04 | 2830.4ms/step | 21.2 samples/s
309
+ Step 3640/50462 | Loss: 3.0067 | LR: 4.96e-04 | 2829.6ms/step | 21.2 samples/s
310
+ Step 3650/50462 | Loss: 2.9829 | LR: 4.96e-04 | 2826.1ms/step | 21.2 samples/s
311
+ Step 3660/50462 | Loss: 3.0274 | LR: 4.96e-04 | 2824.3ms/step | 21.2 samples/s
312
+ Step 3670/50462 | Loss: 3.0084 | LR: 4.96e-04 | 2825.4ms/step | 21.2 samples/s
313
+ Step 3680/50462 | Loss: 3.0090 | LR: 4.96e-04 | 2823.5ms/step | 21.3 samples/s
314
+ Step 3690/50462 | Loss: 3.0014 | LR: 4.96e-04 | 2830.7ms/step | 21.2 samples/s
315
+ Step 3700/50462 | Loss: 3.0171 | LR: 4.96e-04 | 2832.3ms/step | 21.2 samples/s
316
+ Step 3710/50462 | Loss: 2.9566 | LR: 4.96e-04 | 2827.9ms/step | 21.2 samples/s
317
+ Step 3720/50462 | Loss: 2.9820 | LR: 4.96e-04 | 2828.1ms/step | 21.2 samples/s
318
+ Step 3730/50462 | Loss: 2.9866 | LR: 4.96e-04 | 2827.0ms/step | 21.2 samples/s
319
+ Step 3740/50462 | Loss: 2.9623 | LR: 4.96e-04 | 2829.6ms/step | 21.2 samples/s
320
+ Step 3750/50462 | Loss: 3.0068 | LR: 4.96e-04 | 2825.0ms/step | 21.2 samples/s
321
+ Validation at step 3750: Loss = 2.9998, Perplexity = 20.08
322
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-3750
323
+ Step 3760/50462 | Loss: 3.0054 | LR: 4.95e-04 | 4701.9ms/step | 12.8 samples/s
324
+ Step 3770/50462 | Loss: 3.0212 | LR: 4.95e-04 | 2824.0ms/step | 21.2 samples/s
325
+ Step 3780/50462 | Loss: 3.0116 | LR: 4.95e-04 | 2824.1ms/step | 21.2 samples/s
326
+ Step 3790/50462 | Loss: 3.0014 | LR: 4.95e-04 | 2819.7ms/step | 21.3 samples/s
327
+ Step 3800/50462 | Loss: 2.9719 | LR: 4.95e-04 | 2824.0ms/step | 21.2 samples/s
328
+ Step 3810/50462 | Loss: 3.0187 | LR: 4.95e-04 | 2827.6ms/step | 21.2 samples/s
329
+ Step 3820/50462 | Loss: 2.9626 | LR: 4.95e-04 | 2827.9ms/step | 21.2 samples/s
330
+ Step 3830/50462 | Loss: 3.0101 | LR: 4.95e-04 | 2825.0ms/step | 21.2 samples/s
331
+ Step 3840/50462 | Loss: 2.9450 | LR: 4.95e-04 | 2819.9ms/step | 21.3 samples/s
332
+ Step 3850/50462 | Loss: 2.9439 | LR: 4.95e-04 | 2822.3ms/step | 21.3 samples/s
333
+ Step 3860/50462 | Loss: 2.9598 | LR: 4.95e-04 | 2828.4ms/step | 21.2 samples/s
334
+ Step 3870/50462 | Loss: 2.9943 | LR: 4.95e-04 | 2828.8ms/step | 21.2 samples/s
335
+ Step 3880/50462 | Loss: 2.9785 | LR: 4.95e-04 | 2829.3ms/step | 21.2 samples/s
336
+ Step 3890/50462 | Loss: 2.9711 | LR: 4.95e-04 | 2829.3ms/step | 21.2 samples/s
337
+ Step 3900/50462 | Loss: 3.0054 | LR: 4.95e-04 | 2830.9ms/step | 21.2 samples/s
338
+ Step 3910/50462 | Loss: 3.0202 | LR: 4.95e-04 | 2831.5ms/step | 21.2 samples/s
339
+ Step 3920/50462 | Loss: 3.0050 | LR: 4.95e-04 | 2829.1ms/step | 21.2 samples/s
340
+ Step 3930/50462 | Loss: 2.9790 | LR: 4.95e-04 | 2834.8ms/step | 21.2 samples/s
341
+ Step 3940/50462 | Loss: 2.9762 | LR: 4.95e-04 | 2835.1ms/step | 21.2 samples/s
342
+ Step 3950/50462 | Loss: 2.9570 | LR: 4.95e-04 | 2830.0ms/step | 21.2 samples/s
343
+ Step 3960/50462 | Loss: 2.9829 | LR: 4.95e-04 | 2826.5ms/step | 21.2 samples/s
344
+ Step 3970/50462 | Loss: 2.9854 | LR: 4.95e-04 | 2829.6ms/step | 21.2 samples/s
345
+ Step 3980/50462 | Loss: 2.9603 | LR: 4.95e-04 | 2827.0ms/step | 21.2 samples/s
346
+ Step 3990/50462 | Loss: 2.9983 | LR: 4.95e-04 | 2825.7ms/step | 21.2 samples/s
347
+ Step 4000/50462 | Loss: 2.9842 | LR: 4.95e-04 | 2825.1ms/step | 21.2 samples/s
348
+ Validation at step 4000: Loss = 2.9835, Perplexity = 19.76
349
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-4000
350
+ Step 4010/50462 | Loss: 2.9511 | LR: 4.95e-04 | 4804.3ms/step | 12.5 samples/s
351
+ Step 4020/50462 | Loss: 2.9581 | LR: 4.95e-04 | 2825.5ms/step | 21.2 samples/s
352
+ Step 4030/50462 | Loss: 2.9613 | LR: 4.95e-04 | 2825.8ms/step | 21.2 samples/s
353
+ Step 4040/50462 | Loss: 2.9703 | LR: 4.95e-04 | 2824.9ms/step | 21.2 samples/s
354
+ Step 4050/50462 | Loss: 2.9497 | LR: 4.95e-04 | 2826.4ms/step | 21.2 samples/s
355
+ Step 4060/50462 | Loss: 2.9299 | LR: 4.95e-04 | 2821.6ms/step | 21.3 samples/s
356
+ Step 4070/50462 | Loss: 2.9786 | LR: 4.95e-04 | 2830.0ms/step | 21.2 samples/s
357
+ Step 4080/50462 | Loss: 2.9543 | LR: 4.94e-04 | 2830.3ms/step | 21.2 samples/s
358
+ Step 4090/50462 | Loss: 2.9482 | LR: 4.94e-04 | 2828.1ms/step | 21.2 samples/s
359
+ Step 4100/50462 | Loss: 2.9623 | LR: 4.94e-04 | 2828.1ms/step | 21.2 samples/s
360
+ Step 4110/50462 | Loss: 2.9613 | LR: 4.94e-04 | 2829.5ms/step | 21.2 samples/s
361
+ Step 4120/50462 | Loss: 2.9466 | LR: 4.94e-04 | 2832.9ms/step | 21.2 samples/s
362
+ Step 4130/50462 | Loss: 2.9853 | LR: 4.94e-04 | 2830.2ms/step | 21.2 samples/s
363
+ Step 4140/50462 | Loss: 2.9945 | LR: 4.94e-04 | 2830.0ms/step | 21.2 samples/s
364
+ Step 4150/50462 | Loss: 2.9667 | LR: 4.94e-04 | 2834.5ms/step | 21.2 samples/s
365
+ Step 4160/50462 | Loss: 2.9433 | LR: 4.94e-04 | 2831.8ms/step | 21.2 samples/s
366
+ Step 4170/50462 | Loss: 2.9527 | LR: 4.94e-04 | 2831.8ms/step | 21.2 samples/s
367
+ Step 4180/50462 | Loss: 2.9914 | LR: 4.94e-04 | 2834.8ms/step | 21.2 samples/s
368
+ Step 4190/50462 | Loss: 2.9828 | LR: 4.94e-04 | 2833.7ms/step | 21.2 samples/s
369
+ Step 4200/50462 | Loss: 2.9292 | LR: 4.94e-04 | 2834.3ms/step | 21.2 samples/s
370
+ Step 4210/50462 | Loss: 2.9583 | LR: 4.94e-04 | 2831.8ms/step | 21.2 samples/s
371
+ Step 4220/50462 | Loss: 2.9774 | LR: 4.94e-04 | 2831.2ms/step | 21.2 samples/s
372
+ Step 4230/50462 | Loss: 2.9596 | LR: 4.94e-04 | 2831.4ms/step | 21.2 samples/s
373
+ Step 4240/50462 | Loss: 2.9755 | LR: 4.94e-04 | 2827.6ms/step | 21.2 samples/s
374
+ Step 4250/50462 | Loss: 2.9683 | LR: 4.94e-04 | 2822.5ms/step | 21.3 samples/s
375
+ Validation at step 4250: Loss = 2.9662, Perplexity = 19.42
376
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-4250
377
+ Step 4260/50462 | Loss: 2.9682 | LR: 4.94e-04 | 4772.9ms/step | 12.6 samples/s
378
+ Step 4270/50462 | Loss: 2.9680 | LR: 4.94e-04 | 2824.6ms/step | 21.2 samples/s
379
+ Step 4280/50462 | Loss: 2.9415 | LR: 4.94e-04 | 2829.0ms/step | 21.2 samples/s
380
+ Step 4290/50462 | Loss: 2.9117 | LR: 4.94e-04 | 2831.9ms/step | 21.2 samples/s
381
+ Step 4300/50462 | Loss: 2.9448 | LR: 4.94e-04 | 2827.4ms/step | 21.2 samples/s
382
+ Step 4310/50462 | Loss: 2.9895 | LR: 4.94e-04 | 2826.8ms/step | 21.2 samples/s
383
+ Step 4320/50462 | Loss: 2.9625 | LR: 4.94e-04 | 2824.8ms/step | 21.2 samples/s
384
+ Step 4330/50462 | Loss: 2.9566 | LR: 4.94e-04 | 2827.1ms/step | 21.2 samples/s
385
+ Step 4340/50462 | Loss: 2.9272 | LR: 4.94e-04 | 2826.5ms/step | 21.2 samples/s
386
+ Step 4350/50462 | Loss: 2.9196 | LR: 4.94e-04 | 2824.7ms/step | 21.2 samples/s
387
+ Step 4360/50462 | Loss: 2.9507 | LR: 4.94e-04 | 2827.7ms/step | 21.2 samples/s
388
+ Step 4370/50462 | Loss: 2.9366 | LR: 4.93e-04 | 2828.7ms/step | 21.2 samples/s
389
+ Step 4380/50462 | Loss: 2.9304 | LR: 4.93e-04 | 2827.5ms/step | 21.2 samples/s
390
+ Step 4390/50462 | Loss: 2.9474 | LR: 4.93e-04 | 2823.3ms/step | 21.3 samples/s
391
+ Step 4400/50462 | Loss: 2.9487 | LR: 4.93e-04 | 2824.9ms/step | 21.2 samples/s
392
+ Step 4410/50462 | Loss: 2.9371 | LR: 4.93e-04 | 2826.6ms/step | 21.2 samples/s
393
+ Step 4420/50462 | Loss: 2.9730 | LR: 4.93e-04 | 2831.6ms/step | 21.2 samples/s
394
+ Step 4430/50462 | Loss: 2.9602 | LR: 4.93e-04 | 2830.6ms/step | 21.2 samples/s
395
+ Step 4440/50462 | Loss: 2.9637 | LR: 4.93e-04 | 2830.0ms/step | 21.2 samples/s
396
+ Step 4450/50462 | Loss: 2.9654 | LR: 4.93e-04 | 2830.3ms/step | 21.2 samples/s
397
+ Step 4460/50462 | Loss: 2.9312 | LR: 4.93e-04 | 2828.5ms/step | 21.2 samples/s
398
+ Step 4470/50462 | Loss: 2.9378 | LR: 4.93e-04 | 2830.7ms/step | 21.2 samples/s
399
+ Step 4480/50462 | Loss: 2.9645 | LR: 4.93e-04 | 2827.0ms/step | 21.2 samples/s
400
+ Step 4490/50462 | Loss: 2.9449 | LR: 4.93e-04 | 2827.0ms/step | 21.2 samples/s
401
+ Step 4500/50462 | Loss: 2.9202 | LR: 4.93e-04 | 2828.6ms/step | 21.2 samples/s
402
+ Validation at step 4500: Loss = 2.9516, Perplexity = 19.14
403
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-4500
404
+ Step 4510/50462 | Loss: 2.9540 | LR: 4.93e-04 | 4772.5ms/step | 12.6 samples/s
405
+ Step 4520/50462 | Loss: 2.9086 | LR: 4.93e-04 | 2821.9ms/step | 21.3 samples/s
406
+ Step 4530/50462 | Loss: 2.9385 | LR: 4.93e-04 | 2824.3ms/step | 21.2 samples/s
407
+ Step 4540/50462 | Loss: 2.9316 | LR: 4.93e-04 | 2826.6ms/step | 21.2 samples/s
408
+ Step 4550/50462 | Loss: 2.9277 | LR: 4.93e-04 | 2827.8ms/step | 21.2 samples/s
409
+ Step 4560/50462 | Loss: 2.9409 | LR: 4.93e-04 | 2821.7ms/step | 21.3 samples/s
410
+ Step 4570/50462 | Loss: 2.9113 | LR: 4.93e-04 | 2827.5ms/step | 21.2 samples/s
411
+ Step 4580/50462 | Loss: 2.9662 | LR: 4.93e-04 | 2831.0ms/step | 21.2 samples/s
412
+ Step 4590/50462 | Loss: 2.9381 | LR: 4.93e-04 | 2828.3ms/step | 21.2 samples/s
413
+ Step 4600/50462 | Loss: 2.9329 | LR: 4.93e-04 | 2828.5ms/step | 21.2 samples/s
414
+ Step 4610/50462 | Loss: 2.9172 | LR: 4.93e-04 | 2827.7ms/step | 21.2 samples/s
415
+ Step 4620/50462 | Loss: 2.9270 | LR: 4.93e-04 | 2827.1ms/step | 21.2 samples/s
416
+ Step 4630/50462 | Loss: 2.9073 | LR: 4.93e-04 | 2826.0ms/step | 21.2 samples/s
417
+ Step 4640/50462 | Loss: 2.9119 | LR: 4.92e-04 | 2823.2ms/step | 21.3 samples/s
418
+ Step 4650/50462 | Loss: 2.9419 | LR: 4.92e-04 | 2823.0ms/step | 21.3 samples/s
419
+ Step 4660/50462 | Loss: 2.9446 | LR: 4.92e-04 | 2824.9ms/step | 21.2 samples/s
420
+ Step 4670/50462 | Loss: 2.9459 | LR: 4.92e-04 | 2823.2ms/step | 21.3 samples/s
421
+ Step 4680/50462 | Loss: 2.9351 | LR: 4.92e-04 | 2825.8ms/step | 21.2 samples/s
422
+ Step 4690/50462 | Loss: 2.9205 | LR: 4.92e-04 | 2823.9ms/step | 21.2 samples/s
423
+ Step 4700/50462 | Loss: 2.9090 | LR: 4.92e-04 | 2819.4ms/step | 21.3 samples/s
424
+ Step 4710/50462 | Loss: 2.9001 | LR: 4.92e-04 | 2822.0ms/step | 21.3 samples/s
425
+ Step 4720/50462 | Loss: 2.9211 | LR: 4.92e-04 | 2824.6ms/step | 21.2 samples/s
426
+ Step 4730/50462 | Loss: 2.9216 | LR: 4.92e-04 | 2820.6ms/step | 21.3 samples/s
427
+ Step 4740/50462 | Loss: 2.9168 | LR: 4.92e-04 | 2824.8ms/step | 21.2 samples/s
428
+ Step 4750/50462 | Loss: 2.9001 | LR: 4.92e-04 | 2823.3ms/step | 21.3 samples/s
429
+ Validation at step 4750: Loss = 2.9330, Perplexity = 18.78
430
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-4750
431
+ Step 4760/50462 | Loss: 2.9258 | LR: 4.92e-04 | 4762.7ms/step | 12.6 samples/s
432
+ Step 4770/50462 | Loss: 2.9153 | LR: 4.92e-04 | 2818.6ms/step | 21.3 samples/s
433
+ Step 4780/50462 | Loss: 2.9257 | LR: 4.92e-04 | 2825.6ms/step | 21.2 samples/s
434
+ Step 4790/50462 | Loss: 2.9101 | LR: 4.92e-04 | 2826.3ms/step | 21.2 samples/s
435
+ Step 4800/50462 | Loss: 2.9249 | LR: 4.92e-04 | 2824.2ms/step | 21.2 samples/s
436
+ Step 4810/50462 | Loss: 2.9117 | LR: 4.92e-04 | 2819.0ms/step | 21.3 samples/s
437
+ Step 4820/50462 | Loss: 2.8973 | LR: 4.92e-04 | 2824.3ms/step | 21.2 samples/s
438
+ Step 4830/50462 | Loss: 2.8972 | LR: 4.92e-04 | 2825.3ms/step | 21.2 samples/s
439
+ Step 4840/50462 | Loss: 2.9174 | LR: 4.92e-04 | 2825.8ms/step | 21.2 samples/s
440
+ Step 4850/50462 | Loss: 2.9290 | LR: 4.92e-04 | 2825.2ms/step | 21.2 samples/s
441
+ Step 4860/50462 | Loss: 2.9151 | LR: 4.92e-04 | 2821.8ms/step | 21.3 samples/s
442
+ Step 4870/50462 | Loss: 2.9073 | LR: 4.92e-04 | 2820.1ms/step | 21.3 samples/s
443
+ Step 4880/50462 | Loss: 2.9417 | LR: 4.92e-04 | 2824.9ms/step | 21.2 samples/s
444
+ Step 4890/50462 | Loss: 2.9004 | LR: 4.91e-04 | 2821.7ms/step | 21.3 samples/s
445
+ Step 4900/50462 | Loss: 2.9330 | LR: 4.91e-04 | 2824.4ms/step | 21.2 samples/s
446
+ Step 4910/50462 | Loss: 2.9109 | LR: 4.91e-04 | 2826.3ms/step | 21.2 samples/s
447
+ Step 4920/50462 | Loss: 2.9211 | LR: 4.91e-04 | 2821.8ms/step | 21.3 samples/s
448
+ Step 4930/50462 | Loss: 2.9027 | LR: 4.91e-04 | 2822.0ms/step | 21.3 samples/s
449
+ Step 4940/50462 | Loss: 2.9499 | LR: 4.91e-04 | 2824.6ms/step | 21.2 samples/s
450
+ Step 4950/50462 | Loss: 2.9034 | LR: 4.91e-04 | 2824.7ms/step | 21.2 samples/s
451
+ Step 4960/50462 | Loss: 2.8956 | LR: 4.91e-04 | 2823.4ms/step | 21.3 samples/s
452
+ Step 4970/50462 | Loss: 2.8969 | LR: 4.91e-04 | 2824.9ms/step | 21.2 samples/s
453
+ Step 4980/50462 | Loss: 2.9309 | LR: 4.91e-04 | 2821.2ms/step | 21.3 samples/s
454
+ Step 4990/50462 | Loss: 2.9132 | LR: 4.91e-04 | 2824.5ms/step | 21.2 samples/s
455
+ Step 5000/50462 | Loss: 2.8978 | LR: 4.91e-04 | 2822.2ms/step | 21.3 samples/s
456
+ Validation at step 5000: Loss = 2.9191, Perplexity = 18.52
457
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-5000
458
+ Step 5010/50462 | Loss: 2.9316 | LR: 4.91e-04 | 4750.4ms/step | 12.6 samples/s
459
+ Step 5020/50462 | Loss: 2.8803 | LR: 4.91e-04 | 2817.5ms/step | 21.3 samples/s
460
+ Step 5030/50462 | Loss: 2.9282 | LR: 4.91e-04 | 2824.3ms/step | 21.2 samples/s
461
+ Step 5040/50462 | Loss: 2.9254 | LR: 4.91e-04 | 2828.1ms/step | 21.2 samples/s
462
+ Step 5050/50462 | Loss: 2.8792 | LR: 4.91e-04 | 2829.3ms/step | 21.2 samples/s
463
+ Step 5060/50462 | Loss: 2.8882 | LR: 4.91e-04 | 2827.4ms/step | 21.2 samples/s
464
+ Step 5070/50462 | Loss: 2.8775 | LR: 4.91e-04 | 2829.3ms/step | 21.2 samples/s
465
+ Step 5080/50462 | Loss: 2.9056 | LR: 4.91e-04 | 2828.5ms/step | 21.2 samples/s
466
+ Step 5090/50462 | Loss: 2.8806 | LR: 4.91e-04 | 2830.7ms/step | 21.2 samples/s
467
+ Step 5100/50462 | Loss: 2.9018 | LR: 4.91e-04 | 2836.4ms/step | 21.2 samples/s
468
+ Step 5110/50462 | Loss: 2.9334 | LR: 4.91e-04 | 2832.6ms/step | 21.2 samples/s
469
+ Step 5120/50462 | Loss: 2.9076 | LR: 4.91e-04 | 2822.5ms/step | 21.3 samples/s
470
+ Step 5130/50462 | Loss: 2.8508 | LR: 4.90e-04 | 2825.5ms/step | 21.2 samples/s
471
+ Step 5140/50462 | Loss: 2.8856 | LR: 4.90e-04 | 2823.4ms/step | 21.3 samples/s
472
+ Step 5150/50462 | Loss: 2.8929 | LR: 4.90e-04 | 2818.7ms/step | 21.3 samples/s
473
+ Step 5160/50462 | Loss: 2.9245 | LR: 4.90e-04 | 2823.5ms/step | 21.2 samples/s
474
+ Step 5170/50462 | Loss: 2.8828 | LR: 4.90e-04 | 2824.4ms/step | 21.2 samples/s
475
+ Step 5180/50462 | Loss: 2.9112 | LR: 4.90e-04 | 2823.9ms/step | 21.2 samples/s
476
+ Step 5190/50462 | Loss: 2.8910 | LR: 4.90e-04 | 2824.0ms/step | 21.2 samples/s
477
+ Step 5200/50462 | Loss: 2.9031 | LR: 4.90e-04 | 2823.3ms/step | 21.3 samples/s
478
+ Step 5210/50462 | Loss: 2.8920 | LR: 4.90e-04 | 2820.8ms/step | 21.3 samples/s
479
+ Step 5220/50462 | Loss: 2.8773 | LR: 4.90e-04 | 2821.7ms/step | 21.3 samples/s
480
+ Step 5230/50462 | Loss: 2.9029 | LR: 4.90e-04 | 2821.0ms/step | 21.3 samples/s
481
+ Step 5240/50462 | Loss: 2.8666 | LR: 4.90e-04 | 2823.9ms/step | 21.2 samples/s
482
+ Step 5250/50462 | Loss: 2.8929 | LR: 4.90e-04 | 2827.5ms/step | 21.2 samples/s
483
+ Validation at step 5250: Loss = 2.9067, Perplexity = 18.30
484
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-5250
485
+ Step 5260/50462 | Loss: 2.8819 | LR: 4.90e-04 | 4767.7ms/step | 12.6 samples/s
486
+ Step 5270/50462 | Loss: 2.8807 | LR: 4.90e-04 | 2824.3ms/step | 21.2 samples/s
487
+ Step 5280/50462 | Loss: 2.8894 | LR: 4.90e-04 | 2831.5ms/step | 21.2 samples/s
488
+ Step 5290/50462 | Loss: 2.9180 | LR: 4.90e-04 | 2828.8ms/step | 21.2 samples/s
489
+ Step 5300/50462 | Loss: 2.8778 | LR: 4.90e-04 | 2829.2ms/step | 21.2 samples/s
490
+ Step 5310/50462 | Loss: 2.9280 | LR: 4.90e-04 | 2827.9ms/step | 21.2 samples/s
491
+ Step 5320/50462 | Loss: 2.9204 | LR: 4.90e-04 | 2831.6ms/step | 21.2 samples/s
492
+ Step 5330/50462 | Loss: 2.8728 | LR: 4.90e-04 | 2829.1ms/step | 21.2 samples/s
493
+ Step 5340/50462 | Loss: 2.8712 | LR: 4.90e-04 | 2830.1ms/step | 21.2 samples/s
494
+ Step 5350/50462 | Loss: 2.9084 | LR: 4.90e-04 | 2828.1ms/step | 21.2 samples/s
495
+ Step 5360/50462 | Loss: 2.8968 | LR: 4.89e-04 | 2829.3ms/step | 21.2 samples/s
496
+ Step 5370/50462 | Loss: 2.8702 | LR: 4.89e-04 | 2826.8ms/step | 21.2 samples/s
497
+ Step 5380/50462 | Loss: 2.9112 | LR: 4.89e-04 | 2832.5ms/step | 21.2 samples/s
498
+ Step 5390/50462 | Loss: 2.9066 | LR: 4.89e-04 | 2830.3ms/step | 21.2 samples/s
499
+ Step 5400/50462 | Loss: 2.8966 | LR: 4.89e-04 | 2827.6ms/step | 21.2 samples/s
500
+ Step 5410/50462 | Loss: 2.8947 | LR: 4.89e-04 | 2826.6ms/step | 21.2 samples/s
501
+ Step 5420/50462 | Loss: 2.8699 | LR: 4.89e-04 | 2827.3ms/step | 21.2 samples/s
502
+ Step 5430/50462 | Loss: 2.8956 | LR: 4.89e-04 | 2828.5ms/step | 21.2 samples/s
503
+ Step 5440/50462 | Loss: 2.8675 | LR: 4.89e-04 | 2830.4ms/step | 21.2 samples/s
504
+ Step 5450/50462 | Loss: 2.8723 | LR: 4.89e-04 | 2831.2ms/step | 21.2 samples/s
505
+ Step 5460/50462 | Loss: 2.8548 | LR: 4.89e-04 | 2826.4ms/step | 21.2 samples/s
506
+ Step 5470/50462 | Loss: 2.8828 | LR: 4.89e-04 | 2828.1ms/step | 21.2 samples/s
507
+ Step 5480/50462 | Loss: 2.8631 | LR: 4.89e-04 | 2828.7ms/step | 21.2 samples/s
508
+ Step 5490/50462 | Loss: 2.8714 | LR: 4.89e-04 | 2828.4ms/step | 21.2 samples/s
509
+ Step 5500/50462 | Loss: 2.8743 | LR: 4.89e-04 | 2828.7ms/step | 21.2 samples/s
510
+ Validation at step 5500: Loss = 2.8967, Perplexity = 18.11
511
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-5500
512
+ Step 5510/50462 | Loss: 2.8640 | LR: 4.89e-04 | 4749.9ms/step | 12.6 samples/s
513
+ Step 5520/50462 | Loss: 2.8895 | LR: 4.89e-04 | 2826.1ms/step | 21.2 samples/s
514
+ Step 5530/50462 | Loss: 2.8971 | LR: 4.89e-04 | 2825.5ms/step | 21.2 samples/s
515
+ Step 5540/50462 | Loss: 2.8604 | LR: 4.89e-04 | 2826.6ms/step | 21.2 samples/s
516
+ Step 5550/50462 | Loss: 2.8802 | LR: 4.89e-04 | 2827.0ms/step | 21.2 samples/s
517
+ Step 5560/50462 | Loss: 2.8711 | LR: 4.89e-04 | 2827.9ms/step | 21.2 samples/s
518
+ Step 5570/50462 | Loss: 2.8547 | LR: 4.88e-04 | 2827.7ms/step | 21.2 samples/s
519
+ Step 5580/50462 | Loss: 2.8673 | LR: 4.88e-04 | 2824.9ms/step | 21.2 samples/s
520
+ Step 5590/50462 | Loss: 2.9145 | LR: 4.88e-04 | 2825.2ms/step | 21.2 samples/s
521
+ Step 5600/50462 | Loss: 2.8603 | LR: 4.88e-04 | 2829.6ms/step | 21.2 samples/s
522
+ Step 5610/50462 | Loss: 2.8738 | LR: 4.88e-04 | 2831.7ms/step | 21.2 samples/s
523
+ Step 5620/50462 | Loss: 2.8950 | LR: 4.88e-04 | 2824.6ms/step | 21.2 samples/s
524
+ Step 5630/50462 | Loss: 2.8686 | LR: 4.88e-04 | 2824.8ms/step | 21.2 samples/s
525
+ Step 5640/50462 | Loss: 2.8893 | LR: 4.88e-04 | 2825.5ms/step | 21.2 samples/s
526
+ Step 5650/50462 | Loss: 2.8340 | LR: 4.88e-04 | 2832.0ms/step | 21.2 samples/s
527
+ Step 5660/50462 | Loss: 2.8603 | LR: 4.88e-04 | 2829.7ms/step | 21.2 samples/s
528
+ Step 5670/50462 | Loss: 2.8687 | LR: 4.88e-04 | 2829.1ms/step | 21.2 samples/s
529
+ Step 5680/50462 | Loss: 2.8670 | LR: 4.88e-04 | 2825.5ms/step | 21.2 samples/s
530
+ Step 5690/50462 | Loss: 2.8678 | LR: 4.88e-04 | 2828.6ms/step | 21.2 samples/s
531
+ Step 5700/50462 | Loss: 2.8565 | LR: 4.88e-04 | 2826.6ms/step | 21.2 samples/s
532
+ Step 5710/50462 | Loss: 2.8708 | LR: 4.88e-04 | 2825.7ms/step | 21.2 samples/s
533
+ Step 5720/50462 | Loss: 2.8999 | LR: 4.88e-04 | 2826.7ms/step | 21.2 samples/s
534
+ Step 5730/50462 | Loss: 2.8736 | LR: 4.88e-04 | 2831.8ms/step | 21.2 samples/s
535
+ Step 5740/50462 | Loss: 2.8853 | LR: 4.88e-04 | 2832.3ms/step | 21.2 samples/s
536
+ Step 5750/50462 | Loss: 2.8778 | LR: 4.88e-04 | 2826.5ms/step | 21.2 samples/s
537
+ Validation at step 5750: Loss = 2.8842, Perplexity = 17.89
538
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-5750
539
+ Step 5760/50462 | Loss: 2.8811 | LR: 4.88e-04 | 4756.4ms/step | 12.6 samples/s
540
+ Step 5770/50462 | Loss: 2.8951 | LR: 4.88e-04 | 2822.1ms/step | 21.3 samples/s
541
+ Step 5780/50462 | Loss: 2.8813 | LR: 4.87e-04 | 2826.0ms/step | 21.2 samples/s
542
+ Step 5790/50462 | Loss: 2.8404 | LR: 4.87e-04 | 2826.3ms/step | 21.2 samples/s
543
+ Step 5800/50462 | Loss: 2.8783 | LR: 4.87e-04 | 2823.0ms/step | 21.3 samples/s
544
+ Step 5810/50462 | Loss: 2.8830 | LR: 4.87e-04 | 2824.6ms/step | 21.2 samples/s
545
+ Step 5820/50462 | Loss: 2.8514 | LR: 4.87e-04 | 2825.9ms/step | 21.2 samples/s
546
+ Step 5830/50462 | Loss: 2.8776 | LR: 4.87e-04 | 2824.4ms/step | 21.2 samples/s
547
+ Step 5840/50462 | Loss: 2.8238 | LR: 4.87e-04 | 2829.3ms/step | 21.2 samples/s
548
+ Step 5850/50462 | Loss: 2.8744 | LR: 4.87e-04 | 2826.8ms/step | 21.2 samples/s
549
+ Step 5860/50462 | Loss: 2.8967 | LR: 4.87e-04 | 2828.7ms/step | 21.2 samples/s
550
+ Step 5870/50462 | Loss: 2.8746 | LR: 4.87e-04 | 2824.4ms/step | 21.2 samples/s
551
+ Step 5880/50462 | Loss: 2.8556 | LR: 4.87e-04 | 2825.6ms/step | 21.2 samples/s
552
+ Step 5890/50462 | Loss: 2.8700 | LR: 4.87e-04 | 2827.4ms/step | 21.2 samples/s
553
+ Step 5900/50462 | Loss: 2.8365 | LR: 4.87e-04 | 2828.1ms/step | 21.2 samples/s
554
+ Step 5910/50462 | Loss: 2.8588 | LR: 4.87e-04 | 2831.0ms/step | 21.2 samples/s
555
+ Step 5920/50462 | Loss: 2.8830 | LR: 4.87e-04 | 2832.3ms/step | 21.2 samples/s
556
+ Step 5930/50462 | Loss: 2.8810 | LR: 4.87e-04 | 2835.2ms/step | 21.2 samples/s
557
+ Step 5940/50462 | Loss: 2.8708 | LR: 4.87e-04 | 2833.6ms/step | 21.2 samples/s
558
+ Step 5950/50462 | Loss: 2.8651 | LR: 4.87e-04 | 2832.6ms/step | 21.2 samples/s
559
+ Step 5960/50462 | Loss: 2.8942 | LR: 4.87e-04 | 2832.2ms/step | 21.2 samples/s
560
+ Step 5970/50462 | Loss: 2.8955 | LR: 4.87e-04 | 2829.8ms/step | 21.2 samples/s
561
+ Step 5980/50462 | Loss: 2.8690 | LR: 4.86e-04 | 2824.9ms/step | 21.2 samples/s
562
+ Step 5990/50462 | Loss: 2.8288 | LR: 4.86e-04 | 2826.1ms/step | 21.2 samples/s
563
+ Step 6000/50462 | Loss: 2.8741 | LR: 4.86e-04 | 2829.3ms/step | 21.2 samples/s
564
+ Validation at step 6000: Loss = 2.8723, Perplexity = 17.68
565
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-6000
566
+ Step 6010/50462 | Loss: 2.8597 | LR: 4.86e-04 | 4744.6ms/step | 12.6 samples/s
567
+ Step 6020/50462 | Loss: 2.8519 | LR: 4.86e-04 | 2824.0ms/step | 21.2 samples/s
568
+ Step 6030/50462 | Loss: 2.8259 | LR: 4.86e-04 | 2826.1ms/step | 21.2 samples/s
569
+ Step 6040/50462 | Loss: 2.8711 | LR: 4.86e-04 | 2830.3ms/step | 21.2 samples/s
570
+ Step 6050/50462 | Loss: 2.8437 | LR: 4.86e-04 | 2824.0ms/step | 21.2 samples/s
571
+ Step 6060/50462 | Loss: 2.8692 | LR: 4.86e-04 | 2825.0ms/step | 21.2 samples/s
572
+ Step 6070/50462 | Loss: 2.8517 | LR: 4.86e-04 | 2828.4ms/step | 21.2 samples/s
573
+ Step 6080/50462 | Loss: 2.8451 | LR: 4.86e-04 | 2829.4ms/step | 21.2 samples/s
574
+ Step 6090/50462 | Loss: 2.8652 | LR: 4.86e-04 | 2825.3ms/step | 21.2 samples/s
575
+ Step 6100/50462 | Loss: 2.8646 | LR: 4.86e-04 | 2833.5ms/step | 21.2 samples/s
576
+ Step 6110/50462 | Loss: 2.8353 | LR: 4.86e-04 | 2830.6ms/step | 21.2 samples/s
577
+ Step 6120/50462 | Loss: 2.8590 | LR: 4.86e-04 | 2829.5ms/step | 21.2 samples/s
578
+ Step 6130/50462 | Loss: 2.8722 | LR: 4.86e-04 | 2826.8ms/step | 21.2 samples/s
579
+ Step 6140/50462 | Loss: 2.8601 | LR: 4.86e-04 | 2830.5ms/step | 21.2 samples/s
580
+ Step 6150/50462 | Loss: 2.8556 | LR: 4.86e-04 | 2830.8ms/step | 21.2 samples/s
581
+ Step 6160/50462 | Loss: 2.8648 | LR: 4.86e-04 | 2827.6ms/step | 21.2 samples/s
582
+ Step 6170/50462 | Loss: 2.8784 | LR: 4.85e-04 | 2830.2ms/step | 21.2 samples/s
583
+ Step 6180/50462 | Loss: 2.8731 | LR: 4.85e-04 | 2826.2ms/step | 21.2 samples/s
584
+ Step 6190/50462 | Loss: 2.8551 | LR: 4.85e-04 | 2827.7ms/step | 21.2 samples/s
585
+ Step 6200/50462 | Loss: 2.8627 | LR: 4.85e-04 | 2824.1ms/step | 21.2 samples/s
586
+ Step 6210/50462 | Loss: 2.8903 | LR: 4.85e-04 | 2828.8ms/step | 21.2 samples/s
587
+ Step 6220/50462 | Loss: 2.8524 | LR: 4.85e-04 | 2831.8ms/step | 21.2 samples/s
588
+ Step 6230/50462 | Loss: 2.8157 | LR: 4.85e-04 | 2830.2ms/step | 21.2 samples/s
epfl_final/train_logs/p2_1p2b_dense_seed72_4gpu_resume2750_live.log ADDED
@@ -0,0 +1,186 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ W0903 14:12:24.527000 200 site-packages/torch/distributed/run.py:851]
2
+ W0903 14:12:24.527000 200 site-packages/torch/distributed/run.py:851] *****************************************
3
+ W0903 14:12:24.527000 200 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
4
+ W0903 14:12:24.527000 200 site-packages/torch/distributed/run.py:851] *****************************************
5
+ Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72
6
+ Running config:
7
+ {
8
+ "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
9
+ "model_dtype": "bfloat16",
10
+ "init_from_scratch": true,
11
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
12
+ "val_path": null,
13
+ "max_seq_length": 8192,
14
+ "num_workers": 8,
15
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72",
16
+ "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/checkpoint-2750",
17
+ "force_new_run": false,
18
+ "num_epochs": 1,
19
+ "max_steps": 50462,
20
+ "per_device_batch_size": 1,
21
+ "gradient_accumulation_steps": 15,
22
+ "learning_rate": 0.0005,
23
+ "weight_decay": 0.01,
24
+ "warmup_steps": 750,
25
+ "lr_scheduler": "cosine",
26
+ "max_grad_norm": 1.0,
27
+ "save_steps": 250,
28
+ "eval_steps": 250,
29
+ "eval_batches": 50,
30
+ "logging_steps": 10,
31
+ "seed": 72,
32
+ "verbose": true,
33
+ "checkpoint_activations": false,
34
+ "resume_micro_step_override": null,
35
+ "strategy": "ddp",
36
+ "mixed_precision": "bf16",
37
+ "wandb_enabled": true,
38
+ "wandb_project": "stochastic_sparsity",
39
+ "wandb_name": "1.2b dense 8k seed72 4gpu",
40
+ "wandb_tags": [
41
+ "qwen3",
42
+ "1.2b",
43
+ "8k",
44
+ "p2",
45
+ "dense",
46
+ "seed72"
47
+ ],
48
+ "hidden_size": 1536,
49
+ "intermediate_size": 6144,
50
+ "num_attention_heads": 12,
51
+ "num_key_value_heads": 4,
52
+ "num_hidden_layers": 28,
53
+ "head_dim": null,
54
+ "attention_dropout": null,
55
+ "rope_train_position_scale_set": null,
56
+ "attention_type": "sparse_stochastic",
57
+ "sparse_block_size": 128,
58
+ "sparse_num_random_blocks": 31,
59
+ "sparse_mask_pool_size": 1,
60
+ "sparse_val_dense_compare": false,
61
+ "sparse_layer_indices": null,
62
+ "sparse_mask_cache_dir": null,
63
+ "sparse_mask_seed": 0,
64
+ "sparse_mixed_nrb_set": null,
65
+ "dense_mode": true,
66
+ "dense_step_intervals": null
67
+ }
68
+ Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
69
+ [rank3]: Traceback (most recent call last):
70
+ [rank3]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1519, in <module>
71
+ [rank3]: main()
72
+ [rank3]: ~~~~^^
73
+ [rank3]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1516, in main
74
+ [rank3]: train(config)
75
+ [rank3]: ~~~~~^^^^^^^^
76
+ [rank3]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 999, in train
77
+ [rank3]: model_config = Qwen3Config.from_json(checkpoint_config_path)
78
+ [rank3]: File "/mloscratch/homes/navasard/sparse/SparseAtt/sparseatt/config.py", line 79, in from_json
79
+ [rank3]: with open(path, "r") as f:
80
+ [rank3]: ~~~~^^^^^^^^^^^
81
+ [rank3]: FileNotFoundError: [Errno 2] No such file or directory: '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/checkpoint-2750/config.json'
82
+ [rank2]: Traceback (most recent call last):
83
+ [rank2]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1519, in <module>
84
+ [rank2]: main()
85
+ [rank2]: ~~~~^^
86
+ [rank2]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1516, in main
87
+ [rank2]: train(config)
88
+ [rank2]: ~~~~~^^^^^^^^
89
+ [rank2]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 999, in train
90
+ [rank2]: model_config = Qwen3Config.from_json(checkpoint_config_path)
91
+ [rank2]: File "/mloscratch/homes/navasard/sparse/SparseAtt/sparseatt/config.py", line 79, in from_json
92
+ [rank2]: with open(path, "r") as f:
93
+ [rank2]: ~~~~^^^^^^^^^^^
94
+ [rank2]: FileNotFoundError: [Errno 2] No such file or directory: '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/checkpoint-2750/config.json'
95
+ Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/checkpoint-2750
96
+ [rank0]: Traceback (most recent call last):
97
+ [rank0]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1519, in <module>
98
+ [rank0]: main()
99
+ [rank0]: ~~~~^^
100
+ [rank0]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1516, in main
101
+ [rank0]: train(config)
102
+ [rank0]: ~~~~~^^^^^^^^
103
+ [rank0]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 999, in train
104
+ [rank0]: model_config = Qwen3Config.from_json(checkpoint_config_path)
105
+ [rank0]: File "/mloscratch/homes/navasard/sparse/SparseAtt/sparseatt/config.py", line 79, in from_json
106
+ [rank0]: with open(path, "r") as f:
107
+ [rank0]: ~~~~^^^^^^^^^^^
108
+ [rank0]: FileNotFoundError: [Errno 2] No such file or directory: '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/checkpoint-2750/config.json'
109
+ [rank1]: Traceback (most recent call last):
110
+ [rank1]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1519, in <module>
111
+ [rank1]: main()
112
+ [rank1]: ~~~~^^
113
+ [rank1]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1516, in main
114
+ [rank1]: train(config)
115
+ [rank1]: ~~~~~^^^^^^^^
116
+ [rank1]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 999, in train
117
+ [rank1]: model_config = Qwen3Config.from_json(checkpoint_config_path)
118
+ [rank1]: File "/mloscratch/homes/navasard/sparse/SparseAtt/sparseatt/config.py", line 79, in from_json
119
+ [rank1]: with open(path, "r") as f:
120
+ [rank1]: ~~~~^^^^^^^^^^^
121
+ [rank1]: FileNotFoundError: [Errno 2] No such file or directory: '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/checkpoint-2750/config.json'
122
+ [rank0]:[W903 14:12:52.362783876 ProcessGroupNCCL.cpp:1575] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
123
+ W0903 14:12:54.261000 200 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 270 closing signal SIGTERM
124
+ W0903 14:12:54.263000 200 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 271 closing signal SIGTERM
125
+ W0903 14:12:54.263000 200 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 272 closing signal SIGTERM
126
+ E0903 14:12:54.378000 200 site-packages/torch/distributed/elastic/multiprocessing/api.py:986] failed (exitcode: 1) local_rank: 0 (pid: 269) of binary: /mloscratch/homes/navasard/conda/bin/python3.13
127
+ Traceback (most recent call last):
128
+ File "/mloscratch/homes/navasard/conda/bin/torchrun", line 6, in <module>
129
+ sys.exit(main())
130
+ ~~~~^^
131
+ File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper
132
+ return f(*args, **kwargs)
133
+ File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main
134
+ run(args)
135
+ ~~~^^^^^^
136
+ File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run
137
+ elastic_launch(
138
+ ~~~~~~~~~~~~~~~
139
+ config=config,
140
+ ~~~~~~~~~~~~~~
141
+ entrypoint=cmd,
142
+ ~~~~~~~~~~~~~~~
143
+ )(*cmd_args)
144
+ ~^^^^^^^^^^^
145
+ File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__
146
+ return launch_agent(self._config, self._entrypoint, list(args))
147
+ File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 317, in launch_agent
148
+ raise ChildFailedError(
149
+ ...<2 lines>...
150
+ )
151
+ torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
152
+ ============================================================
153
+ train.py FAILED
154
+ ------------------------------------------------------------
155
+ Failures:
156
+ [1]:
157
+ time : 2026-09-03_14:12:54
158
+ host : h100-train-4-gpu-0-53
159
+ rank : 1 (local_rank: 1)
160
+ exitcode : 1 (pid: 270)
161
+ error_file: <N/A>
162
+ traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
163
+ [2]:
164
+ time : 2026-09-03_14:12:54
165
+ host : h100-train-4-gpu-0-53
166
+ rank : 2 (local_rank: 2)
167
+ exitcode : 1 (pid: 271)
168
+ error_file: <N/A>
169
+ traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
170
+ [3]:
171
+ time : 2026-09-03_14:12:54
172
+ host : h100-train-4-gpu-0-53
173
+ rank : 3 (local_rank: 3)
174
+ exitcode : 1 (pid: 272)
175
+ error_file: <N/A>
176
+ traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
177
+ ------------------------------------------------------------
178
+ Root Cause (first observed failure):
179
+ [0]:
180
+ time : 2026-09-03_14:12:54
181
+ host : h100-train-4-gpu-0-53
182
+ rank : 0 (local_rank: 0)
183
+ exitcode : 1 (pid: 269)
184
+ error_file: <N/A>
185
+ traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
186
+ ============================================================
epfl_final/train_logs/p2_1p2b_dense_seed72_6gpu_repair_from9250.log ADDED
@@ -0,0 +1,1136 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ W0906 12:02:29.907000 266 site-packages/torch/distributed/run.py:851]
2
+ W0906 12:02:29.907000 266 site-packages/torch/distributed/run.py:851] *****************************************
3
+ W0906 12:02:29.907000 266 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
4
+ W0906 12:02:29.907000 266 site-packages/torch/distributed/run.py:851] *****************************************
5
+ Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250
6
+ Running config:
7
+ {
8
+ "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
9
+ "model_dtype": "bfloat16",
10
+ "init_from_scratch": true,
11
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
12
+ "val_path": null,
13
+ "max_seq_length": 8192,
14
+ "num_workers": 8,
15
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250",
16
+ "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-9250",
17
+ "force_new_run": false,
18
+ "num_epochs": 1,
19
+ "max_steps": 50462,
20
+ "per_device_batch_size": 1,
21
+ "gradient_accumulation_steps": 10,
22
+ "learning_rate": 0.0005,
23
+ "weight_decay": 0.01,
24
+ "warmup_steps": 750,
25
+ "lr_scheduler": "cosine",
26
+ "max_grad_norm": 1.0,
27
+ "save_steps": 250,
28
+ "eval_steps": 250,
29
+ "eval_batches": 50,
30
+ "logging_steps": 10,
31
+ "seed": 72,
32
+ "verbose": true,
33
+ "checkpoint_activations": false,
34
+ "resume_micro_step_override": 92500,
35
+ "strategy": "ddp",
36
+ "mixed_precision": "bf16",
37
+ "wandb_enabled": false,
38
+ "wandb_project": "stochastic_sparsity",
39
+ "wandb_name": "1.2b dense 8k seed72 6gpu resume9250",
40
+ "wandb_tags": [
41
+ "qwen3",
42
+ "1.2b",
43
+ "8k",
44
+ "p2",
45
+ "dense",
46
+ "seed72"
47
+ ],
48
+ "hidden_size": 1536,
49
+ "intermediate_size": 6144,
50
+ "num_attention_heads": 12,
51
+ "num_key_value_heads": 4,
52
+ "num_hidden_layers": 28,
53
+ "head_dim": null,
54
+ "attention_dropout": null,
55
+ "rope_train_position_scale_set": null,
56
+ "attention_type": "sparse_stochastic",
57
+ "sparse_block_size": 128,
58
+ "sparse_num_random_blocks": 31,
59
+ "sparse_mask_pool_size": 1,
60
+ "sparse_val_dense_compare": false,
61
+ "sparse_layer_indices": null,
62
+ "sparse_mask_cache_dir": null,
63
+ "sparse_mask_seed": 0,
64
+ "sparse_mixed_nrb_set": null,
65
+ "dense_mode": true,
66
+ "dense_step_intervals": null
67
+ }
68
+ Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
69
+ Checkpoint activations: FalseCheckpoint activations: FalseCheckpoint activations: False
70
+
71
+
72
+ Checkpoint activations: False
73
+ Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-9250
74
+ Checkpoint activations: False
75
+ Checkpoint activations: False
76
+ dense_mode=True: sparse attention layers will use standard causal SDPA.
77
+
78
+
79
+ ================================================================================
80
+ MODEL ARCHITECTURE
81
+ ================================================================================
82
+ Model config:
83
+ vocab_size: 151936
84
+ hidden_size: 1536
85
+ num_hidden_layers: 28
86
+ num_attention_heads: 12
87
+ num_key_value_heads: 4
88
+ head_dim: 128
89
+ intermediate_size: 6144
90
+ max_position_embeddings: 32768
91
+ rope_theta: 1000000
92
+ rope_train_position_scale_set: None
93
+ attention_type: sparse_stochastic
94
+ sparse_block_size: 128
95
+ sparse_num_random_blocks: 31
96
+ sparse_mask_pool_size: 1
97
+ sparse_max_seq_length: 8192
98
+
99
+ Parameter count:
100
+ Total parameters: 1,202,352,640
101
+ Trainable parameters: 1,202,352,640
102
+ Non-trainable parameters: 0
103
+ Model size (BF16): 2.40 GB
104
+
105
+ --------------------------------------------------------------------------------
106
+ LAYER-BY-LAYER SUMMARY
107
+ --------------------------------------------------------------------------------
108
+ ================================================================================================================================================================
109
+ Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
110
+ ================================================================================================================================================================
111
+ Qwen3ForCausalLM [1, 8192] -- -- True
112
+ ├─Qwen3Model: 1-1 -- [1, 8192, 1536] -- True
113
+ │ └─Embedding: 2-1 [1, 8192] [1, 8192, 1536] 233,373,696 True
114
+ │ └─ModuleList: 2-2 -- -- -- True
115
+ │ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
116
+ │ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
117
+ │ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
118
+ │ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
119
+ │ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
120
+ │ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
121
+ │ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
122
+ │ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
123
+ │ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
124
+ │ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
125
+ │ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
126
+ │ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
127
+ │ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
128
+ │ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
129
+ │ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
130
+ │ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
131
+ │ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
132
+ │ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
133
+ │ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
134
+ │ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
135
+ │ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
136
+ │ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
137
+ │ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
138
+ │ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
139
+ │ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
140
+ │ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
141
+ │ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
142
+ │ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
143
+ │ └─Qwen3RMSNorm: 2-3 [1, 8192, 1536] [1, 8192, 1536] 1,536 True
144
+ ├─Linear: 1-2 [1, 8192, 1536] [1, 8192, 151936] 233,373,696 True
145
+ ================================================================================================================================================================
146
+ Total params: 1,435,726,336
147
+ Trainable params: 1,435,726,336
148
+ Non-trainable params: 0
149
+ Total mult-adds (Units.GIGABYTES): 1.44
150
+ ================================================================================================================================================================
151
+ Input size (MB): 0.07
152
+ Forward/backward pass size (MB): 26218.59
153
+ Params size (MB): 2871.45
154
+ Estimated Total Size (MB): 29090.11
155
+ ================================================================================================================================================================
156
+ NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
157
+ ================================================================================
158
+
159
+ Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
160
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
161
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
162
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
163
+ Using pre-tokenized dataset (no runtime tokenization)
164
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
165
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
166
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
167
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
168
+
169
+
170
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
171
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
172
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
173
+ Dataset split: 3027764 train, 159356 validation
174
+ Training examples (packed, train split): 3027764
175
+ Validation examples (packed, val split): 159356
176
+ Per-device batch size: 1
177
+ World size (GPUs): 6
178
+ Gradient accumulation steps: 10
179
+ Effective batch size (seqs/step): 60
180
+ Sequence length: 8192
181
+ Tokens per optimizer step: 491,520
182
+ Micro-batches per epoch (per rank): 504628
183
+ Optimizer steps per epoch: 50462
184
+ Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
185
+ Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-9250...
186
+ Preserving checkpoint LR state for exact resume: current=[0.0004647907250034732], schedule_base=[0.0005]
187
+ Resumed from checkpoint at step 9250 (epoch 0, micro_step 138750)
188
+ Overriding resume micro-step 138750 -> 92500 for DDP migration
189
+ Resuming training from step 9250
190
+ Starting training...
191
+ Validation at step 9250: Loss = 2.7934, Perplexity = 16.34
192
+ Switching attention mode for step 9251: dense (dense step interval)
193
+ Step 9260/50462 | Loss: 2.7563 | LR: 4.65e-04 | 20632.5ms/step | 2.9 samples/s
194
+ Step 9270/50462 | Loss: 2.7584 | LR: 4.65e-04 | 1884.6ms/step | 31.8 samples/s
195
+ Step 9280/50462 | Loss: 2.7460 | LR: 4.65e-04 | 1889.9ms/step | 31.7 samples/s
196
+ Step 9290/50462 | Loss: 2.7486 | LR: 4.64e-04 | 1897.9ms/step | 31.6 samples/s
197
+ Step 9300/50462 | Loss: 2.7578 | LR: 4.64e-04 | 1902.6ms/step | 31.5 samples/s
198
+ Step 9310/50462 | Loss: 2.7526 | LR: 4.64e-04 | 1903.7ms/step | 31.5 samples/s
199
+ Step 9320/50462 | Loss: 2.7833 | LR: 4.64e-04 | 1903.4ms/step | 31.5 samples/s
200
+ Step 9330/50462 | Loss: 2.7450 | LR: 4.64e-04 | 1905.5ms/step | 31.5 samples/s
201
+ Step 9340/50462 | Loss: 2.7808 | LR: 4.64e-04 | 1907.3ms/step | 31.5 samples/s
202
+ Step 9350/50462 | Loss: 2.7620 | LR: 4.64e-04 | 1904.5ms/step | 31.5 samples/s
203
+ Step 9360/50462 | Loss: 2.7410 | LR: 4.64e-04 | 1906.9ms/step | 31.5 samples/s
204
+ Step 9370/50462 | Loss: 2.7630 | LR: 4.64e-04 | 1907.7ms/step | 31.5 samples/s
205
+ Step 9380/50462 | Loss: 2.7650 | LR: 4.64e-04 | 1909.9ms/step | 31.4 samples/s
206
+ Step 9390/50462 | Loss: 2.7817 | LR: 4.64e-04 | 1908.1ms/step | 31.4 samples/s
207
+ Step 9400/50462 | Loss: 2.7951 | LR: 4.64e-04 | 1909.4ms/step | 31.4 samples/s
208
+ Step 9410/50462 | Loss: 2.7474 | LR: 4.63e-04 | 1908.6ms/step | 31.4 samples/s
209
+ Step 9420/50462 | Loss: 2.7346 | LR: 4.63e-04 | 1904.9ms/step | 31.5 samples/s
210
+ Step 9430/50462 | Loss: 2.7733 | LR: 4.63e-04 | 1905.4ms/step | 31.5 samples/s
211
+ Step 9440/50462 | Loss: 2.7192 | LR: 4.63e-04 | 1906.3ms/step | 31.5 samples/s
212
+ Step 9450/50462 | Loss: 2.7660 | LR: 4.63e-04 | 1907.5ms/step | 31.5 samples/s
213
+ Step 9460/50462 | Loss: 2.7749 | LR: 4.63e-04 | 1906.8ms/step | 31.5 samples/s
214
+ Step 9470/50462 | Loss: 2.7395 | LR: 4.63e-04 | 1907.9ms/step | 31.4 samples/s
215
+ Step 9480/50462 | Loss: 2.7259 | LR: 4.63e-04 | 1909.2ms/step | 31.4 samples/s
216
+ Step 9490/50462 | Loss: 2.7522 | LR: 4.63e-04 | 1908.0ms/step | 31.4 samples/s
217
+ Step 9500/50462 | Loss: 2.7571 | LR: 4.63e-04 | 1908.6ms/step | 31.4 samples/s
218
+ Validation at step 9500: Loss = 2.7884, Perplexity = 16.26
219
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-9500
220
+ Step 9510/50462 | Loss: 2.7759 | LR: 4.63e-04 | 3270.2ms/step | 18.3 samples/s
221
+ Step 9520/50462 | Loss: 2.7639 | LR: 4.63e-04 | 1902.3ms/step | 31.5 samples/s
222
+ Step 9530/50462 | Loss: 2.7665 | LR: 4.62e-04 | 1908.3ms/step | 31.4 samples/s
223
+ Step 9540/50462 | Loss: 2.7760 | LR: 4.62e-04 | 1911.8ms/step | 31.4 samples/s
224
+ Step 9550/50462 | Loss: 2.7575 | LR: 4.62e-04 | 1911.3ms/step | 31.4 samples/s
225
+ Step 9560/50462 | Loss: 2.7086 | LR: 4.62e-04 | 1915.6ms/step | 31.3 samples/s
226
+ Step 9570/50462 | Loss: 2.7684 | LR: 4.62e-04 | 1911.6ms/step | 31.4 samples/s
227
+ Step 9580/50462 | Loss: 2.7580 | LR: 4.62e-04 | 1917.6ms/step | 31.3 samples/s
228
+ Step 9590/50462 | Loss: 2.7418 | LR: 4.62e-04 | 1917.4ms/step | 31.3 samples/s
229
+ Step 9600/50462 | Loss: 2.7531 | LR: 4.62e-04 | 1909.8ms/step | 31.4 samples/s
230
+ Step 9610/50462 | Loss: 2.7372 | LR: 4.62e-04 | 1911.0ms/step | 31.4 samples/s
231
+ Step 9620/50462 | Loss: 2.7232 | LR: 4.62e-04 | 1910.1ms/step | 31.4 samples/s
232
+ Step 9630/50462 | Loss: 2.7336 | LR: 4.62e-04 | 1911.3ms/step | 31.4 samples/s
233
+ Step 9640/50462 | Loss: 2.7657 | LR: 4.62e-04 | 1914.7ms/step | 31.3 samples/s
234
+ Step 9650/50462 | Loss: 2.8060 | LR: 4.61e-04 | 1913.3ms/step | 31.4 samples/s
235
+ Step 9660/50462 | Loss: 2.7203 | LR: 4.61e-04 | 1910.9ms/step | 31.4 samples/s
236
+ Step 9670/50462 | Loss: 2.7772 | LR: 4.61e-04 | 1912.5ms/step | 31.4 samples/s
237
+ Step 9680/50462 | Loss: 2.7707 | LR: 4.61e-04 | 1913.4ms/step | 31.4 samples/s
238
+ Step 9690/50462 | Loss: 2.7947 | LR: 4.61e-04 | 1913.2ms/step | 31.4 samples/s
239
+ Step 9700/50462 | Loss: 2.7353 | LR: 4.61e-04 | 1911.9ms/step | 31.4 samples/s
240
+ Step 9710/50462 | Loss: 2.7642 | LR: 4.61e-04 | 1910.7ms/step | 31.4 samples/s
241
+ Step 9720/50462 | Loss: 2.7480 | LR: 4.61e-04 | 1912.7ms/step | 31.4 samples/s
242
+ Step 9730/50462 | Loss: 2.7499 | LR: 4.61e-04 | 1911.4ms/step | 31.4 samples/s
243
+ Step 9740/50462 | Loss: 2.7472 | LR: 4.61e-04 | 1913.4ms/step | 31.4 samples/s
244
+ Step 9750/50462 | Loss: 2.7516 | LR: 4.61e-04 | 1911.1ms/step | 31.4 samples/s
245
+ Validation at step 9750: Loss = 2.7825, Perplexity = 16.16
246
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-9750
247
+ Step 9760/50462 | Loss: 2.7461 | LR: 4.61e-04 | 3203.7ms/step | 18.7 samples/s
248
+ Step 9770/50462 | Loss: 2.7727 | LR: 4.60e-04 | 1900.8ms/step | 31.6 samples/s
249
+ Step 9780/50462 | Loss: 2.7404 | LR: 4.60e-04 | 1905.9ms/step | 31.5 samples/s
250
+ Step 9790/50462 | Loss: 2.7550 | LR: 4.60e-04 | 1910.1ms/step | 31.4 samples/s
251
+ Step 9800/50462 | Loss: 2.7153 | LR: 4.60e-04 | 1910.5ms/step | 31.4 samples/s
252
+ Step 9810/50462 | Loss: 2.7620 | LR: 4.60e-04 | 1912.8ms/step | 31.4 samples/s
253
+ Step 9820/50462 | Loss: 2.7638 | LR: 4.60e-04 | 1911.5ms/step | 31.4 samples/s
254
+ Step 9830/50462 | Loss: 2.7192 | LR: 4.60e-04 | 1911.4ms/step | 31.4 samples/s
255
+ Step 9840/50462 | Loss: 2.7618 | LR: 4.60e-04 | 1912.6ms/step | 31.4 samples/s
256
+ Step 9850/50462 | Loss: 2.7840 | LR: 4.60e-04 | 1912.8ms/step | 31.4 samples/s
257
+ Step 9860/50462 | Loss: 2.7727 | LR: 4.60e-04 | 1911.8ms/step | 31.4 samples/s
258
+ Step 9870/50462 | Loss: 2.7409 | LR: 4.60e-04 | 1913.2ms/step | 31.4 samples/s
259
+ Step 9880/50462 | Loss: 2.7519 | LR: 4.60e-04 | 1914.6ms/step | 31.3 samples/s
260
+ Step 9890/50462 | Loss: 2.7726 | LR: 4.59e-04 | 1915.3ms/step | 31.3 samples/s
261
+ Step 9900/50462 | Loss: 2.7774 | LR: 4.59e-04 | 1912.4ms/step | 31.4 samples/s
262
+ Step 9910/50462 | Loss: 2.6663 | LR: 4.59e-04 | 1911.8ms/step | 31.4 samples/s
263
+ Step 9920/50462 | Loss: 2.7485 | LR: 4.59e-04 | 1909.7ms/step | 31.4 samples/s
264
+ Step 9930/50462 | Loss: 2.7122 | LR: 4.59e-04 | 1914.9ms/step | 31.3 samples/s
265
+ Step 9940/50462 | Loss: 2.7709 | LR: 4.59e-04 | 1912.6ms/step | 31.4 samples/s
266
+ Step 9950/50462 | Loss: 2.7396 | LR: 4.59e-04 | 1910.5ms/step | 31.4 samples/s
267
+ Step 9960/50462 | Loss: 2.7573 | LR: 4.59e-04 | 1914.1ms/step | 31.3 samples/s
268
+ Step 9970/50462 | Loss: 2.7128 | LR: 4.59e-04 | 1912.9ms/step | 31.4 samples/s
269
+ Step 9980/50462 | Loss: 2.7121 | LR: 4.59e-04 | 1914.3ms/step | 31.3 samples/s
270
+ Step 9990/50462 | Loss: 2.7237 | LR: 4.59e-04 | 1912.9ms/step | 31.4 samples/s
271
+ Step 10000/50462 | Loss: 2.7523 | LR: 4.58e-04 | 1912.4ms/step | 31.4 samples/s
272
+ Validation at step 10000: Loss = 2.7801, Perplexity = 16.12
273
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-10000
274
+ Step 10010/50462 | Loss: 2.7913 | LR: 4.58e-04 | 3221.4ms/step | 18.6 samples/s
275
+ Step 10020/50462 | Loss: 2.7727 | LR: 4.58e-04 | 1903.6ms/step | 31.5 samples/s
276
+ Step 10030/50462 | Loss: 2.6917 | LR: 4.58e-04 | 1905.1ms/step | 31.5 samples/s
277
+ Step 10040/50462 | Loss: 2.8140 | LR: 4.58e-04 | 1908.6ms/step | 31.4 samples/s
278
+ Step 10050/50462 | Loss: 2.7246 | LR: 4.58e-04 | 1910.3ms/step | 31.4 samples/s
279
+ Step 10060/50462 | Loss: 2.7118 | LR: 4.58e-04 | 1912.9ms/step | 31.4 samples/s
280
+ Step 10070/50462 | Loss: 2.8029 | LR: 4.58e-04 | 1912.9ms/step | 31.4 samples/s
281
+ Step 10080/50462 | Loss: 2.6894 | LR: 4.58e-04 | 1910.6ms/step | 31.4 samples/s
282
+ Step 10090/50462 | Loss: 2.7365 | LR: 4.58e-04 | 1913.4ms/step | 31.4 samples/s
283
+ Step 10100/50462 | Loss: 2.7712 | LR: 4.58e-04 | 1910.7ms/step | 31.4 samples/s
284
+ Step 10110/50462 | Loss: 2.7329 | LR: 4.58e-04 | 1910.2ms/step | 31.4 samples/s
285
+ Step 10120/50462 | Loss: 2.7267 | LR: 4.57e-04 | 1908.6ms/step | 31.4 samples/s
286
+ Step 10130/50462 | Loss: 2.6979 | LR: 4.57e-04 | 1911.7ms/step | 31.4 samples/s
287
+ Step 10140/50462 | Loss: 2.7427 | LR: 4.57e-04 | 1910.0ms/step | 31.4 samples/s
288
+ Step 10150/50462 | Loss: 2.7357 | LR: 4.57e-04 | 1908.7ms/step | 31.4 samples/s
289
+ Step 10160/50462 | Loss: 2.7090 | LR: 4.57e-04 | 1911.9ms/step | 31.4 samples/s
290
+ Step 10170/50462 | Loss: 2.7431 | LR: 4.57e-04 | 1912.3ms/step | 31.4 samples/s
291
+ Step 10180/50462 | Loss: 2.7338 | LR: 4.57e-04 | 1912.0ms/step | 31.4 samples/s
292
+ Step 10190/50462 | Loss: 2.7341 | LR: 4.57e-04 | 1912.1ms/step | 31.4 samples/s
293
+ Step 10200/50462 | Loss: 2.7102 | LR: 4.57e-04 | 1911.9ms/step | 31.4 samples/s
294
+ Step 10210/50462 | Loss: 2.7518 | LR: 4.57e-04 | 1911.8ms/step | 31.4 samples/s
295
+ Step 10220/50462 | Loss: 2.7481 | LR: 4.57e-04 | 1913.3ms/step | 31.4 samples/s
296
+ Step 10230/50462 | Loss: 2.7007 | LR: 4.56e-04 | 1910.7ms/step | 31.4 samples/s
297
+ Step 10240/50462 | Loss: 2.7307 | LR: 4.56e-04 | 1913.6ms/step | 31.4 samples/s
298
+ Step 10250/50462 | Loss: 2.7044 | LR: 4.56e-04 | 1915.8ms/step | 31.3 samples/s
299
+ Validation at step 10250: Loss = 2.7743, Perplexity = 16.03
300
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-10250
301
+ Step 10260/50462 | Loss: 2.7435 | LR: 4.56e-04 | 3200.9ms/step | 18.7 samples/s
302
+ Step 10270/50462 | Loss: 2.7296 | LR: 4.56e-04 | 1900.8ms/step | 31.6 samples/s
303
+ Step 10280/50462 | Loss: 2.7188 | LR: 4.56e-04 | 1909.0ms/step | 31.4 samples/s
304
+ Step 10290/50462 | Loss: 2.7385 | LR: 4.56e-04 | 1910.4ms/step | 31.4 samples/s
305
+ Step 10300/50462 | Loss: 2.7576 | LR: 4.56e-04 | 1910.6ms/step | 31.4 samples/s
306
+ Step 10310/50462 | Loss: 2.7419 | LR: 4.56e-04 | 1913.4ms/step | 31.4 samples/s
307
+ Step 10320/50462 | Loss: 2.7377 | LR: 4.56e-04 | 1913.8ms/step | 31.4 samples/s
308
+ Step 10330/50462 | Loss: 2.7186 | LR: 4.56e-04 | 1915.6ms/step | 31.3 samples/s
309
+ Step 10340/50462 | Loss: 2.7591 | LR: 4.55e-04 | 1912.1ms/step | 31.4 samples/s
310
+ Step 10350/50462 | Loss: 2.6825 | LR: 4.55e-04 | 1912.2ms/step | 31.4 samples/s
311
+ Step 10360/50462 | Loss: 2.7270 | LR: 4.55e-04 | 1912.2ms/step | 31.4 samples/s
312
+ Step 10370/50462 | Loss: 2.7124 | LR: 4.55e-04 | 1916.2ms/step | 31.3 samples/s
313
+ Step 10380/50462 | Loss: 2.7193 | LR: 4.55e-04 | 1918.9ms/step | 31.3 samples/s
314
+ Step 10390/50462 | Loss: 2.7511 | LR: 4.55e-04 | 1917.7ms/step | 31.3 samples/s
315
+ Step 10400/50462 | Loss: 2.7920 | LR: 4.55e-04 | 1916.6ms/step | 31.3 samples/s
316
+ Step 10410/50462 | Loss: 2.7671 | LR: 4.55e-04 | 1917.4ms/step | 31.3 samples/s
317
+ Step 10420/50462 | Loss: 2.7374 | LR: 4.55e-04 | 1915.8ms/step | 31.3 samples/s
318
+ Step 10430/50462 | Loss: 2.7506 | LR: 4.55e-04 | 1916.2ms/step | 31.3 samples/s
319
+ Step 10440/50462 | Loss: 2.7454 | LR: 4.55e-04 | 1912.5ms/step | 31.4 samples/s
320
+ Step 10450/50462 | Loss: 2.7425 | LR: 4.54e-04 | 1913.0ms/step | 31.4 samples/s
321
+ Step 10460/50462 | Loss: 2.7222 | LR: 4.54e-04 | 1918.2ms/step | 31.3 samples/s
322
+ Step 10470/50462 | Loss: 2.6694 | LR: 4.54e-04 | 1915.4ms/step | 31.3 samples/s
323
+ Step 10480/50462 | Loss: 2.7573 | LR: 4.54e-04 | 1915.8ms/step | 31.3 samples/s
324
+ Step 10490/50462 | Loss: 2.7378 | LR: 4.54e-04 | 1913.8ms/step | 31.4 samples/s
325
+ Step 10500/50462 | Loss: 2.7085 | LR: 4.54e-04 | 1914.1ms/step | 31.3 samples/s
326
+ Validation at step 10500: Loss = 2.7691, Perplexity = 15.95
327
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-10500
328
+ Step 10510/50462 | Loss: 2.7403 | LR: 4.54e-04 | 3197.2ms/step | 18.8 samples/s
329
+ Step 10520/50462 | Loss: 2.7228 | LR: 4.54e-04 | 1900.5ms/step | 31.6 samples/s
330
+ Step 10530/50462 | Loss: 2.7078 | LR: 4.54e-04 | 1909.3ms/step | 31.4 samples/s
331
+ Step 10540/50462 | Loss: 2.7489 | LR: 4.54e-04 | 1911.1ms/step | 31.4 samples/s
332
+ Step 10550/50462 | Loss: 2.7265 | LR: 4.54e-04 | 1913.0ms/step | 31.4 samples/s
333
+ Step 10560/50462 | Loss: 2.7649 | LR: 4.53e-04 | 1914.7ms/step | 31.3 samples/s
334
+ Step 10570/50462 | Loss: 2.7231 | LR: 4.53e-04 | 1914.9ms/step | 31.3 samples/s
335
+ Step 10580/50462 | Loss: 2.7137 | LR: 4.53e-04 | 1915.6ms/step | 31.3 samples/s
336
+ Step 10590/50462 | Loss: 2.7147 | LR: 4.53e-04 | 1911.7ms/step | 31.4 samples/s
337
+ Step 10600/50462 | Loss: 2.7245 | LR: 4.53e-04 | 1911.6ms/step | 31.4 samples/s
338
+ Step 10610/50462 | Loss: 2.7628 | LR: 4.53e-04 | 1915.9ms/step | 31.3 samples/s
339
+ Step 10620/50462 | Loss: 2.7002 | LR: 4.53e-04 | 1914.8ms/step | 31.3 samples/s
340
+ Step 10630/50462 | Loss: 2.7250 | LR: 4.53e-04 | 1916.8ms/step | 31.3 samples/s
341
+ Step 10640/50462 | Loss: 2.7296 | LR: 4.53e-04 | 1911.3ms/step | 31.4 samples/s
342
+ Step 10650/50462 | Loss: 2.7128 | LR: 4.53e-04 | 1913.0ms/step | 31.4 samples/s
343
+ Step 10660/50462 | Loss: 2.7083 | LR: 4.53e-04 | 1908.5ms/step | 31.4 samples/s
344
+ Step 10670/50462 | Loss: 2.7208 | LR: 4.52e-04 | 1912.7ms/step | 31.4 samples/s
345
+ Step 10680/50462 | Loss: 2.7488 | LR: 4.52e-04 | 1914.1ms/step | 31.3 samples/s
346
+ Step 10690/50462 | Loss: 2.7191 | LR: 4.52e-04 | 1916.3ms/step | 31.3 samples/s
347
+ Step 10700/50462 | Loss: 2.7207 | LR: 4.52e-04 | 1914.2ms/step | 31.3 samples/s
348
+ Step 10710/50462 | Loss: 2.7563 | LR: 4.52e-04 | 1913.7ms/step | 31.4 samples/s
349
+ Step 10720/50462 | Loss: 2.7111 | LR: 4.52e-04 | 1911.9ms/step | 31.4 samples/s
350
+ Step 10730/50462 | Loss: 2.7579 | LR: 4.52e-04 | 1911.5ms/step | 31.4 samples/s
351
+ Step 10740/50462 | Loss: 2.6897 | LR: 4.52e-04 | 1915.5ms/step | 31.3 samples/s
352
+ Step 10750/50462 | Loss: 2.6540 | LR: 4.52e-04 | 1914.8ms/step | 31.3 samples/s
353
+ Validation at step 10750: Loss = 2.7647, Perplexity = 15.87
354
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-10750
355
+ Step 10760/50462 | Loss: 2.7225 | LR: 4.52e-04 | 3197.7ms/step | 18.8 samples/s
356
+ Step 10770/50462 | Loss: 2.7033 | LR: 4.52e-04 | 1909.6ms/step | 31.4 samples/s
357
+ Step 10780/50462 | Loss: 2.7450 | LR: 4.51e-04 | 1912.0ms/step | 31.4 samples/s
358
+ Step 10790/50462 | Loss: 2.7380 | LR: 4.51e-04 | 1914.5ms/step | 31.3 samples/s
359
+ Step 10800/50462 | Loss: 2.7333 | LR: 4.51e-04 | 1911.6ms/step | 31.4 samples/s
360
+ Step 10810/50462 | Loss: 2.7511 | LR: 4.51e-04 | 1912.8ms/step | 31.4 samples/s
361
+ Step 10820/50462 | Loss: 2.7324 | LR: 4.51e-04 | 1915.3ms/step | 31.3 samples/s
362
+ Step 10830/50462 | Loss: 2.7604 | LR: 4.51e-04 | 1913.0ms/step | 31.4 samples/s
363
+ Step 10840/50462 | Loss: 2.7168 | LR: 4.51e-04 | 1914.6ms/step | 31.3 samples/s
364
+ Step 10850/50462 | Loss: 2.7534 | LR: 4.51e-04 | 1914.8ms/step | 31.3 samples/s
365
+ Step 10860/50462 | Loss: 2.7183 | LR: 4.51e-04 | 1915.2ms/step | 31.3 samples/s
366
+ Step 10870/50462 | Loss: 2.7502 | LR: 4.51e-04 | 1914.3ms/step | 31.3 samples/s
367
+ Step 10880/50462 | Loss: 2.7034 | LR: 4.50e-04 | 1916.7ms/step | 31.3 samples/s
368
+ Step 10890/50462 | Loss: 2.7232 | LR: 4.50e-04 | 1915.2ms/step | 31.3 samples/s
369
+ Step 10900/50462 | Loss: 2.7473 | LR: 4.50e-04 | 1914.9ms/step | 31.3 samples/s
370
+ Step 10910/50462 | Loss: 2.7515 | LR: 4.50e-04 | 1915.2ms/step | 31.3 samples/s
371
+ Step 10920/50462 | Loss: 2.7114 | LR: 4.50e-04 | 1915.4ms/step | 31.3 samples/s
372
+ Step 10930/50462 | Loss: 2.7023 | LR: 4.50e-04 | 1914.3ms/step | 31.3 samples/s
373
+ Step 10940/50462 | Loss: 2.7158 | LR: 4.50e-04 | 1914.5ms/step | 31.3 samples/s
374
+ Step 10950/50462 | Loss: 2.7064 | LR: 4.50e-04 | 1915.9ms/step | 31.3 samples/s
375
+ Step 10960/50462 | Loss: 2.7923 | LR: 4.50e-04 | 1910.9ms/step | 31.4 samples/s
376
+ Step 10970/50462 | Loss: 2.7088 | LR: 4.50e-04 | 1911.9ms/step | 31.4 samples/s
377
+ Step 10980/50462 | Loss: 2.7100 | LR: 4.50e-04 | 1912.9ms/step | 31.4 samples/s
378
+ Step 10990/50462 | Loss: 2.7474 | LR: 4.49e-04 | 1914.8ms/step | 31.3 samples/s
379
+ Step 11000/50462 | Loss: 2.7094 | LR: 4.49e-04 | 1914.6ms/step | 31.3 samples/s
380
+ Validation at step 11000: Loss = 2.7591, Perplexity = 15.79
381
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-11000
382
+ Step 11010/50462 | Loss: 2.7425 | LR: 4.49e-04 | 3199.3ms/step | 18.8 samples/s
383
+ Step 11020/50462 | Loss: 2.7316 | LR: 4.49e-04 | 1904.9ms/step | 31.5 samples/s
384
+ Step 11030/50462 | Loss: 2.7137 | LR: 4.49e-04 | 1908.0ms/step | 31.4 samples/s
385
+ Step 11040/50462 | Loss: 2.7264 | LR: 4.49e-04 | 1906.2ms/step | 31.5 samples/s
386
+ Step 11050/50462 | Loss: 2.7420 | LR: 4.49e-04 | 1906.6ms/step | 31.5 samples/s
387
+ Step 11060/50462 | Loss: 2.7644 | LR: 4.49e-04 | 1909.7ms/step | 31.4 samples/s
388
+ Step 11070/50462 | Loss: 2.7474 | LR: 4.49e-04 | 1913.0ms/step | 31.4 samples/s
389
+ Step 11080/50462 | Loss: 2.7831 | LR: 4.49e-04 | 1913.1ms/step | 31.4 samples/s
390
+ Step 11090/50462 | Loss: 2.7332 | LR: 4.48e-04 | 1915.1ms/step | 31.3 samples/s
391
+ Step 11100/50462 | Loss: 2.7215 | LR: 4.48e-04 | 1912.3ms/step | 31.4 samples/s
392
+ Step 11110/50462 | Loss: 2.7108 | LR: 4.48e-04 | 1914.5ms/step | 31.3 samples/s
393
+ Step 11120/50462 | Loss: 2.7011 | LR: 4.48e-04 | 1912.6ms/step | 31.4 samples/s
394
+ Step 11130/50462 | Loss: 2.7536 | LR: 4.48e-04 | 1913.2ms/step | 31.4 samples/s
395
+ Step 11140/50462 | Loss: 2.7221 | LR: 4.48e-04 | 1912.7ms/step | 31.4 samples/s
396
+ Step 11150/50462 | Loss: 2.6954 | LR: 4.48e-04 | 1914.2ms/step | 31.3 samples/s
397
+ Step 11160/50462 | Loss: 2.7299 | LR: 4.48e-04 | 1913.5ms/step | 31.4 samples/s
398
+ Step 11170/50462 | Loss: 2.7279 | LR: 4.48e-04 | 1912.8ms/step | 31.4 samples/s
399
+ Step 11180/50462 | Loss: 2.7663 | LR: 4.48e-04 | 1911.8ms/step | 31.4 samples/s
400
+ Step 11190/50462 | Loss: 2.7622 | LR: 4.48e-04 | 1912.8ms/step | 31.4 samples/s
401
+ Step 11200/50462 | Loss: 2.7273 | LR: 4.47e-04 | 1913.9ms/step | 31.4 samples/s
402
+ Step 11210/50462 | Loss: 2.7240 | LR: 4.47e-04 | 1914.3ms/step | 31.3 samples/s
403
+ Step 11220/50462 | Loss: 2.7393 | LR: 4.47e-04 | 1912.1ms/step | 31.4 samples/s
404
+ Step 11230/50462 | Loss: 2.6993 | LR: 4.47e-04 | 1911.9ms/step | 31.4 samples/s
405
+ Step 11240/50462 | Loss: 2.6913 | LR: 4.47e-04 | 1913.3ms/step | 31.4 samples/s
406
+ Step 11250/50462 | Loss: 2.7033 | LR: 4.47e-04 | 1910.5ms/step | 31.4 samples/s
407
+ Validation at step 11250: Loss = 2.7556, Perplexity = 15.73
408
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-11250
409
+ Step 11260/50462 | Loss: 2.7065 | LR: 4.47e-04 | 3193.5ms/step | 18.8 samples/s
410
+ Step 11270/50462 | Loss: 2.7224 | LR: 4.47e-04 | 1902.3ms/step | 31.5 samples/s
411
+ Step 11280/50462 | Loss: 2.6910 | LR: 4.47e-04 | 1907.9ms/step | 31.4 samples/s
412
+ Step 11290/50462 | Loss: 2.7148 | LR: 4.47e-04 | 1910.6ms/step | 31.4 samples/s
413
+ Step 11300/50462 | Loss: 2.7347 | LR: 4.46e-04 | 1913.0ms/step | 31.4 samples/s
414
+ Step 11310/50462 | Loss: 2.6865 | LR: 4.46e-04 | 1913.6ms/step | 31.4 samples/s
415
+ Step 11320/50462 | Loss: 2.7046 | LR: 4.46e-04 | 1913.7ms/step | 31.4 samples/s
416
+ Step 11330/50462 | Loss: 2.7322 | LR: 4.46e-04 | 1914.5ms/step | 31.3 samples/s
417
+ Step 11340/50462 | Loss: 2.7464 | LR: 4.46e-04 | 1913.5ms/step | 31.4 samples/s
418
+ Step 11350/50462 | Loss: 2.6853 | LR: 4.46e-04 | 1911.4ms/step | 31.4 samples/s
419
+ Step 11360/50462 | Loss: 2.7316 | LR: 4.46e-04 | 1913.1ms/step | 31.4 samples/s
420
+ Step 11370/50462 | Loss: 2.7573 | LR: 4.46e-04 | 1910.5ms/step | 31.4 samples/s
421
+ Step 11380/50462 | Loss: 2.6803 | LR: 4.46e-04 | 1916.1ms/step | 31.3 samples/s
422
+ Step 11390/50462 | Loss: 2.7326 | LR: 4.46e-04 | 1914.3ms/step | 31.3 samples/s
423
+ Step 11400/50462 | Loss: 2.7317 | LR: 4.45e-04 | 1914.9ms/step | 31.3 samples/s
424
+ Step 11410/50462 | Loss: 2.7144 | LR: 4.45e-04 | 1915.3ms/step | 31.3 samples/s
425
+ Step 11420/50462 | Loss: 2.7495 | LR: 4.45e-04 | 1911.9ms/step | 31.4 samples/s
426
+ Step 11430/50462 | Loss: 2.7424 | LR: 4.45e-04 | 1915.5ms/step | 31.3 samples/s
427
+ Step 11440/50462 | Loss: 2.7325 | LR: 4.45e-04 | 1916.5ms/step | 31.3 samples/s
428
+ Step 11450/50462 | Loss: 2.6828 | LR: 4.45e-04 | 1914.6ms/step | 31.3 samples/s
429
+ Step 11460/50462 | Loss: 2.7551 | LR: 4.45e-04 | 1912.9ms/step | 31.4 samples/s
430
+ Step 11470/50462 | Loss: 2.7107 | LR: 4.45e-04 | 1909.2ms/step | 31.4 samples/s
431
+ Step 11480/50462 | Loss: 2.6940 | LR: 4.45e-04 | 1910.1ms/step | 31.4 samples/s
432
+ Step 11490/50462 | Loss: 2.6941 | LR: 4.45e-04 | 1912.6ms/step | 31.4 samples/s
433
+ Step 11500/50462 | Loss: 2.7055 | LR: 4.44e-04 | 1912.8ms/step | 31.4 samples/s
434
+ Validation at step 11500: Loss = 2.7507, Perplexity = 15.65
435
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-11500
436
+ Step 11510/50462 | Loss: 2.7107 | LR: 4.44e-04 | 3199.3ms/step | 18.8 samples/s
437
+ Step 11520/50462 | Loss: 2.7251 | LR: 4.44e-04 | 1900.0ms/step | 31.6 samples/s
438
+ Step 11530/50462 | Loss: 2.7359 | LR: 4.44e-04 | 1905.0ms/step | 31.5 samples/s
439
+ Step 11540/50462 | Loss: 2.7107 | LR: 4.44e-04 | 1908.8ms/step | 31.4 samples/s
440
+ Step 11550/50462 | Loss: 2.7028 | LR: 4.44e-04 | 1910.6ms/step | 31.4 samples/s
441
+ Step 11560/50462 | Loss: 2.7467 | LR: 4.44e-04 | 1911.5ms/step | 31.4 samples/s
442
+ Step 11570/50462 | Loss: 2.6497 | LR: 4.44e-04 | 1909.5ms/step | 31.4 samples/s
443
+ Step 11580/50462 | Loss: 2.6402 | LR: 4.44e-04 | 1910.9ms/step | 31.4 samples/s
444
+ Step 11590/50462 | Loss: 2.7307 | LR: 4.44e-04 | 1912.0ms/step | 31.4 samples/s
445
+ Step 11600/50462 | Loss: 2.7231 | LR: 4.43e-04 | 1912.0ms/step | 31.4 samples/s
446
+ Step 11610/50462 | Loss: 2.7222 | LR: 4.43e-04 | 1912.3ms/step | 31.4 samples/s
447
+ Step 11620/50462 | Loss: 2.6705 | LR: 4.43e-04 | 1913.2ms/step | 31.4 samples/s
448
+ Step 11630/50462 | Loss: 2.7469 | LR: 4.43e-04 | 1915.9ms/step | 31.3 samples/s
449
+ Step 11640/50462 | Loss: 2.7144 | LR: 4.43e-04 | 1913.3ms/step | 31.4 samples/s
450
+ Step 11650/50462 | Loss: 2.7297 | LR: 4.43e-04 | 1912.7ms/step | 31.4 samples/s
451
+ Step 11660/50462 | Loss: 2.7459 | LR: 4.43e-04 | 1912.0ms/step | 31.4 samples/s
452
+ Step 11670/50462 | Loss: 2.6729 | LR: 4.43e-04 | 1912.0ms/step | 31.4 samples/s
453
+ Step 11680/50462 | Loss: 2.7009 | LR: 4.43e-04 | 1912.0ms/step | 31.4 samples/s
454
+ Step 11690/50462 | Loss: 2.6768 | LR: 4.43e-04 | 1910.9ms/step | 31.4 samples/s
455
+ Step 11700/50462 | Loss: 2.7056 | LR: 4.42e-04 | 1911.7ms/step | 31.4 samples/s
456
+ Step 11710/50462 | Loss: 2.7300 | LR: 4.42e-04 | 1915.2ms/step | 31.3 samples/s
457
+ Step 11720/50462 | Loss: 2.7227 | LR: 4.42e-04 | 1915.0ms/step | 31.3 samples/s
458
+ Step 11730/50462 | Loss: 2.7341 | LR: 4.42e-04 | 1913.1ms/step | 31.4 samples/s
459
+ Step 11740/50462 | Loss: 2.6717 | LR: 4.42e-04 | 1912.1ms/step | 31.4 samples/s
460
+ Step 11750/50462 | Loss: 2.7247 | LR: 4.42e-04 | 1910.6ms/step | 31.4 samples/s
461
+ Validation at step 11750: Loss = 2.7455, Perplexity = 15.57
462
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-11750
463
+ Step 11760/50462 | Loss: 2.7015 | LR: 4.42e-04 | 3206.5ms/step | 18.7 samples/s
464
+ Step 11770/50462 | Loss: 2.7457 | LR: 4.42e-04 | 1905.2ms/step | 31.5 samples/s
465
+ Step 11780/50462 | Loss: 2.7462 | LR: 4.42e-04 | 1909.3ms/step | 31.4 samples/s
466
+ Step 11790/50462 | Loss: 2.6671 | LR: 4.42e-04 | 1907.4ms/step | 31.5 samples/s
467
+ Step 11800/50462 | Loss: 2.7212 | LR: 4.41e-04 | 1909.6ms/step | 31.4 samples/s
468
+ Step 11810/50462 | Loss: 2.7125 | LR: 4.41e-04 | 1911.6ms/step | 31.4 samples/s
469
+ Step 11820/50462 | Loss: 2.7155 | LR: 4.41e-04 | 1910.9ms/step | 31.4 samples/s
470
+ Step 11830/50462 | Loss: 2.7481 | LR: 4.41e-04 | 1908.6ms/step | 31.4 samples/s
471
+ Step 11840/50462 | Loss: 2.7013 | LR: 4.41e-04 | 1911.6ms/step | 31.4 samples/s
472
+ Step 11850/50462 | Loss: 2.6810 | LR: 4.41e-04 | 1911.7ms/step | 31.4 samples/s
473
+ Step 11860/50462 | Loss: 2.7315 | LR: 4.41e-04 | 1915.7ms/step | 31.3 samples/s
474
+ Step 11870/50462 | Loss: 2.7090 | LR: 4.41e-04 | 1913.7ms/step | 31.4 samples/s
475
+ Step 11880/50462 | Loss: 2.7224 | LR: 4.41e-04 | 1912.3ms/step | 31.4 samples/s
476
+ Step 11890/50462 | Loss: 2.6902 | LR: 4.41e-04 | 1914.7ms/step | 31.3 samples/s
477
+ Step 11900/50462 | Loss: 2.6759 | LR: 4.40e-04 | 1914.0ms/step | 31.3 samples/s
478
+ Step 11910/50462 | Loss: 2.6666 | LR: 4.40e-04 | 1912.5ms/step | 31.4 samples/s
479
+ Step 11920/50462 | Loss: 2.7275 | LR: 4.40e-04 | 1915.9ms/step | 31.3 samples/s
480
+ Step 11930/50462 | Loss: 2.7045 | LR: 4.40e-04 | 1912.9ms/step | 31.4 samples/s
481
+ Step 11940/50462 | Loss: 2.6620 | LR: 4.40e-04 | 1914.7ms/step | 31.3 samples/s
482
+ Step 11950/50462 | Loss: 2.7137 | LR: 4.40e-04 | 1912.7ms/step | 31.4 samples/s
483
+ Step 11960/50462 | Loss: 2.7840 | LR: 4.40e-04 | 1914.2ms/step | 31.3 samples/s
484
+ Step 11970/50462 | Loss: 2.7373 | LR: 4.40e-04 | 1911.5ms/step | 31.4 samples/s
485
+ Step 11980/50462 | Loss: 2.7037 | LR: 4.40e-04 | 1913.1ms/step | 31.4 samples/s
486
+ Step 11990/50462 | Loss: 2.7388 | LR: 4.40e-04 | 1914.1ms/step | 31.3 samples/s
487
+ Step 12000/50462 | Loss: 2.7225 | LR: 4.39e-04 | 1913.2ms/step | 31.4 samples/s
488
+ Validation at step 12000: Loss = 2.7427, Perplexity = 15.53
489
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-12000
490
+ Step 12010/50462 | Loss: 2.6783 | LR: 4.39e-04 | 3201.6ms/step | 18.7 samples/s
491
+ Step 12020/50462 | Loss: 2.7055 | LR: 4.39e-04 | 1899.5ms/step | 31.6 samples/s
492
+ Step 12030/50462 | Loss: 2.7482 | LR: 4.39e-04 | 1903.4ms/step | 31.5 samples/s
493
+ Step 12040/50462 | Loss: 2.7038 | LR: 4.39e-04 | 1905.8ms/step | 31.5 samples/s
494
+ Step 12050/50462 | Loss: 2.7121 | LR: 4.39e-04 | 1909.4ms/step | 31.4 samples/s
495
+ Step 12060/50462 | Loss: 2.7261 | LR: 4.39e-04 | 1911.7ms/step | 31.4 samples/s
496
+ Step 12070/50462 | Loss: 2.7242 | LR: 4.39e-04 | 1911.5ms/step | 31.4 samples/s
497
+ Step 12080/50462 | Loss: 2.7136 | LR: 4.39e-04 | 1909.5ms/step | 31.4 samples/s
498
+ Step 12090/50462 | Loss: 2.7385 | LR: 4.39e-04 | 1913.0ms/step | 31.4 samples/s
499
+ Step 12100/50462 | Loss: 2.6778 | LR: 4.38e-04 | 1913.3ms/step | 31.4 samples/s
500
+ Step 12110/50462 | Loss: 2.7345 | LR: 4.38e-04 | 1912.9ms/step | 31.4 samples/s
501
+ Step 12120/50462 | Loss: 2.7177 | LR: 4.38e-04 | 1913.1ms/step | 31.4 samples/s
502
+ Step 12130/50462 | Loss: 2.7601 | LR: 4.38e-04 | 1913.3ms/step | 31.4 samples/s
503
+ Step 12140/50462 | Loss: 2.7144 | LR: 4.38e-04 | 1912.9ms/step | 31.4 samples/s
504
+ Step 12150/50462 | Loss: 2.6947 | LR: 4.38e-04 | 1911.6ms/step | 31.4 samples/s
505
+ Step 12160/50462 | Loss: 2.7361 | LR: 4.38e-04 | 1912.4ms/step | 31.4 samples/s
506
+ Step 12170/50462 | Loss: 2.7258 | LR: 4.38e-04 | 1916.5ms/step | 31.3 samples/s
507
+ Step 12180/50462 | Loss: 2.7035 | LR: 4.38e-04 | 1913.1ms/step | 31.4 samples/s
508
+ Step 12190/50462 | Loss: 2.7248 | LR: 4.37e-04 | 1913.8ms/step | 31.4 samples/s
509
+ Step 12200/50462 | Loss: 2.6995 | LR: 4.37e-04 | 1914.4ms/step | 31.3 samples/s
510
+ Step 12210/50462 | Loss: 2.7403 | LR: 4.37e-04 | 1913.1ms/step | 31.4 samples/s
511
+ Step 12220/50462 | Loss: 2.7144 | LR: 4.37e-04 | 1915.0ms/step | 31.3 samples/s
512
+ Step 12230/50462 | Loss: 2.6881 | LR: 4.37e-04 | 1915.2ms/step | 31.3 samples/s
513
+ Step 12240/50462 | Loss: 2.7114 | LR: 4.37e-04 | 1914.2ms/step | 31.3 samples/s
514
+ Step 12250/50462 | Loss: 2.7199 | LR: 4.37e-04 | 1917.3ms/step | 31.3 samples/s
515
+ Validation at step 12250: Loss = 2.7366, Perplexity = 15.43
516
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-12250
517
+ Step 12260/50462 | Loss: 2.7032 | LR: 4.37e-04 | 3202.1ms/step | 18.7 samples/s
518
+ Step 12270/50462 | Loss: 2.6977 | LR: 4.37e-04 | 1908.0ms/step | 31.4 samples/s
519
+ Step 12280/50462 | Loss: 2.6871 | LR: 4.37e-04 | 1912.1ms/step | 31.4 samples/s
520
+ Step 12290/50462 | Loss: 2.7233 | LR: 4.36e-04 | 1910.4ms/step | 31.4 samples/s
521
+ Step 12300/50462 | Loss: 2.6989 | LR: 4.36e-04 | 1910.8ms/step | 31.4 samples/s
522
+ Step 12310/50462 | Loss: 2.7315 | LR: 4.36e-04 | 1911.9ms/step | 31.4 samples/s
523
+ Step 12320/50462 | Loss: 2.7138 | LR: 4.36e-04 | 1909.9ms/step | 31.4 samples/s
524
+ Step 12330/50462 | Loss: 2.7142 | LR: 4.36e-04 | 1910.0ms/step | 31.4 samples/s
525
+ Step 12340/50462 | Loss: 2.6888 | LR: 4.36e-04 | 1909.6ms/step | 31.4 samples/s
526
+ Step 12350/50462 | Loss: 2.6868 | LR: 4.36e-04 | 1912.0ms/step | 31.4 samples/s
527
+ Step 12360/50462 | Loss: 2.6967 | LR: 4.36e-04 | 1914.6ms/step | 31.3 samples/s
528
+ Step 12370/50462 | Loss: 2.6733 | LR: 4.36e-04 | 1912.8ms/step | 31.4 samples/s
529
+ Step 12380/50462 | Loss: 2.6998 | LR: 4.35e-04 | 1913.2ms/step | 31.4 samples/s
530
+ Step 12390/50462 | Loss: 2.7082 | LR: 4.35e-04 | 1915.5ms/step | 31.3 samples/s
531
+ Step 12400/50462 | Loss: 2.7112 | LR: 4.35e-04 | 1914.8ms/step | 31.3 samples/s
532
+ Step 12410/50462 | Loss: 2.7344 | LR: 4.35e-04 | 1916.0ms/step | 31.3 samples/s
533
+ Step 12420/50462 | Loss: 2.6738 | LR: 4.35e-04 | 1916.6ms/step | 31.3 samples/s
534
+ Step 12430/50462 | Loss: 2.7050 | LR: 4.35e-04 | 1913.6ms/step | 31.4 samples/s
535
+ Step 12440/50462 | Loss: 2.7167 | LR: 4.35e-04 | 1912.6ms/step | 31.4 samples/s
536
+ Step 12450/50462 | Loss: 2.7494 | LR: 4.35e-04 | 1911.6ms/step | 31.4 samples/s
537
+ Step 12460/50462 | Loss: 2.7051 | LR: 4.35e-04 | 1912.1ms/step | 31.4 samples/s
538
+ Step 12470/50462 | Loss: 2.6879 | LR: 4.35e-04 | 1912.4ms/step | 31.4 samples/s
539
+ Step 12480/50462 | Loss: 2.7184 | LR: 4.34e-04 | 1911.7ms/step | 31.4 samples/s
540
+ Step 12490/50462 | Loss: 2.7033 | LR: 4.34e-04 | 1912.2ms/step | 31.4 samples/s
541
+ Step 12500/50462 | Loss: 2.6661 | LR: 4.34e-04 | 1909.5ms/step | 31.4 samples/s
542
+ Validation at step 12500: Loss = 2.7356, Perplexity = 15.42
543
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-12500
544
+ Step 12510/50462 | Loss: 2.6762 | LR: 4.34e-04 | 3188.9ms/step | 18.8 samples/s
545
+ Step 12520/50462 | Loss: 2.6943 | LR: 4.34e-04 | 1899.2ms/step | 31.6 samples/s
546
+ Step 12530/50462 | Loss: 2.6909 | LR: 4.34e-04 | 1901.4ms/step | 31.6 samples/s
547
+ Step 12540/50462 | Loss: 2.7362 | LR: 4.34e-04 | 1906.9ms/step | 31.5 samples/s
548
+ Step 12550/50462 | Loss: 2.6987 | LR: 4.34e-04 | 1911.4ms/step | 31.4 samples/s
549
+ Step 12560/50462 | Loss: 2.7029 | LR: 4.34e-04 | 1912.1ms/step | 31.4 samples/s
550
+ Step 12570/50462 | Loss: 2.6444 | LR: 4.33e-04 | 1913.6ms/step | 31.4 samples/s
551
+ Step 12580/50462 | Loss: 2.7186 | LR: 4.33e-04 | 1913.3ms/step | 31.4 samples/s
552
+ Step 12590/50462 | Loss: 2.6882 | LR: 4.33e-04 | 1911.5ms/step | 31.4 samples/s
553
+ Step 12600/50462 | Loss: 2.7286 | LR: 4.33e-04 | 1913.9ms/step | 31.4 samples/s
554
+ Step 12610/50462 | Loss: 2.7100 | LR: 4.33e-04 | 1914.3ms/step | 31.3 samples/s
555
+ Step 12620/50462 | Loss: 2.6998 | LR: 4.33e-04 | 1914.6ms/step | 31.3 samples/s
556
+ Step 12630/50462 | Loss: 2.7356 | LR: 4.33e-04 | 1912.7ms/step | 31.4 samples/s
557
+ Step 12640/50462 | Loss: 2.6890 | LR: 4.33e-04 | 1911.2ms/step | 31.4 samples/s
558
+ Step 12650/50462 | Loss: 2.6692 | LR: 4.33e-04 | 1913.0ms/step | 31.4 samples/s
559
+ Step 12660/50462 | Loss: 2.6756 | LR: 4.32e-04 | 1913.7ms/step | 31.4 samples/s
560
+ Step 12670/50462 | Loss: 2.7203 | LR: 4.32e-04 | 1915.5ms/step | 31.3 samples/s
561
+ Step 12680/50462 | Loss: 2.6596 | LR: 4.32e-04 | 1913.7ms/step | 31.4 samples/s
562
+ Step 12690/50462 | Loss: 2.7382 | LR: 4.32e-04 | 1915.0ms/step | 31.3 samples/s
563
+ Step 12700/50462 | Loss: 2.7112 | LR: 4.32e-04 | 1914.3ms/step | 31.3 samples/s
564
+ Step 12710/50462 | Loss: 2.7323 | LR: 4.32e-04 | 1911.4ms/step | 31.4 samples/s
565
+ Step 12720/50462 | Loss: 2.7057 | LR: 4.32e-04 | 1911.3ms/step | 31.4 samples/s
566
+ Step 12730/50462 | Loss: 2.6816 | LR: 4.32e-04 | 1911.2ms/step | 31.4 samples/s
567
+ Step 12740/50462 | Loss: 2.6965 | LR: 4.32e-04 | 1913.9ms/step | 31.4 samples/s
568
+ Step 12750/50462 | Loss: 2.6863 | LR: 4.31e-04 | 1911.6ms/step | 31.4 samples/s
569
+ Validation at step 12750: Loss = 2.7294, Perplexity = 15.32
570
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-12750
571
+ Step 12760/50462 | Loss: 2.6660 | LR: 4.31e-04 | 3203.2ms/step | 18.7 samples/s
572
+ Step 12770/50462 | Loss: 2.7157 | LR: 4.31e-04 | 1900.9ms/step | 31.6 samples/s
573
+ Step 12780/50462 | Loss: 2.6925 | LR: 4.31e-04 | 1903.5ms/step | 31.5 samples/s
574
+ Step 12790/50462 | Loss: 2.6927 | LR: 4.31e-04 | 1911.2ms/step | 31.4 samples/s
575
+ Step 12800/50462 | Loss: 2.6571 | LR: 4.31e-04 | 1911.0ms/step | 31.4 samples/s
576
+ Step 12810/50462 | Loss: 2.6663 | LR: 4.31e-04 | 1914.0ms/step | 31.3 samples/s
577
+ Step 12820/50462 | Loss: 2.6817 | LR: 4.31e-04 | 1912.7ms/step | 31.4 samples/s
578
+ Step 12830/50462 | Loss: 2.7099 | LR: 4.31e-04 | 1913.5ms/step | 31.4 samples/s
579
+ Step 12840/50462 | Loss: 2.7000 | LR: 4.31e-04 | 1917.0ms/step | 31.3 samples/s
580
+ Step 12850/50462 | Loss: 2.6435 | LR: 4.30e-04 | 1913.7ms/step | 31.4 samples/s
581
+ Step 12860/50462 | Loss: 2.7115 | LR: 4.30e-04 | 1914.8ms/step | 31.3 samples/s
582
+ Step 12870/50462 | Loss: 2.6861 | LR: 4.30e-04 | 1912.8ms/step | 31.4 samples/s
583
+ Step 12880/50462 | Loss: 2.6829 | LR: 4.30e-04 | 1915.0ms/step | 31.3 samples/s
584
+ Step 12890/50462 | Loss: 2.6829 | LR: 4.30e-04 | 1913.2ms/step | 31.4 samples/s
585
+ Step 12900/50462 | Loss: 2.7061 | LR: 4.30e-04 | 1913.4ms/step | 31.4 samples/s
586
+ Step 12910/50462 | Loss: 2.7188 | LR: 4.30e-04 | 1915.5ms/step | 31.3 samples/s
587
+ Step 12920/50462 | Loss: 2.6689 | LR: 4.30e-04 | 1914.2ms/step | 31.3 samples/s
588
+ Step 12930/50462 | Loss: 2.6708 | LR: 4.30e-04 | 1917.0ms/step | 31.3 samples/s
589
+ Step 12940/50462 | Loss: 2.7180 | LR: 4.29e-04 | 1916.2ms/step | 31.3 samples/s
590
+ Step 12950/50462 | Loss: 2.7349 | LR: 4.29e-04 | 1918.4ms/step | 31.3 samples/s
591
+ Step 12960/50462 | Loss: 2.6835 | LR: 4.29e-04 | 1916.4ms/step | 31.3 samples/s
592
+ Step 12970/50462 | Loss: 2.6893 | LR: 4.29e-04 | 1915.7ms/step | 31.3 samples/s
593
+ Step 12980/50462 | Loss: 2.7146 | LR: 4.29e-04 | 1916.4ms/step | 31.3 samples/s
594
+ Step 12990/50462 | Loss: 2.7052 | LR: 4.29e-04 | 1914.8ms/step | 31.3 samples/s
595
+ Step 13000/50462 | Loss: 2.6984 | LR: 4.29e-04 | 1916.4ms/step | 31.3 samples/s
596
+ Validation at step 13000: Loss = 2.7282, Perplexity = 15.31
597
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-13000
598
+ Step 13010/50462 | Loss: 2.6524 | LR: 4.29e-04 | 3209.7ms/step | 18.7 samples/s
599
+ Step 13020/50462 | Loss: 2.6971 | LR: 4.29e-04 | 1899.5ms/step | 31.6 samples/s
600
+ Step 13030/50462 | Loss: 2.6975 | LR: 4.28e-04 | 1906.8ms/step | 31.5 samples/s
601
+ Step 13040/50462 | Loss: 2.7115 | LR: 4.28e-04 | 1913.1ms/step | 31.4 samples/s
602
+ Step 13050/50462 | Loss: 2.7008 | LR: 4.28e-04 | 1912.5ms/step | 31.4 samples/s
603
+ Step 13060/50462 | Loss: 2.6966 | LR: 4.28e-04 | 1915.4ms/step | 31.3 samples/s
604
+ Step 13070/50462 | Loss: 2.7121 | LR: 4.28e-04 | 1916.6ms/step | 31.3 samples/s
605
+ Step 13080/50462 | Loss: 2.6958 | LR: 4.28e-04 | 1917.9ms/step | 31.3 samples/s
606
+ Step 13090/50462 | Loss: 2.7184 | LR: 4.28e-04 | 1917.2ms/step | 31.3 samples/s
607
+ Step 13100/50462 | Loss: 2.6622 | LR: 4.28e-04 | 1916.9ms/step | 31.3 samples/s
608
+ Step 13110/50462 | Loss: 2.6423 | LR: 4.28e-04 | 1916.9ms/step | 31.3 samples/s
609
+ Step 13120/50462 | Loss: 2.6969 | LR: 4.27e-04 | 1913.6ms/step | 31.4 samples/s
610
+ Step 13130/50462 | Loss: 2.6971 | LR: 4.27e-04 | 1912.5ms/step | 31.4 samples/s
611
+ Step 13140/50462 | Loss: 2.6993 | LR: 4.27e-04 | 1913.9ms/step | 31.3 samples/s
612
+ Step 13150/50462 | Loss: 2.6828 | LR: 4.27e-04 | 1916.5ms/step | 31.3 samples/s
613
+ Step 13160/50462 | Loss: 2.6763 | LR: 4.27e-04 | 1915.5ms/step | 31.3 samples/s
614
+ Step 13170/50462 | Loss: 2.6668 | LR: 4.27e-04 | 1917.1ms/step | 31.3 samples/s
615
+ Step 13180/50462 | Loss: 2.7200 | LR: 4.27e-04 | 1915.7ms/step | 31.3 samples/s
616
+ Step 13190/50462 | Loss: 2.7027 | LR: 4.27e-04 | 1915.4ms/step | 31.3 samples/s
617
+ Step 13200/50462 | Loss: 2.6769 | LR: 4.27e-04 | 1912.6ms/step | 31.4 samples/s
618
+ Step 13210/50462 | Loss: 2.7051 | LR: 4.26e-04 | 1913.2ms/step | 31.4 samples/s
619
+ Step 13220/50462 | Loss: 2.7376 | LR: 4.26e-04 | 1912.9ms/step | 31.4 samples/s
620
+ Step 13230/50462 | Loss: 2.6997 | LR: 4.26e-04 | 1914.7ms/step | 31.3 samples/s
621
+ Step 13240/50462 | Loss: 2.6513 | LR: 4.26e-04 | 1913.2ms/step | 31.4 samples/s
622
+ Step 13250/50462 | Loss: 2.7054 | LR: 4.26e-04 | 1915.0ms/step | 31.3 samples/s
623
+ Validation at step 13250: Loss = 2.7233, Perplexity = 15.23
624
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-13250
625
+ Step 13260/50462 | Loss: 2.7122 | LR: 4.26e-04 | 3189.5ms/step | 18.8 samples/s
626
+ Step 13270/50462 | Loss: 2.6941 | LR: 4.26e-04 | 1906.2ms/step | 31.5 samples/s
627
+ Step 13280/50462 | Loss: 2.6912 | LR: 4.26e-04 | 1910.8ms/step | 31.4 samples/s
628
+ Step 13290/50462 | Loss: 2.6835 | LR: 4.26e-04 | 1910.2ms/step | 31.4 samples/s
629
+ Step 13300/50462 | Loss: 2.7049 | LR: 4.25e-04 | 1912.7ms/step | 31.4 samples/s
630
+ Step 13310/50462 | Loss: 2.6858 | LR: 4.25e-04 | 1915.3ms/step | 31.3 samples/s
631
+ Step 13320/50462 | Loss: 2.6502 | LR: 4.25e-04 | 1912.8ms/step | 31.4 samples/s
632
+ Step 13330/50462 | Loss: 2.6757 | LR: 4.25e-04 | 1914.5ms/step | 31.3 samples/s
633
+ Step 13340/50462 | Loss: 2.6894 | LR: 4.25e-04 | 1915.4ms/step | 31.3 samples/s
634
+ Step 13350/50462 | Loss: 2.6880 | LR: 4.25e-04 | 1912.5ms/step | 31.4 samples/s
635
+ Step 13360/50462 | Loss: 2.6856 | LR: 4.25e-04 | 1914.8ms/step | 31.3 samples/s
636
+ Step 13370/50462 | Loss: 2.6631 | LR: 4.25e-04 | 1916.8ms/step | 31.3 samples/s
637
+ Step 13380/50462 | Loss: 2.6875 | LR: 4.25e-04 | 1914.1ms/step | 31.3 samples/s
638
+ Step 13390/50462 | Loss: 2.6430 | LR: 4.24e-04 | 1915.7ms/step | 31.3 samples/s
639
+ Step 13400/50462 | Loss: 2.7012 | LR: 4.24e-04 | 1913.6ms/step | 31.4 samples/s
640
+ Step 13410/50462 | Loss: 2.6610 | LR: 4.24e-04 | 1909.5ms/step | 31.4 samples/s
641
+ Step 13420/50462 | Loss: 2.6879 | LR: 4.24e-04 | 1909.7ms/step | 31.4 samples/s
642
+ Step 13430/50462 | Loss: 2.6853 | LR: 4.24e-04 | 1912.3ms/step | 31.4 samples/s
643
+ Step 13440/50462 | Loss: 2.6519 | LR: 4.24e-04 | 1908.6ms/step | 31.4 samples/s
644
+ Step 13450/50462 | Loss: 2.6682 | LR: 4.24e-04 | 1914.1ms/step | 31.3 samples/s
645
+ Step 13460/50462 | Loss: 2.6733 | LR: 4.24e-04 | 1911.3ms/step | 31.4 samples/s
646
+ Step 13470/50462 | Loss: 2.6739 | LR: 4.23e-04 | 1910.8ms/step | 31.4 samples/s
647
+ Step 13480/50462 | Loss: 2.6618 | LR: 4.23e-04 | 1914.8ms/step | 31.3 samples/s
648
+ Step 13490/50462 | Loss: 2.6822 | LR: 4.23e-04 | 1913.5ms/step | 31.4 samples/s
649
+ Step 13500/50462 | Loss: 2.7068 | LR: 4.23e-04 | 1914.9ms/step | 31.3 samples/s
650
+ Validation at step 13500: Loss = 2.7205, Perplexity = 15.19
651
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-13500
652
+ Step 13510/50462 | Loss: 2.6906 | LR: 4.23e-04 | 3204.8ms/step | 18.7 samples/s
653
+ Step 13520/50462 | Loss: 2.6896 | LR: 4.23e-04 | 1900.9ms/step | 31.6 samples/s
654
+ Step 13530/50462 | Loss: 2.7128 | LR: 4.23e-04 | 1905.8ms/step | 31.5 samples/s
655
+ Step 13540/50462 | Loss: 2.6486 | LR: 4.23e-04 | 1908.2ms/step | 31.4 samples/s
656
+ Step 13550/50462 | Loss: 2.6337 | LR: 4.23e-04 | 1910.8ms/step | 31.4 samples/s
657
+ Step 13560/50462 | Loss: 2.6807 | LR: 4.22e-04 | 1913.7ms/step | 31.4 samples/s
658
+ Step 13570/50462 | Loss: 2.6283 | LR: 4.22e-04 | 1916.5ms/step | 31.3 samples/s
659
+ Step 13580/50462 | Loss: 2.6981 | LR: 4.22e-04 | 1917.8ms/step | 31.3 samples/s
660
+ Step 13590/50462 | Loss: 2.6838 | LR: 4.22e-04 | 1916.0ms/step | 31.3 samples/s
661
+ Step 13600/50462 | Loss: 2.7035 | LR: 4.22e-04 | 1914.8ms/step | 31.3 samples/s
662
+ Step 13610/50462 | Loss: 2.6917 | LR: 4.22e-04 | 1916.9ms/step | 31.3 samples/s
663
+ Step 13620/50462 | Loss: 2.6834 | LR: 4.22e-04 | 1912.5ms/step | 31.4 samples/s
664
+ Step 13630/50462 | Loss: 2.6860 | LR: 4.22e-04 | 1912.7ms/step | 31.4 samples/s
665
+ Step 13640/50462 | Loss: 2.6428 | LR: 4.22e-04 | 1911.3ms/step | 31.4 samples/s
666
+ Step 13650/50462 | Loss: 2.6693 | LR: 4.21e-04 | 1914.6ms/step | 31.3 samples/s
667
+ Step 13660/50462 | Loss: 2.6956 | LR: 4.21e-04 | 1912.1ms/step | 31.4 samples/s
668
+ Step 13670/50462 | Loss: 2.6838 | LR: 4.21e-04 | 1915.5ms/step | 31.3 samples/s
669
+ Step 13680/50462 | Loss: 2.6918 | LR: 4.21e-04 | 1914.7ms/step | 31.3 samples/s
670
+ Step 13690/50462 | Loss: 2.6567 | LR: 4.21e-04 | 1918.4ms/step | 31.3 samples/s
671
+ Step 13700/50462 | Loss: 2.7002 | LR: 4.21e-04 | 1916.5ms/step | 31.3 samples/s
672
+ Step 13710/50462 | Loss: 2.6810 | LR: 4.21e-04 | 1915.0ms/step | 31.3 samples/s
673
+ Step 13720/50462 | Loss: 2.6709 | LR: 4.21e-04 | 1914.9ms/step | 31.3 samples/s
674
+ Step 13730/50462 | Loss: 2.7270 | LR: 4.21e-04 | 1916.8ms/step | 31.3 samples/s
675
+ Step 13740/50462 | Loss: 2.6647 | LR: 4.20e-04 | 1915.8ms/step | 31.3 samples/s
676
+ Step 13750/50462 | Loss: 2.6902 | LR: 4.20e-04 | 1914.9ms/step | 31.3 samples/s
677
+ Validation at step 13750: Loss = 2.7173, Perplexity = 15.14
678
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-13750
679
+ Step 13760/50462 | Loss: 2.6547 | LR: 4.20e-04 | 3207.6ms/step | 18.7 samples/s
680
+ Step 13770/50462 | Loss: 2.6929 | LR: 4.20e-04 | 1901.4ms/step | 31.6 samples/s
681
+ Step 13780/50462 | Loss: 2.6881 | LR: 4.20e-04 | 1902.6ms/step | 31.5 samples/s
682
+ Step 13790/50462 | Loss: 2.6969 | LR: 4.20e-04 | 1906.0ms/step | 31.5 samples/s
683
+ Step 13800/50462 | Loss: 2.6454 | LR: 4.20e-04 | 1907.6ms/step | 31.5 samples/s
684
+ Step 13810/50462 | Loss: 2.6982 | LR: 4.20e-04 | 1908.2ms/step | 31.4 samples/s
685
+ Step 13820/50462 | Loss: 2.6554 | LR: 4.19e-04 | 1913.0ms/step | 31.4 samples/s
686
+ Step 13830/50462 | Loss: 2.6668 | LR: 4.19e-04 | 1909.1ms/step | 31.4 samples/s
687
+ Step 13840/50462 | Loss: 2.6959 | LR: 4.19e-04 | 1907.8ms/step | 31.5 samples/s
688
+ Step 13850/50462 | Loss: 2.7010 | LR: 4.19e-04 | 1905.3ms/step | 31.5 samples/s
689
+ Step 13860/50462 | Loss: 2.6976 | LR: 4.19e-04 | 1907.5ms/step | 31.5 samples/s
690
+ Step 13870/50462 | Loss: 2.6865 | LR: 4.19e-04 | 1910.0ms/step | 31.4 samples/s
691
+ Step 13880/50462 | Loss: 2.7005 | LR: 4.19e-04 | 1911.9ms/step | 31.4 samples/s
692
+ Step 13890/50462 | Loss: 2.6974 | LR: 4.19e-04 | 1909.1ms/step | 31.4 samples/s
693
+ Step 13900/50462 | Loss: 2.7208 | LR: 4.19e-04 | 1912.1ms/step | 31.4 samples/s
694
+ Step 13910/50462 | Loss: 2.7100 | LR: 4.18e-04 | 1907.3ms/step | 31.5 samples/s
695
+ Step 13920/50462 | Loss: 2.6604 | LR: 4.18e-04 | 1909.2ms/step | 31.4 samples/s
696
+ Step 13930/50462 | Loss: 2.7042 | LR: 4.18e-04 | 1910.5ms/step | 31.4 samples/s
697
+ Step 13940/50462 | Loss: 2.7074 | LR: 4.18e-04 | 1911.1ms/step | 31.4 samples/s
698
+ Step 13950/50462 | Loss: 2.6926 | LR: 4.18e-04 | 1910.2ms/step | 31.4 samples/s
699
+ Step 13960/50462 | Loss: 2.6976 | LR: 4.18e-04 | 1912.8ms/step | 31.4 samples/s
700
+ Step 13970/50462 | Loss: 2.6666 | LR: 4.18e-04 | 1909.0ms/step | 31.4 samples/s
701
+ Step 13980/50462 | Loss: 2.7165 | LR: 4.18e-04 | 1910.7ms/step | 31.4 samples/s
702
+ Step 13990/50462 | Loss: 2.7115 | LR: 4.17e-04 | 1911.5ms/step | 31.4 samples/s
703
+ Step 14000/50462 | Loss: 2.6927 | LR: 4.17e-04 | 1913.0ms/step | 31.4 samples/s
704
+ Validation at step 14000: Loss = 2.7137, Perplexity = 15.09
705
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-14000
706
+ Step 14010/50462 | Loss: 2.6740 | LR: 4.17e-04 | 3184.2ms/step | 18.8 samples/s
707
+ Step 14020/50462 | Loss: 2.6292 | LR: 4.17e-04 | 1903.0ms/step | 31.5 samples/s
708
+ Step 14030/50462 | Loss: 2.6820 | LR: 4.17e-04 | 1906.4ms/step | 31.5 samples/s
709
+ Step 14040/50462 | Loss: 2.7082 | LR: 4.17e-04 | 1909.1ms/step | 31.4 samples/s
710
+ Step 14050/50462 | Loss: 2.6605 | LR: 4.17e-04 | 1909.6ms/step | 31.4 samples/s
711
+ Step 14060/50462 | Loss: 2.6862 | LR: 4.17e-04 | 1908.1ms/step | 31.4 samples/s
712
+ Step 14070/50462 | Loss: 2.6654 | LR: 4.17e-04 | 1911.2ms/step | 31.4 samples/s
713
+ Step 14080/50462 | Loss: 2.6585 | LR: 4.16e-04 | 1908.3ms/step | 31.4 samples/s
714
+ Step 14090/50462 | Loss: 2.6621 | LR: 4.16e-04 | 1912.6ms/step | 31.4 samples/s
715
+ Step 14100/50462 | Loss: 2.6814 | LR: 4.16e-04 | 1914.5ms/step | 31.3 samples/s
716
+ Step 14110/50462 | Loss: 2.7083 | LR: 4.16e-04 | 1911.4ms/step | 31.4 samples/s
717
+ Step 14120/50462 | Loss: 2.6735 | LR: 4.16e-04 | 1908.2ms/step | 31.4 samples/s
718
+ Step 14130/50462 | Loss: 2.6540 | LR: 4.16e-04 | 1908.3ms/step | 31.4 samples/s
719
+ Step 14140/50462 | Loss: 2.7194 | LR: 4.16e-04 | 1913.4ms/step | 31.4 samples/s
720
+ Step 14150/50462 | Loss: 2.6836 | LR: 4.16e-04 | 1912.9ms/step | 31.4 samples/s
721
+ Step 14160/50462 | Loss: 2.6628 | LR: 4.15e-04 | 1911.5ms/step | 31.4 samples/s
722
+ Step 14170/50462 | Loss: 2.6913 | LR: 4.15e-04 | 1910.0ms/step | 31.4 samples/s
723
+ Step 14180/50462 | Loss: 2.6542 | LR: 4.15e-04 | 1913.4ms/step | 31.4 samples/s
724
+ Step 14190/50462 | Loss: 2.6464 | LR: 4.15e-04 | 1911.5ms/step | 31.4 samples/s
725
+ Step 14200/50462 | Loss: 2.6778 | LR: 4.15e-04 | 1911.9ms/step | 31.4 samples/s
726
+ Step 14210/50462 | Loss: 2.6375 | LR: 4.15e-04 | 1909.9ms/step | 31.4 samples/s
727
+ Step 14220/50462 | Loss: 2.6365 | LR: 4.15e-04 | 1911.2ms/step | 31.4 samples/s
728
+ Step 14230/50462 | Loss: 2.6862 | LR: 4.15e-04 | 1909.7ms/step | 31.4 samples/s
729
+ Step 14240/50462 | Loss: 2.6074 | LR: 4.15e-04 | 1910.7ms/step | 31.4 samples/s
730
+ Step 14250/50462 | Loss: 2.6910 | LR: 4.14e-04 | 1912.7ms/step | 31.4 samples/s
731
+ Validation at step 14250: Loss = 2.7109, Perplexity = 15.04
732
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-14250
733
+ Step 14260/50462 | Loss: 2.6611 | LR: 4.14e-04 | 3201.6ms/step | 18.7 samples/s
734
+ Step 14270/50462 | Loss: 2.6560 | LR: 4.14e-04 | 1899.0ms/step | 31.6 samples/s
735
+ Step 14280/50462 | Loss: 2.6245 | LR: 4.14e-04 | 1903.5ms/step | 31.5 samples/s
736
+ Step 14290/50462 | Loss: 2.6882 | LR: 4.14e-04 | 1903.6ms/step | 31.5 samples/s
737
+ Step 14300/50462 | Loss: 2.6580 | LR: 4.14e-04 | 1904.9ms/step | 31.5 samples/s
738
+ Step 14310/50462 | Loss: 2.6707 | LR: 4.14e-04 | 1907.6ms/step | 31.5 samples/s
739
+ Step 14320/50462 | Loss: 2.6650 | LR: 4.14e-04 | 1907.4ms/step | 31.5 samples/s
740
+ Step 14330/50462 | Loss: 2.6752 | LR: 4.13e-04 | 1909.5ms/step | 31.4 samples/s
741
+ Step 14340/50462 | Loss: 2.6627 | LR: 4.13e-04 | 1911.0ms/step | 31.4 samples/s
742
+ Step 14350/50462 | Loss: 2.7035 | LR: 4.13e-04 | 1912.7ms/step | 31.4 samples/s
743
+ Step 14360/50462 | Loss: 2.6225 | LR: 4.13e-04 | 1914.4ms/step | 31.3 samples/s
744
+ Step 14370/50462 | Loss: 2.7040 | LR: 4.13e-04 | 1913.9ms/step | 31.3 samples/s
745
+ Step 14380/50462 | Loss: 2.6905 | LR: 4.13e-04 | 1914.2ms/step | 31.3 samples/s
746
+ Step 14390/50462 | Loss: 2.6100 | LR: 4.13e-04 | 1913.9ms/step | 31.3 samples/s
747
+ Step 14400/50462 | Loss: 2.6579 | LR: 4.13e-04 | 1914.2ms/step | 31.3 samples/s
748
+ Step 14410/50462 | Loss: 2.6557 | LR: 4.12e-04 | 1913.9ms/step | 31.4 samples/s
749
+ Step 14420/50462 | Loss: 2.6335 | LR: 4.12e-04 | 1912.7ms/step | 31.4 samples/s
750
+ Step 14430/50462 | Loss: 2.6551 | LR: 4.12e-04 | 1914.5ms/step | 31.3 samples/s
751
+ Step 14440/50462 | Loss: 2.7152 | LR: 4.12e-04 | 1912.6ms/step | 31.4 samples/s
752
+ Step 14450/50462 | Loss: 2.6986 | LR: 4.12e-04 | 1912.2ms/step | 31.4 samples/s
753
+ Step 14460/50462 | Loss: 2.6495 | LR: 4.12e-04 | 1913.7ms/step | 31.4 samples/s
754
+ Step 14470/50462 | Loss: 2.6475 | LR: 4.12e-04 | 1912.1ms/step | 31.4 samples/s
755
+ Step 14480/50462 | Loss: 2.6867 | LR: 4.12e-04 | 1911.3ms/step | 31.4 samples/s
756
+ Step 14490/50462 | Loss: 2.6974 | LR: 4.12e-04 | 1912.7ms/step | 31.4 samples/s
757
+ Step 14500/50462 | Loss: 2.6967 | LR: 4.11e-04 | 1913.3ms/step | 31.4 samples/s
758
+ Validation at step 14500: Loss = 2.7071, Perplexity = 14.99
759
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-14500
760
+ Step 14510/50462 | Loss: 2.6509 | LR: 4.11e-04 | 3179.4ms/step | 18.9 samples/s
761
+ Step 14520/50462 | Loss: 2.6361 | LR: 4.11e-04 | 1901.0ms/step | 31.6 samples/s
762
+ Step 14530/50462 | Loss: 2.6914 | LR: 4.11e-04 | 1904.4ms/step | 31.5 samples/s
763
+ Step 14540/50462 | Loss: 2.7034 | LR: 4.11e-04 | 1909.6ms/step | 31.4 samples/s
764
+ Step 14550/50462 | Loss: 2.6993 | LR: 4.11e-04 | 1910.0ms/step | 31.4 samples/s
765
+ Step 14560/50462 | Loss: 2.6832 | LR: 4.11e-04 | 1910.7ms/step | 31.4 samples/s
766
+ Step 14570/50462 | Loss: 2.6842 | LR: 4.11e-04 | 1912.0ms/step | 31.4 samples/s
767
+ Step 14580/50462 | Loss: 2.6528 | LR: 4.10e-04 | 1911.8ms/step | 31.4 samples/s
768
+ Step 14590/50462 | Loss: 2.6850 | LR: 4.10e-04 | 1911.6ms/step | 31.4 samples/s
769
+ Step 14600/50462 | Loss: 2.6936 | LR: 4.10e-04 | 1909.7ms/step | 31.4 samples/s
770
+ Step 14610/50462 | Loss: 2.6512 | LR: 4.10e-04 | 1910.9ms/step | 31.4 samples/s
771
+ Step 14620/50462 | Loss: 2.6708 | LR: 4.10e-04 | 1911.7ms/step | 31.4 samples/s
772
+ Step 14630/50462 | Loss: 2.6954 | LR: 4.10e-04 | 1912.7ms/step | 31.4 samples/s
773
+ Step 14640/50462 | Loss: 2.6237 | LR: 4.10e-04 | 1912.8ms/step | 31.4 samples/s
774
+ Step 14650/50462 | Loss: 2.6866 | LR: 4.10e-04 | 1913.5ms/step | 31.4 samples/s
775
+ Step 14660/50462 | Loss: 2.6868 | LR: 4.09e-04 | 1912.0ms/step | 31.4 samples/s
776
+ Step 14670/50462 | Loss: 2.6510 | LR: 4.09e-04 | 1913.1ms/step | 31.4 samples/s
777
+ Step 14680/50462 | Loss: 2.6189 | LR: 4.09e-04 | 1912.4ms/step | 31.4 samples/s
778
+ Step 14690/50462 | Loss: 2.6779 | LR: 4.09e-04 | 1913.1ms/step | 31.4 samples/s
779
+ Step 14700/50462 | Loss: 2.6607 | LR: 4.09e-04 | 1912.7ms/step | 31.4 samples/s
780
+ Step 14710/50462 | Loss: 2.6549 | LR: 4.09e-04 | 1911.8ms/step | 31.4 samples/s
781
+ Step 14720/50462 | Loss: 2.6382 | LR: 4.09e-04 | 1911.6ms/step | 31.4 samples/s
782
+ Step 14730/50462 | Loss: 2.7062 | LR: 4.09e-04 | 1913.2ms/step | 31.4 samples/s
783
+ Step 14740/50462 | Loss: 2.7230 | LR: 4.08e-04 | 1913.6ms/step | 31.4 samples/s
784
+ Step 14750/50462 | Loss: 2.6619 | LR: 4.08e-04 | 1913.3ms/step | 31.4 samples/s
785
+ Validation at step 14750: Loss = 2.7049, Perplexity = 14.95
786
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-14750
787
+ Step 14760/50462 | Loss: 2.6801 | LR: 4.08e-04 | 3219.9ms/step | 18.6 samples/s
788
+ Step 14770/50462 | Loss: 2.6920 | LR: 4.08e-04 | 1898.3ms/step | 31.6 samples/s
789
+ Step 14780/50462 | Loss: 2.6952 | LR: 4.08e-04 | 1905.1ms/step | 31.5 samples/s
790
+ Step 14790/50462 | Loss: 2.6883 | LR: 4.08e-04 | 1904.9ms/step | 31.5 samples/s
791
+ Step 14800/50462 | Loss: 2.7011 | LR: 4.08e-04 | 1907.4ms/step | 31.5 samples/s
792
+ Step 14810/50462 | Loss: 2.6661 | LR: 4.08e-04 | 1909.2ms/step | 31.4 samples/s
793
+ Step 14820/50462 | Loss: 2.6562 | LR: 4.08e-04 | 1910.9ms/step | 31.4 samples/s
794
+ Step 14830/50462 | Loss: 2.6673 | LR: 4.07e-04 | 1914.6ms/step | 31.3 samples/s
795
+ Step 14840/50462 | Loss: 2.6783 | LR: 4.07e-04 | 1911.8ms/step | 31.4 samples/s
796
+ Step 14850/50462 | Loss: 2.6597 | LR: 4.07e-04 | 1912.3ms/step | 31.4 samples/s
797
+ Step 14860/50462 | Loss: 2.6744 | LR: 4.07e-04 | 1914.6ms/step | 31.3 samples/s
798
+ Step 14870/50462 | Loss: 2.6471 | LR: 4.07e-04 | 1909.8ms/step | 31.4 samples/s
799
+ Step 14880/50462 | Loss: 2.6481 | LR: 4.07e-04 | 1913.0ms/step | 31.4 samples/s
800
+ Step 14890/50462 | Loss: 2.6410 | LR: 4.07e-04 | 1912.0ms/step | 31.4 samples/s
801
+ Step 14900/50462 | Loss: 2.6649 | LR: 4.07e-04 | 1910.7ms/step | 31.4 samples/s
802
+ Step 14910/50462 | Loss: 2.6797 | LR: 4.06e-04 | 1910.8ms/step | 31.4 samples/s
803
+ Step 14920/50462 | Loss: 2.6324 | LR: 4.06e-04 | 1916.0ms/step | 31.3 samples/s
804
+ Step 14930/50462 | Loss: 2.6967 | LR: 4.06e-04 | 1916.8ms/step | 31.3 samples/s
805
+ Step 14940/50462 | Loss: 2.6230 | LR: 4.06e-04 | 1917.6ms/step | 31.3 samples/s
806
+ Step 14950/50462 | Loss: 2.6910 | LR: 4.06e-04 | 1917.1ms/step | 31.3 samples/s
807
+ Step 14960/50462 | Loss: 2.6437 | LR: 4.06e-04 | 1919.1ms/step | 31.3 samples/s
808
+ Step 14970/50462 | Loss: 2.6574 | LR: 4.06e-04 | 1919.4ms/step | 31.3 samples/s
809
+ Step 14980/50462 | Loss: 2.6871 | LR: 4.06e-04 | 1919.6ms/step | 31.3 samples/s
810
+ Step 14990/50462 | Loss: 2.6959 | LR: 4.05e-04 | 1920.4ms/step | 31.2 samples/s
811
+ Step 15000/50462 | Loss: 2.6484 | LR: 4.05e-04 | 1917.5ms/step | 31.3 samples/s
812
+ Validation at step 15000: Loss = 2.6996, Perplexity = 14.87
813
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-15000
814
+ Step 15010/50462 | Loss: 2.6783 | LR: 4.05e-04 | 3205.8ms/step | 18.7 samples/s
815
+ Step 15020/50462 | Loss: 2.6700 | LR: 4.05e-04 | 1904.2ms/step | 31.5 samples/s
816
+ Step 15030/50462 | Loss: 2.6489 | LR: 4.05e-04 | 1906.0ms/step | 31.5 samples/s
817
+ Step 15040/50462 | Loss: 2.6601 | LR: 4.05e-04 | 1910.1ms/step | 31.4 samples/s
818
+ Step 15050/50462 | Loss: 2.6580 | LR: 4.05e-04 | 1912.0ms/step | 31.4 samples/s
819
+ Step 15060/50462 | Loss: 2.6321 | LR: 4.05e-04 | 1912.4ms/step | 31.4 samples/s
820
+ Step 15070/50462 | Loss: 2.6320 | LR: 4.04e-04 | 1912.7ms/step | 31.4 samples/s
821
+ Step 15080/50462 | Loss: 2.6830 | LR: 4.04e-04 | 1914.5ms/step | 31.3 samples/s
822
+ Step 15090/50462 | Loss: 2.6338 | LR: 4.04e-04 | 1911.9ms/step | 31.4 samples/s
823
+ Step 15100/50462 | Loss: 2.6304 | LR: 4.04e-04 | 1914.8ms/step | 31.3 samples/s
824
+ Step 15110/50462 | Loss: 2.6362 | LR: 4.04e-04 | 1917.4ms/step | 31.3 samples/s
825
+ Step 15120/50462 | Loss: 2.6865 | LR: 4.04e-04 | 1915.1ms/step | 31.3 samples/s
826
+ Step 15130/50462 | Loss: 2.6253 | LR: 4.04e-04 | 1910.8ms/step | 31.4 samples/s
827
+ Step 15140/50462 | Loss: 2.6700 | LR: 4.04e-04 | 1912.3ms/step | 31.4 samples/s
828
+ Step 15150/50462 | Loss: 2.6351 | LR: 4.03e-04 | 1911.7ms/step | 31.4 samples/s
829
+ Step 15160/50462 | Loss: 2.6988 | LR: 4.03e-04 | 1912.1ms/step | 31.4 samples/s
830
+ Step 15170/50462 | Loss: 2.6241 | LR: 4.03e-04 | 1911.9ms/step | 31.4 samples/s
831
+ Step 15180/50462 | Loss: 2.6740 | LR: 4.03e-04 | 1913.2ms/step | 31.4 samples/s
832
+ Step 15190/50462 | Loss: 2.6586 | LR: 4.03e-04 | 1914.0ms/step | 31.3 samples/s
833
+ Step 15200/50462 | Loss: 2.6356 | LR: 4.03e-04 | 1915.6ms/step | 31.3 samples/s
834
+ Step 15210/50462 | Loss: 2.6496 | LR: 4.03e-04 | 1914.2ms/step | 31.3 samples/s
835
+ Step 15220/50462 | Loss: 2.6904 | LR: 4.03e-04 | 1911.7ms/step | 31.4 samples/s
836
+ Step 15230/50462 | Loss: 2.6454 | LR: 4.02e-04 | 1911.7ms/step | 31.4 samples/s
837
+ Step 15240/50462 | Loss: 2.6358 | LR: 4.02e-04 | 1914.0ms/step | 31.3 samples/s
838
+ Step 15250/50462 | Loss: 2.6155 | LR: 4.02e-04 | 1911.4ms/step | 31.4 samples/s
839
+ Validation at step 15250: Loss = 2.6976, Perplexity = 14.84
840
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-15250
841
+ Step 15260/50462 | Loss: 2.6134 | LR: 4.02e-04 | 3216.2ms/step | 18.7 samples/s
842
+ Step 15270/50462 | Loss: 2.6697 | LR: 4.02e-04 | 1905.5ms/step | 31.5 samples/s
843
+ Step 15280/50462 | Loss: 2.6893 | LR: 4.02e-04 | 1906.8ms/step | 31.5 samples/s
844
+ Step 15290/50462 | Loss: 2.6172 | LR: 4.02e-04 | 1911.9ms/step | 31.4 samples/s
845
+ Step 15300/50462 | Loss: 2.6762 | LR: 4.02e-04 | 1911.6ms/step | 31.4 samples/s
846
+ Step 15310/50462 | Loss: 2.6682 | LR: 4.01e-04 | 1911.5ms/step | 31.4 samples/s
847
+ Step 15320/50462 | Loss: 2.6100 | LR: 4.01e-04 | 1915.4ms/step | 31.3 samples/s
848
+ Step 15330/50462 | Loss: 2.6370 | LR: 4.01e-04 | 1916.2ms/step | 31.3 samples/s
849
+ Step 15340/50462 | Loss: 2.6661 | LR: 4.01e-04 | 1914.7ms/step | 31.3 samples/s
850
+ Step 15350/50462 | Loss: 2.6439 | LR: 4.01e-04 | 1915.9ms/step | 31.3 samples/s
851
+ Step 15360/50462 | Loss: 2.6900 | LR: 4.01e-04 | 1915.1ms/step | 31.3 samples/s
852
+ Step 15370/50462 | Loss: 2.6213 | LR: 4.01e-04 | 1913.1ms/step | 31.4 samples/s
853
+ Step 15380/50462 | Loss: 2.6312 | LR: 4.01e-04 | 1914.6ms/step | 31.3 samples/s
854
+ Step 15390/50462 | Loss: 2.6672 | LR: 4.00e-04 | 1914.5ms/step | 31.3 samples/s
855
+ Step 15400/50462 | Loss: 2.6633 | LR: 4.00e-04 | 1913.7ms/step | 31.4 samples/s
856
+ Step 15410/50462 | Loss: 2.6103 | LR: 4.00e-04 | 1910.2ms/step | 31.4 samples/s
857
+ Step 15420/50462 | Loss: 2.6698 | LR: 4.00e-04 | 1910.1ms/step | 31.4 samples/s
858
+ Step 15430/50462 | Loss: 2.6666 | LR: 4.00e-04 | 1913.4ms/step | 31.4 samples/s
859
+ Step 15440/50462 | Loss: 2.6589 | LR: 4.00e-04 | 1911.3ms/step | 31.4 samples/s
860
+ Step 15450/50462 | Loss: 2.6234 | LR: 4.00e-04 | 1915.4ms/step | 31.3 samples/s
861
+ Step 15460/50462 | Loss: 2.6191 | LR: 4.00e-04 | 1913.7ms/step | 31.4 samples/s
862
+ Step 15470/50462 | Loss: 2.6319 | LR: 3.99e-04 | 1912.8ms/step | 31.4 samples/s
863
+ Step 15480/50462 | Loss: 2.6755 | LR: 3.99e-04 | 1915.4ms/step | 31.3 samples/s
864
+ Step 15490/50462 | Loss: 2.6253 | LR: 3.99e-04 | 1912.6ms/step | 31.4 samples/s
865
+ Step 15500/50462 | Loss: 2.6614 | LR: 3.99e-04 | 1914.0ms/step | 31.3 samples/s
866
+ Validation at step 15500: Loss = 2.6952, Perplexity = 14.81
867
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-15500
868
+ Step 15510/50462 | Loss: 2.6635 | LR: 3.99e-04 | 3200.9ms/step | 18.7 samples/s
869
+ Step 15520/50462 | Loss: 2.6076 | LR: 3.99e-04 | 1906.0ms/step | 31.5 samples/s
870
+ Step 15530/50462 | Loss: 2.6667 | LR: 3.99e-04 | 1910.8ms/step | 31.4 samples/s
871
+ Step 15540/50462 | Loss: 2.6513 | LR: 3.99e-04 | 1910.3ms/step | 31.4 samples/s
872
+ Step 15550/50462 | Loss: 2.6761 | LR: 3.98e-04 | 1913.3ms/step | 31.4 samples/s
873
+ Step 15560/50462 | Loss: 2.6650 | LR: 3.98e-04 | 1912.8ms/step | 31.4 samples/s
874
+ Step 15570/50462 | Loss: 2.6634 | LR: 3.98e-04 | 1913.8ms/step | 31.4 samples/s
875
+ Step 15580/50462 | Loss: 2.6501 | LR: 3.98e-04 | 1914.8ms/step | 31.3 samples/s
876
+ Step 15590/50462 | Loss: 2.6576 | LR: 3.98e-04 | 1914.6ms/step | 31.3 samples/s
877
+ Step 15600/50462 | Loss: 2.6431 | LR: 3.98e-04 | 1914.4ms/step | 31.3 samples/s
878
+ Step 15610/50462 | Loss: 2.6713 | LR: 3.98e-04 | 1914.5ms/step | 31.3 samples/s
879
+ Step 15620/50462 | Loss: 2.6503 | LR: 3.98e-04 | 1913.4ms/step | 31.4 samples/s
880
+ Step 15630/50462 | Loss: 2.6564 | LR: 3.97e-04 | 1918.1ms/step | 31.3 samples/s
881
+ Step 15640/50462 | Loss: 2.6888 | LR: 3.97e-04 | 1912.3ms/step | 31.4 samples/s
882
+ Step 15650/50462 | Loss: 2.6377 | LR: 3.97e-04 | 1912.7ms/step | 31.4 samples/s
883
+ Step 15660/50462 | Loss: 2.6429 | LR: 3.97e-04 | 1915.6ms/step | 31.3 samples/s
884
+ Step 15670/50462 | Loss: 2.6838 | LR: 3.97e-04 | 1915.1ms/step | 31.3 samples/s
885
+ Step 15680/50462 | Loss: 2.6529 | LR: 3.97e-04 | 1915.7ms/step | 31.3 samples/s
886
+ Step 15690/50462 | Loss: 2.6563 | LR: 3.97e-04 | 1913.9ms/step | 31.3 samples/s
887
+ Step 15700/50462 | Loss: 2.6655 | LR: 3.96e-04 | 1912.3ms/step | 31.4 samples/s
888
+ Step 15710/50462 | Loss: 2.6753 | LR: 3.96e-04 | 1908.9ms/step | 31.4 samples/s
889
+ Step 15720/50462 | Loss: 2.6310 | LR: 3.96e-04 | 1909.6ms/step | 31.4 samples/s
890
+ Step 15730/50462 | Loss: 2.6873 | LR: 3.96e-04 | 1909.7ms/step | 31.4 samples/s
891
+ Step 15740/50462 | Loss: 2.6180 | LR: 3.96e-04 | 1911.2ms/step | 31.4 samples/s
892
+ Step 15750/50462 | Loss: 2.6462 | LR: 3.96e-04 | 1912.5ms/step | 31.4 samples/s
893
+ Validation at step 15750: Loss = 2.6903, Perplexity = 14.74
894
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-15750
895
+ Step 15760/50462 | Loss: 2.6774 | LR: 3.96e-04 | 3203.3ms/step | 18.7 samples/s
896
+ Step 15770/50462 | Loss: 2.5979 | LR: 3.96e-04 | 1898.8ms/step | 31.6 samples/s
897
+ Step 15780/50462 | Loss: 2.6195 | LR: 3.95e-04 | 1903.5ms/step | 31.5 samples/s
898
+ Step 15790/50462 | Loss: 2.6282 | LR: 3.95e-04 | 1908.0ms/step | 31.4 samples/s
899
+ Step 15800/50462 | Loss: 2.6344 | LR: 3.95e-04 | 1910.4ms/step | 31.4 samples/s
900
+ Step 15810/50462 | Loss: 2.6690 | LR: 3.95e-04 | 1911.4ms/step | 31.4 samples/s
901
+ Step 15820/50462 | Loss: 2.6354 | LR: 3.95e-04 | 1910.7ms/step | 31.4 samples/s
902
+ Step 15830/50462 | Loss: 2.6326 | LR: 3.95e-04 | 1911.0ms/step | 31.4 samples/s
903
+ Step 15840/50462 | Loss: 2.6607 | LR: 3.95e-04 | 1912.5ms/step | 31.4 samples/s
904
+ Step 15850/50462 | Loss: 2.6456 | LR: 3.95e-04 | 1909.9ms/step | 31.4 samples/s
905
+ Step 15860/50462 | Loss: 2.6608 | LR: 3.94e-04 | 1913.7ms/step | 31.4 samples/s
906
+ Step 15870/50462 | Loss: 2.6227 | LR: 3.94e-04 | 1914.9ms/step | 31.3 samples/s
907
+ Step 15880/50462 | Loss: 2.6725 | LR: 3.94e-04 | 1911.7ms/step | 31.4 samples/s
908
+ Step 15890/50462 | Loss: 2.6542 | LR: 3.94e-04 | 1911.8ms/step | 31.4 samples/s
909
+ Step 15900/50462 | Loss: 2.6325 | LR: 3.94e-04 | 1911.4ms/step | 31.4 samples/s
910
+ Step 15910/50462 | Loss: 2.6732 | LR: 3.94e-04 | 1911.9ms/step | 31.4 samples/s
911
+ Step 15920/50462 | Loss: 2.6539 | LR: 3.94e-04 | 1913.3ms/step | 31.4 samples/s
912
+ Step 15930/50462 | Loss: 2.6846 | LR: 3.94e-04 | 1911.3ms/step | 31.4 samples/s
913
+ Step 15940/50462 | Loss: 2.6198 | LR: 3.93e-04 | 1913.3ms/step | 31.4 samples/s
914
+ Step 15950/50462 | Loss: 2.6687 | LR: 3.93e-04 | 1911.4ms/step | 31.4 samples/s
915
+ Step 15960/50462 | Loss: 2.6272 | LR: 3.93e-04 | 1910.5ms/step | 31.4 samples/s
916
+ Step 15970/50462 | Loss: 2.6533 | LR: 3.93e-04 | 1911.2ms/step | 31.4 samples/s
917
+ Step 15980/50462 | Loss: 2.6433 | LR: 3.93e-04 | 1912.6ms/step | 31.4 samples/s
918
+ Step 15990/50462 | Loss: 2.6214 | LR: 3.93e-04 | 1913.6ms/step | 31.4 samples/s
919
+ Step 16000/50462 | Loss: 2.6431 | LR: 3.93e-04 | 1913.7ms/step | 31.4 samples/s
920
+ Validation at step 16000: Loss = 2.6889, Perplexity = 14.72
921
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-16000
922
+ Step 16010/50462 | Loss: 2.6661 | LR: 3.92e-04 | 3199.0ms/step | 18.8 samples/s
923
+ Step 16020/50462 | Loss: 2.6490 | LR: 3.92e-04 | 1910.0ms/step | 31.4 samples/s
924
+ Step 16030/50462 | Loss: 2.6933 | LR: 3.92e-04 | 1910.2ms/step | 31.4 samples/s
925
+ Step 16040/50462 | Loss: 2.6825 | LR: 3.92e-04 | 1914.8ms/step | 31.3 samples/s
926
+ Step 16050/50462 | Loss: 2.6783 | LR: 3.92e-04 | 1911.5ms/step | 31.4 samples/s
927
+ Step 16060/50462 | Loss: 2.6879 | LR: 3.92e-04 | 1911.6ms/step | 31.4 samples/s
928
+ Step 16070/50462 | Loss: 2.6411 | LR: 3.92e-04 | 1914.0ms/step | 31.3 samples/s
929
+ Step 16080/50462 | Loss: 2.6398 | LR: 3.92e-04 | 1911.6ms/step | 31.4 samples/s
930
+ Step 16090/50462 | Loss: 2.6282 | LR: 3.91e-04 | 1911.7ms/step | 31.4 samples/s
931
+ Step 16100/50462 | Loss: 2.6156 | LR: 3.91e-04 | 1910.6ms/step | 31.4 samples/s
932
+ Step 16110/50462 | Loss: 2.6778 | LR: 3.91e-04 | 1912.1ms/step | 31.4 samples/s
933
+ Step 16120/50462 | Loss: 2.6644 | LR: 3.91e-04 | 1911.7ms/step | 31.4 samples/s
934
+ Step 16130/50462 | Loss: 2.6283 | LR: 3.91e-04 | 1913.5ms/step | 31.4 samples/s
935
+ Step 16140/50462 | Loss: 2.6540 | LR: 3.91e-04 | 1912.0ms/step | 31.4 samples/s
936
+ Step 16150/50462 | Loss: 2.6605 | LR: 3.91e-04 | 1912.4ms/step | 31.4 samples/s
937
+ Step 16160/50462 | Loss: 2.6176 | LR: 3.91e-04 | 1913.2ms/step | 31.4 samples/s
938
+ Step 16170/50462 | Loss: 2.6288 | LR: 3.90e-04 | 1911.8ms/step | 31.4 samples/s
939
+ Step 16180/50462 | Loss: 2.6310 | LR: 3.90e-04 | 1912.0ms/step | 31.4 samples/s
940
+ Step 16190/50462 | Loss: 2.6370 | LR: 3.90e-04 | 1912.1ms/step | 31.4 samples/s
941
+ Step 16200/50462 | Loss: 2.6460 | LR: 3.90e-04 | 1910.7ms/step | 31.4 samples/s
942
+ Step 16210/50462 | Loss: 2.6041 | LR: 3.90e-04 | 1913.4ms/step | 31.4 samples/s
943
+ Step 16220/50462 | Loss: 2.6752 | LR: 3.90e-04 | 1910.7ms/step | 31.4 samples/s
944
+ Step 16230/50462 | Loss: 2.6639 | LR: 3.90e-04 | 1915.1ms/step | 31.3 samples/s
945
+ Step 16240/50462 | Loss: 2.6690 | LR: 3.89e-04 | 1910.1ms/step | 31.4 samples/s
946
+ Step 16250/50462 | Loss: 2.6424 | LR: 3.89e-04 | 1912.1ms/step | 31.4 samples/s
947
+ Validation at step 16250: Loss = 2.6856, Perplexity = 14.67
948
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-16250
949
+ Step 16260/50462 | Loss: 2.6184 | LR: 3.89e-04 | 3197.0ms/step | 18.8 samples/s
950
+ Step 16270/50462 | Loss: 2.6411 | LR: 3.89e-04 | 1902.8ms/step | 31.5 samples/s
951
+ Step 16280/50462 | Loss: 2.6515 | LR: 3.89e-04 | 1906.6ms/step | 31.5 samples/s
952
+ Step 16290/50462 | Loss: 2.6494 | LR: 3.89e-04 | 1907.0ms/step | 31.5 samples/s
953
+ Step 16300/50462 | Loss: 2.6717 | LR: 3.89e-04 | 1908.2ms/step | 31.4 samples/s
954
+ Step 16310/50462 | Loss: 2.6502 | LR: 3.89e-04 | 1911.1ms/step | 31.4 samples/s
955
+ Step 16320/50462 | Loss: 2.6320 | LR: 3.88e-04 | 1909.0ms/step | 31.4 samples/s
956
+ Step 16330/50462 | Loss: 2.6809 | LR: 3.88e-04 | 1909.3ms/step | 31.4 samples/s
957
+ Step 16340/50462 | Loss: 2.6990 | LR: 3.88e-04 | 1912.4ms/step | 31.4 samples/s
958
+ Step 16350/50462 | Loss: 2.6621 | LR: 3.88e-04 | 1914.3ms/step | 31.3 samples/s
959
+ Step 16360/50462 | Loss: 2.6546 | LR: 3.88e-04 | 1911.7ms/step | 31.4 samples/s
960
+ Step 16370/50462 | Loss: 2.6557 | LR: 3.88e-04 | 1913.0ms/step | 31.4 samples/s
961
+ Step 16380/50462 | Loss: 2.6172 | LR: 3.88e-04 | 1910.9ms/step | 31.4 samples/s
962
+ Step 16390/50462 | Loss: 2.6517 | LR: 3.88e-04 | 1912.1ms/step | 31.4 samples/s
963
+ Step 16400/50462 | Loss: 2.6405 | LR: 3.87e-04 | 1915.6ms/step | 31.3 samples/s
964
+ Step 16410/50462 | Loss: 2.5805 | LR: 3.87e-04 | 1912.8ms/step | 31.4 samples/s
965
+ Step 16420/50462 | Loss: 2.6689 | LR: 3.87e-04 | 1911.9ms/step | 31.4 samples/s
966
+ Step 16430/50462 | Loss: 2.6536 | LR: 3.87e-04 | 1912.6ms/step | 31.4 samples/s
967
+ Step 16440/50462 | Loss: 2.6773 | LR: 3.87e-04 | 1914.4ms/step | 31.3 samples/s
968
+ Step 16450/50462 | Loss: 2.6743 | LR: 3.87e-04 | 1914.1ms/step | 31.3 samples/s
969
+ Step 16460/50462 | Loss: 2.6503 | LR: 3.87e-04 | 1909.7ms/step | 31.4 samples/s
970
+ Step 16470/50462 | Loss: 2.6584 | LR: 3.86e-04 | 1912.9ms/step | 31.4 samples/s
971
+ Step 16480/50462 | Loss: 2.6128 | LR: 3.86e-04 | 1914.7ms/step | 31.3 samples/s
972
+ Step 16490/50462 | Loss: 2.6443 | LR: 3.86e-04 | 1915.8ms/step | 31.3 samples/s
973
+ Step 16500/50462 | Loss: 2.6465 | LR: 3.86e-04 | 1916.2ms/step | 31.3 samples/s
974
+ Validation at step 16500: Loss = 2.6810, Perplexity = 14.60
975
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-16500
976
+ Step 16510/50462 | Loss: 2.6361 | LR: 3.86e-04 | 3204.5ms/step | 18.7 samples/s
977
+ Step 16520/50462 | Loss: 2.6308 | LR: 3.86e-04 | 1903.1ms/step | 31.5 samples/s
978
+ Step 16530/50462 | Loss: 2.6520 | LR: 3.86e-04 | 1907.3ms/step | 31.5 samples/s
979
+ Step 16540/50462 | Loss: 2.6415 | LR: 3.86e-04 | 1908.5ms/step | 31.4 samples/s
980
+ Step 16550/50462 | Loss: 2.6590 | LR: 3.85e-04 | 1908.0ms/step | 31.4 samples/s
981
+ Step 16560/50462 | Loss: 2.6669 | LR: 3.85e-04 | 1911.9ms/step | 31.4 samples/s
982
+ Step 16570/50462 | Loss: 2.6632 | LR: 3.85e-04 | 1911.8ms/step | 31.4 samples/s
983
+ Step 16580/50462 | Loss: 2.6244 | LR: 3.85e-04 | 1912.6ms/step | 31.4 samples/s
984
+ Step 16590/50462 | Loss: 2.6568 | LR: 3.85e-04 | 1914.1ms/step | 31.3 samples/s
985
+ Step 16600/50462 | Loss: 2.6961 | LR: 3.85e-04 | 1914.3ms/step | 31.3 samples/s
986
+ Step 16610/50462 | Loss: 2.6171 | LR: 3.85e-04 | 1913.8ms/step | 31.4 samples/s
987
+ Step 16620/50462 | Loss: 2.6379 | LR: 3.84e-04 | 1915.2ms/step | 31.3 samples/s
988
+ Step 16630/50462 | Loss: 2.6187 | LR: 3.84e-04 | 1915.0ms/step | 31.3 samples/s
989
+ Step 16640/50462 | Loss: 2.6535 | LR: 3.84e-04 | 1914.5ms/step | 31.3 samples/s
990
+ Step 16650/50462 | Loss: 2.6662 | LR: 3.84e-04 | 1916.2ms/step | 31.3 samples/s
991
+ Step 16660/50462 | Loss: 2.6238 | LR: 3.84e-04 | 1912.5ms/step | 31.4 samples/s
992
+ Step 16670/50462 | Loss: 2.6149 | LR: 3.84e-04 | 1914.0ms/step | 31.3 samples/s
993
+ Step 16680/50462 | Loss: 2.6602 | LR: 3.84e-04 | 1913.3ms/step | 31.4 samples/s
994
+ Step 16690/50462 | Loss: 2.6606 | LR: 3.84e-04 | 1913.2ms/step | 31.4 samples/s
995
+ Step 16700/50462 | Loss: 2.6494 | LR: 3.83e-04 | 1912.4ms/step | 31.4 samples/s
996
+ Step 16710/50462 | Loss: 2.6794 | LR: 3.83e-04 | 1912.5ms/step | 31.4 samples/s
997
+ Step 16720/50462 | Loss: 2.6857 | LR: 3.83e-04 | 1912.1ms/step | 31.4 samples/s
998
+ Step 16730/50462 | Loss: 2.6243 | LR: 3.83e-04 | 1914.8ms/step | 31.3 samples/s
999
+ Step 16740/50462 | Loss: 2.6275 | LR: 3.83e-04 | 1914.2ms/step | 31.3 samples/s
1000
+ Step 16750/50462 | Loss: 2.6379 | LR: 3.83e-04 | 1916.0ms/step | 31.3 samples/s
1001
+ Validation at step 16750: Loss = 2.6772, Perplexity = 14.54
1002
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-16750
1003
+ Step 16760/50462 | Loss: 2.6664 | LR: 3.83e-04 | 3202.8ms/step | 18.7 samples/s
1004
+ Step 16770/50462 | Loss: 2.6339 | LR: 3.82e-04 | 1910.2ms/step | 31.4 samples/s
1005
+ Step 16780/50462 | Loss: 2.6433 | LR: 3.82e-04 | 1912.6ms/step | 31.4 samples/s
1006
+ Step 16790/50462 | Loss: 2.6144 | LR: 3.82e-04 | 1911.2ms/step | 31.4 samples/s
1007
+ Step 16800/50462 | Loss: 2.6741 | LR: 3.82e-04 | 1913.5ms/step | 31.4 samples/s
1008
+ Step 16810/50462 | Loss: 2.6283 | LR: 3.82e-04 | 1911.1ms/step | 31.4 samples/s
1009
+ Step 16820/50462 | Loss: 2.6325 | LR: 3.82e-04 | 1914.4ms/step | 31.3 samples/s
1010
+ Step 16830/50462 | Loss: 2.6055 | LR: 3.82e-04 | 1912.8ms/step | 31.4 samples/s
1011
+ Step 16840/50462 | Loss: 2.6469 | LR: 3.82e-04 | 1912.4ms/step | 31.4 samples/s
1012
+ Step 16850/50462 | Loss: 2.6149 | LR: 3.81e-04 | 1910.4ms/step | 31.4 samples/s
1013
+ Step 16860/50462 | Loss: 2.6369 | LR: 3.81e-04 | 1913.6ms/step | 31.4 samples/s
1014
+ Step 16870/50462 | Loss: 2.6799 | LR: 3.81e-04 | 1914.4ms/step | 31.3 samples/s
1015
+ Step 16880/50462 | Loss: 2.6215 | LR: 3.81e-04 | 1912.7ms/step | 31.4 samples/s
1016
+ Step 16890/50462 | Loss: 2.6555 | LR: 3.81e-04 | 1912.1ms/step | 31.4 samples/s
1017
+ Step 16900/50462 | Loss: 2.6329 | LR: 3.81e-04 | 1913.4ms/step | 31.4 samples/s
1018
+ Step 16910/50462 | Loss: 2.6186 | LR: 3.81e-04 | 1911.5ms/step | 31.4 samples/s
1019
+ Step 16920/50462 | Loss: 2.6605 | LR: 3.80e-04 | 1912.6ms/step | 31.4 samples/s
1020
+ Step 16930/50462 | Loss: 2.6096 | LR: 3.80e-04 | 1913.3ms/step | 31.4 samples/s
1021
+ Step 16940/50462 | Loss: 2.6460 | LR: 3.80e-04 | 1913.8ms/step | 31.4 samples/s
1022
+ Step 16950/50462 | Loss: 2.5978 | LR: 3.80e-04 | 1914.5ms/step | 31.3 samples/s
1023
+ Step 16960/50462 | Loss: 2.6277 | LR: 3.80e-04 | 1913.6ms/step | 31.4 samples/s
1024
+ Step 16970/50462 | Loss: 2.5980 | LR: 3.80e-04 | 1913.1ms/step | 31.4 samples/s
1025
+ Step 16980/50462 | Loss: 2.6326 | LR: 3.80e-04 | 1914.3ms/step | 31.3 samples/s
1026
+ Step 16990/50462 | Loss: 2.6603 | LR: 3.79e-04 | 1914.9ms/step | 31.3 samples/s
1027
+ Step 17000/50462 | Loss: 2.6313 | LR: 3.79e-04 | 1914.7ms/step | 31.3 samples/s
1028
+ Validation at step 17000: Loss = 2.6774, Perplexity = 14.55
1029
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-17000
1030
+ Step 17010/50462 | Loss: 2.5948 | LR: 3.79e-04 | 3195.7ms/step | 18.8 samples/s
1031
+ Step 17020/50462 | Loss: 2.6628 | LR: 3.79e-04 | 1896.4ms/step | 31.6 samples/s
1032
+ Step 17030/50462 | Loss: 2.6819 | LR: 3.79e-04 | 1901.1ms/step | 31.6 samples/s
1033
+ Step 17040/50462 | Loss: 2.6539 | LR: 3.79e-04 | 1905.2ms/step | 31.5 samples/s
1034
+ Step 17050/50462 | Loss: 2.6412 | LR: 3.79e-04 | 1910.9ms/step | 31.4 samples/s
1035
+ Step 17060/50462 | Loss: 2.6464 | LR: 3.79e-04 | 1912.8ms/step | 31.4 samples/s
1036
+ Step 17070/50462 | Loss: 2.6259 | LR: 3.78e-04 | 1913.9ms/step | 31.3 samples/s
1037
+ Step 17080/50462 | Loss: 2.6206 | LR: 3.78e-04 | 1915.0ms/step | 31.3 samples/s
1038
+ Step 17090/50462 | Loss: 2.6531 | LR: 3.78e-04 | 1915.1ms/step | 31.3 samples/s
1039
+ Step 17100/50462 | Loss: 2.6547 | LR: 3.78e-04 | 1913.6ms/step | 31.4 samples/s
1040
+ Step 17110/50462 | Loss: 2.6061 | LR: 3.78e-04 | 1914.1ms/step | 31.3 samples/s
1041
+ Step 17120/50462 | Loss: 2.6243 | LR: 3.78e-04 | 1915.3ms/step | 31.3 samples/s
1042
+ Step 17130/50462 | Loss: 2.6554 | LR: 3.78e-04 | 1914.7ms/step | 31.3 samples/s
1043
+ Step 17140/50462 | Loss: 2.5879 | LR: 3.77e-04 | 1913.8ms/step | 31.4 samples/s
1044
+ Step 17150/50462 | Loss: 2.6015 | LR: 3.77e-04 | 1915.9ms/step | 31.3 samples/s
1045
+ Step 17160/50462 | Loss: 2.6634 | LR: 3.77e-04 | 1918.4ms/step | 31.3 samples/s
1046
+ Step 17170/50462 | Loss: 2.5761 | LR: 3.77e-04 | 1914.7ms/step | 31.3 samples/s
1047
+ Step 17180/50462 | Loss: 2.6439 | LR: 3.77e-04 | 1919.2ms/step | 31.3 samples/s
1048
+ Step 17190/50462 | Loss: 2.6769 | LR: 3.77e-04 | 1916.8ms/step | 31.3 samples/s
1049
+ Step 17200/50462 | Loss: 2.6475 | LR: 3.77e-04 | 1915.8ms/step | 31.3 samples/s
1050
+ Step 17210/50462 | Loss: 2.6304 | LR: 3.77e-04 | 1914.5ms/step | 31.3 samples/s
1051
+ Step 17220/50462 | Loss: 2.6548 | LR: 3.76e-04 | 1913.6ms/step | 31.4 samples/s
1052
+ Step 17230/50462 | Loss: 2.7022 | LR: 3.76e-04 | 1911.0ms/step | 31.4 samples/s
1053
+ Step 17240/50462 | Loss: 2.6152 | LR: 3.76e-04 | 1911.3ms/step | 31.4 samples/s
1054
+ Step 17250/50462 | Loss: 2.6317 | LR: 3.76e-04 | 1912.0ms/step | 31.4 samples/s
1055
+ Validation at step 17250: Loss = 2.6709, Perplexity = 14.45
1056
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-17250
1057
+ Step 17260/50462 | Loss: 2.6489 | LR: 3.76e-04 | 3191.1ms/step | 18.8 samples/s
1058
+ Step 17270/50462 | Loss: 2.6606 | LR: 3.76e-04 | 1900.3ms/step | 31.6 samples/s
1059
+ Step 17280/50462 | Loss: 2.6245 | LR: 3.76e-04 | 1901.1ms/step | 31.6 samples/s
1060
+ Step 17290/50462 | Loss: 2.6588 | LR: 3.75e-04 | 1906.6ms/step | 31.5 samples/s
1061
+ Step 17300/50462 | Loss: 2.6502 | LR: 3.75e-04 | 1911.0ms/step | 31.4 samples/s
1062
+ Step 17310/50462 | Loss: 2.6474 | LR: 3.75e-04 | 1909.4ms/step | 31.4 samples/s
1063
+ Step 17320/50462 | Loss: 2.6170 | LR: 3.75e-04 | 1908.6ms/step | 31.4 samples/s
1064
+ Step 17330/50462 | Loss: 2.6274 | LR: 3.75e-04 | 1913.6ms/step | 31.4 samples/s
1065
+ Step 17340/50462 | Loss: 2.6461 | LR: 3.75e-04 | 1912.9ms/step | 31.4 samples/s
1066
+ Step 17350/50462 | Loss: 2.6593 | LR: 3.75e-04 | 1914.0ms/step | 31.3 samples/s
1067
+ Step 17360/50462 | Loss: 2.6249 | LR: 3.74e-04 | 1911.4ms/step | 31.4 samples/s
1068
+ Step 17370/50462 | Loss: 2.6439 | LR: 3.74e-04 | 1914.5ms/step | 31.3 samples/s
1069
+ Step 17380/50462 | Loss: 2.6546 | LR: 3.74e-04 | 1914.0ms/step | 31.3 samples/s
1070
+ Step 17390/50462 | Loss: 2.6214 | LR: 3.74e-04 | 1912.7ms/step | 31.4 samples/s
1071
+ Step 17400/50462 | Loss: 2.6166 | LR: 3.74e-04 | 1914.9ms/step | 31.3 samples/s
1072
+ Step 17410/50462 | Loss: 2.6761 | LR: 3.74e-04 | 1914.3ms/step | 31.3 samples/s
1073
+ Step 17420/50462 | Loss: 2.6243 | LR: 3.74e-04 | 1913.0ms/step | 31.4 samples/s
1074
+ Step 17430/50462 | Loss: 2.6448 | LR: 3.74e-04 | 1913.3ms/step | 31.4 samples/s
1075
+ Step 17440/50462 | Loss: 2.6530 | LR: 3.73e-04 | 1910.7ms/step | 31.4 samples/s
1076
+ Step 17450/50462 | Loss: 2.6464 | LR: 3.73e-04 | 1913.8ms/step | 31.4 samples/s
1077
+ Step 17460/50462 | Loss: 2.6197 | LR: 3.73e-04 | 1913.1ms/step | 31.4 samples/s
1078
+ Step 17470/50462 | Loss: 2.6214 | LR: 3.73e-04 | 1917.4ms/step | 31.3 samples/s
1079
+ Step 17480/50462 | Loss: 2.6412 | LR: 3.73e-04 | 1913.2ms/step | 31.4 samples/s
1080
+ Step 17490/50462 | Loss: 2.6328 | LR: 3.73e-04 | 1911.5ms/step | 31.4 samples/s
1081
+ Step 17500/50462 | Loss: 2.6211 | LR: 3.73e-04 | 1912.9ms/step | 31.4 samples/s
1082
+ Validation at step 17500: Loss = 2.6688, Perplexity = 14.42
1083
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-17500
1084
+ Step 17510/50462 | Loss: 2.6199 | LR: 3.72e-04 | 3194.4ms/step | 18.8 samples/s
1085
+ Step 17520/50462 | Loss: 2.6083 | LR: 3.72e-04 | 1901.1ms/step | 31.6 samples/s
1086
+ Step 17530/50462 | Loss: 2.6108 | LR: 3.72e-04 | 1905.3ms/step | 31.5 samples/s
1087
+ Step 17540/50462 | Loss: 2.6196 | LR: 3.72e-04 | 1910.6ms/step | 31.4 samples/s
1088
+ Step 17550/50462 | Loss: 2.6308 | LR: 3.72e-04 | 1910.8ms/step | 31.4 samples/s
1089
+ Step 17560/50462 | Loss: 2.6255 | LR: 3.72e-04 | 1911.6ms/step | 31.4 samples/s
1090
+ Step 17570/50462 | Loss: 2.6174 | LR: 3.72e-04 | 1909.7ms/step | 31.4 samples/s
1091
+ Step 17580/50462 | Loss: 2.6132 | LR: 3.71e-04 | 1911.0ms/step | 31.4 samples/s
1092
+ Step 17590/50462 | Loss: 2.6711 | LR: 3.71e-04 | 1911.2ms/step | 31.4 samples/s
1093
+ Step 17600/50462 | Loss: 2.6518 | LR: 3.71e-04 | 1912.3ms/step | 31.4 samples/s
1094
+ Step 17610/50462 | Loss: 2.6585 | LR: 3.71e-04 | 1913.9ms/step | 31.3 samples/s
1095
+ Step 17620/50462 | Loss: 2.6520 | LR: 3.71e-04 | 1916.4ms/step | 31.3 samples/s
1096
+ Step 17630/50462 | Loss: 2.6200 | LR: 3.71e-04 | 1916.8ms/step | 31.3 samples/s
1097
+ Step 17640/50462 | Loss: 2.6119 | LR: 3.71e-04 | 1915.0ms/step | 31.3 samples/s
1098
+ Step 17650/50462 | Loss: 2.6327 | LR: 3.70e-04 | 1916.6ms/step | 31.3 samples/s
1099
+ Step 17660/50462 | Loss: 2.6256 | LR: 3.70e-04 | 1916.1ms/step | 31.3 samples/s
1100
+ Step 17670/50462 | Loss: 2.6297 | LR: 3.70e-04 | 1917.4ms/step | 31.3 samples/s
1101
+ Step 17680/50462 | Loss: 2.6425 | LR: 3.70e-04 | 1915.7ms/step | 31.3 samples/s
1102
+ Step 17690/50462 | Loss: 2.6443 | LR: 3.70e-04 | 1916.8ms/step | 31.3 samples/s
1103
+ Step 17700/50462 | Loss: 2.6181 | LR: 3.70e-04 | 1916.8ms/step | 31.3 samples/s
1104
+ Step 17710/50462 | Loss: 2.5879 | LR: 3.70e-04 | 1915.9ms/step | 31.3 samples/s
1105
+ Step 17720/50462 | Loss: 2.6108 | LR: 3.69e-04 | 1913.4ms/step | 31.4 samples/s
1106
+ Step 17730/50462 | Loss: 2.6255 | LR: 3.69e-04 | 1912.7ms/step | 31.4 samples/s
1107
+ Step 17740/50462 | Loss: 2.6339 | LR: 3.69e-04 | 1915.5ms/step | 31.3 samples/s
1108
+ Step 17750/50462 | Loss: 2.6387 | LR: 3.69e-04 | 1911.4ms/step | 31.4 samples/s
1109
+ Validation at step 17750: Loss = 2.6664, Perplexity = 14.39
1110
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-17750
1111
+ Step 17760/50462 | Loss: 2.6366 | LR: 3.69e-04 | 3196.3ms/step | 18.8 samples/s
1112
+ Step 17770/50462 | Loss: 2.5889 | LR: 3.69e-04 | 1900.3ms/step | 31.6 samples/s
1113
+ Step 17780/50462 | Loss: 2.6140 | LR: 3.69e-04 | 1904.0ms/step | 31.5 samples/s
1114
+ Step 17790/50462 | Loss: 2.6487 | LR: 3.69e-04 | 1909.5ms/step | 31.4 samples/s
1115
+ Step 17800/50462 | Loss: 2.6494 | LR: 3.68e-04 | 1911.9ms/step | 31.4 samples/s
1116
+ Step 17810/50462 | Loss: 2.6110 | LR: 3.68e-04 | 1911.7ms/step | 31.4 samples/s
1117
+ Step 17820/50462 | Loss: 2.6365 | LR: 3.68e-04 | 1911.8ms/step | 31.4 samples/s
1118
+ Step 17830/50462 | Loss: 2.6052 | LR: 3.68e-04 | 1911.7ms/step | 31.4 samples/s
1119
+ Step 17840/50462 | Loss: 2.6462 | LR: 3.68e-04 | 1913.8ms/step | 31.4 samples/s
1120
+ Step 17850/50462 | Loss: 2.6287 | LR: 3.68e-04 | 1913.1ms/step | 31.4 samples/s
1121
+ Step 17860/50462 | Loss: 2.6000 | LR: 3.68e-04 | 1912.9ms/step | 31.4 samples/s
1122
+ Step 17870/50462 | Loss: 2.5992 | LR: 3.67e-04 | 1914.1ms/step | 31.3 samples/s
1123
+ Step 17880/50462 | Loss: 2.6234 | LR: 3.67e-04 | 1912.3ms/step | 31.4 samples/s
1124
+ Step 17890/50462 | Loss: 2.6426 | LR: 3.67e-04 | 1913.0ms/step | 31.4 samples/s
1125
+ Step 17900/50462 | Loss: 2.6368 | LR: 3.67e-04 | 1912.8ms/step | 31.4 samples/s
1126
+ Step 17910/50462 | Loss: 2.6429 | LR: 3.67e-04 | 1911.9ms/step | 31.4 samples/s
1127
+ Step 17920/50462 | Loss: 2.5897 | LR: 3.67e-04 | 1910.6ms/step | 31.4 samples/s
1128
+ Step 17930/50462 | Loss: 2.6470 | LR: 3.67e-04 | 1911.8ms/step | 31.4 samples/s
1129
+ Step 17940/50462 | Loss: 2.5928 | LR: 3.66e-04 | 1913.0ms/step | 31.4 samples/s
1130
+ Step 17950/50462 | Loss: 2.5939 | LR: 3.66e-04 | 1913.7ms/step | 31.4 samples/s
1131
+ Step 17960/50462 | Loss: 2.6251 | LR: 3.66e-04 | 1911.0ms/step | 31.4 samples/s
1132
+ Step 17970/50462 | Loss: 2.6430 | LR: 3.66e-04 | 1915.0ms/step | 31.3 samples/s
1133
+ Step 17980/50462 | Loss: 2.6352 | LR: 3.66e-04 | 1913.6ms/step | 31.4 samples/s
1134
+ Step 17990/50462 | Loss: 2.6428 | LR: 3.66e-04 | 1913.7ms/step | 31.4 samples/s
1135
+ Step 18000/50462 | Loss: 2.5948 | LR: 3.66e-04 | 1912.7ms/step | 31.4 samples/s
1136
+ Validation at step 18000: Loss = 2.6612, Perplexity = 14.31
epfl_final/train_logs/p2_1p2b_dense_seed72_6gpu_resume18000.log ADDED
@@ -0,0 +1,496 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ *****************************************
3
+ Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
4
+ *****************************************
5
+ Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250
6
+ Running config:
7
+ {
8
+ "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
9
+ "model_dtype": "bfloat16",
10
+ "init_from_scratch": true,
11
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
12
+ "val_path": null,
13
+ "max_seq_length": 8192,
14
+ "num_workers": 8,
15
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250",
16
+ "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-18000",
17
+ "force_new_run": false,
18
+ "num_epochs": 1,
19
+ "max_steps": 50462,
20
+ "per_device_batch_size": 1,
21
+ "gradient_accumulation_steps": 10,
22
+ "learning_rate": 0.0005,
23
+ "weight_decay": 0.01,
24
+ "warmup_steps": 750,
25
+ "lr_scheduler": "cosine",
26
+ "max_grad_norm": 1.0,
27
+ "save_steps": 250,
28
+ "eval_steps": 250,
29
+ "eval_batches": 50,
30
+ "logging_steps": 10,
31
+ "seed": 72,
32
+ "verbose": true,
33
+ "checkpoint_activations": false,
34
+ "resume_micro_step_override": 180000,
35
+ "strategy": "ddp",
36
+ "mixed_precision": "bf16",
37
+ "wandb_enabled": false,
38
+ "wandb_project": "stochastic_sparsity",
39
+ "wandb_name": "1.2b dense 8k seed72 6gpu resume18000",
40
+ "wandb_tags": [
41
+ "qwen3",
42
+ "1.2b",
43
+ "8k",
44
+ "p2",
45
+ "dense",
46
+ "seed72"
47
+ ],
48
+ "hidden_size": 1536,
49
+ "intermediate_size": 6144,
50
+ "num_attention_heads": 12,
51
+ "num_key_value_heads": 4,
52
+ "num_hidden_layers": 28,
53
+ "head_dim": null,
54
+ "attention_dropout": null,
55
+ "rope_train_position_scale_set": null,
56
+ "attention_type": "sparse_stochastic",
57
+ "sparse_block_size": 128,
58
+ "sparse_num_random_blocks": 31,
59
+ "sparse_mask_pool_size": 1,
60
+ "sparse_val_dense_compare": false,
61
+ "sparse_layer_indices": null,
62
+ "sparse_mask_cache_dir": null,
63
+ "sparse_mask_seed": 0,
64
+ "sparse_mixed_nrb_set": null,
65
+ "dense_mode": true,
66
+ "dense_step_intervals": null
67
+ }
68
+ Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
69
+ Checkpoint activations: False
70
+ Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-18000
71
+ Checkpoint activations: False
72
+ Checkpoint activations: False
73
+ Checkpoint activations: False
74
+ Checkpoint activations: False
75
+ Checkpoint activations: False
76
+ [rank5]: Traceback (most recent call last):
77
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 173, in getuser
78
+ [rank5]: return pwd.getpwuid(os.getuid())[0]
79
+ [rank5]: ~~~~~~~~~~~~^^^^^^^^^^^^^
80
+ [rank5]: KeyError: 'getpwuid(): uid not found: 285818'
81
+
82
+ [rank5]: The above exception was the direct cause of the following exception:
83
+
84
+ [rank5]: Traceback (most recent call last):
85
+ [rank5]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1519, in <module>
86
+ [rank5]: main()
87
+ [rank5]: ~~~~^^
88
+ [rank5]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1516, in main
89
+ [rank5]: train(config)
90
+ [rank5]: ~~~~~^^^^^^^^
91
+ [rank5]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1107, in train
92
+ [rank5]: model = torch.compile(model)
93
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/__init__.py", line 2758, in compile
94
+ [rank5]: return torch._dynamo.optimize(
95
+ [rank5]: ~~~~~~~~~~~~~~~~~~~~~~^
96
+ [rank5]: backend=backend,
97
+ [rank5]: ^^^^^^^^^^^^^^^^
98
+ [rank5]: ...<3 lines>...
99
+ [rank5]: guard_filter_fn=guard_filter_fn,
100
+ [rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
101
+ [rank5]: )(model) # type: ignore[return-value]
102
+ [rank5]: ^
103
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1488, in optimize
104
+ [rank5]: return _optimize(rebuild_ctx, *args, **kwargs)
105
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1564, in _optimize
106
+ [rank5]: backend = get_compiler_fn(backend)
107
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1325, in get_compiler_fn
108
+ [rank5]: from .repro.after_dynamo import wrap_backend_debug
109
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/repro/after_dynamo.py", line 33, in <module>
110
+ [rank5]: from torch._dynamo.debug_utils import (
111
+ [rank5]: ...<15 lines>...
112
+ [rank5]: )
113
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/debug_utils.py", line 43, in <module>
114
+ [rank5]: from torch._dynamo.testing import rand_strided
115
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/testing.py", line 33, in <module>
116
+ [rank5]: from torch._dynamo.backends.debugging import aot_eager
117
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/backends/debugging.py", line 34, in <module>
118
+ [rank5]: from functorch.compile import min_cut_rematerialization_partition
119
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/functorch/compile/__init__.py", line 2, in <module>
120
+ [rank5]: from torch._functorch.aot_autograd import (
121
+ [rank5]: ...<10 lines>...
122
+ [rank5]: )
123
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/aot_autograd.py", line 38, in <module>
124
+ [rank5]: from ._aot_autograd.autograd_cache import ( # noqa: F401
125
+ [rank5]: ...<4 lines>...
126
+ [rank5]: )
127
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/_aot_autograd/autograd_cache.py", line 27, in <module>
128
+ [rank5]: from torch._inductor.codecache import (
129
+ [rank5]: ...<11 lines>...
130
+ [rank5]: )
131
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/codecache.py", line 2753, in <module>
132
+ [rank5]: _HEADER_DIR = os.path.join(default_cache_dir(), "precompiled_headers")
133
+ [rank5]: ~~~~~~~~~~~~~~~~~^^
134
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/runtime/cache_dir_utils.py", line 23, in default_cache_dir
135
+ [rank5]: sanitized_username = re.sub(r'[\\/:*?"<>|]', "_", getpass.getuser())
136
+ [rank5]: ~~~~~~~~~~~~~~~^^
137
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 175, in getuser
138
+ [rank5]: raise OSError('No username set in the environment') from e
139
+ [rank5]: OSError: No username set in the environment
140
+ [rank3]: Traceback (most recent call last):
141
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 173, in getuser
142
+ [rank3]: return pwd.getpwuid(os.getuid())[0]
143
+ [rank3]: ~~~~~~~~~~~~^^^^^^^^^^^^^
144
+ [rank3]: KeyError: 'getpwuid(): uid not found: 285818'
145
+
146
+ [rank3]: The above exception was the direct cause of the following exception:
147
+
148
+ [rank3]: Traceback (most recent call last):
149
+ [rank3]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1519, in <module>
150
+ [rank3]: main()
151
+ [rank3]: ~~~~^^
152
+ [rank3]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1516, in main
153
+ [rank3]: train(config)
154
+ [rank3]: ~~~~~^^^^^^^^
155
+ [rank3]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1107, in train
156
+ [rank3]: model = torch.compile(model)
157
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/__init__.py", line 2758, in compile
158
+ [rank3]: return torch._dynamo.optimize(
159
+ [rank3]: ~~~~~~~~~~~~~~~~~~~~~~^
160
+ [rank3]: backend=backend,
161
+ [rank3]: ^^^^^^^^^^^^^^^^
162
+ [rank3]: ...<3 lines>...
163
+ [rank3]: guard_filter_fn=guard_filter_fn,
164
+ [rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
165
+ [rank3]: )(model) # type: ignore[return-value]
166
+ [rank3]: ^
167
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1488, in optimize
168
+ [rank3]: return _optimize(rebuild_ctx, *args, **kwargs)
169
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1564, in _optimize
170
+ [rank3]: backend = get_compiler_fn(backend)
171
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1325, in get_compiler_fn
172
+ [rank3]: from .repro.after_dynamo import wrap_backend_debug
173
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/repro/after_dynamo.py", line 33, in <module>
174
+ [rank3]: from torch._dynamo.debug_utils import (
175
+ [rank3]: ...<15 lines>...
176
+ [rank3]: )
177
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/debug_utils.py", line 43, in <module>
178
+ [rank3]: from torch._dynamo.testing import rand_strided
179
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/testing.py", line 33, in <module>
180
+ [rank3]: from torch._dynamo.backends.debugging import aot_eager
181
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/backends/debugging.py", line 34, in <module>
182
+ [rank3]: from functorch.compile import min_cut_rematerialization_partition
183
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/functorch/compile/__init__.py", line 2, in <module>
184
+ [rank3]: from torch._functorch.aot_autograd import (
185
+ [rank3]: ...<10 lines>...
186
+ [rank3]: )
187
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/aot_autograd.py", line 38, in <module>
188
+ [rank3]: from ._aot_autograd.autograd_cache import ( # noqa: F401
189
+ [rank3]: ...<4 lines>...
190
+ [rank3]: )
191
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/_aot_autograd/autograd_cache.py", line 27, in <module>
192
+ [rank3]: from torch._inductor.codecache import (
193
+ [rank3]: ...<11 lines>...
194
+ [rank3]: )
195
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/codecache.py", line 2753, in <module>
196
+ [rank3]: _HEADER_DIR = os.path.join(default_cache_dir(), "precompiled_headers")
197
+ [rank3]: ~~~~~~~~~~~~~~~~~^^
198
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/runtime/cache_dir_utils.py", line 23, in default_cache_dir
199
+ [rank3]: sanitized_username = re.sub(r'[\\/:*?"<>|]', "_", getpass.getuser())
200
+ [rank3]: ~~~~~~~~~~~~~~~^^
201
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 175, in getuser
202
+ [rank3]: raise OSError('No username set in the environment') from e
203
+ [rank3]: OSError: No username set in the environment
204
+ dense_mode=True: sparse attention layers will use standard causal SDPA.
205
+
206
+
207
+ ================================================================================
208
+ MODEL ARCHITECTURE
209
+ ================================================================================
210
+ Model config:
211
+ vocab_size: 151936
212
+ hidden_size: 1536
213
+ num_hidden_layers: 28
214
+ num_attention_heads: 12
215
+ num_key_value_heads: 4
216
+ head_dim: 128
217
+ intermediate_size: 6144
218
+ max_position_embeddings: 32768
219
+ rope_theta: 1000000
220
+ rope_train_position_scale_set: None
221
+ attention_type: sparse_stochastic
222
+ sparse_block_size: 128
223
+ sparse_num_random_blocks: 31
224
+ sparse_mask_pool_size: 1
225
+ sparse_max_seq_length: 8192
226
+
227
+ Parameter count:
228
+ Total parameters: 1,202,352,640
229
+ Trainable parameters: 1,202,352,640
230
+ Non-trainable parameters: 0
231
+ Model size (BF16): 2.40 GB
232
+
233
+ --------------------------------------------------------------------------------
234
+ LAYER-BY-LAYER SUMMARY
235
+ --------------------------------------------------------------------------------
236
+ [rank1]: Traceback (most recent call last):
237
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 173, in getuser
238
+ [rank1]: return pwd.getpwuid(os.getuid())[0]
239
+ [rank1]: ~~~~~~~~~~~~^^^^^^^^^^^^^
240
+ [rank1]: KeyError: 'getpwuid(): uid not found: 285818'
241
+
242
+ [rank1]: The above exception was the direct cause of the following exception:
243
+
244
+ [rank1]: Traceback (most recent call last):
245
+ [rank1]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1519, in <module>
246
+ [rank1]: main()
247
+ [rank1]: ~~~~^^
248
+ [rank1]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1516, in main
249
+ [rank1]: train(config)
250
+ [rank1]: ~~~~~^^^^^^^^
251
+ [rank1]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1107, in train
252
+ [rank1]: model = torch.compile(model)
253
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/__init__.py", line 2758, in compile
254
+ [rank1]: return torch._dynamo.optimize(
255
+ [rank1]: ~~~~~~~~~~~~~~~~~~~~~~^
256
+ [rank1]: backend=backend,
257
+ [rank1]: ^^^^^^^^^^^^^^^^
258
+ [rank1]: ...<3 lines>...
259
+ [rank1]: guard_filter_fn=guard_filter_fn,
260
+ [rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
261
+ [rank1]: )(model) # type: ignore[return-value]
262
+ [rank1]: ^
263
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1488, in optimize
264
+ [rank1]: return _optimize(rebuild_ctx, *args, **kwargs)
265
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1564, in _optimize
266
+ [rank1]: backend = get_compiler_fn(backend)
267
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1325, in get_compiler_fn
268
+ [rank1]: from .repro.after_dynamo import wrap_backend_debug
269
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/repro/after_dynamo.py", line 33, in <module>
270
+ [rank1]: from torch._dynamo.debug_utils import (
271
+ [rank1]: ...<15 lines>...
272
+ [rank1]: )
273
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/debug_utils.py", line 43, in <module>
274
+ [rank1]: from torch._dynamo.testing import rand_strided
275
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/testing.py", line 33, in <module>
276
+ [rank1]: from torch._dynamo.backends.debugging import aot_eager
277
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/backends/debugging.py", line 34, in <module>
278
+ [rank1]: from functorch.compile import min_cut_rematerialization_partition
279
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/functorch/compile/__init__.py", line 2, in <module>
280
+ [rank1]: from torch._functorch.aot_autograd import (
281
+ [rank1]: ...<10 lines>...
282
+ [rank1]: )
283
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/aot_autograd.py", line 38, in <module>
284
+ [rank1]: from ._aot_autograd.autograd_cache import ( # noqa: F401
285
+ [rank1]: ...<4 lines>...
286
+ [rank1]: )
287
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/_aot_autograd/autograd_cache.py", line 27, in <module>
288
+ [rank1]: from torch._inductor.codecache import (
289
+ [rank1]: ...<11 lines>...
290
+ [rank1]: )
291
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/codecache.py", line 2753, in <module>
292
+ [rank1]: _HEADER_DIR = os.path.join(default_cache_dir(), "precompiled_headers")
293
+ [rank1]: ~~~~~~~~~~~~~~~~~^^
294
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/runtime/cache_dir_utils.py", line 23, in default_cache_dir
295
+ [rank1]: sanitized_username = re.sub(r'[\\/:*?"<>|]', "_", getpass.getuser())
296
+ [rank1]: ~~~~~~~~~~~~~~~^^
297
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 175, in getuser
298
+ [rank1]: raise OSError('No username set in the environment') from e
299
+ [rank1]: OSError: No username set in the environment
300
+ ================================================================================================================================================================
301
+ Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
302
+ ================================================================================================================================================================
303
+ Qwen3ForCausalLM [1, 8192] -- -- True
304
+ ├─Qwen3Model: 1-1 -- [1, 8192, 1536] -- True
305
+ │ └─Embedding: 2-1 [1, 8192] [1, 8192, 1536] 233,373,696 True
306
+ │ └─ModuleList: 2-2 -- -- -- True
307
+ │ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
308
+ │ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
309
+ │ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
310
+ │ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
311
+ │ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
312
+ │ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
313
+ │ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
314
+ │ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
315
+ │ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
316
+ │ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
317
+ │ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
318
+ │ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
319
+ │ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
320
+ │ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
321
+ │ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
322
+ │ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
323
+ │ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
324
+ │ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
325
+ │ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
326
+ │ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
327
+ │ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
328
+ │ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
329
+ │ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
330
+ │ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
331
+ │ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
332
+ │ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
333
+ │ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
334
+ │ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
335
+ │ └─Qwen3RMSNorm: 2-3 [1, 8192, 1536] [1, 8192, 1536] 1,536 True
336
+ ├─Linear: 1-2 [1, 8192, 1536] [1, 8192, 151936] 233,373,696 True
337
+ ================================================================================================================================================================
338
+ Total params: 1,435,726,336
339
+ Trainable params: 1,435,726,336
340
+ Non-trainable params: 0
341
+ Total mult-adds (Units.GIGABYTES): 1.44
342
+ ================================================================================================================================================================
343
+ Input size (MB): 0.07
344
+ Forward/backward pass size (MB): 26218.59
345
+ Params size (MB): 2871.45
346
+ Estimated Total Size (MB): 29090.11
347
+ ================================================================================================================================================================
348
+ NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
349
+ ================================================================================
350
+
351
+ [rank0]: Traceback (most recent call last):
352
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 173, in getuser
353
+ [rank0]: return pwd.getpwuid(os.getuid())[0]
354
+ [rank0]: ~~~~~~~~~~~~^^^^^^^^^^^^^
355
+ [rank0]: KeyError: 'getpwuid(): uid not found: 285818'
356
+
357
+ [rank0]: The above exception was the direct cause of the following exception:
358
+
359
+ [rank0]: Traceback (most recent call last):
360
+ [rank0]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1519, in <module>
361
+ [rank0]: main()
362
+ [rank0]: ~~~~^^
363
+ [rank0]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1516, in main
364
+ [rank0]: train(config)
365
+ [rank0]: ~~~~~^^^^^^^^
366
+ [rank0]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1107, in train
367
+ [rank0]: model = torch.compile(model)
368
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/__init__.py", line 2758, in compile
369
+ [rank0]: return torch._dynamo.optimize(
370
+ [rank0]: ~~~~~~~~~~~~~~~~~~~~~~^
371
+ [rank0]: backend=backend,
372
+ [rank0]: ^^^^^^^^^^^^^^^^
373
+ [rank0]: ...<3 lines>...
374
+ [rank0]: guard_filter_fn=guard_filter_fn,
375
+ [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
376
+ [rank0]: )(model) # type: ignore[return-value]
377
+ [rank0]: ^
378
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1488, in optimize
379
+ [rank0]: return _optimize(rebuild_ctx, *args, **kwargs)
380
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1564, in _optimize
381
+ [rank0]: backend = get_compiler_fn(backend)
382
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1325, in get_compiler_fn
383
+ [rank0]: from .repro.after_dynamo import wrap_backend_debug
384
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/repro/after_dynamo.py", line 33, in <module>
385
+ [rank0]: from torch._dynamo.debug_utils import (
386
+ [rank0]: ...<15 lines>...
387
+ [rank0]: )
388
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/debug_utils.py", line 43, in <module>
389
+ [rank0]: from torch._dynamo.testing import rand_strided
390
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/testing.py", line 33, in <module>
391
+ [rank0]: from torch._dynamo.backends.debugging import aot_eager
392
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/backends/debugging.py", line 34, in <module>
393
+ [rank0]: from functorch.compile import min_cut_rematerialization_partition
394
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/functorch/compile/__init__.py", line 2, in <module>
395
+ [rank0]: from torch._functorch.aot_autograd import (
396
+ [rank0]: ...<10 lines>...
397
+ [rank0]: )
398
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/aot_autograd.py", line 38, in <module>
399
+ [rank0]: from ._aot_autograd.autograd_cache import ( # noqa: F401
400
+ [rank0]: ...<4 lines>...
401
+ [rank0]: )
402
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/_aot_autograd/autograd_cache.py", line 27, in <module>
403
+ [rank0]: from torch._inductor.codecache import (
404
+ [rank0]: ...<11 lines>...
405
+ [rank0]: )
406
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/codecache.py", line 2753, in <module>
407
+ [rank0]: _HEADER_DIR = os.path.join(default_cache_dir(), "precompiled_headers")
408
+ [rank0]: ~~~~~~~~~~~~~~~~~^^
409
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/runtime/cache_dir_utils.py", line 23, in default_cache_dir
410
+ [rank0]: sanitized_username = re.sub(r'[\\/:*?"<>|]', "_", getpass.getuser())
411
+ [rank0]: ~~~~~~~~~~~~~~~^^
412
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 175, in getuser
413
+ [rank0]: raise OSError('No username set in the environment') from e
414
+ [rank0]: OSError: No username set in the environment
415
+ [rank0]:[W906 20:51:07.982844041 ProcessGroupNCCL.cpp:1575] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
416
+ W0906 20:51:08.022000 5655 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 5721 closing signal SIGTERM
417
+ W0906 20:51:08.023000 5655 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 5722 closing signal SIGTERM
418
+ W0906 20:51:08.024000 5655 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 5723 closing signal SIGTERM
419
+ W0906 20:51:08.031000 5655 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 5725 closing signal SIGTERM
420
+ E0906 20:51:08.828000 5655 site-packages/torch/distributed/elastic/multiprocessing/api.py:986] failed (exitcode: 1) local_rank: 5 (pid: 5726) of binary: /mloscratch/homes/navasard/conda/bin/python3.13
421
+ Traceback (most recent call last):
422
+ File "<frozen runpy>", line 198, in _run_module_as_main
423
+ File "<frozen runpy>", line 88, in _run_code
424
+ File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 994, in <module>
425
+ main()
426
+ ~~~~^^
427
+ File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper
428
+ return f(*args, **kwargs)
429
+ File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main
430
+ run(args)
431
+ ~~~^^^^^^
432
+ File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run
433
+ elastic_launch(
434
+ ~~~~~~~~~~~~~~~
435
+ config=config,
436
+ ~~~~~~~~~~~~~~
437
+ entrypoint=cmd,
438
+ ~~~~~~~~~~~~~~~
439
+ )(*cmd_args)
440
+ ~^^^^^^^^^^^
441
+ File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__
442
+ return launch_agent(self._config, self._entrypoint, list(args))
443
+ File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 317, in launch_agent
444
+ raise ChildFailedError(
445
+ ...<2 lines>...
446
+ )
447
+ torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
448
+ ============================================================
449
+ train.py FAILED
450
+ ------------------------------------------------------------
451
+ Failures:
452
+ [1]:
453
+ time : 2026-09-06_20:51:08
454
+ host : h100-workspace-6-gpu-0-27
455
+ rank : 0 (local_rank: 0)
456
+ exitcode : -15 (pid: 5721)
457
+ error_file: <N/A>
458
+ traceback : Signal 15 (SIGTERM) received by PID 5721
459
+ [2]:
460
+ time : 2026-09-06_20:51:08
461
+ host : h100-workspace-6-gpu-0-27
462
+ rank : 1 (local_rank: 1)
463
+ exitcode : 1 (pid: 5722)
464
+ error_file: <N/A>
465
+ traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
466
+ [3]:
467
+ time : 2026-09-06_20:51:08
468
+ host : h100-workspace-6-gpu-0-27
469
+ rank : 2 (local_rank: 2)
470
+ exitcode : -15 (pid: 5723)
471
+ error_file: <N/A>
472
+ traceback : Signal 15 (SIGTERM) received by PID 5723
473
+ [4]:
474
+ time : 2026-09-06_20:51:08
475
+ host : h100-workspace-6-gpu-0-27
476
+ rank : 3 (local_rank: 3)
477
+ exitcode : 1 (pid: 5724)
478
+ error_file: <N/A>
479
+ traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
480
+ [5]:
481
+ time : 2026-09-06_20:51:08
482
+ host : h100-workspace-6-gpu-0-27
483
+ rank : 4 (local_rank: 4)
484
+ exitcode : -15 (pid: 5725)
485
+ error_file: <N/A>
486
+ traceback : Signal 15 (SIGTERM) received by PID 5725
487
+ ------------------------------------------------------------
488
+ Root Cause (first observed failure):
489
+ [0]:
490
+ time : 2026-09-06_20:51:08
491
+ host : h100-workspace-6-gpu-0-27
492
+ rank : 5 (local_rank: 5)
493
+ exitcode : 1 (pid: 5726)
494
+ error_file: <N/A>
495
+ traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
496
+ ============================================================
epfl_final/train_logs/p2_1p2b_dense_seed72_6gpu_resume18000_retry.log ADDED
@@ -0,0 +1,789 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ *****************************************
3
+ Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
4
+ *****************************************
5
+ Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250
6
+ Running config:
7
+ {
8
+ "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
9
+ "model_dtype": "bfloat16",
10
+ "init_from_scratch": true,
11
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
12
+ "val_path": null,
13
+ "max_seq_length": 8192,
14
+ "num_workers": 8,
15
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250",
16
+ "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-18000",
17
+ "force_new_run": false,
18
+ "num_epochs": 1,
19
+ "max_steps": 50462,
20
+ "per_device_batch_size": 1,
21
+ "gradient_accumulation_steps": 10,
22
+ "learning_rate": 0.0005,
23
+ "weight_decay": 0.01,
24
+ "warmup_steps": 750,
25
+ "lr_scheduler": "cosine",
26
+ "max_grad_norm": 1.0,
27
+ "save_steps": 250,
28
+ "eval_steps": 250,
29
+ "eval_batches": 50,
30
+ "logging_steps": 10,
31
+ "seed": 72,
32
+ "verbose": true,
33
+ "checkpoint_activations": false,
34
+ "resume_micro_step_override": 180000,
35
+ "strategy": "ddp",
36
+ "mixed_precision": "bf16",
37
+ "wandb_enabled": false,
38
+ "wandb_project": "stochastic_sparsity",
39
+ "wandb_name": "1.2b dense 8k seed72 6gpu resume18000",
40
+ "wandb_tags": [
41
+ "qwen3",
42
+ "1.2b",
43
+ "8k",
44
+ "p2",
45
+ "dense",
46
+ "seed72"
47
+ ],
48
+ "hidden_size": 1536,
49
+ "intermediate_size": 6144,
50
+ "num_attention_heads": 12,
51
+ "num_key_value_heads": 4,
52
+ "num_hidden_layers": 28,
53
+ "head_dim": null,
54
+ "attention_dropout": null,
55
+ "rope_train_position_scale_set": null,
56
+ "attention_type": "sparse_stochastic",
57
+ "sparse_block_size": 128,
58
+ "sparse_num_random_blocks": 31,
59
+ "sparse_mask_pool_size": 1,
60
+ "sparse_val_dense_compare": false,
61
+ "sparse_layer_indices": null,
62
+ "sparse_mask_cache_dir": null,
63
+ "sparse_mask_seed": 0,
64
+ "sparse_mixed_nrb_set": null,
65
+ "dense_mode": true,
66
+ "dense_step_intervals": null
67
+ }
68
+ Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
69
+ Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-18000
70
+ Checkpoint activations: False
71
+ Checkpoint activations: False
72
+ Checkpoint activations: False
73
+ Checkpoint activations: False
74
+ Checkpoint activations: False
75
+ Checkpoint activations: False
76
+ dense_mode=True: sparse attention layers will use standard causal SDPA.
77
+
78
+
79
+ ================================================================================
80
+ MODEL ARCHITECTURE
81
+ ================================================================================
82
+ Model config:
83
+ vocab_size: 151936
84
+ hidden_size: 1536
85
+ num_hidden_layers: 28
86
+ num_attention_heads: 12
87
+ num_key_value_heads: 4
88
+ head_dim: 128
89
+ intermediate_size: 6144
90
+ max_position_embeddings: 32768
91
+ rope_theta: 1000000
92
+ rope_train_position_scale_set: None
93
+ attention_type: sparse_stochastic
94
+ sparse_block_size: 128
95
+ sparse_num_random_blocks: 31
96
+ sparse_mask_pool_size: 1
97
+ sparse_max_seq_length: 8192
98
+
99
+ Parameter count:
100
+ Total parameters: 1,202,352,640
101
+ Trainable parameters: 1,202,352,640
102
+ Non-trainable parameters: 0
103
+ Model size (BF16): 2.40 GB
104
+
105
+ --------------------------------------------------------------------------------
106
+ LAYER-BY-LAYER SUMMARY
107
+ --------------------------------------------------------------------------------
108
+ ================================================================================================================================================================
109
+ Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
110
+ ================================================================================================================================================================
111
+ Qwen3ForCausalLM [1, 8192] -- -- True
112
+ ├─Qwen3Model: 1-1 -- [1, 8192, 1536] -- True
113
+ │ └─Embedding: 2-1 [1, 8192] [1, 8192, 1536] 233,373,696 True
114
+ │ └─ModuleList: 2-2 -- -- -- True
115
+ │ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
116
+ │ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
117
+ │ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
118
+ │ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
119
+ │ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
120
+ │ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
121
+ │ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
122
+ │ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
123
+ │ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
124
+ │ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
125
+ │ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
126
+ │ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
127
+ │ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
128
+ │ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
129
+ │ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
130
+ │ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
131
+ │ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
132
+ │ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
133
+ │ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
134
+ │ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
135
+ │ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
136
+ │ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
137
+ │ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
138
+ │ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
139
+ │ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
140
+ │ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
141
+ │ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
142
+ │ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
143
+ │ └─Qwen3RMSNorm: 2-3 [1, 8192, 1536] [1, 8192, 1536] 1,536 True
144
+ ├─Linear: 1-2 [1, 8192, 1536] [1, 8192, 151936] 233,373,696 True
145
+ ================================================================================================================================================================
146
+ Total params: 1,435,726,336
147
+ Trainable params: 1,435,726,336
148
+ Non-trainable params: 0
149
+ Total mult-adds (Units.GIGABYTES): 1.44
150
+ ================================================================================================================================================================
151
+ Input size (MB): 0.07
152
+ Forward/backward pass size (MB): 26218.59
153
+ Params size (MB): 2871.45
154
+ Estimated Total Size (MB): 29090.11
155
+ ================================================================================================================================================================
156
+ NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
157
+ ================================================================================
158
+
159
+ Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
160
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
161
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
162
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
163
+ Using pre-tokenized dataset (no runtime tokenization)
164
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
165
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
166
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
167
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
168
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
169
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
170
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
171
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
172
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
173
+ Dataset split: 3027764 train, 159356 validation
174
+ Training examples (packed, train split): 3027764
175
+ Validation examples (packed, val split): 159356
176
+ Per-device batch size: 1
177
+ World size (GPUs): 6
178
+ Gradient accumulation steps: 10
179
+ Effective batch size (seqs/step): 60
180
+ Sequence length: 8192
181
+ Tokens per optimizer step: 491,520
182
+ Micro-batches per epoch (per rank): 504628
183
+ Optimizer steps per epoch: 50462
184
+ Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
185
+ Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-18000...
186
+ Preserving checkpoint LR state for exact resume: current=[0.00036559283431974993], schedule_base=[0.0005]
187
+ Resumed from checkpoint at step 18000 (epoch 0, micro_step 180000)
188
+ Overriding resume micro-step 180000 -> 180000 for DDP migration
189
+ Resuming training from step 18000
190
+ Starting training...
191
+ Validation at step 18000: Loss = 2.6612, Perplexity = 14.31
192
+ Switching attention mode for step 18001: dense (dense step interval)
193
+ Step 18010/50462 | Loss: 2.5910 | LR: 3.65e-04 | 21606.9ms/step | 2.8 samples/s
194
+ Step 18020/50462 | Loss: 2.6355 | LR: 3.65e-04 | 1887.0ms/step | 31.8 samples/s
195
+ Step 18030/50462 | Loss: 2.5903 | LR: 3.65e-04 | 1892.2ms/step | 31.7 samples/s
196
+ Step 18040/50462 | Loss: 2.6196 | LR: 3.65e-04 | 1891.7ms/step | 31.7 samples/s
197
+ Step 18050/50462 | Loss: 2.6288 | LR: 3.65e-04 | 1899.2ms/step | 31.6 samples/s
198
+ Step 18060/50462 | Loss: 2.6145 | LR: 3.65e-04 | 1901.6ms/step | 31.6 samples/s
199
+ Step 18070/50462 | Loss: 2.6270 | LR: 3.65e-04 | 1903.4ms/step | 31.5 samples/s
200
+ Step 18080/50462 | Loss: 2.6557 | LR: 3.64e-04 | 1902.9ms/step | 31.5 samples/s
201
+ Step 18090/50462 | Loss: 2.6014 | LR: 3.64e-04 | 1906.2ms/step | 31.5 samples/s
202
+ Step 18100/50462 | Loss: 2.6457 | LR: 3.64e-04 | 1908.7ms/step | 31.4 samples/s
203
+ Step 18110/50462 | Loss: 2.6302 | LR: 3.64e-04 | 1907.0ms/step | 31.5 samples/s
204
+ Step 18120/50462 | Loss: 2.6305 | LR: 3.64e-04 | 1906.3ms/step | 31.5 samples/s
205
+ Step 18130/50462 | Loss: 2.5993 | LR: 3.64e-04 | 1907.5ms/step | 31.5 samples/s
206
+ Step 18140/50462 | Loss: 2.6265 | LR: 3.64e-04 | 1904.0ms/step | 31.5 samples/s
207
+ Step 18150/50462 | Loss: 2.6629 | LR: 3.63e-04 | 1908.3ms/step | 31.4 samples/s
208
+ Step 18160/50462 | Loss: 2.6925 | LR: 3.63e-04 | 1906.4ms/step | 31.5 samples/s
209
+ Step 18170/50462 | Loss: 2.6133 | LR: 3.63e-04 | 1910.0ms/step | 31.4 samples/s
210
+ Step 18180/50462 | Loss: 2.6319 | LR: 3.63e-04 | 1910.6ms/step | 31.4 samples/s
211
+ Step 18190/50462 | Loss: 2.5892 | LR: 3.63e-04 | 1910.7ms/step | 31.4 samples/s
212
+ Step 18200/50462 | Loss: 2.6347 | LR: 3.63e-04 | 1912.1ms/step | 31.4 samples/s
213
+ Step 18210/50462 | Loss: 2.6112 | LR: 3.63e-04 | 1910.8ms/step | 31.4 samples/s
214
+ Step 18220/50462 | Loss: 2.6560 | LR: 3.62e-04 | 1909.9ms/step | 31.4 samples/s
215
+ Step 18230/50462 | Loss: 2.6303 | LR: 3.62e-04 | 1911.3ms/step | 31.4 samples/s
216
+ Step 18240/50462 | Loss: 2.6216 | LR: 3.62e-04 | 1912.5ms/step | 31.4 samples/s
217
+ Step 18250/50462 | Loss: 2.6485 | LR: 3.62e-04 | 1914.1ms/step | 31.3 samples/s
218
+ Validation at step 18250: Loss = 2.6587, Perplexity = 14.28
219
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-18250
220
+ Step 18260/50462 | Loss: 2.6117 | LR: 3.62e-04 | 3280.0ms/step | 18.3 samples/s
221
+ Step 18270/50462 | Loss: 2.6335 | LR: 3.62e-04 | 1896.9ms/step | 31.6 samples/s
222
+ Step 18280/50462 | Loss: 2.6223 | LR: 3.62e-04 | 1902.4ms/step | 31.5 samples/s
223
+ Step 18290/50462 | Loss: 2.5832 | LR: 3.62e-04 | 1906.9ms/step | 31.5 samples/s
224
+ Step 18300/50462 | Loss: 2.5971 | LR: 3.61e-04 | 1911.0ms/step | 31.4 samples/s
225
+ Step 18310/50462 | Loss: 2.6034 | LR: 3.61e-04 | 1913.4ms/step | 31.4 samples/s
226
+ Step 18320/50462 | Loss: 2.6681 | LR: 3.61e-04 | 1913.4ms/step | 31.4 samples/s
227
+ Step 18330/50462 | Loss: 2.6223 | LR: 3.61e-04 | 1911.8ms/step | 31.4 samples/s
228
+ Step 18340/50462 | Loss: 2.6323 | LR: 3.61e-04 | 1912.5ms/step | 31.4 samples/s
229
+ Step 18350/50462 | Loss: 2.6457 | LR: 3.61e-04 | 1910.5ms/step | 31.4 samples/s
230
+ Step 18360/50462 | Loss: 2.6311 | LR: 3.61e-04 | 1910.8ms/step | 31.4 samples/s
231
+ Step 18370/50462 | Loss: 2.6195 | LR: 3.60e-04 | 1912.3ms/step | 31.4 samples/s
232
+ Step 18380/50462 | Loss: 2.6143 | LR: 3.60e-04 | 1912.7ms/step | 31.4 samples/s
233
+ Step 18390/50462 | Loss: 2.6431 | LR: 3.60e-04 | 1911.5ms/step | 31.4 samples/s
234
+ Step 18400/50462 | Loss: 2.6542 | LR: 3.60e-04 | 1910.9ms/step | 31.4 samples/s
235
+ Step 18410/50462 | Loss: 2.6265 | LR: 3.60e-04 | 1908.8ms/step | 31.4 samples/s
236
+ Step 18420/50462 | Loss: 2.6270 | LR: 3.60e-04 | 1912.3ms/step | 31.4 samples/s
237
+ Step 18430/50462 | Loss: 2.6580 | LR: 3.60e-04 | 1915.7ms/step | 31.3 samples/s
238
+ Step 18440/50462 | Loss: 2.6507 | LR: 3.59e-04 | 1914.1ms/step | 31.3 samples/s
239
+ Step 18450/50462 | Loss: 2.6144 | LR: 3.59e-04 | 1915.1ms/step | 31.3 samples/s
240
+ Step 18460/50462 | Loss: 2.6081 | LR: 3.59e-04 | 1913.9ms/step | 31.3 samples/s
241
+ Step 18470/50462 | Loss: 2.6627 | LR: 3.59e-04 | 1915.1ms/step | 31.3 samples/s
242
+ Step 18480/50462 | Loss: 2.6097 | LR: 3.59e-04 | 1915.6ms/step | 31.3 samples/s
243
+ Step 18490/50462 | Loss: 2.6525 | LR: 3.59e-04 | 1917.3ms/step | 31.3 samples/s
244
+ Step 18500/50462 | Loss: 2.6050 | LR: 3.59e-04 | 1911.7ms/step | 31.4 samples/s
245
+ Validation at step 18500: Loss = 2.6565, Perplexity = 14.25
246
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-18500
247
+ Step 18510/50462 | Loss: 2.6120 | LR: 3.58e-04 | 3254.0ms/step | 18.4 samples/s
248
+ Step 18520/50462 | Loss: 2.6169 | LR: 3.58e-04 | 1896.3ms/step | 31.6 samples/s
249
+ Step 18530/50462 | Loss: 2.6385 | LR: 3.58e-04 | 1903.0ms/step | 31.5 samples/s
250
+ Step 18540/50462 | Loss: 2.6696 | LR: 3.58e-04 | 1906.6ms/step | 31.5 samples/s
251
+ Step 18550/50462 | Loss: 2.5863 | LR: 3.58e-04 | 1907.5ms/step | 31.5 samples/s
252
+ Step 18560/50462 | Loss: 2.6505 | LR: 3.58e-04 | 1908.7ms/step | 31.4 samples/s
253
+ Step 18570/50462 | Loss: 2.6068 | LR: 3.58e-04 | 1909.6ms/step | 31.4 samples/s
254
+ Step 18580/50462 | Loss: 2.6266 | LR: 3.57e-04 | 1910.1ms/step | 31.4 samples/s
255
+ Step 18590/50462 | Loss: 2.6362 | LR: 3.57e-04 | 1909.8ms/step | 31.4 samples/s
256
+ Step 18600/50462 | Loss: 2.6442 | LR: 3.57e-04 | 1909.3ms/step | 31.4 samples/s
257
+ Step 18610/50462 | Loss: 2.6351 | LR: 3.57e-04 | 1910.4ms/step | 31.4 samples/s
258
+ Step 18620/50462 | Loss: 2.6031 | LR: 3.57e-04 | 1908.1ms/step | 31.4 samples/s
259
+ Step 18630/50462 | Loss: 2.5842 | LR: 3.57e-04 | 1911.3ms/step | 31.4 samples/s
260
+ Step 18640/50462 | Loss: 2.6047 | LR: 3.57e-04 | 1911.5ms/step | 31.4 samples/s
261
+ Step 18650/50462 | Loss: 2.6561 | LR: 3.56e-04 | 1915.9ms/step | 31.3 samples/s
262
+ Step 18660/50462 | Loss: 2.6499 | LR: 3.56e-04 | 1912.3ms/step | 31.4 samples/s
263
+ Step 18670/50462 | Loss: 2.5908 | LR: 3.56e-04 | 1915.6ms/step | 31.3 samples/s
264
+ Step 18680/50462 | Loss: 2.6114 | LR: 3.56e-04 | 1915.7ms/step | 31.3 samples/s
265
+ Step 18690/50462 | Loss: 2.6575 | LR: 3.56e-04 | 1912.5ms/step | 31.4 samples/s
266
+ Step 18700/50462 | Loss: 2.6305 | LR: 3.56e-04 | 1913.6ms/step | 31.4 samples/s
267
+ Step 18710/50462 | Loss: 2.5891 | LR: 3.56e-04 | 1911.1ms/step | 31.4 samples/s
268
+ Step 18720/50462 | Loss: 2.5792 | LR: 3.55e-04 | 1911.9ms/step | 31.4 samples/s
269
+ Step 18730/50462 | Loss: 2.6205 | LR: 3.55e-04 | 1910.7ms/step | 31.4 samples/s
270
+ Step 18740/50462 | Loss: 2.6577 | LR: 3.55e-04 | 1911.4ms/step | 31.4 samples/s
271
+ Step 18750/50462 | Loss: 2.6335 | LR: 3.55e-04 | 1910.6ms/step | 31.4 samples/s
272
+ Validation at step 18750: Loss = 2.6544, Perplexity = 14.22
273
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-18750
274
+ Step 18760/50462 | Loss: 2.6205 | LR: 3.55e-04 | 3256.2ms/step | 18.4 samples/s
275
+ Step 18770/50462 | Loss: 2.6322 | LR: 3.55e-04 | 1906.1ms/step | 31.5 samples/s
276
+ Step 18780/50462 | Loss: 2.5991 | LR: 3.55e-04 | 1909.6ms/step | 31.4 samples/s
277
+ Step 18790/50462 | Loss: 2.6373 | LR: 3.54e-04 | 1910.5ms/step | 31.4 samples/s
278
+ Step 18800/50462 | Loss: 2.6339 | LR: 3.54e-04 | 1911.1ms/step | 31.4 samples/s
279
+ Step 18810/50462 | Loss: 2.6174 | LR: 3.54e-04 | 1909.0ms/step | 31.4 samples/s
280
+ Step 18820/50462 | Loss: 2.5873 | LR: 3.54e-04 | 1912.0ms/step | 31.4 samples/s
281
+ Step 18830/50462 | Loss: 2.6261 | LR: 3.54e-04 | 1909.8ms/step | 31.4 samples/s
282
+ Step 18840/50462 | Loss: 2.6000 | LR: 3.54e-04 | 1911.0ms/step | 31.4 samples/s
283
+ Step 18850/50462 | Loss: 2.6259 | LR: 3.54e-04 | 1912.0ms/step | 31.4 samples/s
284
+ Step 18860/50462 | Loss: 2.6163 | LR: 3.53e-04 | 1910.4ms/step | 31.4 samples/s
285
+ Step 18870/50462 | Loss: 2.6101 | LR: 3.53e-04 | 1911.8ms/step | 31.4 samples/s
286
+ Step 18880/50462 | Loss: 2.6605 | LR: 3.53e-04 | 1914.1ms/step | 31.3 samples/s
287
+ Step 18890/50462 | Loss: 2.6609 | LR: 3.53e-04 | 1914.3ms/step | 31.3 samples/s
288
+ Step 18900/50462 | Loss: 2.6544 | LR: 3.53e-04 | 1912.0ms/step | 31.4 samples/s
289
+ Step 18910/50462 | Loss: 2.6347 | LR: 3.53e-04 | 1914.4ms/step | 31.3 samples/s
290
+ Step 18920/50462 | Loss: 2.6209 | LR: 3.53e-04 | 1912.3ms/step | 31.4 samples/s
291
+ Step 18930/50462 | Loss: 2.5533 | LR: 3.52e-04 | 1911.8ms/step | 31.4 samples/s
292
+ Step 18940/50462 | Loss: 2.5966 | LR: 3.52e-04 | 1913.1ms/step | 31.4 samples/s
293
+ Step 18950/50462 | Loss: 2.6445 | LR: 3.52e-04 | 1911.3ms/step | 31.4 samples/s
294
+ Step 18960/50462 | Loss: 2.6424 | LR: 3.52e-04 | 1910.6ms/step | 31.4 samples/s
295
+ Step 18970/50462 | Loss: 2.6139 | LR: 3.52e-04 | 1913.2ms/step | 31.4 samples/s
296
+ Step 18980/50462 | Loss: 2.6561 | LR: 3.52e-04 | 1911.3ms/step | 31.4 samples/s
297
+ Step 18990/50462 | Loss: 2.6385 | LR: 3.52e-04 | 1912.4ms/step | 31.4 samples/s
298
+ Step 19000/50462 | Loss: 2.6576 | LR: 3.51e-04 | 1912.3ms/step | 31.4 samples/s
299
+ Validation at step 19000: Loss = 2.6527, Perplexity = 14.19
300
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-19000
301
+ Step 19010/50462 | Loss: 2.6492 | LR: 3.51e-04 | 3261.3ms/step | 18.4 samples/s
302
+ Step 19020/50462 | Loss: 2.6157 | LR: 3.51e-04 | 1899.4ms/step | 31.6 samples/s
303
+ Step 19030/50462 | Loss: 2.5824 | LR: 3.51e-04 | 1905.5ms/step | 31.5 samples/s
304
+ Step 19040/50462 | Loss: 2.5805 | LR: 3.51e-04 | 1908.0ms/step | 31.4 samples/s
305
+ Step 19050/50462 | Loss: 2.5887 | LR: 3.51e-04 | 1910.3ms/step | 31.4 samples/s
306
+ Step 19060/50462 | Loss: 2.6211 | LR: 3.50e-04 | 1912.0ms/step | 31.4 samples/s
307
+ Step 19070/50462 | Loss: 2.6398 | LR: 3.50e-04 | 1912.5ms/step | 31.4 samples/s
308
+ Step 19080/50462 | Loss: 2.6417 | LR: 3.50e-04 | 1908.9ms/step | 31.4 samples/s
309
+ Step 19090/50462 | Loss: 2.6060 | LR: 3.50e-04 | 1910.4ms/step | 31.4 samples/s
310
+ Step 19100/50462 | Loss: 2.5939 | LR: 3.50e-04 | 1912.5ms/step | 31.4 samples/s
311
+ Step 19110/50462 | Loss: 2.5928 | LR: 3.50e-04 | 1911.5ms/step | 31.4 samples/s
312
+ Step 19120/50462 | Loss: 2.5716 | LR: 3.50e-04 | 1913.9ms/step | 31.3 samples/s
313
+ Step 19130/50462 | Loss: 2.5661 | LR: 3.49e-04 | 1914.8ms/step | 31.3 samples/s
314
+ Step 19140/50462 | Loss: 2.6397 | LR: 3.49e-04 | 1910.6ms/step | 31.4 samples/s
315
+ Step 19150/50462 | Loss: 2.6445 | LR: 3.49e-04 | 1913.3ms/step | 31.4 samples/s
316
+ Step 19160/50462 | Loss: 2.5597 | LR: 3.49e-04 | 1917.0ms/step | 31.3 samples/s
317
+ Step 19170/50462 | Loss: 2.6252 | LR: 3.49e-04 | 1911.2ms/step | 31.4 samples/s
318
+ Step 19180/50462 | Loss: 2.6363 | LR: 3.49e-04 | 1913.8ms/step | 31.4 samples/s
319
+ Step 19190/50462 | Loss: 2.6512 | LR: 3.49e-04 | 1913.0ms/step | 31.4 samples/s
320
+ Step 19200/50462 | Loss: 2.6225 | LR: 3.48e-04 | 1913.2ms/step | 31.4 samples/s
321
+ Step 19210/50462 | Loss: 2.6175 | LR: 3.48e-04 | 1912.1ms/step | 31.4 samples/s
322
+ Step 19220/50462 | Loss: 2.6019 | LR: 3.48e-04 | 1911.9ms/step | 31.4 samples/s
323
+ Step 19230/50462 | Loss: 2.6175 | LR: 3.48e-04 | 1913.0ms/step | 31.4 samples/s
324
+ Step 19240/50462 | Loss: 2.6266 | LR: 3.48e-04 | 1914.4ms/step | 31.3 samples/s
325
+ Step 19250/50462 | Loss: 2.5794 | LR: 3.48e-04 | 1913.1ms/step | 31.4 samples/s
326
+ Validation at step 19250: Loss = 2.6484, Perplexity = 14.13
327
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-19250
328
+ Step 19260/50462 | Loss: 2.6225 | LR: 3.48e-04 | 3256.9ms/step | 18.4 samples/s
329
+ Step 19270/50462 | Loss: 2.6104 | LR: 3.47e-04 | 1903.7ms/step | 31.5 samples/s
330
+ Step 19280/50462 | Loss: 2.6102 | LR: 3.47e-04 | 1907.6ms/step | 31.5 samples/s
331
+ Step 19290/50462 | Loss: 2.5918 | LR: 3.47e-04 | 1908.9ms/step | 31.4 samples/s
332
+ Step 19300/50462 | Loss: 2.6563 | LR: 3.47e-04 | 1913.7ms/step | 31.4 samples/s
333
+ Step 19310/50462 | Loss: 2.6250 | LR: 3.47e-04 | 1912.9ms/step | 31.4 samples/s
334
+ Step 19320/50462 | Loss: 2.6022 | LR: 3.47e-04 | 1912.1ms/step | 31.4 samples/s
335
+ Step 19330/50462 | Loss: 2.5846 | LR: 3.47e-04 | 1916.3ms/step | 31.3 samples/s
336
+ Step 19340/50462 | Loss: 2.6111 | LR: 3.46e-04 | 1913.5ms/step | 31.4 samples/s
337
+ Step 19350/50462 | Loss: 2.6194 | LR: 3.46e-04 | 1913.7ms/step | 31.4 samples/s
338
+ Step 19360/50462 | Loss: 2.6053 | LR: 3.46e-04 | 1912.9ms/step | 31.4 samples/s
339
+ Step 19370/50462 | Loss: 2.6049 | LR: 3.46e-04 | 1913.9ms/step | 31.3 samples/s
340
+ Step 19380/50462 | Loss: 2.6261 | LR: 3.46e-04 | 1911.6ms/step | 31.4 samples/s
341
+ Step 19390/50462 | Loss: 2.5998 | LR: 3.46e-04 | 1912.0ms/step | 31.4 samples/s
342
+ Step 19400/50462 | Loss: 2.6079 | LR: 3.46e-04 | 1913.6ms/step | 31.4 samples/s
343
+ Step 19410/50462 | Loss: 2.5657 | LR: 3.45e-04 | 1913.3ms/step | 31.4 samples/s
344
+ Step 19420/50462 | Loss: 2.5844 | LR: 3.45e-04 | 1911.9ms/step | 31.4 samples/s
345
+ Step 19430/50462 | Loss: 2.6536 | LR: 3.45e-04 | 1911.5ms/step | 31.4 samples/s
346
+ Step 19440/50462 | Loss: 2.6145 | LR: 3.45e-04 | 1913.1ms/step | 31.4 samples/s
347
+ Step 19450/50462 | Loss: 2.6120 | LR: 3.45e-04 | 1912.3ms/step | 31.4 samples/s
348
+ Step 19460/50462 | Loss: 2.5891 | LR: 3.45e-04 | 1913.9ms/step | 31.3 samples/s
349
+ Step 19470/50462 | Loss: 2.6217 | LR: 3.45e-04 | 1909.5ms/step | 31.4 samples/s
350
+ Step 19480/50462 | Loss: 2.6218 | LR: 3.44e-04 | 1911.7ms/step | 31.4 samples/s
351
+ Step 19490/50462 | Loss: 2.5934 | LR: 3.44e-04 | 1912.6ms/step | 31.4 samples/s
352
+ Step 19500/50462 | Loss: 2.6275 | LR: 3.44e-04 | 1914.6ms/step | 31.3 samples/s
353
+ Validation at step 19500: Loss = 2.6459, Perplexity = 14.10
354
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-19500
355
+ Step 19510/50462 | Loss: 2.5604 | LR: 3.44e-04 | 3235.7ms/step | 18.5 samples/s
356
+ Step 19520/50462 | Loss: 2.6779 | LR: 3.44e-04 | 1903.8ms/step | 31.5 samples/s
357
+ Step 19530/50462 | Loss: 2.6006 | LR: 3.44e-04 | 1905.1ms/step | 31.5 samples/s
358
+ Step 19540/50462 | Loss: 2.6599 | LR: 3.44e-04 | 1911.2ms/step | 31.4 samples/s
359
+ Step 19550/50462 | Loss: 2.6343 | LR: 3.43e-04 | 1909.9ms/step | 31.4 samples/s
360
+ Step 19560/50462 | Loss: 2.5957 | LR: 3.43e-04 | 1910.3ms/step | 31.4 samples/s
361
+ Step 19570/50462 | Loss: 2.6059 | LR: 3.43e-04 | 1911.7ms/step | 31.4 samples/s
362
+ Step 19580/50462 | Loss: 2.5893 | LR: 3.43e-04 | 1912.1ms/step | 31.4 samples/s
363
+ Step 19590/50462 | Loss: 2.6353 | LR: 3.43e-04 | 1910.0ms/step | 31.4 samples/s
364
+ Step 19600/50462 | Loss: 2.6282 | LR: 3.43e-04 | 1914.4ms/step | 31.3 samples/s
365
+ Step 19610/50462 | Loss: 2.6362 | LR: 3.42e-04 | 1914.7ms/step | 31.3 samples/s
366
+ Step 19620/50462 | Loss: 2.6148 | LR: 3.42e-04 | 1913.1ms/step | 31.4 samples/s
367
+ Step 19630/50462 | Loss: 2.6079 | LR: 3.42e-04 | 1909.7ms/step | 31.4 samples/s
368
+ Step 19640/50462 | Loss: 2.6007 | LR: 3.42e-04 | 1910.6ms/step | 31.4 samples/s
369
+ Step 19650/50462 | Loss: 2.5746 | LR: 3.42e-04 | 1910.9ms/step | 31.4 samples/s
370
+ Step 19660/50462 | Loss: 2.6372 | LR: 3.42e-04 | 1910.8ms/step | 31.4 samples/s
371
+ Step 19670/50462 | Loss: 2.6233 | LR: 3.42e-04 | 1914.0ms/step | 31.3 samples/s
372
+ Step 19680/50462 | Loss: 2.6114 | LR: 3.41e-04 | 1911.0ms/step | 31.4 samples/s
373
+ Step 19690/50462 | Loss: 2.6008 | LR: 3.41e-04 | 1910.6ms/step | 31.4 samples/s
374
+ Step 19700/50462 | Loss: 2.5953 | LR: 3.41e-04 | 1912.5ms/step | 31.4 samples/s
375
+ Step 19710/50462 | Loss: 2.6111 | LR: 3.41e-04 | 1911.1ms/step | 31.4 samples/s
376
+ Step 19720/50462 | Loss: 2.6409 | LR: 3.41e-04 | 1909.7ms/step | 31.4 samples/s
377
+ Step 19730/50462 | Loss: 2.5840 | LR: 3.41e-04 | 1914.1ms/step | 31.3 samples/s
378
+ Step 19740/50462 | Loss: 2.6308 | LR: 3.41e-04 | 1914.0ms/step | 31.3 samples/s
379
+ Step 19750/50462 | Loss: 2.6077 | LR: 3.40e-04 | 1913.2ms/step | 31.4 samples/s
380
+ Validation at step 19750: Loss = 2.6430, Perplexity = 14.06
381
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-19750
382
+ Step 19760/50462 | Loss: 2.5989 | LR: 3.40e-04 | 3260.3ms/step | 18.4 samples/s
383
+ Step 19770/50462 | Loss: 2.6005 | LR: 3.40e-04 | 1903.8ms/step | 31.5 samples/s
384
+ Step 19780/50462 | Loss: 2.5689 | LR: 3.40e-04 | 1905.4ms/step | 31.5 samples/s
385
+ Step 19790/50462 | Loss: 2.6813 | LR: 3.40e-04 | 1912.0ms/step | 31.4 samples/s
386
+ Step 19800/50462 | Loss: 2.5807 | LR: 3.40e-04 | 1911.3ms/step | 31.4 samples/s
387
+ Step 19810/50462 | Loss: 2.5938 | LR: 3.40e-04 | 1909.5ms/step | 31.4 samples/s
388
+ Step 19820/50462 | Loss: 2.5810 | LR: 3.39e-04 | 1909.5ms/step | 31.4 samples/s
389
+ Step 19830/50462 | Loss: 2.5649 | LR: 3.39e-04 | 1910.6ms/step | 31.4 samples/s
390
+ Step 19840/50462 | Loss: 2.6501 | LR: 3.39e-04 | 1912.3ms/step | 31.4 samples/s
391
+ Step 19850/50462 | Loss: 2.6020 | LR: 3.39e-04 | 1911.8ms/step | 31.4 samples/s
392
+ Step 19860/50462 | Loss: 2.6337 | LR: 3.39e-04 | 1914.7ms/step | 31.3 samples/s
393
+ Step 19870/50462 | Loss: 2.6218 | LR: 3.39e-04 | 1914.0ms/step | 31.3 samples/s
394
+ Step 19880/50462 | Loss: 2.5932 | LR: 3.39e-04 | 1911.3ms/step | 31.4 samples/s
395
+ Step 19890/50462 | Loss: 2.6111 | LR: 3.38e-04 | 1912.8ms/step | 31.4 samples/s
396
+ Step 19900/50462 | Loss: 2.6320 | LR: 3.38e-04 | 1914.1ms/step | 31.3 samples/s
397
+ Step 19910/50462 | Loss: 2.6025 | LR: 3.38e-04 | 1912.6ms/step | 31.4 samples/s
398
+ Step 19920/50462 | Loss: 2.6350 | LR: 3.38e-04 | 1912.0ms/step | 31.4 samples/s
399
+ Step 19930/50462 | Loss: 2.6080 | LR: 3.38e-04 | 1914.2ms/step | 31.3 samples/s
400
+ Step 19940/50462 | Loss: 2.6464 | LR: 3.38e-04 | 1914.3ms/step | 31.3 samples/s
401
+ Step 19950/50462 | Loss: 2.6031 | LR: 3.37e-04 | 1915.4ms/step | 31.3 samples/s
402
+ Step 19960/50462 | Loss: 2.5793 | LR: 3.37e-04 | 1916.0ms/step | 31.3 samples/s
403
+ Step 19970/50462 | Loss: 2.6396 | LR: 3.37e-04 | 1914.2ms/step | 31.3 samples/s
404
+ Step 19980/50462 | Loss: 2.6336 | LR: 3.37e-04 | 1911.0ms/step | 31.4 samples/s
405
+ Step 19990/50462 | Loss: 2.6424 | LR: 3.37e-04 | 1915.2ms/step | 31.3 samples/s
406
+ Step 20000/50462 | Loss: 2.6295 | LR: 3.37e-04 | 1912.1ms/step | 31.4 samples/s
407
+ Validation at step 20000: Loss = 2.6413, Perplexity = 14.03
408
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-20000
409
+ Step 20010/50462 | Loss: 2.5677 | LR: 3.37e-04 | 3236.0ms/step | 18.5 samples/s
410
+ Step 20020/50462 | Loss: 2.5907 | LR: 3.36e-04 | 1896.1ms/step | 31.6 samples/s
411
+ Step 20030/50462 | Loss: 2.5801 | LR: 3.36e-04 | 1902.5ms/step | 31.5 samples/s
412
+ Step 20040/50462 | Loss: 2.6322 | LR: 3.36e-04 | 1905.5ms/step | 31.5 samples/s
413
+ Step 20050/50462 | Loss: 2.6093 | LR: 3.36e-04 | 1908.9ms/step | 31.4 samples/s
414
+ Step 20060/50462 | Loss: 2.6349 | LR: 3.36e-04 | 1911.1ms/step | 31.4 samples/s
415
+ Step 20070/50462 | Loss: 2.6036 | LR: 3.36e-04 | 1914.4ms/step | 31.3 samples/s
416
+ Step 20080/50462 | Loss: 2.6037 | LR: 3.36e-04 | 1913.5ms/step | 31.4 samples/s
417
+ Step 20090/50462 | Loss: 2.6222 | LR: 3.35e-04 | 1910.8ms/step | 31.4 samples/s
418
+ Step 20100/50462 | Loss: 2.5881 | LR: 3.35e-04 | 1913.8ms/step | 31.4 samples/s
419
+ Step 20110/50462 | Loss: 2.6135 | LR: 3.35e-04 | 1914.9ms/step | 31.3 samples/s
420
+ Step 20120/50462 | Loss: 2.6334 | LR: 3.35e-04 | 1916.1ms/step | 31.3 samples/s
421
+ Step 20130/50462 | Loss: 2.6022 | LR: 3.35e-04 | 1913.7ms/step | 31.4 samples/s
422
+ Step 20140/50462 | Loss: 2.6003 | LR: 3.35e-04 | 1916.0ms/step | 31.3 samples/s
423
+ Step 20150/50462 | Loss: 2.5629 | LR: 3.35e-04 | 1916.3ms/step | 31.3 samples/s
424
+ Step 20160/50462 | Loss: 2.6348 | LR: 3.34e-04 | 1913.7ms/step | 31.4 samples/s
425
+ Step 20170/50462 | Loss: 2.6346 | LR: 3.34e-04 | 1912.2ms/step | 31.4 samples/s
426
+ Step 20180/50462 | Loss: 2.6052 | LR: 3.34e-04 | 1914.2ms/step | 31.3 samples/s
427
+ Step 20190/50462 | Loss: 2.6035 | LR: 3.34e-04 | 1912.8ms/step | 31.4 samples/s
428
+ Step 20200/50462 | Loss: 2.5824 | LR: 3.34e-04 | 1909.4ms/step | 31.4 samples/s
429
+ Step 20210/50462 | Loss: 2.6271 | LR: 3.34e-04 | 1909.5ms/step | 31.4 samples/s
430
+ Step 20220/50462 | Loss: 2.5822 | LR: 3.33e-04 | 1910.4ms/step | 31.4 samples/s
431
+ Step 20230/50462 | Loss: 2.5888 | LR: 3.33e-04 | 1911.3ms/step | 31.4 samples/s
432
+ Step 20240/50462 | Loss: 2.5953 | LR: 3.33e-04 | 1910.6ms/step | 31.4 samples/s
433
+ Step 20250/50462 | Loss: 2.5406 | LR: 3.33e-04 | 1910.3ms/step | 31.4 samples/s
434
+ Validation at step 20250: Loss = 2.6383, Perplexity = 13.99
435
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-20250
436
+ Step 20260/50462 | Loss: 2.5164 | LR: 3.33e-04 | 3282.8ms/step | 18.3 samples/s
437
+ Step 20270/50462 | Loss: 2.5542 | LR: 3.33e-04 | 1898.1ms/step | 31.6 samples/s
438
+ Step 20280/50462 | Loss: 2.6074 | LR: 3.33e-04 | 1903.8ms/step | 31.5 samples/s
439
+ Step 20290/50462 | Loss: 2.6013 | LR: 3.32e-04 | 1908.0ms/step | 31.4 samples/s
440
+ Step 20300/50462 | Loss: 2.6301 | LR: 3.32e-04 | 1906.7ms/step | 31.5 samples/s
441
+ Step 20310/50462 | Loss: 2.6353 | LR: 3.32e-04 | 1908.9ms/step | 31.4 samples/s
442
+ Step 20320/50462 | Loss: 2.5831 | LR: 3.32e-04 | 1912.2ms/step | 31.4 samples/s
443
+ Step 20330/50462 | Loss: 2.6414 | LR: 3.32e-04 | 1911.6ms/step | 31.4 samples/s
444
+ Step 20340/50462 | Loss: 2.6375 | LR: 3.32e-04 | 1911.8ms/step | 31.4 samples/s
445
+ Step 20350/50462 | Loss: 2.5959 | LR: 3.32e-04 | 1912.2ms/step | 31.4 samples/s
446
+ Step 20360/50462 | Loss: 2.6019 | LR: 3.31e-04 | 1911.0ms/step | 31.4 samples/s
447
+ Step 20370/50462 | Loss: 2.5541 | LR: 3.31e-04 | 1911.2ms/step | 31.4 samples/s
448
+ Step 20380/50462 | Loss: 2.6114 | LR: 3.31e-04 | 1913.6ms/step | 31.4 samples/s
449
+ Step 20390/50462 | Loss: 2.6026 | LR: 3.31e-04 | 1910.7ms/step | 31.4 samples/s
450
+ Step 20400/50462 | Loss: 2.6232 | LR: 3.31e-04 | 1909.9ms/step | 31.4 samples/s
451
+ Step 20410/50462 | Loss: 2.5703 | LR: 3.31e-04 | 1911.3ms/step | 31.4 samples/s
452
+ Step 20420/50462 | Loss: 2.6439 | LR: 3.30e-04 | 1913.1ms/step | 31.4 samples/s
453
+ Step 20430/50462 | Loss: 2.5944 | LR: 3.30e-04 | 1912.3ms/step | 31.4 samples/s
454
+ Step 20440/50462 | Loss: 2.6130 | LR: 3.30e-04 | 1913.9ms/step | 31.3 samples/s
455
+ Step 20450/50462 | Loss: 2.6189 | LR: 3.30e-04 | 1911.3ms/step | 31.4 samples/s
456
+ Step 20460/50462 | Loss: 2.6417 | LR: 3.30e-04 | 1911.3ms/step | 31.4 samples/s
457
+ Step 20470/50462 | Loss: 2.5918 | LR: 3.30e-04 | 1913.5ms/step | 31.4 samples/s
458
+ Step 20480/50462 | Loss: 2.6264 | LR: 3.30e-04 | 1911.9ms/step | 31.4 samples/s
459
+ Step 20490/50462 | Loss: 2.5639 | LR: 3.29e-04 | 1912.6ms/step | 31.4 samples/s
460
+ Step 20500/50462 | Loss: 2.5957 | LR: 3.29e-04 | 1910.0ms/step | 31.4 samples/s
461
+ Validation at step 20500: Loss = 2.6355, Perplexity = 13.95
462
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-20500
463
+ Step 20510/50462 | Loss: 2.6133 | LR: 3.29e-04 | 3261.0ms/step | 18.4 samples/s
464
+ Step 20520/50462 | Loss: 2.6043 | LR: 3.29e-04 | 1903.4ms/step | 31.5 samples/s
465
+ Step 20530/50462 | Loss: 2.6151 | LR: 3.29e-04 | 1906.8ms/step | 31.5 samples/s
466
+ Step 20540/50462 | Loss: 2.6205 | LR: 3.29e-04 | 1908.3ms/step | 31.4 samples/s
467
+ Step 20550/50462 | Loss: 2.5878 | LR: 3.29e-04 | 1909.8ms/step | 31.4 samples/s
468
+ Step 20560/50462 | Loss: 2.5958 | LR: 3.28e-04 | 1909.3ms/step | 31.4 samples/s
469
+ Step 20570/50462 | Loss: 2.6529 | LR: 3.28e-04 | 1911.5ms/step | 31.4 samples/s
470
+ Step 20580/50462 | Loss: 2.5929 | LR: 3.28e-04 | 1911.5ms/step | 31.4 samples/s
471
+ Step 20590/50462 | Loss: 2.5995 | LR: 3.28e-04 | 1912.8ms/step | 31.4 samples/s
472
+ Step 20600/50462 | Loss: 2.6384 | LR: 3.28e-04 | 1914.8ms/step | 31.3 samples/s
473
+ Step 20610/50462 | Loss: 2.6430 | LR: 3.28e-04 | 1915.1ms/step | 31.3 samples/s
474
+ Step 20620/50462 | Loss: 2.5655 | LR: 3.27e-04 | 1912.2ms/step | 31.4 samples/s
475
+ Step 20630/50462 | Loss: 2.5936 | LR: 3.27e-04 | 1915.5ms/step | 31.3 samples/s
476
+ Step 20640/50462 | Loss: 2.6308 | LR: 3.27e-04 | 1913.0ms/step | 31.4 samples/s
477
+ Step 20650/50462 | Loss: 2.6178 | LR: 3.27e-04 | 1915.0ms/step | 31.3 samples/s
478
+ Step 20660/50462 | Loss: 2.6010 | LR: 3.27e-04 | 1912.3ms/step | 31.4 samples/s
479
+ Step 20670/50462 | Loss: 2.6205 | LR: 3.27e-04 | 1912.0ms/step | 31.4 samples/s
480
+ Step 20680/50462 | Loss: 2.5740 | LR: 3.27e-04 | 1914.0ms/step | 31.3 samples/s
481
+ Step 20690/50462 | Loss: 2.6138 | LR: 3.26e-04 | 1913.1ms/step | 31.4 samples/s
482
+ Step 20700/50462 | Loss: 2.6325 | LR: 3.26e-04 | 1913.2ms/step | 31.4 samples/s
483
+ Step 20710/50462 | Loss: 2.6136 | LR: 3.26e-04 | 1911.7ms/step | 31.4 samples/s
484
+ Step 20720/50462 | Loss: 2.6035 | LR: 3.26e-04 | 1913.1ms/step | 31.4 samples/s
485
+ Step 20730/50462 | Loss: 2.5756 | LR: 3.26e-04 | 1912.8ms/step | 31.4 samples/s
486
+ Step 20740/50462 | Loss: 2.6052 | LR: 3.26e-04 | 1912.5ms/step | 31.4 samples/s
487
+ Step 20750/50462 | Loss: 2.5800 | LR: 3.26e-04 | 1912.4ms/step | 31.4 samples/s
488
+ Validation at step 20750: Loss = 2.6351, Perplexity = 13.95
489
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-20750
490
+ Step 20760/50462 | Loss: 2.5755 | LR: 3.25e-04 | 3258.4ms/step | 18.4 samples/s
491
+ Step 20770/50462 | Loss: 2.6335 | LR: 3.25e-04 | 1899.1ms/step | 31.6 samples/s
492
+ Step 20780/50462 | Loss: 2.5925 | LR: 3.25e-04 | 1903.5ms/step | 31.5 samples/s
493
+ Step 20790/50462 | Loss: 2.5616 | LR: 3.25e-04 | 1908.2ms/step | 31.4 samples/s
494
+ Step 20800/50462 | Loss: 2.6030 | LR: 3.25e-04 | 1910.4ms/step | 31.4 samples/s
495
+ Step 20810/50462 | Loss: 2.6394 | LR: 3.25e-04 | 1909.4ms/step | 31.4 samples/s
496
+ Step 20820/50462 | Loss: 2.5853 | LR: 3.24e-04 | 1909.5ms/step | 31.4 samples/s
497
+ Step 20830/50462 | Loss: 2.6072 | LR: 3.24e-04 | 1910.7ms/step | 31.4 samples/s
498
+ Step 20840/50462 | Loss: 2.5875 | LR: 3.24e-04 | 1909.7ms/step | 31.4 samples/s
499
+ Step 20850/50462 | Loss: 2.6089 | LR: 3.24e-04 | 1914.0ms/step | 31.3 samples/s
500
+ Step 20860/50462 | Loss: 2.6371 | LR: 3.24e-04 | 1915.1ms/step | 31.3 samples/s
501
+ Step 20870/50462 | Loss: 2.5738 | LR: 3.24e-04 | 1916.6ms/step | 31.3 samples/s
502
+ Step 20880/50462 | Loss: 2.6104 | LR: 3.24e-04 | 1916.5ms/step | 31.3 samples/s
503
+ Step 20890/50462 | Loss: 2.5983 | LR: 3.23e-04 | 1913.0ms/step | 31.4 samples/s
504
+ Step 20900/50462 | Loss: 2.5691 | LR: 3.23e-04 | 1913.9ms/step | 31.4 samples/s
505
+ Step 20910/50462 | Loss: 2.5705 | LR: 3.23e-04 | 1911.8ms/step | 31.4 samples/s
506
+ Step 20920/50462 | Loss: 2.6291 | LR: 3.23e-04 | 1911.6ms/step | 31.4 samples/s
507
+ Step 20930/50462 | Loss: 2.5896 | LR: 3.23e-04 | 1912.1ms/step | 31.4 samples/s
508
+ Step 20940/50462 | Loss: 2.6207 | LR: 3.23e-04 | 1914.4ms/step | 31.3 samples/s
509
+ Step 20950/50462 | Loss: 2.6257 | LR: 3.23e-04 | 1914.5ms/step | 31.3 samples/s
510
+ Step 20960/50462 | Loss: 2.5747 | LR: 3.22e-04 | 1915.0ms/step | 31.3 samples/s
511
+ Step 20970/50462 | Loss: 2.6017 | LR: 3.22e-04 | 1913.4ms/step | 31.4 samples/s
512
+ Step 20980/50462 | Loss: 2.6056 | LR: 3.22e-04 | 1911.7ms/step | 31.4 samples/s
513
+ Step 20990/50462 | Loss: 2.5909 | LR: 3.22e-04 | 1909.4ms/step | 31.4 samples/s
514
+ Step 21000/50462 | Loss: 2.6135 | LR: 3.22e-04 | 1911.4ms/step | 31.4 samples/s
515
+ Validation at step 21000: Loss = 2.6301, Perplexity = 13.87
516
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-21000
517
+ Step 21010/50462 | Loss: 2.6196 | LR: 3.22e-04 | 3273.2ms/step | 18.3 samples/s
518
+ Step 21020/50462 | Loss: 2.5684 | LR: 3.21e-04 | 1905.4ms/step | 31.5 samples/s
519
+ Step 21030/50462 | Loss: 2.5986 | LR: 3.21e-04 | 1907.8ms/step | 31.5 samples/s
520
+ Step 21040/50462 | Loss: 2.5889 | LR: 3.21e-04 | 1908.7ms/step | 31.4 samples/s
521
+ Step 21050/50462 | Loss: 2.6080 | LR: 3.21e-04 | 1910.9ms/step | 31.4 samples/s
522
+ Step 21060/50462 | Loss: 2.6136 | LR: 3.21e-04 | 1909.6ms/step | 31.4 samples/s
523
+ Step 21070/50462 | Loss: 2.5853 | LR: 3.21e-04 | 1911.1ms/step | 31.4 samples/s
524
+ Step 21080/50462 | Loss: 2.6048 | LR: 3.21e-04 | 1913.3ms/step | 31.4 samples/s
525
+ Step 21090/50462 | Loss: 2.6307 | LR: 3.20e-04 | 1910.7ms/step | 31.4 samples/s
526
+ Step 21100/50462 | Loss: 2.5744 | LR: 3.20e-04 | 1909.5ms/step | 31.4 samples/s
527
+ Step 21110/50462 | Loss: 2.6053 | LR: 3.20e-04 | 1912.6ms/step | 31.4 samples/s
528
+ Step 21120/50462 | Loss: 2.5595 | LR: 3.20e-04 | 1913.6ms/step | 31.4 samples/s
529
+ Step 21130/50462 | Loss: 2.6161 | LR: 3.20e-04 | 1913.9ms/step | 31.3 samples/s
530
+ Step 21140/50462 | Loss: 2.5931 | LR: 3.20e-04 | 1913.0ms/step | 31.4 samples/s
531
+ Step 21150/50462 | Loss: 2.5475 | LR: 3.19e-04 | 1914.9ms/step | 31.3 samples/s
532
+ Step 21160/50462 | Loss: 2.5743 | LR: 3.19e-04 | 1915.6ms/step | 31.3 samples/s
533
+ Step 21170/50462 | Loss: 2.5890 | LR: 3.19e-04 | 1911.9ms/step | 31.4 samples/s
534
+ Step 21180/50462 | Loss: 2.5737 | LR: 3.19e-04 | 1912.8ms/step | 31.4 samples/s
535
+ Step 21190/50462 | Loss: 2.5743 | LR: 3.19e-04 | 1910.4ms/step | 31.4 samples/s
536
+ Step 21200/50462 | Loss: 2.5958 | LR: 3.19e-04 | 1910.2ms/step | 31.4 samples/s
537
+ Step 21210/50462 | Loss: 2.6054 | LR: 3.19e-04 | 1913.2ms/step | 31.4 samples/s
538
+ Step 21220/50462 | Loss: 2.6172 | LR: 3.18e-04 | 1912.2ms/step | 31.4 samples/s
539
+ Step 21230/50462 | Loss: 2.5785 | LR: 3.18e-04 | 1915.1ms/step | 31.3 samples/s
540
+ Step 21240/50462 | Loss: 2.5606 | LR: 3.18e-04 | 1913.6ms/step | 31.4 samples/s
541
+ Step 21250/50462 | Loss: 2.6152 | LR: 3.18e-04 | 1914.3ms/step | 31.3 samples/s
542
+ Validation at step 21250: Loss = 2.6296, Perplexity = 13.87
543
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-21250
544
+ Step 21260/50462 | Loss: 2.6061 | LR: 3.18e-04 | 3265.5ms/step | 18.4 samples/s
545
+ Step 21270/50462 | Loss: 2.5476 | LR: 3.18e-04 | 1897.7ms/step | 31.6 samples/s
546
+ Step 21280/50462 | Loss: 2.6041 | LR: 3.17e-04 | 1900.9ms/step | 31.6 samples/s
547
+ Step 21290/50462 | Loss: 2.6026 | LR: 3.17e-04 | 1906.4ms/step | 31.5 samples/s
548
+ Step 21300/50462 | Loss: 2.5826 | LR: 3.17e-04 | 1911.5ms/step | 31.4 samples/s
549
+ Step 21310/50462 | Loss: 2.5948 | LR: 3.17e-04 | 1911.0ms/step | 31.4 samples/s
550
+ Step 21320/50462 | Loss: 2.5679 | LR: 3.17e-04 | 1909.6ms/step | 31.4 samples/s
551
+ Step 21330/50462 | Loss: 2.6007 | LR: 3.17e-04 | 1911.2ms/step | 31.4 samples/s
552
+ Step 21340/50462 | Loss: 2.5999 | LR: 3.17e-04 | 1910.8ms/step | 31.4 samples/s
553
+ Step 21350/50462 | Loss: 2.6052 | LR: 3.16e-04 | 1912.4ms/step | 31.4 samples/s
554
+ Step 21360/50462 | Loss: 2.6335 | LR: 3.16e-04 | 1915.5ms/step | 31.3 samples/s
555
+ Step 21370/50462 | Loss: 2.5865 | LR: 3.16e-04 | 1913.3ms/step | 31.4 samples/s
556
+ Step 21380/50462 | Loss: 2.5503 | LR: 3.16e-04 | 1908.6ms/step | 31.4 samples/s
557
+ Step 21390/50462 | Loss: 2.6118 | LR: 3.16e-04 | 1911.9ms/step | 31.4 samples/s
558
+ Step 21400/50462 | Loss: 2.5877 | LR: 3.16e-04 | 1912.0ms/step | 31.4 samples/s
559
+ Step 21410/50462 | Loss: 2.5927 | LR: 3.16e-04 | 1912.7ms/step | 31.4 samples/s
560
+ Step 21420/50462 | Loss: 2.6199 | LR: 3.15e-04 | 1914.0ms/step | 31.3 samples/s
561
+ Step 21430/50462 | Loss: 2.6150 | LR: 3.15e-04 | 1913.2ms/step | 31.4 samples/s
562
+ Step 21440/50462 | Loss: 2.6397 | LR: 3.15e-04 | 1912.5ms/step | 31.4 samples/s
563
+ Step 21450/50462 | Loss: 2.6114 | LR: 3.15e-04 | 1914.9ms/step | 31.3 samples/s
564
+ Step 21460/50462 | Loss: 2.6277 | LR: 3.15e-04 | 1913.2ms/step | 31.4 samples/s
565
+ Step 21470/50462 | Loss: 2.5673 | LR: 3.15e-04 | 1914.2ms/step | 31.3 samples/s
566
+ Step 21480/50462 | Loss: 2.6101 | LR: 3.14e-04 | 1915.5ms/step | 31.3 samples/s
567
+ Step 21490/50462 | Loss: 2.5593 | LR: 3.14e-04 | 1916.1ms/step | 31.3 samples/s
568
+ Step 21500/50462 | Loss: 2.6084 | LR: 3.14e-04 | 1915.7ms/step | 31.3 samples/s
569
+ Validation at step 21500: Loss = 2.6263, Perplexity = 13.82
570
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-21500
571
+ Step 21510/50462 | Loss: 2.5897 | LR: 3.14e-04 | 3285.0ms/step | 18.3 samples/s
572
+ Step 21520/50462 | Loss: 2.6148 | LR: 3.14e-04 | 1904.0ms/step | 31.5 samples/s
573
+ Step 21530/50462 | Loss: 2.6033 | LR: 3.14e-04 | 1908.3ms/step | 31.4 samples/s
574
+ Step 21540/50462 | Loss: 2.5645 | LR: 3.14e-04 | 1909.9ms/step | 31.4 samples/s
575
+ Step 21550/50462 | Loss: 2.5595 | LR: 3.13e-04 | 1909.9ms/step | 31.4 samples/s
576
+ Step 21560/50462 | Loss: 2.6155 | LR: 3.13e-04 | 1911.5ms/step | 31.4 samples/s
577
+ Step 21570/50462 | Loss: 2.5999 | LR: 3.13e-04 | 1911.0ms/step | 31.4 samples/s
578
+ Step 21580/50462 | Loss: 2.5943 | LR: 3.13e-04 | 1910.6ms/step | 31.4 samples/s
579
+ Step 21590/50462 | Loss: 2.6093 | LR: 3.13e-04 | 1914.8ms/step | 31.3 samples/s
580
+ Step 21600/50462 | Loss: 2.5589 | LR: 3.13e-04 | 1909.6ms/step | 31.4 samples/s
581
+ Step 21610/50462 | Loss: 2.6043 | LR: 3.12e-04 | 1913.1ms/step | 31.4 samples/s
582
+ Step 21620/50462 | Loss: 2.5748 | LR: 3.12e-04 | 1911.6ms/step | 31.4 samples/s
583
+ Step 21630/50462 | Loss: 2.5820 | LR: 3.12e-04 | 1913.8ms/step | 31.4 samples/s
584
+ Step 21640/50462 | Loss: 2.5787 | LR: 3.12e-04 | 1912.0ms/step | 31.4 samples/s
585
+ Step 21650/50462 | Loss: 2.6078 | LR: 3.12e-04 | 1910.2ms/step | 31.4 samples/s
586
+ Step 21660/50462 | Loss: 2.5807 | LR: 3.12e-04 | 1911.8ms/step | 31.4 samples/s
587
+ Step 21670/50462 | Loss: 2.5712 | LR: 3.12e-04 | 1913.9ms/step | 31.3 samples/s
588
+ Step 21680/50462 | Loss: 2.5636 | LR: 3.11e-04 | 1914.7ms/step | 31.3 samples/s
589
+ Step 21690/50462 | Loss: 2.5645 | LR: 3.11e-04 | 1909.4ms/step | 31.4 samples/s
590
+ Step 21700/50462 | Loss: 2.5814 | LR: 3.11e-04 | 1913.8ms/step | 31.4 samples/s
591
+ Step 21710/50462 | Loss: 2.5667 | LR: 3.11e-04 | 1910.2ms/step | 31.4 samples/s
592
+ Step 21720/50462 | Loss: 2.6321 | LR: 3.11e-04 | 1910.0ms/step | 31.4 samples/s
593
+ Step 21730/50462 | Loss: 2.6153 | LR: 3.11e-04 | 1912.2ms/step | 31.4 samples/s
594
+ Step 21740/50462 | Loss: 2.5707 | LR: 3.10e-04 | 1914.2ms/step | 31.3 samples/s
595
+ Step 21750/50462 | Loss: 2.6216 | LR: 3.10e-04 | 1914.5ms/step | 31.3 samples/s
596
+ Validation at step 21750: Loss = 2.6228, Perplexity = 13.77
597
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-21750
598
+ Step 21760/50462 | Loss: 2.5854 | LR: 3.10e-04 | 3260.8ms/step | 18.4 samples/s
599
+ Step 21770/50462 | Loss: 2.6213 | LR: 3.10e-04 | 1903.0ms/step | 31.5 samples/s
600
+ Step 21780/50462 | Loss: 2.5668 | LR: 3.10e-04 | 1906.3ms/step | 31.5 samples/s
601
+ Step 21790/50462 | Loss: 2.6246 | LR: 3.10e-04 | 1907.8ms/step | 31.4 samples/s
602
+ Step 21800/50462 | Loss: 2.5686 | LR: 3.10e-04 | 1913.2ms/step | 31.4 samples/s
603
+ Step 21810/50462 | Loss: 2.5830 | LR: 3.09e-04 | 1912.2ms/step | 31.4 samples/s
604
+ Step 21820/50462 | Loss: 2.6037 | LR: 3.09e-04 | 1914.5ms/step | 31.3 samples/s
605
+ Step 21830/50462 | Loss: 2.5710 | LR: 3.09e-04 | 1913.2ms/step | 31.4 samples/s
606
+ Step 21840/50462 | Loss: 2.6223 | LR: 3.09e-04 | 1915.5ms/step | 31.3 samples/s
607
+ Step 21850/50462 | Loss: 2.6049 | LR: 3.09e-04 | 1915.3ms/step | 31.3 samples/s
608
+ Step 21860/50462 | Loss: 2.6264 | LR: 3.09e-04 | 1916.7ms/step | 31.3 samples/s
609
+ Step 21870/50462 | Loss: 2.6100 | LR: 3.08e-04 | 1914.8ms/step | 31.3 samples/s
610
+ Step 21880/50462 | Loss: 2.6026 | LR: 3.08e-04 | 1912.3ms/step | 31.4 samples/s
611
+ Step 21890/50462 | Loss: 2.5795 | LR: 3.08e-04 | 1913.4ms/step | 31.4 samples/s
612
+ Step 21900/50462 | Loss: 2.6781 | LR: 3.08e-04 | 1915.3ms/step | 31.3 samples/s
613
+ Step 21910/50462 | Loss: 2.5953 | LR: 3.08e-04 | 1911.7ms/step | 31.4 samples/s
614
+ Step 21920/50462 | Loss: 2.5793 | LR: 3.08e-04 | 1916.8ms/step | 31.3 samples/s
615
+ Step 21930/50462 | Loss: 2.5426 | LR: 3.08e-04 | 1914.9ms/step | 31.3 samples/s
616
+ Step 21940/50462 | Loss: 2.6012 | LR: 3.07e-04 | 1913.4ms/step | 31.4 samples/s
617
+ Step 21950/50462 | Loss: 2.5855 | LR: 3.07e-04 | 1913.1ms/step | 31.4 samples/s
618
+ Step 21960/50462 | Loss: 2.5717 | LR: 3.07e-04 | 1911.8ms/step | 31.4 samples/s
619
+ Step 21970/50462 | Loss: 2.5940 | LR: 3.07e-04 | 1913.5ms/step | 31.4 samples/s
620
+ Step 21980/50462 | Loss: 2.6124 | LR: 3.07e-04 | 1914.2ms/step | 31.3 samples/s
621
+ Step 21990/50462 | Loss: 2.6038 | LR: 3.07e-04 | 1914.0ms/step | 31.3 samples/s
622
+ Step 22000/50462 | Loss: 2.5808 | LR: 3.06e-04 | 1916.5ms/step | 31.3 samples/s
623
+ Validation at step 22000: Loss = 2.6206, Perplexity = 13.74
624
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-22000
625
+ Step 22010/50462 | Loss: 2.5784 | LR: 3.06e-04 | 3264.7ms/step | 18.4 samples/s
626
+ Step 22020/50462 | Loss: 2.6000 | LR: 3.06e-04 | 1901.3ms/step | 31.6 samples/s
627
+ Step 22030/50462 | Loss: 2.5627 | LR: 3.06e-04 | 1903.7ms/step | 31.5 samples/s
628
+ Step 22040/50462 | Loss: 2.6183 | LR: 3.06e-04 | 1908.0ms/step | 31.4 samples/s
629
+ Step 22050/50462 | Loss: 2.5786 | LR: 3.06e-04 | 1908.6ms/step | 31.4 samples/s
630
+ Step 22060/50462 | Loss: 2.5998 | LR: 3.06e-04 | 1912.2ms/step | 31.4 samples/s
631
+ Step 22070/50462 | Loss: 2.5676 | LR: 3.05e-04 | 1911.1ms/step | 31.4 samples/s
632
+ Step 22080/50462 | Loss: 2.5750 | LR: 3.05e-04 | 1912.2ms/step | 31.4 samples/s
633
+ Step 22090/50462 | Loss: 2.5975 | LR: 3.05e-04 | 1912.8ms/step | 31.4 samples/s
634
+ Step 22100/50462 | Loss: 2.5609 | LR: 3.05e-04 | 1911.8ms/step | 31.4 samples/s
635
+ Step 22110/50462 | Loss: 2.5555 | LR: 3.05e-04 | 1908.6ms/step | 31.4 samples/s
636
+ Step 22120/50462 | Loss: 2.6211 | LR: 3.05e-04 | 1913.2ms/step | 31.4 samples/s
637
+ Step 22130/50462 | Loss: 2.5927 | LR: 3.04e-04 | 1913.8ms/step | 31.4 samples/s
638
+ Step 22140/50462 | Loss: 2.5989 | LR: 3.04e-04 | 1912.5ms/step | 31.4 samples/s
639
+ Step 22150/50462 | Loss: 2.6044 | LR: 3.04e-04 | 1910.8ms/step | 31.4 samples/s
640
+ Step 22160/50462 | Loss: 2.5758 | LR: 3.04e-04 | 1912.2ms/step | 31.4 samples/s
641
+ Step 22170/50462 | Loss: 2.5628 | LR: 3.04e-04 | 1910.7ms/step | 31.4 samples/s
642
+ Step 22180/50462 | Loss: 2.6219 | LR: 3.04e-04 | 1912.4ms/step | 31.4 samples/s
643
+ Step 22190/50462 | Loss: 2.5603 | LR: 3.04e-04 | 1913.1ms/step | 31.4 samples/s
644
+ Step 22200/50462 | Loss: 2.5867 | LR: 3.03e-04 | 1912.6ms/step | 31.4 samples/s
645
+ Step 22210/50462 | Loss: 2.5994 | LR: 3.03e-04 | 1911.7ms/step | 31.4 samples/s
646
+ Step 22220/50462 | Loss: 2.5797 | LR: 3.03e-04 | 1911.5ms/step | 31.4 samples/s
647
+ Step 22230/50462 | Loss: 2.6290 | LR: 3.03e-04 | 1907.3ms/step | 31.5 samples/s
648
+ Step 22240/50462 | Loss: 2.5986 | LR: 3.03e-04 | 1909.6ms/step | 31.4 samples/s
649
+ Step 22250/50462 | Loss: 2.5890 | LR: 3.03e-04 | 1910.1ms/step | 31.4 samples/s
650
+ Validation at step 22250: Loss = 2.6200, Perplexity = 13.74
651
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-22250
652
+ Step 22260/50462 | Loss: 2.5626 | LR: 3.02e-04 | 3257.4ms/step | 18.4 samples/s
653
+ Step 22270/50462 | Loss: 2.5336 | LR: 3.02e-04 | 1898.4ms/step | 31.6 samples/s
654
+ Step 22280/50462 | Loss: 2.5648 | LR: 3.02e-04 | 1904.2ms/step | 31.5 samples/s
655
+ Step 22290/50462 | Loss: 2.5743 | LR: 3.02e-04 | 1907.0ms/step | 31.5 samples/s
656
+ Step 22300/50462 | Loss: 2.5389 | LR: 3.02e-04 | 1907.3ms/step | 31.5 samples/s
657
+ Step 22310/50462 | Loss: 2.6319 | LR: 3.02e-04 | 1910.4ms/step | 31.4 samples/s
658
+ Step 22320/50462 | Loss: 2.6025 | LR: 3.02e-04 | 1909.3ms/step | 31.4 samples/s
659
+ Step 22330/50462 | Loss: 2.5956 | LR: 3.01e-04 | 1912.2ms/step | 31.4 samples/s
660
+ Step 22340/50462 | Loss: 2.6151 | LR: 3.01e-04 | 1911.7ms/step | 31.4 samples/s
661
+ Step 22350/50462 | Loss: 2.6025 | LR: 3.01e-04 | 1913.8ms/step | 31.4 samples/s
662
+ Step 22360/50462 | Loss: 2.6332 | LR: 3.01e-04 | 1913.5ms/step | 31.4 samples/s
663
+ Step 22370/50462 | Loss: 2.5640 | LR: 3.01e-04 | 1912.1ms/step | 31.4 samples/s
664
+ Step 22380/50462 | Loss: 2.6128 | LR: 3.01e-04 | 1909.0ms/step | 31.4 samples/s
665
+ Step 22390/50462 | Loss: 2.6334 | LR: 3.00e-04 | 1911.7ms/step | 31.4 samples/s
666
+ Step 22400/50462 | Loss: 2.6103 | LR: 3.00e-04 | 1909.3ms/step | 31.4 samples/s
667
+ Step 22410/50462 | Loss: 2.5963 | LR: 3.00e-04 | 1911.9ms/step | 31.4 samples/s
668
+ Step 22420/50462 | Loss: 2.5977 | LR: 3.00e-04 | 1911.5ms/step | 31.4 samples/s
669
+ Step 22430/50462 | Loss: 2.6197 | LR: 3.00e-04 | 1912.5ms/step | 31.4 samples/s
670
+ Step 22440/50462 | Loss: 2.5731 | LR: 3.00e-04 | 1910.8ms/step | 31.4 samples/s
671
+ Step 22450/50462 | Loss: 2.6005 | LR: 3.00e-04 | 1911.5ms/step | 31.4 samples/s
672
+ Step 22460/50462 | Loss: 2.5848 | LR: 2.99e-04 | 1912.8ms/step | 31.4 samples/s
673
+ Step 22470/50462 | Loss: 2.5765 | LR: 2.99e-04 | 1911.0ms/step | 31.4 samples/s
674
+ Step 22480/50462 | Loss: 2.6070 | LR: 2.99e-04 | 1912.7ms/step | 31.4 samples/s
675
+ Step 22490/50462 | Loss: 2.5783 | LR: 2.99e-04 | 1913.1ms/step | 31.4 samples/s
676
+ Step 22500/50462 | Loss: 2.5851 | LR: 2.99e-04 | 1912.9ms/step | 31.4 samples/s
677
+ Validation at step 22500: Loss = 2.6157, Perplexity = 13.68
678
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-22500
679
+ Step 22510/50462 | Loss: 2.6120 | LR: 2.99e-04 | 3260.0ms/step | 18.4 samples/s
680
+ Step 22520/50462 | Loss: 2.5879 | LR: 2.98e-04 | 1899.8ms/step | 31.6 samples/s
681
+ Step 22530/50462 | Loss: 2.5628 | LR: 2.98e-04 | 1902.3ms/step | 31.5 samples/s
682
+ Step 22540/50462 | Loss: 2.5831 | LR: 2.98e-04 | 1905.7ms/step | 31.5 samples/s
683
+ Step 22550/50462 | Loss: 2.5618 | LR: 2.98e-04 | 1909.7ms/step | 31.4 samples/s
684
+ Step 22560/50462 | Loss: 2.5788 | LR: 2.98e-04 | 1910.7ms/step | 31.4 samples/s
685
+ Step 22570/50462 | Loss: 2.6069 | LR: 2.98e-04 | 1911.2ms/step | 31.4 samples/s
686
+ Step 22580/50462 | Loss: 2.5584 | LR: 2.98e-04 | 1912.0ms/step | 31.4 samples/s
687
+ Step 22590/50462 | Loss: 2.5822 | LR: 2.97e-04 | 1911.5ms/step | 31.4 samples/s
688
+ Step 22600/50462 | Loss: 2.5532 | LR: 2.97e-04 | 1912.1ms/step | 31.4 samples/s
689
+ Step 22610/50462 | Loss: 2.5660 | LR: 2.97e-04 | 1909.7ms/step | 31.4 samples/s
690
+ Step 22620/50462 | Loss: 2.5894 | LR: 2.97e-04 | 1910.6ms/step | 31.4 samples/s
691
+ Step 22630/50462 | Loss: 2.5561 | LR: 2.97e-04 | 1909.3ms/step | 31.4 samples/s
692
+ Step 22640/50462 | Loss: 2.5704 | LR: 2.97e-04 | 1910.9ms/step | 31.4 samples/s
693
+ Step 22650/50462 | Loss: 2.6014 | LR: 2.96e-04 | 1910.8ms/step | 31.4 samples/s
694
+ Step 22660/50462 | Loss: 2.5827 | LR: 2.96e-04 | 1912.0ms/step | 31.4 samples/s
695
+ Step 22670/50462 | Loss: 2.5833 | LR: 2.96e-04 | 1913.6ms/step | 31.4 samples/s
696
+ Step 22680/50462 | Loss: 2.5563 | LR: 2.96e-04 | 1909.4ms/step | 31.4 samples/s
697
+ Step 22690/50462 | Loss: 2.5963 | LR: 2.96e-04 | 1912.2ms/step | 31.4 samples/s
698
+ Step 22700/50462 | Loss: 2.5539 | LR: 2.96e-04 | 1907.7ms/step | 31.5 samples/s
699
+ Step 22710/50462 | Loss: 2.6261 | LR: 2.95e-04 | 1905.1ms/step | 31.5 samples/s
700
+ Step 22720/50462 | Loss: 2.5782 | LR: 2.95e-04 | 1906.8ms/step | 31.5 samples/s
701
+ Step 22730/50462 | Loss: 2.5867 | LR: 2.95e-04 | 1911.9ms/step | 31.4 samples/s
702
+ Step 22740/50462 | Loss: 2.6066 | LR: 2.95e-04 | 1911.4ms/step | 31.4 samples/s
703
+ Step 22750/50462 | Loss: 2.5653 | LR: 2.95e-04 | 1910.0ms/step | 31.4 samples/s
704
+ Validation at step 22750: Loss = 2.6133, Perplexity = 13.64
705
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-22750
706
+ Step 22760/50462 | Loss: 2.5910 | LR: 2.95e-04 | 3267.4ms/step | 18.4 samples/s
707
+ Step 22770/50462 | Loss: 2.5562 | LR: 2.95e-04 | 1897.4ms/step | 31.6 samples/s
708
+ Step 22780/50462 | Loss: 2.5692 | LR: 2.94e-04 | 1903.7ms/step | 31.5 samples/s
709
+ Step 22790/50462 | Loss: 2.5828 | LR: 2.94e-04 | 1909.4ms/step | 31.4 samples/s
710
+ Step 22800/50462 | Loss: 2.5835 | LR: 2.94e-04 | 1906.8ms/step | 31.5 samples/s
711
+ Step 22810/50462 | Loss: 2.5724 | LR: 2.94e-04 | 1908.8ms/step | 31.4 samples/s
712
+ Step 22820/50462 | Loss: 2.5116 | LR: 2.94e-04 | 1909.6ms/step | 31.4 samples/s
713
+ Step 22830/50462 | Loss: 2.5620 | LR: 2.94e-04 | 1910.9ms/step | 31.4 samples/s
714
+ Step 22840/50462 | Loss: 2.5921 | LR: 2.93e-04 | 1910.9ms/step | 31.4 samples/s
715
+ Step 22850/50462 | Loss: 2.5268 | LR: 2.93e-04 | 1908.5ms/step | 31.4 samples/s
716
+ Step 22860/50462 | Loss: 2.5401 | LR: 2.93e-04 | 1910.3ms/step | 31.4 samples/s
717
+ Step 22870/50462 | Loss: 2.6028 | LR: 2.93e-04 | 1910.9ms/step | 31.4 samples/s
718
+ Step 22880/50462 | Loss: 2.5955 | LR: 2.93e-04 | 1911.5ms/step | 31.4 samples/s
719
+ Step 22890/50462 | Loss: 2.5594 | LR: 2.93e-04 | 1912.3ms/step | 31.4 samples/s
720
+ Step 22900/50462 | Loss: 2.6080 | LR: 2.93e-04 | 1912.5ms/step | 31.4 samples/s
721
+ Step 22910/50462 | Loss: 2.5633 | LR: 2.92e-04 | 1910.1ms/step | 31.4 samples/s
722
+ Step 22920/50462 | Loss: 2.5698 | LR: 2.92e-04 | 1913.7ms/step | 31.4 samples/s
723
+ Step 22930/50462 | Loss: 2.5954 | LR: 2.92e-04 | 1908.6ms/step | 31.4 samples/s
724
+ Step 22940/50462 | Loss: 2.5652 | LR: 2.92e-04 | 1912.4ms/step | 31.4 samples/s
725
+ Step 22950/50462 | Loss: 2.5481 | LR: 2.92e-04 | 1912.8ms/step | 31.4 samples/s
726
+ Step 22960/50462 | Loss: 2.5894 | LR: 2.92e-04 | 1914.7ms/step | 31.3 samples/s
727
+ Step 22970/50462 | Loss: 2.5522 | LR: 2.91e-04 | 1911.7ms/step | 31.4 samples/s
728
+ Step 22980/50462 | Loss: 2.5783 | LR: 2.91e-04 | 1909.8ms/step | 31.4 samples/s
729
+ Step 22990/50462 | Loss: 2.5354 | LR: 2.91e-04 | 1909.9ms/step | 31.4 samples/s
730
+ Step 23000/50462 | Loss: 2.5598 | LR: 2.91e-04 | 1912.4ms/step | 31.4 samples/s
731
+ Validation at step 23000: Loss = 2.6102, Perplexity = 13.60
732
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-23000
733
+ Step 23010/50462 | Loss: 2.5902 | LR: 2.91e-04 | 3269.2ms/step | 18.4 samples/s
734
+ Step 23020/50462 | Loss: 2.5613 | LR: 2.91e-04 | 1903.0ms/step | 31.5 samples/s
735
+ Step 23030/50462 | Loss: 2.5312 | LR: 2.91e-04 | 1907.9ms/step | 31.4 samples/s
736
+ Step 23040/50462 | Loss: 2.5884 | LR: 2.90e-04 | 1911.5ms/step | 31.4 samples/s
737
+ Step 23050/50462 | Loss: 2.5547 | LR: 2.90e-04 | 1912.4ms/step | 31.4 samples/s
738
+ Step 23060/50462 | Loss: 2.5554 | LR: 2.90e-04 | 1911.7ms/step | 31.4 samples/s
739
+ Step 23070/50462 | Loss: 2.6217 | LR: 2.90e-04 | 1909.4ms/step | 31.4 samples/s
740
+ Step 23080/50462 | Loss: 2.6150 | LR: 2.90e-04 | 1908.8ms/step | 31.4 samples/s
741
+ Step 23090/50462 | Loss: 2.5773 | LR: 2.90e-04 | 1909.4ms/step | 31.4 samples/s
742
+ Step 23100/50462 | Loss: 2.5547 | LR: 2.89e-04 | 1910.2ms/step | 31.4 samples/s
743
+ Step 23110/50462 | Loss: 2.5634 | LR: 2.89e-04 | 1912.2ms/step | 31.4 samples/s
744
+ Step 23120/50462 | Loss: 2.5377 | LR: 2.89e-04 | 1912.0ms/step | 31.4 samples/s
745
+ Step 23130/50462 | Loss: 2.5744 | LR: 2.89e-04 | 1912.5ms/step | 31.4 samples/s
746
+ Step 23140/50462 | Loss: 2.5763 | LR: 2.89e-04 | 1909.5ms/step | 31.4 samples/s
747
+ Step 23150/50462 | Loss: 2.5559 | LR: 2.89e-04 | 1911.5ms/step | 31.4 samples/s
748
+ Step 23160/50462 | Loss: 2.6127 | LR: 2.88e-04 | 1909.1ms/step | 31.4 samples/s
749
+ Step 23170/50462 | Loss: 2.5841 | LR: 2.88e-04 | 1911.0ms/step | 31.4 samples/s
750
+ Step 23180/50462 | Loss: 2.5839 | LR: 2.88e-04 | 1912.1ms/step | 31.4 samples/s
751
+ Step 23190/50462 | Loss: 2.5858 | LR: 2.88e-04 | 1915.0ms/step | 31.3 samples/s
752
+ Step 23200/50462 | Loss: 2.5739 | LR: 2.88e-04 | 1910.9ms/step | 31.4 samples/s
753
+ Step 23210/50462 | Loss: 2.5684 | LR: 2.88e-04 | 1913.2ms/step | 31.4 samples/s
754
+ Step 23220/50462 | Loss: 2.5615 | LR: 2.88e-04 | 1914.9ms/step | 31.3 samples/s
755
+ Step 23230/50462 | Loss: 2.6217 | LR: 2.87e-04 | 1910.8ms/step | 31.4 samples/s
756
+ Step 23240/50462 | Loss: 2.5796 | LR: 2.87e-04 | 1913.4ms/step | 31.4 samples/s
757
+ Step 23250/50462 | Loss: 2.5750 | LR: 2.87e-04 | 1911.5ms/step | 31.4 samples/s
758
+ Validation at step 23250: Loss = 2.6094, Perplexity = 13.59
759
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-23250
760
+ Step 23260/50462 | Loss: 2.5647 | LR: 2.87e-04 | 3254.7ms/step | 18.4 samples/s
761
+ Step 23270/50462 | Loss: 2.5711 | LR: 2.87e-04 | 1899.4ms/step | 31.6 samples/s
762
+ Step 23280/50462 | Loss: 2.5946 | LR: 2.87e-04 | 1904.2ms/step | 31.5 samples/s
763
+ Step 23290/50462 | Loss: 2.5627 | LR: 2.86e-04 | 1905.5ms/step | 31.5 samples/s
764
+ Step 23300/50462 | Loss: 2.5248 | LR: 2.86e-04 | 1911.3ms/step | 31.4 samples/s
765
+ Step 23310/50462 | Loss: 2.5599 | LR: 2.86e-04 | 1912.7ms/step | 31.4 samples/s
766
+ Step 23320/50462 | Loss: 2.5500 | LR: 2.86e-04 | 1913.3ms/step | 31.4 samples/s
767
+ Step 23330/50462 | Loss: 2.5628 | LR: 2.86e-04 | 1912.1ms/step | 31.4 samples/s
768
+ Step 23340/50462 | Loss: 2.5104 | LR: 2.86e-04 | 1913.8ms/step | 31.4 samples/s
769
+ Step 23350/50462 | Loss: 2.5524 | LR: 2.86e-04 | 1911.7ms/step | 31.4 samples/s
770
+ Step 23360/50462 | Loss: 2.6158 | LR: 2.85e-04 | 1912.3ms/step | 31.4 samples/s
771
+ Step 23370/50462 | Loss: 2.5232 | LR: 2.85e-04 | 1914.0ms/step | 31.3 samples/s
772
+ Step 23380/50462 | Loss: 2.6036 | LR: 2.85e-04 | 1913.0ms/step | 31.4 samples/s
773
+ Step 23390/50462 | Loss: 2.5839 | LR: 2.85e-04 | 1912.2ms/step | 31.4 samples/s
774
+ Step 23400/50462 | Loss: 2.5861 | LR: 2.85e-04 | 1913.6ms/step | 31.4 samples/s
775
+ Step 23410/50462 | Loss: 2.5523 | LR: 2.85e-04 | 1913.7ms/step | 31.4 samples/s
776
+ Step 23420/50462 | Loss: 2.5103 | LR: 2.84e-04 | 1911.1ms/step | 31.4 samples/s
777
+ Step 23430/50462 | Loss: 2.5597 | LR: 2.84e-04 | 1910.1ms/step | 31.4 samples/s
778
+ Step 23440/50462 | Loss: 2.5224 | LR: 2.84e-04 | 1911.9ms/step | 31.4 samples/s
779
+ Step 23450/50462 | Loss: 2.5733 | LR: 2.84e-04 | 1913.9ms/step | 31.3 samples/s
780
+ Step 23460/50462 | Loss: 2.5448 | LR: 2.84e-04 | 1914.0ms/step | 31.3 samples/s
781
+ Step 23470/50462 | Loss: 2.5773 | LR: 2.84e-04 | 1914.4ms/step | 31.3 samples/s
782
+ Step 23480/50462 | Loss: 2.5813 | LR: 2.83e-04 | 1913.9ms/step | 31.3 samples/s
783
+ Step 23490/50462 | Loss: 2.5477 | LR: 2.83e-04 | 1914.2ms/step | 31.3 samples/s
784
+ Step 23500/50462 | Loss: 2.5776 | LR: 2.83e-04 | 1913.8ms/step | 31.4 samples/s
785
+ Validation at step 23500: Loss = 2.6068, Perplexity = 13.56
786
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-23500
787
+ Step 23510/50462 | Loss: 2.6061 | LR: 2.83e-04 | 3258.1ms/step | 18.4 samples/s
788
+ Step 23520/50462 | Loss: 2.5557 | LR: 2.83e-04 | 1896.9ms/step | 31.6 samples/s
789
+ Step 23530/50462 | Loss: 2.5842 | LR: 2.83e-04 | 1902.4ms/step | 31.5 samples/s
epfl_final/train_logs/p2_1p2b_dense_seed72_6gpu_resume23500.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/train_logs/p2_1p2b_dense_seed72_6gpu_resume23500.pid ADDED
@@ -0,0 +1 @@
 
 
1
+ 210
epfl_final/train_logs/p2_1p2b_dense_seed72_6gpu_resume31000.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/train_logs/p2_1p2b_dense_seed72_6gpu_resume44250.log ADDED
@@ -0,0 +1,606 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ *****************************************
3
+ Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
4
+ *****************************************
5
+ Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250
6
+ Running config:
7
+ {
8
+ "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
9
+ "model_dtype": "bfloat16",
10
+ "init_from_scratch": true,
11
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
12
+ "val_path": null,
13
+ "max_seq_length": 8192,
14
+ "num_workers": 8,
15
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250",
16
+ "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-44250",
17
+ "force_new_run": false,
18
+ "num_epochs": 1,
19
+ "max_steps": 50462,
20
+ "per_device_batch_size": 1,
21
+ "gradient_accumulation_steps": 10,
22
+ "learning_rate": 0.0005,
23
+ "weight_decay": 0.01,
24
+ "warmup_steps": 750,
25
+ "lr_scheduler": "cosine",
26
+ "max_grad_norm": 1.0,
27
+ "save_steps": 250,
28
+ "eval_steps": 250,
29
+ "eval_batches": 50,
30
+ "logging_steps": 10,
31
+ "seed": 72,
32
+ "verbose": true,
33
+ "checkpoint_activations": false,
34
+ "resume_micro_step_override": 442500,
35
+ "strategy": "ddp",
36
+ "mixed_precision": "bf16",
37
+ "wandb_enabled": false,
38
+ "wandb_project": "stochastic_sparsity",
39
+ "wandb_name": "1.2b dense 8k seed72 6gpu resume44250",
40
+ "wandb_tags": [
41
+ "qwen3",
42
+ "1.2b",
43
+ "8k",
44
+ "p2",
45
+ "dense",
46
+ "seed72"
47
+ ],
48
+ "hidden_size": 1536,
49
+ "intermediate_size": 6144,
50
+ "num_attention_heads": 12,
51
+ "num_key_value_heads": 4,
52
+ "num_hidden_layers": 28,
53
+ "head_dim": null,
54
+ "attention_dropout": null,
55
+ "rope_train_position_scale_set": null,
56
+ "attention_type": "sparse_stochastic",
57
+ "sparse_block_size": 128,
58
+ "sparse_num_random_blocks": 31,
59
+ "sparse_mask_pool_size": 1,
60
+ "sparse_val_dense_compare": false,
61
+ "sparse_layer_indices": null,
62
+ "sparse_mask_cache_dir": null,
63
+ "sparse_mask_seed": 0,
64
+ "sparse_mixed_nrb_set": null,
65
+ "dense_mode": true,
66
+ "dense_step_intervals": null
67
+ }
68
+ Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
69
+ Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-44250
70
+ Checkpoint activations: FalseCheckpoint activations: FalseCheckpoint activations: FalseCheckpoint activations: False
71
+
72
+
73
+
74
+ Checkpoint activations: False
75
+ Checkpoint activations: False
76
+ dense_mode=True: sparse attention layers will use standard causal SDPA.
77
+
78
+
79
+ ================================================================================
80
+ MODEL ARCHITECTURE
81
+ ================================================================================
82
+ Model config:
83
+ vocab_size: 151936
84
+ hidden_size: 1536
85
+ num_hidden_layers: 28
86
+ num_attention_heads: 12
87
+ num_key_value_heads: 4
88
+ head_dim: 128
89
+ intermediate_size: 6144
90
+ max_position_embeddings: 32768
91
+ rope_theta: 1000000
92
+ rope_train_position_scale_set: None
93
+ attention_type: sparse_stochastic
94
+ sparse_block_size: 128
95
+ sparse_num_random_blocks: 31
96
+ sparse_mask_pool_size: 1
97
+ sparse_max_seq_length: 8192
98
+
99
+ Parameter count:
100
+ Total parameters: 1,202,352,640
101
+ Trainable parameters: 1,202,352,640
102
+ Non-trainable parameters: 0
103
+ Model size (BF16): 2.40 GB
104
+
105
+ --------------------------------------------------------------------------------
106
+ LAYER-BY-LAYER SUMMARY
107
+ --------------------------------------------------------------------------------
108
+ ================================================================================================================================================================
109
+ Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
110
+ ================================================================================================================================================================
111
+ Qwen3ForCausalLM [1, 8192] -- -- True
112
+ ├─Qwen3Model: 1-1 -- [1, 8192, 1536] -- True
113
+ │ └─Embedding: 2-1 [1, 8192] [1, 8192, 1536] 233,373,696 True
114
+ │ └─ModuleList: 2-2 -- -- -- True
115
+ │ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
116
+ │ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
117
+ │ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
118
+ │ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
119
+ │ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
120
+ │ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
121
+ │ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
122
+ │ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
123
+ │ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
124
+ │ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
125
+ │ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
126
+ │ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
127
+ │ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
128
+ │ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
129
+ │ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
130
+ │ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
131
+ │ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
132
+ │ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
133
+ │ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
134
+ │ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
135
+ │ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
136
+ │ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
137
+ │ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
138
+ │ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
139
+ │ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
140
+ │ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
141
+ │ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
142
+ │ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
143
+ │ └─Qwen3RMSNorm: 2-3 [1, 8192, 1536] [1, 8192, 1536] 1,536 True
144
+ ├─Linear: 1-2 [1, 8192, 1536] [1, 8192, 151936] 233,373,696 True
145
+ ================================================================================================================================================================
146
+ Total params: 1,435,726,336
147
+ Trainable params: 1,435,726,336
148
+ Non-trainable params: 0
149
+ Total mult-adds (Units.GIGABYTES): 1.44
150
+ ================================================================================================================================================================
151
+ Input size (MB): 0.07
152
+ Forward/backward pass size (MB): 26218.59
153
+ Params size (MB): 2871.45
154
+ Estimated Total Size (MB): 29090.11
155
+ ================================================================================================================================================================
156
+ NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
157
+ ================================================================================
158
+
159
+ Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
160
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
161
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
162
+ Using pre-tokenized dataset (no runtime tokenization)
163
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
164
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
165
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
166
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
167
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
168
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
169
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
170
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
171
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
172
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
173
+ Dataset split: 3027764 train, 159356 validation
174
+ Training examples (packed, train split): 3027764
175
+ Validation examples (packed, val split): 159356
176
+ Per-device batch size: 1
177
+ World size (GPUs): 6
178
+ Gradient accumulation steps: 10
179
+ Effective batch size (seqs/step): 60
180
+ Sequence length: 8192
181
+ Tokens per optimizer step: 491,520
182
+ Micro-batches per epoch (per rank): 504628
183
+ Optimizer steps per epoch: 50462
184
+ Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
185
+ Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-44250...
186
+ Preserving checkpoint LR state for exact resume: current=[1.9018026712523283e-05], schedule_base=[0.0005]
187
+ Resumed from checkpoint at step 44250 (epoch 0, micro_step 442500)
188
+ Overriding resume micro-step 442500 -> 442500 for DDP migration
189
+ Resuming training from step 44250
190
+ Starting training...
191
+ Validation at step 44250: Loss = 2.5311, Perplexity = 12.57
192
+ Switching attention mode for step 44251: dense (dense step interval)
193
+ Step 44260/50462 | Loss: 2.4875 | LR: 1.90e-05 | 62466.2ms/step | 1.0 samples/s
194
+ Step 44270/50462 | Loss: 2.4649 | LR: 1.89e-05 | 1898.4ms/step | 31.6 samples/s
195
+ Step 44280/50462 | Loss: 2.5022 | LR: 1.88e-05 | 1897.8ms/step | 31.6 samples/s
196
+ Step 44290/50462 | Loss: 2.4772 | LR: 1.88e-05 | 1899.5ms/step | 31.6 samples/s
197
+ Step 44300/50462 | Loss: 2.4839 | LR: 1.87e-05 | 1903.4ms/step | 31.5 samples/s
198
+ Step 44310/50462 | Loss: 2.4613 | LR: 1.87e-05 | 1905.5ms/step | 31.5 samples/s
199
+ Step 44320/50462 | Loss: 2.4983 | LR: 1.86e-05 | 1905.3ms/step | 31.5 samples/s
200
+ Step 44330/50462 | Loss: 2.5037 | LR: 1.85e-05 | 1906.2ms/step | 31.5 samples/s
201
+ Step 44340/50462 | Loss: 2.4928 | LR: 1.85e-05 | 1907.7ms/step | 31.5 samples/s
202
+ Step 44350/50462 | Loss: 2.4678 | LR: 1.84e-05 | 1906.1ms/step | 31.5 samples/s
203
+ Step 44360/50462 | Loss: 2.5009 | LR: 1.84e-05 | 1907.7ms/step | 31.5 samples/s
204
+ Step 44370/50462 | Loss: 2.4825 | LR: 1.83e-05 | 1908.3ms/step | 31.4 samples/s
205
+ Step 44380/50462 | Loss: 2.5132 | LR: 1.82e-05 | 1908.0ms/step | 31.4 samples/s
206
+ Step 44390/50462 | Loss: 2.5091 | LR: 1.82e-05 | 1910.0ms/step | 31.4 samples/s
207
+ Step 44400/50462 | Loss: 2.5162 | LR: 1.81e-05 | 1910.0ms/step | 31.4 samples/s
208
+ Step 44410/50462 | Loss: 2.5072 | LR: 1.81e-05 | 1910.0ms/step | 31.4 samples/s
209
+ Step 44420/50462 | Loss: 2.5269 | LR: 1.80e-05 | 1908.2ms/step | 31.4 samples/s
210
+ Step 44430/50462 | Loss: 2.5012 | LR: 1.79e-05 | 1909.6ms/step | 31.4 samples/s
211
+ Step 44440/50462 | Loss: 2.5153 | LR: 1.79e-05 | 1908.7ms/step | 31.4 samples/s
212
+ Step 44450/50462 | Loss: 2.4780 | LR: 1.78e-05 | 1911.6ms/step | 31.4 samples/s
213
+ Step 44460/50462 | Loss: 2.4922 | LR: 1.78e-05 | 1912.5ms/step | 31.4 samples/s
214
+ Step 44470/50462 | Loss: 2.4626 | LR: 1.77e-05 | 1912.7ms/step | 31.4 samples/s
215
+ Step 44480/50462 | Loss: 2.4807 | LR: 1.77e-05 | 1911.3ms/step | 31.4 samples/s
216
+ Step 44490/50462 | Loss: 2.4760 | LR: 1.76e-05 | 1912.6ms/step | 31.4 samples/s
217
+ Step 44500/50462 | Loss: 2.5024 | LR: 1.75e-05 | 1910.7ms/step | 31.4 samples/s
218
+ Validation at step 44500: Loss = 2.5311, Perplexity = 12.57
219
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-44500
220
+ Step 44510/50462 | Loss: 2.4874 | LR: 1.75e-05 | 3281.2ms/step | 18.3 samples/s
221
+ Step 44520/50462 | Loss: 2.5196 | LR: 1.74e-05 | 1909.0ms/step | 31.4 samples/s
222
+ Step 44530/50462 | Loss: 2.5086 | LR: 1.74e-05 | 1911.7ms/step | 31.4 samples/s
223
+ Step 44540/50462 | Loss: 2.5274 | LR: 1.73e-05 | 1915.1ms/step | 31.3 samples/s
224
+ Step 44550/50462 | Loss: 2.4965 | LR: 1.72e-05 | 1915.0ms/step | 31.3 samples/s
225
+ Step 44560/50462 | Loss: 2.4621 | LR: 1.72e-05 | 1913.9ms/step | 31.4 samples/s
226
+ Step 44570/50462 | Loss: 2.4800 | LR: 1.71e-05 | 1914.5ms/step | 31.3 samples/s
227
+ Step 44580/50462 | Loss: 2.4736 | LR: 1.71e-05 | 1915.3ms/step | 31.3 samples/s
228
+ Step 44590/50462 | Loss: 2.5194 | LR: 1.70e-05 | 1916.2ms/step | 31.3 samples/s
229
+ Step 44600/50462 | Loss: 2.4914 | LR: 1.70e-05 | 1915.0ms/step | 31.3 samples/s
230
+ Step 44610/50462 | Loss: 2.4897 | LR: 1.69e-05 | 1914.3ms/step | 31.3 samples/s
231
+ Step 44620/50462 | Loss: 2.4309 | LR: 1.68e-05 | 1916.1ms/step | 31.3 samples/s
232
+ Step 44630/50462 | Loss: 2.4868 | LR: 1.68e-05 | 1914.5ms/step | 31.3 samples/s
233
+ Step 44640/50462 | Loss: 2.4832 | LR: 1.67e-05 | 1914.1ms/step | 31.3 samples/s
234
+ Step 44650/50462 | Loss: 2.5239 | LR: 1.67e-05 | 1913.8ms/step | 31.4 samples/s
235
+ Step 44660/50462 | Loss: 2.5081 | LR: 1.66e-05 | 1916.4ms/step | 31.3 samples/s
236
+ Step 44670/50462 | Loss: 2.4998 | LR: 1.66e-05 | 1915.7ms/step | 31.3 samples/s
237
+ Step 44680/50462 | Loss: 2.4819 | LR: 1.65e-05 | 1918.2ms/step | 31.3 samples/s
238
+ Step 44690/50462 | Loss: 2.4999 | LR: 1.64e-05 | 1916.6ms/step | 31.3 samples/s
239
+ Step 44700/50462 | Loss: 2.4991 | LR: 1.64e-05 | 1916.5ms/step | 31.3 samples/s
240
+ Step 44710/50462 | Loss: 2.5074 | LR: 1.63e-05 | 1914.3ms/step | 31.3 samples/s
241
+ Step 44720/50462 | Loss: 2.4654 | LR: 1.63e-05 | 1916.3ms/step | 31.3 samples/s
242
+ Step 44730/50462 | Loss: 2.4763 | LR: 1.62e-05 | 1914.7ms/step | 31.3 samples/s
243
+ Step 44740/50462 | Loss: 2.4709 | LR: 1.62e-05 | 1914.2ms/step | 31.3 samples/s
244
+ Step 44750/50462 | Loss: 2.4729 | LR: 1.61e-05 | 1915.1ms/step | 31.3 samples/s
245
+ Validation at step 44750: Loss = 2.5309, Perplexity = 12.57
246
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-44750
247
+ Step 44760/50462 | Loss: 2.4883 | LR: 1.61e-05 | 3239.9ms/step | 18.5 samples/s
248
+ Step 44770/50462 | Loss: 2.4270 | LR: 1.60e-05 | 1908.8ms/step | 31.4 samples/s
249
+ Step 44780/50462 | Loss: 2.5161 | LR: 1.59e-05 | 1909.3ms/step | 31.4 samples/s
250
+ Step 44790/50462 | Loss: 2.4766 | LR: 1.59e-05 | 1911.8ms/step | 31.4 samples/s
251
+ Step 44800/50462 | Loss: 2.4866 | LR: 1.58e-05 | 1914.6ms/step | 31.3 samples/s
252
+ Step 44810/50462 | Loss: 2.4742 | LR: 1.58e-05 | 1915.7ms/step | 31.3 samples/s
253
+ Step 44820/50462 | Loss: 2.5088 | LR: 1.57e-05 | 1914.8ms/step | 31.3 samples/s
254
+ Step 44830/50462 | Loss: 2.4567 | LR: 1.57e-05 | 1914.3ms/step | 31.3 samples/s
255
+ Step 44840/50462 | Loss: 2.4769 | LR: 1.56e-05 | 1914.3ms/step | 31.3 samples/s
256
+ Step 44850/50462 | Loss: 2.4991 | LR: 1.56e-05 | 1917.3ms/step | 31.3 samples/s
257
+ Step 44860/50462 | Loss: 2.4622 | LR: 1.55e-05 | 1916.7ms/step | 31.3 samples/s
258
+ Step 44870/50462 | Loss: 2.5112 | LR: 1.54e-05 | 1918.2ms/step | 31.3 samples/s
259
+ Step 44880/50462 | Loss: 2.5396 | LR: 1.54e-05 | 1913.5ms/step | 31.4 samples/s
260
+ Step 44890/50462 | Loss: 2.4590 | LR: 1.53e-05 | 1917.0ms/step | 31.3 samples/s
261
+ Step 44900/50462 | Loss: 2.4542 | LR: 1.53e-05 | 1914.9ms/step | 31.3 samples/s
262
+ Step 44910/50462 | Loss: 2.5461 | LR: 1.52e-05 | 1915.8ms/step | 31.3 samples/s
263
+ Step 44920/50462 | Loss: 2.4996 | LR: 1.52e-05 | 1914.3ms/step | 31.3 samples/s
264
+ Step 44930/50462 | Loss: 2.4740 | LR: 1.51e-05 | 1915.1ms/step | 31.3 samples/s
265
+ Step 44940/50462 | Loss: 2.4627 | LR: 1.51e-05 | 1916.2ms/step | 31.3 samples/s
266
+ Step 44950/50462 | Loss: 2.5079 | LR: 1.50e-05 | 1915.4ms/step | 31.3 samples/s
267
+ Step 44960/50462 | Loss: 2.5162 | LR: 1.50e-05 | 1914.7ms/step | 31.3 samples/s
268
+ Step 44970/50462 | Loss: 2.4912 | LR: 1.49e-05 | 1916.2ms/step | 31.3 samples/s
269
+ Step 44980/50462 | Loss: 2.5217 | LR: 1.49e-05 | 1916.0ms/step | 31.3 samples/s
270
+ Step 44990/50462 | Loss: 2.4743 | LR: 1.48e-05 | 1915.5ms/step | 31.3 samples/s
271
+ Step 45000/50462 | Loss: 2.5047 | LR: 1.47e-05 | 1915.6ms/step | 31.3 samples/s
272
+ Validation at step 45000: Loss = 2.5310, Perplexity = 12.57
273
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-45000
274
+ Step 45010/50462 | Loss: 2.4862 | LR: 1.47e-05 | 3227.1ms/step | 18.6 samples/s
275
+ Step 45020/50462 | Loss: 2.5085 | LR: 1.46e-05 | 1908.5ms/step | 31.4 samples/s
276
+ Step 45030/50462 | Loss: 2.4635 | LR: 1.46e-05 | 1909.8ms/step | 31.4 samples/s
277
+ Step 45040/50462 | Loss: 2.5551 | LR: 1.45e-05 | 1910.7ms/step | 31.4 samples/s
278
+ Step 45050/50462 | Loss: 2.5438 | LR: 1.45e-05 | 1909.9ms/step | 31.4 samples/s
279
+ Step 45060/50462 | Loss: 2.4936 | LR: 1.44e-05 | 1912.8ms/step | 31.4 samples/s
280
+ Step 45070/50462 | Loss: 2.5248 | LR: 1.44e-05 | 1911.8ms/step | 31.4 samples/s
281
+ Step 45080/50462 | Loss: 2.5043 | LR: 1.43e-05 | 1915.2ms/step | 31.3 samples/s
282
+ Step 45090/50462 | Loss: 2.5272 | LR: 1.43e-05 | 1916.9ms/step | 31.3 samples/s
283
+ Step 45100/50462 | Loss: 2.5303 | LR: 1.42e-05 | 1914.3ms/step | 31.3 samples/s
284
+ Step 45110/50462 | Loss: 2.5453 | LR: 1.42e-05 | 1914.3ms/step | 31.3 samples/s
285
+ Step 45120/50462 | Loss: 2.5147 | LR: 1.41e-05 | 1916.0ms/step | 31.3 samples/s
286
+ Step 45130/50462 | Loss: 2.5259 | LR: 1.41e-05 | 1913.0ms/step | 31.4 samples/s
287
+ Step 45140/50462 | Loss: 2.4699 | LR: 1.40e-05 | 1912.5ms/step | 31.4 samples/s
288
+ Step 45150/50462 | Loss: 2.5238 | LR: 1.40e-05 | 1915.1ms/step | 31.3 samples/s
289
+ Step 45160/50462 | Loss: 2.4760 | LR: 1.39e-05 | 1914.2ms/step | 31.3 samples/s
290
+ Step 45170/50462 | Loss: 2.5286 | LR: 1.39e-05 | 1914.0ms/step | 31.3 samples/s
291
+ Step 45180/50462 | Loss: 2.4917 | LR: 1.38e-05 | 1916.1ms/step | 31.3 samples/s
292
+ Step 45190/50462 | Loss: 2.5112 | LR: 1.37e-05 | 1917.9ms/step | 31.3 samples/s
293
+ Step 45200/50462 | Loss: 2.5001 | LR: 1.37e-05 | 1915.1ms/step | 31.3 samples/s
294
+ Step 45210/50462 | Loss: 2.4910 | LR: 1.36e-05 | 1916.3ms/step | 31.3 samples/s
295
+ Step 45220/50462 | Loss: 2.4571 | LR: 1.36e-05 | 1916.7ms/step | 31.3 samples/s
296
+ Step 45230/50462 | Loss: 2.5052 | LR: 1.35e-05 | 1914.5ms/step | 31.3 samples/s
297
+ Step 45240/50462 | Loss: 2.4740 | LR: 1.35e-05 | 1917.3ms/step | 31.3 samples/s
298
+ Step 45250/50462 | Loss: 2.5015 | LR: 1.34e-05 | 1915.3ms/step | 31.3 samples/s
299
+ Validation at step 45250: Loss = 2.5309, Perplexity = 12.56
300
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-45250
301
+ Step 45260/50462 | Loss: 2.4924 | LR: 1.34e-05 | 3232.6ms/step | 18.6 samples/s
302
+ Step 45270/50462 | Loss: 2.5233 | LR: 1.33e-05 | 1904.3ms/step | 31.5 samples/s
303
+ Step 45280/50462 | Loss: 2.5402 | LR: 1.33e-05 | 1909.8ms/step | 31.4 samples/s
304
+ Step 45290/50462 | Loss: 2.4720 | LR: 1.32e-05 | 1911.7ms/step | 31.4 samples/s
305
+ Step 45300/50462 | Loss: 2.5193 | LR: 1.32e-05 | 1913.0ms/step | 31.4 samples/s
306
+ Step 45310/50462 | Loss: 2.4674 | LR: 1.31e-05 | 1916.6ms/step | 31.3 samples/s
307
+ Step 45320/50462 | Loss: 2.4957 | LR: 1.31e-05 | 1912.5ms/step | 31.4 samples/s
308
+ Step 45330/50462 | Loss: 2.4991 | LR: 1.30e-05 | 1913.2ms/step | 31.4 samples/s
309
+ Step 45340/50462 | Loss: 2.5115 | LR: 1.30e-05 | 1913.9ms/step | 31.3 samples/s
310
+ Step 45350/50462 | Loss: 2.4899 | LR: 1.29e-05 | 1914.7ms/step | 31.3 samples/s
311
+ Step 45360/50462 | Loss: 2.5496 | LR: 1.29e-05 | 1915.3ms/step | 31.3 samples/s
312
+ Step 45370/50462 | Loss: 2.5390 | LR: 1.28e-05 | 1915.7ms/step | 31.3 samples/s
313
+ Step 45380/50462 | Loss: 2.4794 | LR: 1.28e-05 | 1914.0ms/step | 31.3 samples/s
314
+ Step 45390/50462 | Loss: 2.5373 | LR: 1.27e-05 | 1914.4ms/step | 31.3 samples/s
315
+ Step 45400/50462 | Loss: 2.5242 | LR: 1.27e-05 | 1916.0ms/step | 31.3 samples/s
316
+ Step 45410/50462 | Loss: 2.5106 | LR: 1.26e-05 | 1914.0ms/step | 31.3 samples/s
317
+ Step 45420/50462 | Loss: 2.5266 | LR: 1.26e-05 | 1916.2ms/step | 31.3 samples/s
318
+ Step 45430/50462 | Loss: 2.4788 | LR: 1.25e-05 | 1914.0ms/step | 31.3 samples/s
319
+ Step 45440/50462 | Loss: 2.5020 | LR: 1.25e-05 | 1915.3ms/step | 31.3 samples/s
320
+ Step 45450/50462 | Loss: 2.5041 | LR: 1.24e-05 | 1914.6ms/step | 31.3 samples/s
321
+ Step 45460/50462 | Loss: 2.4367 | LR: 1.24e-05 | 1917.0ms/step | 31.3 samples/s
322
+ Step 45470/50462 | Loss: 2.4667 | LR: 1.23e-05 | 1917.2ms/step | 31.3 samples/s
323
+ Step 45480/50462 | Loss: 2.4886 | LR: 1.23e-05 | 1915.9ms/step | 31.3 samples/s
324
+ Step 45490/50462 | Loss: 2.5107 | LR: 1.22e-05 | 1911.2ms/step | 31.4 samples/s
325
+ Step 45500/50462 | Loss: 2.4713 | LR: 1.22e-05 | 1913.9ms/step | 31.3 samples/s
326
+ Validation at step 45500: Loss = 2.5308, Perplexity = 12.56
327
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-45500
328
+ Step 45510/50462 | Loss: 2.4943 | LR: 1.21e-05 | 3238.9ms/step | 18.5 samples/s
329
+ Step 45520/50462 | Loss: 2.5403 | LR: 1.21e-05 | 1910.8ms/step | 31.4 samples/s
330
+ Step 45530/50462 | Loss: 2.5173 | LR: 1.20e-05 | 1911.5ms/step | 31.4 samples/s
331
+ Step 45540/50462 | Loss: 2.4642 | LR: 1.20e-05 | 1914.0ms/step | 31.3 samples/s
332
+ Step 45550/50462 | Loss: 2.5093 | LR: 1.19e-05 | 1916.1ms/step | 31.3 samples/s
333
+ Step 45560/50462 | Loss: 2.4897 | LR: 1.19e-05 | 1914.9ms/step | 31.3 samples/s
334
+ Step 45570/50462 | Loss: 2.4857 | LR: 1.19e-05 | 1918.8ms/step | 31.3 samples/s
335
+ Step 45580/50462 | Loss: 2.4939 | LR: 1.18e-05 | 1916.8ms/step | 31.3 samples/s
336
+ Step 45590/50462 | Loss: 2.4843 | LR: 1.18e-05 | 1915.0ms/step | 31.3 samples/s
337
+ Step 45600/50462 | Loss: 2.4878 | LR: 1.17e-05 | 1916.9ms/step | 31.3 samples/s
338
+ Step 45610/50462 | Loss: 2.5188 | LR: 1.17e-05 | 1916.5ms/step | 31.3 samples/s
339
+ Step 45620/50462 | Loss: 2.5333 | LR: 1.16e-05 | 1915.4ms/step | 31.3 samples/s
340
+ Step 45630/50462 | Loss: 2.5229 | LR: 1.16e-05 | 1916.8ms/step | 31.3 samples/s
341
+ Step 45640/50462 | Loss: 2.4912 | LR: 1.15e-05 | 1915.3ms/step | 31.3 samples/s
342
+ Step 45650/50462 | Loss: 2.4455 | LR: 1.15e-05 | 1914.9ms/step | 31.3 samples/s
343
+ Step 45660/50462 | Loss: 2.4912 | LR: 1.14e-05 | 1917.1ms/step | 31.3 samples/s
344
+ Step 45670/50462 | Loss: 2.4671 | LR: 1.14e-05 | 1915.8ms/step | 31.3 samples/s
345
+ Step 45680/50462 | Loss: 2.4898 | LR: 1.13e-05 | 1917.5ms/step | 31.3 samples/s
346
+ Step 45690/50462 | Loss: 2.4747 | LR: 1.13e-05 | 1913.6ms/step | 31.4 samples/s
347
+ Step 45700/50462 | Loss: 2.5201 | LR: 1.12e-05 | 1916.4ms/step | 31.3 samples/s
348
+ Step 45710/50462 | Loss: 2.4833 | LR: 1.12e-05 | 1914.3ms/step | 31.3 samples/s
349
+ Step 45720/50462 | Loss: 2.4639 | LR: 1.11e-05 | 1915.8ms/step | 31.3 samples/s
350
+ Step 45730/50462 | Loss: 2.4568 | LR: 1.11e-05 | 1913.6ms/step | 31.4 samples/s
351
+ Step 45740/50462 | Loss: 2.4802 | LR: 1.10e-05 | 1914.8ms/step | 31.3 samples/s
352
+ Step 45750/50462 | Loss: 2.5080 | LR: 1.10e-05 | 1915.7ms/step | 31.3 samples/s
353
+ Validation at step 45750: Loss = 2.5309, Perplexity = 12.56
354
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-45750
355
+ Step 45760/50462 | Loss: 2.5235 | LR: 1.10e-05 | 3246.4ms/step | 18.5 samples/s
356
+ Step 45770/50462 | Loss: 2.4801 | LR: 1.09e-05 | 1908.0ms/step | 31.4 samples/s
357
+ Step 45780/50462 | Loss: 2.5055 | LR: 1.09e-05 | 1909.5ms/step | 31.4 samples/s
358
+ Step 45790/50462 | Loss: 2.4944 | LR: 1.08e-05 | 1908.7ms/step | 31.4 samples/s
359
+ Step 45800/50462 | Loss: 2.5210 | LR: 1.08e-05 | 1912.3ms/step | 31.4 samples/s
360
+ Step 45810/50462 | Loss: 2.4879 | LR: 1.07e-05 | 1914.2ms/step | 31.3 samples/s
361
+ Step 45820/50462 | Loss: 2.5103 | LR: 1.07e-05 | 1913.4ms/step | 31.4 samples/s
362
+ Step 45830/50462 | Loss: 2.5560 | LR: 1.06e-05 | 1913.6ms/step | 31.4 samples/s
363
+ Step 45840/50462 | Loss: 2.4799 | LR: 1.06e-05 | 1916.2ms/step | 31.3 samples/s
364
+ Step 45850/50462 | Loss: 2.4942 | LR: 1.05e-05 | 1915.3ms/step | 31.3 samples/s
365
+ Step 45860/50462 | Loss: 2.5119 | LR: 1.05e-05 | 1916.6ms/step | 31.3 samples/s
366
+ Step 45870/50462 | Loss: 2.4986 | LR: 1.05e-05 | 1917.5ms/step | 31.3 samples/s
367
+ Step 45880/50462 | Loss: 2.5765 | LR: 1.04e-05 | 1914.5ms/step | 31.3 samples/s
368
+ Step 45890/50462 | Loss: 2.4850 | LR: 1.04e-05 | 1914.8ms/step | 31.3 samples/s
369
+ Step 45900/50462 | Loss: 2.4717 | LR: 1.03e-05 | 1916.1ms/step | 31.3 samples/s
370
+ Step 45910/50462 | Loss: 2.4851 | LR: 1.03e-05 | 1913.3ms/step | 31.4 samples/s
371
+ Step 45920/50462 | Loss: 2.5123 | LR: 1.02e-05 | 1913.7ms/step | 31.4 samples/s
372
+ Step 45930/50462 | Loss: 2.4652 | LR: 1.02e-05 | 1913.3ms/step | 31.4 samples/s
373
+ Step 45940/50462 | Loss: 2.4927 | LR: 1.01e-05 | 1913.6ms/step | 31.4 samples/s
374
+ Step 45950/50462 | Loss: 2.4882 | LR: 1.01e-05 | 1915.1ms/step | 31.3 samples/s
375
+ Step 45960/50462 | Loss: 2.5271 | LR: 1.01e-05 | 1914.5ms/step | 31.3 samples/s
376
+ Step 45970/50462 | Loss: 2.4620 | LR: 1.00e-05 | 1916.0ms/step | 31.3 samples/s
377
+ Step 45980/50462 | Loss: 2.4599 | LR: 9.96e-06 | 1917.7ms/step | 31.3 samples/s
378
+ Step 45990/50462 | Loss: 2.5113 | LR: 9.92e-06 | 1915.4ms/step | 31.3 samples/s
379
+ Step 46000/50462 | Loss: 2.4799 | LR: 9.87e-06 | 1913.2ms/step | 31.4 samples/s
380
+ Validation at step 46000: Loss = 2.5309, Perplexity = 12.56
381
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-46000
382
+ Step 46010/50462 | Loss: 2.4929 | LR: 9.83e-06 | 3226.7ms/step | 18.6 samples/s
383
+ Step 46020/50462 | Loss: 2.4749 | LR: 9.79e-06 | 1907.0ms/step | 31.5 samples/s
384
+ Step 46030/50462 | Loss: 2.4832 | LR: 9.74e-06 | 1909.9ms/step | 31.4 samples/s
385
+ Step 46040/50462 | Loss: 2.5271 | LR: 9.70e-06 | 1912.6ms/step | 31.4 samples/s
386
+ Step 46050/50462 | Loss: 2.4858 | LR: 9.65e-06 | 1913.2ms/step | 31.4 samples/s
387
+ Step 46060/50462 | Loss: 2.4920 | LR: 9.61e-06 | 1914.0ms/step | 31.3 samples/s
388
+ Step 46070/50462 | Loss: 2.5206 | LR: 9.57e-06 | 1915.9ms/step | 31.3 samples/s
389
+ Step 46080/50462 | Loss: 2.5000 | LR: 9.52e-06 | 1914.7ms/step | 31.3 samples/s
390
+ Step 46090/50462 | Loss: 2.5039 | LR: 9.48e-06 | 1914.7ms/step | 31.3 samples/s
391
+ Step 46100/50462 | Loss: 2.4993 | LR: 9.44e-06 | 1913.5ms/step | 31.4 samples/s
392
+ Step 46110/50462 | Loss: 2.4911 | LR: 9.40e-06 | 1913.6ms/step | 31.4 samples/s
393
+ Step 46120/50462 | Loss: 2.4740 | LR: 9.35e-06 | 1913.1ms/step | 31.4 samples/s
394
+ Step 46130/50462 | Loss: 2.4695 | LR: 9.31e-06 | 1914.8ms/step | 31.3 samples/s
395
+ Step 46140/50462 | Loss: 2.4818 | LR: 9.27e-06 | 1913.6ms/step | 31.4 samples/s
396
+ Step 46150/50462 | Loss: 2.4727 | LR: 9.22e-06 | 1914.9ms/step | 31.3 samples/s
397
+ Step 46160/50462 | Loss: 2.4915 | LR: 9.18e-06 | 1913.7ms/step | 31.4 samples/s
398
+ Step 46170/50462 | Loss: 2.4995 | LR: 9.14e-06 | 1914.5ms/step | 31.3 samples/s
399
+ Step 46180/50462 | Loss: 2.4968 | LR: 9.10e-06 | 1917.2ms/step | 31.3 samples/s
400
+ Step 46190/50462 | Loss: 2.4414 | LR: 9.06e-06 | 1915.3ms/step | 31.3 samples/s
401
+ Step 46200/50462 | Loss: 2.4790 | LR: 9.01e-06 | 1917.8ms/step | 31.3 samples/s
402
+ Step 46210/50462 | Loss: 2.4653 | LR: 8.97e-06 | 1916.3ms/step | 31.3 samples/s
403
+ Step 46220/50462 | Loss: 2.5318 | LR: 8.93e-06 | 1915.0ms/step | 31.3 samples/s
404
+ Step 46230/50462 | Loss: 2.4859 | LR: 8.89e-06 | 1916.3ms/step | 31.3 samples/s
405
+ Step 46240/50462 | Loss: 2.4667 | LR: 8.85e-06 | 1915.6ms/step | 31.3 samples/s
406
+ Step 46250/50462 | Loss: 2.4915 | LR: 8.80e-06 | 1915.5ms/step | 31.3 samples/s
407
+ Validation at step 46250: Loss = 2.5309, Perplexity = 12.56
408
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-46250
409
+ Step 46260/50462 | Loss: 2.4865 | LR: 8.76e-06 | 3220.7ms/step | 18.6 samples/s
410
+ Step 46270/50462 | Loss: 2.4951 | LR: 8.72e-06 | 1906.9ms/step | 31.5 samples/s
411
+ Step 46280/50462 | Loss: 2.4695 | LR: 8.68e-06 | 1906.1ms/step | 31.5 samples/s
412
+ Step 46290/50462 | Loss: 2.4927 | LR: 8.64e-06 | 1911.2ms/step | 31.4 samples/s
413
+ Step 46300/50462 | Loss: 2.4301 | LR: 8.60e-06 | 1911.8ms/step | 31.4 samples/s
414
+ Step 46310/50462 | Loss: 2.4800 | LR: 8.56e-06 | 1914.0ms/step | 31.3 samples/s
415
+ Step 46320/50462 | Loss: 2.5049 | LR: 8.52e-06 | 1911.7ms/step | 31.4 samples/s
416
+ Step 46330/50462 | Loss: 2.5201 | LR: 8.47e-06 | 1913.6ms/step | 31.4 samples/s
417
+ Step 46340/50462 | Loss: 2.4823 | LR: 8.43e-06 | 1913.2ms/step | 31.4 samples/s
418
+ Step 46350/50462 | Loss: 2.5083 | LR: 8.39e-06 | 1913.5ms/step | 31.4 samples/s
419
+ Step 46360/50462 | Loss: 2.4970 | LR: 8.35e-06 | 1913.3ms/step | 31.4 samples/s
420
+ Step 46370/50462 | Loss: 2.5434 | LR: 8.31e-06 | 1913.5ms/step | 31.4 samples/s
421
+ Step 46380/50462 | Loss: 2.4829 | LR: 8.27e-06 | 1913.2ms/step | 31.4 samples/s
422
+ Step 46390/50462 | Loss: 2.4522 | LR: 8.23e-06 | 1913.5ms/step | 31.4 samples/s
423
+ Step 46400/50462 | Loss: 2.5336 | LR: 8.19e-06 | 1911.5ms/step | 31.4 samples/s
424
+ Step 46410/50462 | Loss: 2.4784 | LR: 8.15e-06 | 1912.5ms/step | 31.4 samples/s
425
+ Step 46420/50462 | Loss: 2.4940 | LR: 8.11e-06 | 1912.9ms/step | 31.4 samples/s
426
+ Step 46430/50462 | Loss: 2.4748 | LR: 8.07e-06 | 1917.0ms/step | 31.3 samples/s
427
+ Step 46440/50462 | Loss: 2.4878 | LR: 8.03e-06 | 1914.2ms/step | 31.3 samples/s
428
+ Step 46450/50462 | Loss: 2.5019 | LR: 7.99e-06 | 1917.0ms/step | 31.3 samples/s
429
+ Step 46460/50462 | Loss: 2.4940 | LR: 7.95e-06 | 1915.5ms/step | 31.3 samples/s
430
+ Step 46470/50462 | Loss: 2.4810 | LR: 7.91e-06 | 1913.8ms/step | 31.4 samples/s
431
+ Step 46480/50462 | Loss: 2.4756 | LR: 7.87e-06 | 1915.6ms/step | 31.3 samples/s
432
+ Step 46490/50462 | Loss: 2.4944 | LR: 7.83e-06 | 1915.6ms/step | 31.3 samples/s
433
+ Step 46500/50462 | Loss: 2.4812 | LR: 7.80e-06 | 1914.7ms/step | 31.3 samples/s
434
+ Validation at step 46500: Loss = 2.5308, Perplexity = 12.56
435
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-46500
436
+ Step 46510/50462 | Loss: 2.4620 | LR: 7.76e-06 | 3251.9ms/step | 18.5 samples/s
437
+ Step 46520/50462 | Loss: 2.5068 | LR: 7.72e-06 | 1905.2ms/step | 31.5 samples/s
438
+ Step 46530/50462 | Loss: 2.4683 | LR: 7.68e-06 | 1909.0ms/step | 31.4 samples/s
439
+ Step 46540/50462 | Loss: 2.4691 | LR: 7.64e-06 | 1912.9ms/step | 31.4 samples/s
440
+ Step 46550/50462 | Loss: 2.4695 | LR: 7.60e-06 | 1912.2ms/step | 31.4 samples/s
441
+ Step 46560/50462 | Loss: 2.4714 | LR: 7.56e-06 | 1913.1ms/step | 31.4 samples/s
442
+ Step 46570/50462 | Loss: 2.4782 | LR: 7.52e-06 | 1913.0ms/step | 31.4 samples/s
443
+ Step 46580/50462 | Loss: 2.4929 | LR: 7.49e-06 | 1914.6ms/step | 31.3 samples/s
444
+ Step 46590/50462 | Loss: 2.4906 | LR: 7.45e-06 | 1911.8ms/step | 31.4 samples/s
445
+ Step 46600/50462 | Loss: 2.4802 | LR: 7.41e-06 | 1912.8ms/step | 31.4 samples/s
446
+ Step 46610/50462 | Loss: 2.4910 | LR: 7.37e-06 | 1913.5ms/step | 31.4 samples/s
447
+ Step 46620/50462 | Loss: 2.5157 | LR: 7.33e-06 | 1917.8ms/step | 31.3 samples/s
448
+ Step 46630/50462 | Loss: 2.5073 | LR: 7.29e-06 | 1916.2ms/step | 31.3 samples/s
449
+ Step 46640/50462 | Loss: 2.4821 | LR: 7.26e-06 | 1916.6ms/step | 31.3 samples/s
450
+ Step 46650/50462 | Loss: 2.4770 | LR: 7.22e-06 | 1914.1ms/step | 31.3 samples/s
451
+ Step 46660/50462 | Loss: 2.4859 | LR: 7.18e-06 | 1914.8ms/step | 31.3 samples/s
452
+ Step 46670/50462 | Loss: 2.5090 | LR: 7.14e-06 | 1913.9ms/step | 31.3 samples/s
453
+ Step 46680/50462 | Loss: 2.5153 | LR: 7.11e-06 | 1914.0ms/step | 31.3 samples/s
454
+ Step 46690/50462 | Loss: 2.4791 | LR: 7.07e-06 | 1914.0ms/step | 31.3 samples/s
455
+ Step 46700/50462 | Loss: 2.5013 | LR: 7.03e-06 | 1914.5ms/step | 31.3 samples/s
456
+ Step 46710/50462 | Loss: 2.5231 | LR: 6.99e-06 | 1916.0ms/step | 31.3 samples/s
457
+ Step 46720/50462 | Loss: 2.5342 | LR: 6.96e-06 | 1914.7ms/step | 31.3 samples/s
458
+ Step 46730/50462 | Loss: 2.4785 | LR: 6.92e-06 | 1914.8ms/step | 31.3 samples/s
459
+ Step 46740/50462 | Loss: 2.4883 | LR: 6.88e-06 | 1916.0ms/step | 31.3 samples/s
460
+ Step 46750/50462 | Loss: 2.5016 | LR: 6.85e-06 | 1914.0ms/step | 31.3 samples/s
461
+ Validation at step 46750: Loss = 2.5308, Perplexity = 12.56
462
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-46750
463
+ Step 46760/50462 | Loss: 2.5177 | LR: 6.81e-06 | 3236.2ms/step | 18.5 samples/s
464
+ Step 46770/50462 | Loss: 2.4800 | LR: 6.77e-06 | 1908.2ms/step | 31.4 samples/s
465
+ Step 46780/50462 | Loss: 2.4518 | LR: 6.74e-06 | 1912.4ms/step | 31.4 samples/s
466
+ Step 46790/50462 | Loss: 2.4601 | LR: 6.70e-06 | 1910.7ms/step | 31.4 samples/s
467
+ Step 46800/50462 | Loss: 2.5264 | LR: 6.66e-06 | 1914.2ms/step | 31.3 samples/s
468
+ Step 46810/50462 | Loss: 2.4911 | LR: 6.63e-06 | 1912.6ms/step | 31.4 samples/s
469
+ Step 46820/50462 | Loss: 2.4904 | LR: 6.59e-06 | 1915.4ms/step | 31.3 samples/s
470
+ Step 46830/50462 | Loss: 2.5061 | LR: 6.56e-06 | 1915.5ms/step | 31.3 samples/s
471
+ Step 46840/50462 | Loss: 2.4911 | LR: 6.52e-06 | 1915.4ms/step | 31.3 samples/s
472
+ Step 46850/50462 | Loss: 2.4823 | LR: 6.48e-06 | 1915.8ms/step | 31.3 samples/s
473
+ Step 46860/50462 | Loss: 2.5089 | LR: 6.45e-06 | 1915.2ms/step | 31.3 samples/s
474
+ Step 46870/50462 | Loss: 2.4952 | LR: 6.41e-06 | 1912.5ms/step | 31.4 samples/s
475
+ Step 46880/50462 | Loss: 2.5246 | LR: 6.38e-06 | 1912.9ms/step | 31.4 samples/s
476
+ Step 46890/50462 | Loss: 2.4621 | LR: 6.34e-06 | 1914.1ms/step | 31.3 samples/s
477
+ Step 46900/50462 | Loss: 2.4367 | LR: 6.31e-06 | 1912.7ms/step | 31.4 samples/s
478
+ Step 46910/50462 | Loss: 2.4977 | LR: 6.27e-06 | 1916.6ms/step | 31.3 samples/s
479
+ Step 46920/50462 | Loss: 2.5336 | LR: 6.24e-06 | 1912.9ms/step | 31.4 samples/s
480
+ Step 46930/50462 | Loss: 2.5069 | LR: 6.20e-06 | 1916.4ms/step | 31.3 samples/s
481
+ Step 46940/50462 | Loss: 2.4656 | LR: 6.17e-06 | 1917.4ms/step | 31.3 samples/s
482
+ Step 46950/50462 | Loss: 2.5048 | LR: 6.13e-06 | 1916.2ms/step | 31.3 samples/s
483
+ Step 46960/50462 | Loss: 2.5027 | LR: 6.10e-06 | 1917.3ms/step | 31.3 samples/s
484
+ Step 46970/50462 | Loss: 2.5118 | LR: 6.06e-06 | 1915.0ms/step | 31.3 samples/s
485
+ Step 46980/50462 | Loss: 2.4776 | LR: 6.03e-06 | 1914.7ms/step | 31.3 samples/s
486
+ Step 46990/50462 | Loss: 2.5030 | LR: 5.99e-06 | 1913.6ms/step | 31.4 samples/s
487
+ Step 47000/50462 | Loss: 2.5190 | LR: 5.96e-06 | 1915.5ms/step | 31.3 samples/s
488
+ Validation at step 47000: Loss = 2.5309, Perplexity = 12.56
489
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-47000
490
+ Step 47010/50462 | Loss: 2.4761 | LR: 5.93e-06 | 3240.9ms/step | 18.5 samples/s
491
+ Step 47020/50462 | Loss: 2.4598 | LR: 5.89e-06 | 1905.5ms/step | 31.5 samples/s
492
+ Step 47030/50462 | Loss: 2.5364 | LR: 5.86e-06 | 1908.4ms/step | 31.4 samples/s
493
+ Step 47040/50462 | Loss: 2.5209 | LR: 5.82e-06 | 1912.2ms/step | 31.4 samples/s
494
+ Step 47050/50462 | Loss: 2.5351 | LR: 5.79e-06 | 1912.5ms/step | 31.4 samples/s
495
+ Step 47060/50462 | Loss: 2.4932 | LR: 5.76e-06 | 1913.1ms/step | 31.4 samples/s
496
+ Step 47070/50462 | Loss: 2.4777 | LR: 5.72e-06 | 1914.8ms/step | 31.3 samples/s
497
+ Step 47080/50462 | Loss: 2.4569 | LR: 5.69e-06 | 1913.9ms/step | 31.4 samples/s
498
+ Step 47090/50462 | Loss: 2.5621 | LR: 5.65e-06 | 1914.7ms/step | 31.3 samples/s
499
+ Step 47100/50462 | Loss: 2.4979 | LR: 5.62e-06 | 1915.7ms/step | 31.3 samples/s
500
+ Step 47110/50462 | Loss: 2.5095 | LR: 5.59e-06 | 1914.4ms/step | 31.3 samples/s
501
+ Step 47120/50462 | Loss: 2.4699 | LR: 5.56e-06 | 1912.9ms/step | 31.4 samples/s
502
+ Step 47130/50462 | Loss: 2.4756 | LR: 5.52e-06 | 1911.9ms/step | 31.4 samples/s
503
+ Step 47140/50462 | Loss: 2.4591 | LR: 5.49e-06 | 1914.0ms/step | 31.3 samples/s
504
+ Step 47150/50462 | Loss: 2.5278 | LR: 5.46e-06 | 1915.7ms/step | 31.3 samples/s
505
+ Step 47160/50462 | Loss: 2.4688 | LR: 5.42e-06 | 1915.5ms/step | 31.3 samples/s
506
+ Step 47170/50462 | Loss: 2.5370 | LR: 5.39e-06 | 1912.9ms/step | 31.4 samples/s
507
+ Step 47180/50462 | Loss: 2.4628 | LR: 5.36e-06 | 1912.9ms/step | 31.4 samples/s
508
+ Step 47190/50462 | Loss: 2.5077 | LR: 5.33e-06 | 1914.0ms/step | 31.3 samples/s
509
+ Step 47200/50462 | Loss: 2.4704 | LR: 5.29e-06 | 1913.7ms/step | 31.4 samples/s
510
+ Step 47210/50462 | Loss: 2.4855 | LR: 5.26e-06 | 1913.0ms/step | 31.4 samples/s
511
+ Step 47220/50462 | Loss: 2.5158 | LR: 5.23e-06 | 1913.6ms/step | 31.4 samples/s
512
+ Step 47230/50462 | Loss: 2.4976 | LR: 5.20e-06 | 1910.5ms/step | 31.4 samples/s
513
+ Step 47240/50462 | Loss: 2.5172 | LR: 5.16e-06 | 1913.4ms/step | 31.4 samples/s
514
+ Step 47250/50462 | Loss: 2.4661 | LR: 5.13e-06 | 1914.0ms/step | 31.3 samples/s
515
+ Validation at step 47250: Loss = 2.5308, Perplexity = 12.56
516
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-47250
517
+ Step 47260/50462 | Loss: 2.4899 | LR: 5.10e-06 | 3222.9ms/step | 18.6 samples/s
518
+ Step 47270/50462 | Loss: 2.4847 | LR: 5.07e-06 | 1903.9ms/step | 31.5 samples/s
519
+ Step 47280/50462 | Loss: 2.4743 | LR: 5.04e-06 | 1909.8ms/step | 31.4 samples/s
520
+ Step 47290/50462 | Loss: 2.4642 | LR: 5.01e-06 | 1913.0ms/step | 31.4 samples/s
521
+ Step 47300/50462 | Loss: 2.4418 | LR: 4.97e-06 | 1911.8ms/step | 31.4 samples/s
522
+ Step 47310/50462 | Loss: 2.4786 | LR: 4.94e-06 | 1912.7ms/step | 31.4 samples/s
523
+ Step 47320/50462 | Loss: 2.5015 | LR: 4.91e-06 | 1912.7ms/step | 31.4 samples/s
524
+ Step 47330/50462 | Loss: 2.5046 | LR: 4.88e-06 | 1914.3ms/step | 31.3 samples/s
525
+ Step 47340/50462 | Loss: 2.4394 | LR: 4.85e-06 | 1912.7ms/step | 31.4 samples/s
526
+ Step 47350/50462 | Loss: 2.5383 | LR: 4.82e-06 | 1913.9ms/step | 31.4 samples/s
527
+ Step 47360/50462 | Loss: 2.5352 | LR: 4.79e-06 | 1915.7ms/step | 31.3 samples/s
528
+ Step 47370/50462 | Loss: 2.4830 | LR: 4.76e-06 | 1913.8ms/step | 31.4 samples/s
529
+ Step 47380/50462 | Loss: 2.4550 | LR: 4.73e-06 | 1914.5ms/step | 31.3 samples/s
530
+ Step 47390/50462 | Loss: 2.5305 | LR: 4.70e-06 | 1915.0ms/step | 31.3 samples/s
531
+ Step 47400/50462 | Loss: 2.4938 | LR: 4.67e-06 | 1914.4ms/step | 31.3 samples/s
532
+ Step 47410/50462 | Loss: 2.4807 | LR: 4.64e-06 | 1914.3ms/step | 31.3 samples/s
533
+ Step 47420/50462 | Loss: 2.5109 | LR: 4.61e-06 | 1914.3ms/step | 31.3 samples/s
534
+ Step 47430/50462 | Loss: 2.5044 | LR: 4.58e-06 | 1915.4ms/step | 31.3 samples/s
535
+ Step 47440/50462 | Loss: 2.4789 | LR: 4.55e-06 | 1914.1ms/step | 31.3 samples/s
536
+ Step 47450/50462 | Loss: 2.5265 | LR: 4.52e-06 | 1918.1ms/step | 31.3 samples/s
537
+ Step 47460/50462 | Loss: 2.5115 | LR: 4.49e-06 | 1917.7ms/step | 31.3 samples/s
538
+ Step 47470/50462 | Loss: 2.4846 | LR: 4.46e-06 | 1915.5ms/step | 31.3 samples/s
539
+ Step 47480/50462 | Loss: 2.4638 | LR: 4.43e-06 | 1919.3ms/step | 31.3 samples/s
540
+ Step 47490/50462 | Loss: 2.4710 | LR: 4.40e-06 | 1917.0ms/step | 31.3 samples/s
541
+ Step 47500/50462 | Loss: 2.5075 | LR: 4.37e-06 | 1917.4ms/step | 31.3 samples/s
542
+ Validation at step 47500: Loss = 2.5308, Perplexity = 12.56
543
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-47500
544
+ Step 47510/50462 | Loss: 2.5275 | LR: 4.34e-06 | 3237.3ms/step | 18.5 samples/s
545
+ Step 47520/50462 | Loss: 2.5001 | LR: 4.31e-06 | 1909.3ms/step | 31.4 samples/s
546
+ Step 47530/50462 | Loss: 2.4687 | LR: 4.28e-06 | 1914.7ms/step | 31.3 samples/s
547
+ Step 47540/50462 | Loss: 2.5010 | LR: 4.25e-06 | 1914.5ms/step | 31.3 samples/s
548
+ Step 47550/50462 | Loss: 2.4718 | LR: 4.22e-06 | 1917.2ms/step | 31.3 samples/s
549
+ Step 47560/50462 | Loss: 2.5223 | LR: 4.19e-06 | 1916.5ms/step | 31.3 samples/s
550
+ Step 47570/50462 | Loss: 2.5270 | LR: 4.16e-06 | 1915.1ms/step | 31.3 samples/s
551
+ Step 47580/50462 | Loss: 2.5523 | LR: 4.13e-06 | 1915.3ms/step | 31.3 samples/s
552
+ Step 47590/50462 | Loss: 2.4717 | LR: 4.11e-06 | 1913.4ms/step | 31.4 samples/s
553
+ Step 47600/50462 | Loss: 2.5337 | LR: 4.08e-06 | 1916.5ms/step | 31.3 samples/s
554
+ Step 47610/50462 | Loss: 2.4628 | LR: 4.05e-06 | 1915.2ms/step | 31.3 samples/s
555
+ Step 47620/50462 | Loss: 2.4790 | LR: 4.02e-06 | 1914.0ms/step | 31.3 samples/s
556
+ Step 47630/50462 | Loss: 2.5043 | LR: 3.99e-06 | 1913.0ms/step | 31.4 samples/s
557
+ Step 47640/50462 | Loss: 2.4790 | LR: 3.97e-06 | 1915.8ms/step | 31.3 samples/s
558
+ Step 47650/50462 | Loss: 2.4905 | LR: 3.94e-06 | 1914.1ms/step | 31.3 samples/s
559
+ Step 47660/50462 | Loss: 2.5135 | LR: 3.91e-06 | 1917.5ms/step | 31.3 samples/s
560
+ Step 47670/50462 | Loss: 2.4927 | LR: 3.88e-06 | 1915.4ms/step | 31.3 samples/s
561
+ Step 47680/50462 | Loss: 2.4909 | LR: 3.85e-06 | 1912.3ms/step | 31.4 samples/s
562
+ Step 47690/50462 | Loss: 2.4665 | LR: 3.83e-06 | 1914.5ms/step | 31.3 samples/s
563
+ Step 47700/50462 | Loss: 2.4916 | LR: 3.80e-06 | 1916.0ms/step | 31.3 samples/s
564
+ Step 47710/50462 | Loss: 2.4908 | LR: 3.77e-06 | 1915.1ms/step | 31.3 samples/s
565
+ Step 47720/50462 | Loss: 2.4569 | LR: 3.74e-06 | 1913.6ms/step | 31.4 samples/s
566
+ Step 47730/50462 | Loss: 2.4885 | LR: 3.72e-06 | 1913.5ms/step | 31.4 samples/s
567
+ Step 47740/50462 | Loss: 2.4840 | LR: 3.69e-06 | 1913.3ms/step | 31.4 samples/s
568
+ Step 47750/50462 | Loss: 2.5015 | LR: 3.66e-06 | 1914.3ms/step | 31.3 samples/s
569
+ Validation at step 47750: Loss = 2.5308, Perplexity = 12.56
570
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-47750
571
+ Step 47760/50462 | Loss: 2.4937 | LR: 3.64e-06 | 3245.7ms/step | 18.5 samples/s
572
+ Step 47770/50462 | Loss: 2.4367 | LR: 3.61e-06 | 1906.6ms/step | 31.5 samples/s
573
+ Step 47780/50462 | Loss: 2.4823 | LR: 3.58e-06 | 1909.5ms/step | 31.4 samples/s
574
+ Step 47790/50462 | Loss: 2.5063 | LR: 3.56e-06 | 1914.4ms/step | 31.3 samples/s
575
+ Step 47800/50462 | Loss: 2.4812 | LR: 3.53e-06 | 1914.5ms/step | 31.3 samples/s
576
+ Step 47810/50462 | Loss: 2.5136 | LR: 3.50e-06 | 1916.1ms/step | 31.3 samples/s
577
+ Step 47820/50462 | Loss: 2.5004 | LR: 3.48e-06 | 1913.3ms/step | 31.4 samples/s
578
+ Step 47830/50462 | Loss: 2.4921 | LR: 3.45e-06 | 1914.8ms/step | 31.3 samples/s
579
+ Step 47840/50462 | Loss: 2.4957 | LR: 3.42e-06 | 1914.9ms/step | 31.3 samples/s
580
+ Step 47850/50462 | Loss: 2.4720 | LR: 3.40e-06 | 1915.2ms/step | 31.3 samples/s
581
+ Step 47860/50462 | Loss: 2.4834 | LR: 3.37e-06 | 1913.4ms/step | 31.4 samples/s
582
+ Step 47870/50462 | Loss: 2.5159 | LR: 3.35e-06 | 1914.9ms/step | 31.3 samples/s
583
+ Step 47880/50462 | Loss: 2.4781 | LR: 3.32e-06 | 1916.5ms/step | 31.3 samples/s
584
+ Step 47890/50462 | Loss: 2.4729 | LR: 3.30e-06 | 1915.6ms/step | 31.3 samples/s
585
+ Step 47900/50462 | Loss: 2.4669 | LR: 3.27e-06 | 1915.4ms/step | 31.3 samples/s
586
+ Step 47910/50462 | Loss: 2.4426 | LR: 3.24e-06 | 1914.1ms/step | 31.3 samples/s
587
+ Step 47920/50462 | Loss: 2.4623 | LR: 3.22e-06 | 1915.3ms/step | 31.3 samples/s
588
+ Step 47930/50462 | Loss: 2.5007 | LR: 3.19e-06 | 1914.1ms/step | 31.3 samples/s
589
+ Step 47940/50462 | Loss: 2.5137 | LR: 3.17e-06 | 1914.2ms/step | 31.3 samples/s
590
+ Step 47950/50462 | Loss: 2.5016 | LR: 3.14e-06 | 1915.6ms/step | 31.3 samples/s
591
+ Step 47960/50462 | Loss: 2.4992 | LR: 3.12e-06 | 1915.8ms/step | 31.3 samples/s
592
+ Step 47970/50462 | Loss: 2.5044 | LR: 3.09e-06 | 1914.9ms/step | 31.3 samples/s
593
+ Step 47980/50462 | Loss: 2.5101 | LR: 3.07e-06 | 1913.9ms/step | 31.4 samples/s
594
+ Step 47990/50462 | Loss: 2.4759 | LR: 3.04e-06 | 1915.8ms/step | 31.3 samples/s
595
+ Step 48000/50462 | Loss: 2.3804 | LR: 3.02e-06 | 1917.7ms/step | 31.3 samples/s
596
+ Validation at step 48000: Loss = 2.5308, Perplexity = 12.56
597
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72_repaired/run_20260906_repair_from9250/checkpoint-48000
598
+ Step 48010/50462 | Loss: 2.5038 | LR: 3.00e-06 | 3240.9ms/step | 18.5 samples/s
599
+ Step 48020/50462 | Loss: 2.4784 | LR: 2.97e-06 | 1906.2ms/step | 31.5 samples/s
600
+ Step 48030/50462 | Loss: 2.4670 | LR: 2.95e-06 | 1909.9ms/step | 31.4 samples/s
601
+ Step 48040/50462 | Loss: 2.4977 | LR: 2.92e-06 | 1911.7ms/step | 31.4 samples/s
602
+ Step 48050/50462 | Loss: 2.5003 | LR: 2.90e-06 | 1914.8ms/step | 31.3 samples/s
603
+ Step 48060/50462 | Loss: 2.4906 | LR: 2.87e-06 | 1916.6ms/step | 31.3 samples/s
604
+ Step 48070/50462 | Loss: 2.4854 | LR: 2.85e-06 | 1917.4ms/step | 31.3 samples/s
605
+ Step 48080/50462 | Loss: 2.4770 | LR: 2.83e-06 | 1915.5ms/step | 31.3 samples/s
606
+ Step 48090/50462 | Loss: 2.4894 | LR: 2.80e-06 | 1912.7ms/step | 31.4 samples/s
epfl_final/train_logs/p2_1p2b_dense_seed72_6gpu_resume44250.pid ADDED
@@ -0,0 +1 @@
 
 
1
+ 355
epfl_final/train_logs/p2_1p2b_dense_seed72_6gpu_resume9250.log ADDED
@@ -0,0 +1,623 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ W0905 13:01:37.443000 157 site-packages/torch/distributed/run.py:851]
2
+ W0905 13:01:37.443000 157 site-packages/torch/distributed/run.py:851] *****************************************
3
+ W0905 13:01:37.443000 157 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
4
+ W0905 13:01:37.443000 157 site-packages/torch/distributed/run.py:851] *****************************************
5
+ Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216
6
+ Running config:
7
+ {
8
+ "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
9
+ "model_dtype": "bfloat16",
10
+ "init_from_scratch": true,
11
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
12
+ "val_path": null,
13
+ "max_seq_length": 8192,
14
+ "num_workers": 8,
15
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216",
16
+ "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-9250",
17
+ "force_new_run": false,
18
+ "num_epochs": 1,
19
+ "max_steps": 50462,
20
+ "per_device_batch_size": 1,
21
+ "gradient_accumulation_steps": 10,
22
+ "learning_rate": 0.0005,
23
+ "weight_decay": 0.01,
24
+ "warmup_steps": 750,
25
+ "lr_scheduler": "cosine",
26
+ "max_grad_norm": 1.0,
27
+ "save_steps": 250,
28
+ "eval_steps": 250,
29
+ "eval_batches": 50,
30
+ "logging_steps": 10,
31
+ "seed": 72,
32
+ "verbose": true,
33
+ "checkpoint_activations": false,
34
+ "resume_micro_step_override": null,
35
+ "strategy": "ddp",
36
+ "mixed_precision": "bf16",
37
+ "wandb_enabled": false,
38
+ "wandb_project": "stochastic_sparsity",
39
+ "wandb_name": "1.2b dense 8k seed72 6gpu resume9250",
40
+ "wandb_tags": [
41
+ "qwen3",
42
+ "1.2b",
43
+ "8k",
44
+ "p2",
45
+ "dense",
46
+ "seed72"
47
+ ],
48
+ "hidden_size": 1536,
49
+ "intermediate_size": 6144,
50
+ "num_attention_heads": 12,
51
+ "num_key_value_heads": 4,
52
+ "num_hidden_layers": 28,
53
+ "head_dim": null,
54
+ "attention_dropout": null,
55
+ "rope_train_position_scale_set": null,
56
+ "attention_type": "sparse_stochastic",
57
+ "sparse_block_size": 128,
58
+ "sparse_num_random_blocks": 31,
59
+ "sparse_mask_pool_size": 1,
60
+ "sparse_val_dense_compare": false,
61
+ "sparse_layer_indices": null,
62
+ "sparse_mask_cache_dir": null,
63
+ "sparse_mask_seed": 0,
64
+ "sparse_mixed_nrb_set": null,
65
+ "dense_mode": true,
66
+ "dense_step_intervals": null
67
+ }
68
+ Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
69
+ Checkpoint activations: FalseCheckpoint activations: FalseCheckpoint activations: False
70
+
71
+ Checkpoint activations: False
72
+
73
+ Checkpoint activations: False
74
+ Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-9250
75
+ Checkpoint activations: False
76
+ [rank3]: Traceback (most recent call last):
77
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 173, in getuser
78
+ [rank3]: return pwd.getpwuid(os.getuid())[0]
79
+ [rank3]: ~~~~~~~~~~~~^^^^^^^^^^^^^
80
+ [rank3]: KeyError: 'getpwuid(): uid not found: 285818'
81
+
82
+ [rank3]: The above exception was the direct cause of the following exception:
83
+
84
+ [rank3]: Traceback (most recent call last):
85
+ [rank3]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1519, in <module>
86
+ [rank3]: main()
87
+ [rank3]: ~~~~^^
88
+ [rank3]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1516, in main
89
+ [rank3]: train(config)
90
+ [rank3]: ~~~~~^^^^^^^^
91
+ [rank3]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1107, in train
92
+ [rank3]: model = torch.compile(model)
93
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/__init__.py", line 2758, in compile
94
+ [rank3]: return torch._dynamo.optimize(
95
+ [rank3]: ~~~~~~~~~~~~~~~~~~~~~~^
96
+ [rank3]: backend=backend,
97
+ [rank3]: ^^^^^^^^^^^^^^^^
98
+ [rank3]: ...<3 lines>...
99
+ [rank3]: guard_filter_fn=guard_filter_fn,
100
+ [rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
101
+ [rank3]: )(model) # type: ignore[return-value]
102
+ [rank3]: ^
103
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1488, in optimize
104
+ [rank3]: return _optimize(rebuild_ctx, *args, **kwargs)
105
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1564, in _optimize
106
+ [rank3]: backend = get_compiler_fn(backend)
107
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1325, in get_compiler_fn
108
+ [rank3]: from .repro.after_dynamo import wrap_backend_debug
109
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/repro/after_dynamo.py", line 33, in <module>
110
+ [rank3]: from torch._dynamo.debug_utils import (
111
+ [rank3]: ...<15 lines>...
112
+ [rank3]: )
113
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/debug_utils.py", line 43, in <module>
114
+ [rank3]: from torch._dynamo.testing import rand_strided
115
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/testing.py", line 33, in <module>
116
+ [rank3]: from torch._dynamo.backends.debugging import aot_eager
117
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/backends/debugging.py", line 34, in <module>
118
+ [rank3]: from functorch.compile import min_cut_rematerialization_partition
119
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/functorch/compile/__init__.py", line 2, in <module>
120
+ [rank3]: from torch._functorch.aot_autograd import (
121
+ [rank3]: ...<10 lines>...
122
+ [rank3]: )
123
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/aot_autograd.py", line 38, in <module>
124
+ [rank3]: from ._aot_autograd.autograd_cache import ( # noqa: F401
125
+ [rank3]: ...<4 lines>...
126
+ [rank3]: )
127
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/_aot_autograd/autograd_cache.py", line 27, in <module>
128
+ [rank3]: from torch._inductor.codecache import (
129
+ [rank3]: ...<11 lines>...
130
+ [rank3]: )
131
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/codecache.py", line 2753, in <module>
132
+ [rank3]: _HEADER_DIR = os.path.join(default_cache_dir(), "precompiled_headers")
133
+ [rank3]: ~~~~~~~~~~~~~~~~~^^
134
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/runtime/cache_dir_utils.py", line 23, in default_cache_dir
135
+ [rank3]: sanitized_username = re.sub(r'[\\/:*?"<>|]', "_", getpass.getuser())
136
+ [rank3]: ~~~~~~~~~~~~~~~^^
137
+ [rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 175, in getuser
138
+ [rank3]: raise OSError('No username set in the environment') from e
139
+ [rank3]: OSError: No username set in the environment
140
+ [rank1]: Traceback (most recent call last):
141
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 173, in getuser
142
+ [rank1]: return pwd.getpwuid(os.getuid())[0]
143
+ [rank1]: ~~~~~~~~~~~~^^^^^^^^^^^^^
144
+ [rank1]: KeyError: 'getpwuid(): uid not found: 285818'
145
+
146
+ [rank1]: The above exception was the direct cause of the following exception:
147
+
148
+ [rank1]: Traceback (most recent call last):
149
+ [rank1]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1519, in <module>
150
+ [rank1]: main()
151
+ [rank1]: ~~~~^^
152
+ [rank1]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1516, in main
153
+ [rank1]: train(config)
154
+ [rank1]: ~~~~~^^^^^^^^
155
+ [rank1]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1107, in train
156
+ [rank1]: model = torch.compile(model)
157
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/__init__.py", line 2758, in compile
158
+ [rank1]: return torch._dynamo.optimize(
159
+ [rank1]: ~~~~~~~~~~~~~~~~~~~~~~^
160
+ [rank1]: backend=backend,
161
+ [rank1]: ^^^^^^^^^^^^^^^^
162
+ [rank1]: ...<3 lines>...
163
+ [rank1]: guard_filter_fn=guard_filter_fn,
164
+ [rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
165
+ [rank1]: )(model) # type: ignore[return-value]
166
+ [rank1]: ^
167
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1488, in optimize
168
+ [rank1]: return _optimize(rebuild_ctx, *args, **kwargs)
169
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1564, in _optimize
170
+ [rank1]: backend = get_compiler_fn(backend)
171
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1325, in get_compiler_fn
172
+ [rank1]: from .repro.after_dynamo import wrap_backend_debug
173
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/repro/after_dynamo.py", line 33, in <module>
174
+ [rank1]: from torch._dynamo.debug_utils import (
175
+ [rank1]: ...<15 lines>...
176
+ [rank1]: )
177
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/debug_utils.py", line 43, in <module>
178
+ [rank1]: from torch._dynamo.testing import rand_strided
179
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/testing.py", line 33, in <module>
180
+ [rank1]: from torch._dynamo.backends.debugging import aot_eager
181
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/backends/debugging.py", line 34, in <module>
182
+ [rank1]: from functorch.compile import min_cut_rematerialization_partition
183
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/functorch/compile/__init__.py", line 2, in <module>
184
+ [rank1]: from torch._functorch.aot_autograd import (
185
+ [rank1]: ...<10 lines>...
186
+ [rank1]: )
187
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/aot_autograd.py", line 38, in <module>
188
+ [rank1]: from ._aot_autograd.autograd_cache import ( # noqa: F401
189
+ [rank1]: ...<4 lines>...
190
+ [rank1]: )
191
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/_aot_autograd/autograd_cache.py", line 27, in <module>
192
+ [rank1]: from torch._inductor.codecache import (
193
+ [rank1]: ...<11 lines>...
194
+ [rank1]: )
195
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/codecache.py", line 2753, in <module>
196
+ [rank1]: _HEADER_DIR = os.path.join(default_cache_dir(), "precompiled_headers")
197
+ [rank1]: ~~~~~~~~~~~~~~~~~^^
198
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/runtime/cache_dir_utils.py", line 23, in default_cache_dir
199
+ [rank1]: sanitized_username = re.sub(r'[\\/:*?"<>|]', "_", getpass.getuser())
200
+ [rank1]: ~~~~~~~~~~~~~~~^^
201
+ [rank1]: File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 175, in getuser
202
+ [rank1]: raise OSError('No username set in the environment') from e
203
+ [rank1]: OSError: No username set in the environment
204
+ dense_mode=True: sparse attention layers will use standard causal SDPA.
205
+
206
+
207
+ ================================================================================
208
+ MODEL ARCHITECTURE
209
+ ================================================================================
210
+ Model config:
211
+ vocab_size: 151936
212
+ hidden_size: 1536
213
+ num_hidden_layers: 28
214
+ num_attention_heads: 12
215
+ num_key_value_heads: 4
216
+ head_dim: 128
217
+ intermediate_size: 6144
218
+ max_position_embeddings: 32768
219
+ rope_theta: 1000000
220
+ rope_train_position_scale_set: None
221
+ attention_type: sparse_stochastic
222
+ sparse_block_size: 128
223
+ sparse_num_random_blocks: 31
224
+ sparse_mask_pool_size: 1
225
+ sparse_max_seq_length: 8192
226
+
227
+ Parameter count:
228
+ Total parameters: 1,202,352,640
229
+ Trainable parameters: 1,202,352,640
230
+ Non-trainable parameters: 0
231
+ Model size (BF16): 2.40 GB
232
+
233
+ --------------------------------------------------------------------------------
234
+ LAYER-BY-LAYER SUMMARY
235
+ --------------------------------------------------------------------------------
236
+ [rank4]: Traceback (most recent call last):
237
+ [rank4]: File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 173, in getuser
238
+ [rank4]: return pwd.getpwuid(os.getuid())[0]
239
+ [rank4]: ~~~~~~~~~~~~^^^^^^^^^^^^^
240
+ [rank4]: KeyError: 'getpwuid(): uid not found: 285818'
241
+
242
+ [rank4]: The above exception was the direct cause of the following exception:
243
+
244
+ [rank4]: Traceback (most recent call last):
245
+ [rank4]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1519, in <module>
246
+ [rank4]: main()
247
+ [rank4]: ~~~~^^
248
+ [rank4]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1516, in main
249
+ [rank4]: train(config)
250
+ [rank4]: ~~~~~^^^^^^^^
251
+ [rank4]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1107, in train
252
+ [rank4]: model = torch.compile(model)
253
+ [rank4]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/__init__.py", line 2758, in compile
254
+ [rank4]: return torch._dynamo.optimize(
255
+ [rank4]: ~~~~~~~~~~~~~~~~~~~~~~^
256
+ [rank4]: backend=backend,
257
+ [rank4]: ^^^^^^^^^^^^^^^^
258
+ [rank4]: ...<3 lines>...
259
+ [rank4]: guard_filter_fn=guard_filter_fn,
260
+ [rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
261
+ [rank4]: )(model) # type: ignore[return-value]
262
+ [rank4]: ^
263
+ [rank4]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1488, in optimize
264
+ [rank4]: return _optimize(rebuild_ctx, *args, **kwargs)
265
+ [rank4]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1564, in _optimize
266
+ [rank4]: backend = get_compiler_fn(backend)
267
+ [rank4]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1325, in get_compiler_fn
268
+ [rank4]: from .repro.after_dynamo import wrap_backend_debug
269
+ [rank4]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/repro/after_dynamo.py", line 33, in <module>
270
+ [rank4]: from torch._dynamo.debug_utils import (
271
+ [rank4]: ...<15 lines>...
272
+ [rank4]: )
273
+ [rank4]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/debug_utils.py", line 43, in <module>
274
+ [rank4]: from torch._dynamo.testing import rand_strided
275
+ [rank4]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/testing.py", line 33, in <module>
276
+ [rank4]: from torch._dynamo.backends.debugging import aot_eager
277
+ [rank4]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/backends/debugging.py", line 34, in <module>
278
+ [rank4]: from functorch.compile import min_cut_rematerialization_partition
279
+ [rank4]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/functorch/compile/__init__.py", line 2, in <module>
280
+ [rank4]: from torch._functorch.aot_autograd import (
281
+ [rank4]: ...<10 lines>...
282
+ [rank4]: )
283
+ [rank4]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/aot_autograd.py", line 38, in <module>
284
+ [rank4]: from ._aot_autograd.autograd_cache import ( # noqa: F401
285
+ [rank4]: ...<4 lines>...
286
+ [rank4]: )
287
+ [rank4]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/_aot_autograd/autograd_cache.py", line 27, in <module>
288
+ [rank4]: from torch._inductor.codecache import (
289
+ [rank4]: ...<11 lines>...
290
+ [rank4]: )
291
+ [rank4]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/codecache.py", line 2753, in <module>
292
+ [rank4]: _HEADER_DIR = os.path.join(default_cache_dir(), "precompiled_headers")
293
+ [rank4]: ~~~~~~~~~~~~~~~~~^^
294
+ [rank4]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/runtime/cache_dir_utils.py", line 23, in default_cache_dir
295
+ [rank4]: sanitized_username = re.sub(r'[\\/:*?"<>|]', "_", getpass.getuser())
296
+ [rank4]: ~~~~~~~~~~~~~~~^^
297
+ [rank4]: File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 175, in getuser
298
+ [rank4]: raise OSError('No username set in the environment') from e
299
+ [rank4]: OSError: No username set in the environment
300
+ [rank2]: Traceback (most recent call last):
301
+ [rank2]: File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 173, in getuser
302
+ [rank2]: return pwd.getpwuid(os.getuid())[0]
303
+ [rank2]: ~~~~~~~~~~~~^^^^^^^^^^^^^
304
+ [rank2]: KeyError: 'getpwuid(): uid not found: 285818'
305
+
306
+ [rank2]: The above exception was the direct cause of the following exception:
307
+
308
+ [rank2]: Traceback (most recent call last):
309
+ [rank2]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1519, in <module>
310
+ [rank2]: main()
311
+ [rank2]: ~~~~^^
312
+ [rank2]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1516, in main
313
+ [rank2]: train(config)
314
+ [rank2]: ~~~~~^^^^^^^^
315
+ [rank2]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1107, in train
316
+ [rank2]: model = torch.compile(model)
317
+ [rank2]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/__init__.py", line 2758, in compile
318
+ [rank2]: return torch._dynamo.optimize(
319
+ [rank2]: ~~~~~~~~~~~~~~~~~~~~~~^
320
+ [rank2]: backend=backend,
321
+ [rank2]: ^^^^^^^^^^^^^^^^
322
+ [rank2]: ...<3 lines>...
323
+ [rank2]: guard_filter_fn=guard_filter_fn,
324
+ [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
325
+ [rank2]: )(model) # type: ignore[return-value]
326
+ [rank2]: ^
327
+ [rank2]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1488, in optimize
328
+ [rank2]: return _optimize(rebuild_ctx, *args, **kwargs)
329
+ [rank2]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1564, in _optimize
330
+ [rank2]: backend = get_compiler_fn(backend)
331
+ [rank2]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1325, in get_compiler_fn
332
+ [rank2]: from .repro.after_dynamo import wrap_backend_debug
333
+ [rank2]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/repro/after_dynamo.py", line 33, in <module>
334
+ [rank2]: from torch._dynamo.debug_utils import (
335
+ [rank2]: ...<15 lines>...
336
+ [rank2]: )
337
+ [rank2]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/debug_utils.py", line 43, in <module>
338
+ [rank2]: from torch._dynamo.testing import rand_strided
339
+ [rank2]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/testing.py", line 33, in <module>
340
+ [rank2]: from torch._dynamo.backends.debugging import aot_eager
341
+ [rank2]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/backends/debugging.py", line 34, in <module>
342
+ [rank2]: from functorch.compile import min_cut_rematerialization_partition
343
+ [rank2]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/functorch/compile/__init__.py", line 2, in <module>
344
+ [rank2]: from torch._functorch.aot_autograd import (
345
+ [rank2]: ...<10 lines>...
346
+ [rank2]: )
347
+ [rank2]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/aot_autograd.py", line 38, in <module>
348
+ [rank2]: from ._aot_autograd.autograd_cache import ( # noqa: F401
349
+ [rank2]: ...<4 lines>...
350
+ [rank2]: )
351
+ [rank2]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/_aot_autograd/autograd_cache.py", line 27, in <module>
352
+ [rank2]: from torch._inductor.codecache import (
353
+ [rank2]: ...<11 lines>...
354
+ [rank2]: )
355
+ [rank2]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/codecache.py", line 2753, in <module>
356
+ [rank2]: _HEADER_DIR = os.path.join(default_cache_dir(), "precompiled_headers")
357
+ [rank2]: ~~~~~~~~~~~~~~~~~^^
358
+ [rank2]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/runtime/cache_dir_utils.py", line 23, in default_cache_dir
359
+ [rank2]: sanitized_username = re.sub(r'[\\/:*?"<>|]', "_", getpass.getuser())
360
+ [rank2]: ~~~~~~~~~~~~~~~^^
361
+ [rank2]: File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 175, in getuser
362
+ [rank2]: raise OSError('No username set in the environment') from e
363
+ [rank2]: OSError: No username set in the environment
364
+ ================================================================================================================================================================
365
+ Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
366
+ ================================================================================================================================================================
367
+ Qwen3ForCausalLM [1, 8192] -- -- True
368
+ ├─Qwen3Model: 1-1 -- [1, 8192, 1536] -- True
369
+ │ └─Embedding: 2-1 [1, 8192] [1, 8192, 1536] 233,373,696 True
370
+ │ └─ModuleList: 2-2 -- -- -- True
371
+ │ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
372
+ │ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
373
+ │ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
374
+ │ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
375
+ │ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
376
+ │ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
377
+ │ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
378
+ │ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
379
+ │ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
380
+ │ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
381
+ │ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
382
+ │ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
383
+ │ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
384
+ │ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
385
+ │ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
386
+ │ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
387
+ │ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
388
+ │ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
389
+ │ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
390
+ │ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
391
+ │ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
392
+ │ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
393
+ │ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
394
+ │ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
395
+ │ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
396
+ │ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
397
+ │ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
398
+ │ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
399
+ │ └─Qwen3RMSNorm: 2-3 [1, 8192, 1536] [1, 8192, 1536] 1,536 True
400
+ ├─Linear: 1-2 [1, 8192, 1536] [1, 8192, 151936] 233,373,696 True
401
+ ================================================================================================================================================================
402
+ Total params: 1,435,726,336
403
+ Trainable params: 1,435,726,336
404
+ Non-trainable params: 0
405
+ Total mult-adds (Units.GIGABYTES): 1.44
406
+ ================================================================================================================================================================
407
+ Input size (MB): 0.07
408
+ Forward/backward pass size (MB): 26218.59
409
+ Params size (MB): 2871.45
410
+ Estimated Total Size (MB): 29090.11
411
+ ================================================================================================================================================================
412
+ NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
413
+ ================================================================================
414
+
415
+ [rank0]: Traceback (most recent call last):
416
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 173, in getuser
417
+ [rank0]: return pwd.getpwuid(os.getuid())[0]
418
+ [rank0]: ~~~~~~~~~~~~^^^^^^^^^^^^^
419
+ [rank0]: KeyError: 'getpwuid(): uid not found: 285818'
420
+
421
+ [rank0]: The above exception was the direct cause of the following exception:
422
+
423
+ [rank0]: Traceback (most recent call last):
424
+ [rank0]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1519, in <module>
425
+ [rank0]: main()
426
+ [rank0]: ~~~~^^
427
+ [rank0]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1516, in main
428
+ [rank0]: train(config)
429
+ [rank0]: ~~~~~^^^^^^^^
430
+ [rank0]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1107, in train
431
+ [rank0]: model = torch.compile(model)
432
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/__init__.py", line 2758, in compile
433
+ [rank0]: return torch._dynamo.optimize(
434
+ [rank0]: ~~~~~~~~~~~~~~~~~~~~~~^
435
+ [rank0]: backend=backend,
436
+ [rank0]: ^^^^^^^^^^^^^^^^
437
+ [rank0]: ...<3 lines>...
438
+ [rank0]: guard_filter_fn=guard_filter_fn,
439
+ [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
440
+ [rank0]: )(model) # type: ignore[return-value]
441
+ [rank0]: ^
442
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1488, in optimize
443
+ [rank0]: return _optimize(rebuild_ctx, *args, **kwargs)
444
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1564, in _optimize
445
+ [rank0]: backend = get_compiler_fn(backend)
446
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1325, in get_compiler_fn
447
+ [rank0]: from .repro.after_dynamo import wrap_backend_debug
448
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/repro/after_dynamo.py", line 33, in <module>
449
+ [rank0]: from torch._dynamo.debug_utils import (
450
+ [rank0]: ...<15 lines>...
451
+ [rank0]: )
452
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/debug_utils.py", line 43, in <module>
453
+ [rank0]: from torch._dynamo.testing import rand_strided
454
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/testing.py", line 33, in <module>
455
+ [rank0]: from torch._dynamo.backends.debugging import aot_eager
456
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/backends/debugging.py", line 34, in <module>
457
+ [rank0]: from functorch.compile import min_cut_rematerialization_partition
458
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/functorch/compile/__init__.py", line 2, in <module>
459
+ [rank0]: from torch._functorch.aot_autograd import (
460
+ [rank0]: ...<10 lines>...
461
+ [rank0]: )
462
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/aot_autograd.py", line 38, in <module>
463
+ [rank0]: from ._aot_autograd.autograd_cache import ( # noqa: F401
464
+ [rank0]: ...<4 lines>...
465
+ [rank0]: )
466
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/_aot_autograd/autograd_cache.py", line 27, in <module>
467
+ [rank0]: from torch._inductor.codecache import (
468
+ [rank0]: ...<11 lines>...
469
+ [rank0]: )
470
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/codecache.py", line 2753, in <module>
471
+ [rank0]: _HEADER_DIR = os.path.join(default_cache_dir(), "precompiled_headers")
472
+ [rank0]: ~~~~~~~~~~~~~~~~~^^
473
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/runtime/cache_dir_utils.py", line 23, in default_cache_dir
474
+ [rank0]: sanitized_username = re.sub(r'[\\/:*?"<>|]', "_", getpass.getuser())
475
+ [rank0]: ~~~~~~~~~~~~~~~^^
476
+ [rank0]: File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 175, in getuser
477
+ [rank0]: raise OSError('No username set in the environment') from e
478
+ [rank0]: OSError: No username set in the environment
479
+ [rank5]: Traceback (most recent call last):
480
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 173, in getuser
481
+ [rank5]: return pwd.getpwuid(os.getuid())[0]
482
+ [rank5]: ~~~~~~~~~~~~^^^^^^^^^^^^^
483
+ [rank5]: KeyError: 'getpwuid(): uid not found: 285818'
484
+
485
+ [rank5]: The above exception was the direct cause of the following exception:
486
+
487
+ [rank5]: Traceback (most recent call last):
488
+ [rank5]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1519, in <module>
489
+ [rank5]: main()
490
+ [rank5]: ~~~~^^
491
+ [rank5]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1516, in main
492
+ [rank5]: train(config)
493
+ [rank5]: ~~~~~^^^^^^^^
494
+ [rank5]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1107, in train
495
+ [rank5]: model = torch.compile(model)
496
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/__init__.py", line 2758, in compile
497
+ [rank5]: return torch._dynamo.optimize(
498
+ [rank5]: ~~~~~~~~~~~~~~~~~~~~~~^
499
+ [rank5]: backend=backend,
500
+ [rank5]: ^^^^^^^^^^^^^^^^
501
+ [rank5]: ...<3 lines>...
502
+ [rank5]: guard_filter_fn=guard_filter_fn,
503
+ [rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
504
+ [rank5]: )(model) # type: ignore[return-value]
505
+ [rank5]: ^
506
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1488, in optimize
507
+ [rank5]: return _optimize(rebuild_ctx, *args, **kwargs)
508
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1564, in _optimize
509
+ [rank5]: backend = get_compiler_fn(backend)
510
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1325, in get_compiler_fn
511
+ [rank5]: from .repro.after_dynamo import wrap_backend_debug
512
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/repro/after_dynamo.py", line 33, in <module>
513
+ [rank5]: from torch._dynamo.debug_utils import (
514
+ [rank5]: ...<15 lines>...
515
+ [rank5]: )
516
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/debug_utils.py", line 43, in <module>
517
+ [rank5]: from torch._dynamo.testing import rand_strided
518
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/testing.py", line 33, in <module>
519
+ [rank5]: from torch._dynamo.backends.debugging import aot_eager
520
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/backends/debugging.py", line 34, in <module>
521
+ [rank5]: from functorch.compile import min_cut_rematerialization_partition
522
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/functorch/compile/__init__.py", line 2, in <module>
523
+ [rank5]: from torch._functorch.aot_autograd import (
524
+ [rank5]: ...<10 lines>...
525
+ [rank5]: )
526
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/aot_autograd.py", line 38, in <module>
527
+ [rank5]: from ._aot_autograd.autograd_cache import ( # noqa: F401
528
+ [rank5]: ...<4 lines>...
529
+ [rank5]: )
530
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/_aot_autograd/autograd_cache.py", line 27, in <module>
531
+ [rank5]: from torch._inductor.codecache import (
532
+ [rank5]: ...<11 lines>...
533
+ [rank5]: )
534
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/codecache.py", line 2753, in <module>
535
+ [rank5]: _HEADER_DIR = os.path.join(default_cache_dir(), "precompiled_headers")
536
+ [rank5]: ~~~~~~~~~~~~~~~~~^^
537
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/runtime/cache_dir_utils.py", line 23, in default_cache_dir
538
+ [rank5]: sanitized_username = re.sub(r'[\\/:*?"<>|]', "_", getpass.getuser())
539
+ [rank5]: ~~~~~~~~~~~~~~~^^
540
+ [rank5]: File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 175, in getuser
541
+ [rank5]: raise OSError('No username set in the environment') from e
542
+ [rank5]: OSError: No username set in the environment
543
+ [rank0]:[W905 13:02:21.135808580 ProcessGroupNCCL.cpp:1575] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
544
+ W0905 13:02:22.231000 157 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 225 closing signal SIGTERM
545
+ W0905 13:02:22.232000 157 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 226 closing signal SIGTERM
546
+ W0905 13:02:22.232000 157 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 227 closing signal SIGTERM
547
+ W0905 13:02:22.233000 157 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 229 closing signal SIGTERM
548
+ W0905 13:02:22.234000 157 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 230 closing signal SIGTERM
549
+ E0905 13:02:22.801000 157 site-packages/torch/distributed/elastic/multiprocessing/api.py:986] failed (exitcode: 1) local_rank: 3 (pid: 228) of binary: /mloscratch/homes/navasard/conda/bin/python3.13
550
+ Traceback (most recent call last):
551
+ File "/mloscratch/homes/navasard/conda/bin/torchrun", line 6, in <module>
552
+ sys.exit(main())
553
+ ~~~~^^
554
+ File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper
555
+ return f(*args, **kwargs)
556
+ File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main
557
+ run(args)
558
+ ~~~^^^^^^
559
+ File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run
560
+ elastic_launch(
561
+ ~~~~~~~~~~~~~~~
562
+ config=config,
563
+ ~~~~~~~~~~~~~~
564
+ entrypoint=cmd,
565
+ ~~~~~~~~~~~~~~~
566
+ )(*cmd_args)
567
+ ~^^^^^^^^^^^
568
+ File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__
569
+ return launch_agent(self._config, self._entrypoint, list(args))
570
+ File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 317, in launch_agent
571
+ raise ChildFailedError(
572
+ ...<2 lines>...
573
+ )
574
+ torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
575
+ ============================================================
576
+ train.py FAILED
577
+ ------------------------------------------------------------
578
+ Failures:
579
+ [1]:
580
+ time : 2026-09-05_13:02:22
581
+ host : h100-workspace-6-gpu-0-25
582
+ rank : 0 (local_rank: 0)
583
+ exitcode : -15 (pid: 225)
584
+ error_file: <N/A>
585
+ traceback : Signal 15 (SIGTERM) received by PID 225
586
+ [2]:
587
+ time : 2026-09-05_13:02:22
588
+ host : h100-workspace-6-gpu-0-25
589
+ rank : 1 (local_rank: 1)
590
+ exitcode : 1 (pid: 226)
591
+ error_file: <N/A>
592
+ traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
593
+ [3]:
594
+ time : 2026-09-05_13:02:22
595
+ host : h100-workspace-6-gpu-0-25
596
+ rank : 2 (local_rank: 2)
597
+ exitcode : 1 (pid: 227)
598
+ error_file: <N/A>
599
+ traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
600
+ [4]:
601
+ time : 2026-09-05_13:02:22
602
+ host : h100-workspace-6-gpu-0-25
603
+ rank : 4 (local_rank: 4)
604
+ exitcode : 1 (pid: 229)
605
+ error_file: <N/A>
606
+ traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
607
+ [5]:
608
+ time : 2026-09-05_13:02:22
609
+ host : h100-workspace-6-gpu-0-25
610
+ rank : 5 (local_rank: 5)
611
+ exitcode : -15 (pid: 230)
612
+ error_file: <N/A>
613
+ traceback : Signal 15 (SIGTERM) received by PID 230
614
+ ------------------------------------------------------------
615
+ Root Cause (first observed failure):
616
+ [0]:
617
+ time : 2026-09-05_13:02:22
618
+ host : h100-workspace-6-gpu-0-25
619
+ rank : 3 (local_rank: 3)
620
+ exitcode : 1 (pid: 228)
621
+ error_file: <N/A>
622
+ traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
623
+ ============================================================
epfl_final/train_logs/p2_1p2b_dense_seed72_6gpu_resume9250_retry.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/train_logs/p2_1p2b_dense_seed72_resume6000.log ADDED
@@ -0,0 +1,515 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ W0903 19:13:00.509000 299 site-packages/torch/distributed/run.py:851]
2
+ W0903 19:13:00.509000 299 site-packages/torch/distributed/run.py:851] *****************************************
3
+ W0903 19:13:00.509000 299 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
4
+ W0903 19:13:00.509000 299 site-packages/torch/distributed/run.py:851] *****************************************
5
+ Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216
6
+ Running config:
7
+ {
8
+ "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
9
+ "model_dtype": "bfloat16",
10
+ "init_from_scratch": true,
11
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
12
+ "val_path": null,
13
+ "max_seq_length": 8192,
14
+ "num_workers": 8,
15
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216",
16
+ "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-6000",
17
+ "force_new_run": false,
18
+ "num_epochs": 1,
19
+ "max_steps": 50462,
20
+ "per_device_batch_size": 1,
21
+ "gradient_accumulation_steps": 15,
22
+ "learning_rate": 0.0005,
23
+ "weight_decay": 0.01,
24
+ "warmup_steps": 750,
25
+ "lr_scheduler": "cosine",
26
+ "max_grad_norm": 1.0,
27
+ "save_steps": 250,
28
+ "eval_steps": 250,
29
+ "eval_batches": 50,
30
+ "logging_steps": 10,
31
+ "seed": 72,
32
+ "verbose": true,
33
+ "checkpoint_activations": false,
34
+ "resume_micro_step_override": null,
35
+ "strategy": "ddp",
36
+ "mixed_precision": "bf16",
37
+ "wandb_enabled": true,
38
+ "wandb_project": "stochastic_sparsity",
39
+ "wandb_name": "1.2b dense 8k seed72 4gpu",
40
+ "wandb_tags": [
41
+ "qwen3",
42
+ "1.2b",
43
+ "8k",
44
+ "p2",
45
+ "dense",
46
+ "seed72"
47
+ ],
48
+ "hidden_size": 1536,
49
+ "intermediate_size": 6144,
50
+ "num_attention_heads": 12,
51
+ "num_key_value_heads": 4,
52
+ "num_hidden_layers": 28,
53
+ "head_dim": null,
54
+ "attention_dropout": null,
55
+ "rope_train_position_scale_set": null,
56
+ "attention_type": "sparse_stochastic",
57
+ "sparse_block_size": 128,
58
+ "sparse_num_random_blocks": 31,
59
+ "sparse_mask_pool_size": 1,
60
+ "sparse_val_dense_compare": false,
61
+ "sparse_layer_indices": null,
62
+ "sparse_mask_cache_dir": null,
63
+ "sparse_mask_seed": 0,
64
+ "sparse_mixed_nrb_set": null,
65
+ "dense_mode": true,
66
+ "dense_step_intervals": null
67
+ }
68
+ Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
69
+ Checkpoint activations: False
70
+ Checkpoint activations: False
71
+ Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-6000
72
+ Checkpoint activations: False
73
+ Checkpoint activations: False
74
+ dense_mode=True: sparse attention layers will use standard causal SDPA.
75
+
76
+
77
+ ================================================================================
78
+ MODEL ARCHITECTURE
79
+ ================================================================================
80
+ Model config:
81
+ vocab_size: 151936
82
+ hidden_size: 1536
83
+ num_hidden_layers: 28
84
+ num_attention_heads: 12
85
+ num_key_value_heads: 4
86
+ head_dim: 128
87
+ intermediate_size: 6144
88
+ max_position_embeddings: 32768
89
+ rope_theta: 1000000
90
+ rope_train_position_scale_set: None
91
+ attention_type: sparse_stochastic
92
+ sparse_block_size: 128
93
+ sparse_num_random_blocks: 31
94
+ sparse_mask_pool_size: 1
95
+ sparse_max_seq_length: 8192
96
+
97
+ Parameter count:
98
+ Total parameters: 1,202,352,640
99
+ Trainable parameters: 1,202,352,640
100
+ Non-trainable parameters: 0
101
+ Model size (BF16): 2.40 GB
102
+
103
+ --------------------------------------------------------------------------------
104
+ LAYER-BY-LAYER SUMMARY
105
+ --------------------------------------------------------------------------------
106
+ ================================================================================================================================================================
107
+ Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
108
+ ================================================================================================================================================================
109
+ Qwen3ForCausalLM [1, 8192] -- -- True
110
+ ├─Qwen3Model: 1-1 -- [1, 8192, 1536] -- True
111
+ │ └─Embedding: 2-1 [1, 8192] [1, 8192, 1536] 233,373,696 True
112
+ │ └─ModuleList: 2-2 -- -- -- True
113
+ │ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
114
+ │ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
115
+ │ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
116
+ │ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
117
+ │ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
118
+ │ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
119
+ │ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
120
+ │ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
121
+ │ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
122
+ │ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
123
+ │ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
124
+ │ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
125
+ │ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
126
+ │ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
127
+ │ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
128
+ │ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
129
+ │ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
130
+ │ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
131
+ │ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
132
+ │ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
133
+ │ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
134
+ │ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
135
+ │ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
136
+ │ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
137
+ │ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
138
+ │ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
139
+ │ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
140
+ │ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
141
+ │ └─Qwen3RMSNorm: 2-3 [1, 8192, 1536] [1, 8192, 1536] 1,536 True
142
+ ├─Linear: 1-2 [1, 8192, 1536] [1, 8192, 151936] 233,373,696 True
143
+ ================================================================================================================================================================
144
+ Total params: 1,435,726,336
145
+ Trainable params: 1,435,726,336
146
+ Non-trainable params: 0
147
+ Total mult-adds (Units.GIGABYTES): 1.44
148
+ ================================================================================================================================================================
149
+ Input size (MB): 0.07
150
+ Forward/backward pass size (MB): 26218.59
151
+ Params size (MB): 2871.45
152
+ Estimated Total Size (MB): 29090.11
153
+ ================================================================================================================================================================
154
+ NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
155
+ ================================================================================
156
+
157
+ Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
158
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
159
+ Using pre-tokenized dataset (no runtime tokenization)
160
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
161
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
162
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
163
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
164
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
165
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
166
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
167
+ Dataset split: 3027764 train, 159356 validation
168
+ Training examples (packed, train split): 3027764
169
+ Validation examples (packed, val split): 159356
170
+ Per-device batch size: 1
171
+ World size (GPUs): 4
172
+ Gradient accumulation steps: 15
173
+ Effective batch size (seqs/step): 60
174
+ Sequence length: 8192
175
+ Tokens per optimizer step: 491,520
176
+ Micro-batches per epoch (per rank): 756941
177
+ Optimizer steps per epoch: 50462
178
+ Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
179
+ Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-6000...
180
+ Preserving checkpoint LR state for exact resume: current=[0.0004863661528872283], schedule_base=[0.0005]
181
+ Resumed from checkpoint at step 6000 (epoch 0, micro_step 90000)
182
+ Resuming training from step 6000
183
+ Resuming wandb run 7tfq6nwm
184
+ wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
185
+ wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
186
+ wandb: Tracking run with wandb version 0.26.1
187
+ wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260903_191410-7tfq6nwm
188
+ wandb: Run `wandb offline` to turn off syncing.
189
+ wandb: Resuming run 1.2b dense 8k seed72 4gpu
190
+ wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
191
+ wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/7tfq6nwm
192
+ Starting training...
193
+ Validation at step 6000: Loss = 2.8723, Perplexity = 17.68
194
+ wandb: WARNING Tried to log to step 6000 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
195
+ Switching attention mode for step 6001: dense (dense step interval)
196
+ Step 6010/50462 | Loss: 2.8597 | LR: 4.86e-04 | 21236.9ms/step | 2.8 samples/s
197
+ wandb: WARNING Tried to log to step 6010 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
198
+ Step 6020/50462 | Loss: 2.8519 | LR: 4.86e-04 | 2809.8ms/step | 21.4 samples/s
199
+ wandb: WARNING Tried to log to step 6020 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
200
+ Step 6030/50462 | Loss: 2.8259 | LR: 4.86e-04 | 2816.6ms/step | 21.3 samples/s
201
+ wandb: WARNING Tried to log to step 6030 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
202
+ Step 6040/50462 | Loss: 2.8712 | LR: 4.86e-04 | 2816.1ms/step | 21.3 samples/s
203
+ wandb: WARNING Tried to log to step 6040 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
204
+ Step 6050/50462 | Loss: 2.8437 | LR: 4.86e-04 | 2822.8ms/step | 21.3 samples/s
205
+ wandb: WARNING Tried to log to step 6050 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
206
+ Step 6060/50462 | Loss: 2.8693 | LR: 4.86e-04 | 2818.7ms/step | 21.3 samples/s
207
+ wandb: WARNING Tried to log to step 6060 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
208
+ Step 6070/50462 | Loss: 2.8517 | LR: 4.86e-04 | 2820.4ms/step | 21.3 samples/s
209
+ wandb: WARNING Tried to log to step 6070 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
210
+ Step 6080/50462 | Loss: 2.8451 | LR: 4.86e-04 | 2817.7ms/step | 21.3 samples/s
211
+ wandb: WARNING Tried to log to step 6080 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
212
+ Step 6090/50462 | Loss: 2.8651 | LR: 4.86e-04 | 2822.0ms/step | 21.3 samples/s
213
+ wandb: WARNING Tried to log to step 6090 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
214
+ Step 6100/50462 | Loss: 2.8645 | LR: 4.86e-04 | 2818.9ms/step | 21.3 samples/s
215
+ wandb: WARNING Tried to log to step 6100 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
216
+ Step 6110/50462 | Loss: 2.8353 | LR: 4.86e-04 | 2822.6ms/step | 21.3 samples/s
217
+ wandb: WARNING Tried to log to step 6110 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
218
+ Step 6120/50462 | Loss: 2.8590 | LR: 4.86e-04 | 2822.6ms/step | 21.3 samples/s
219
+ wandb: WARNING Tried to log to step 6120 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
220
+ Step 6130/50462 | Loss: 2.8722 | LR: 4.86e-04 | 2819.8ms/step | 21.3 samples/s
221
+ wandb: WARNING Tried to log to step 6130 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
222
+ Step 6140/50462 | Loss: 2.8600 | LR: 4.86e-04 | 2818.8ms/step | 21.3 samples/s
223
+ wandb: WARNING Tried to log to step 6140 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
224
+ Step 6150/50462 | Loss: 2.8555 | LR: 4.86e-04 | 2819.9ms/step | 21.3 samples/s
225
+ wandb: WARNING Tried to log to step 6150 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
226
+ Step 6160/50462 | Loss: 2.8649 | LR: 4.86e-04 | 2818.8ms/step | 21.3 samples/s
227
+ wandb: WARNING Tried to log to step 6160 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
228
+ Step 6170/50462 | Loss: 2.8784 | LR: 4.85e-04 | 2822.2ms/step | 21.3 samples/s
229
+ wandb: WARNING Tried to log to step 6170 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
230
+ Step 6180/50462 | Loss: 2.8731 | LR: 4.85e-04 | 2821.1ms/step | 21.3 samples/s
231
+ wandb: WARNING Tried to log to step 6180 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
232
+ Step 6190/50462 | Loss: 2.8551 | LR: 4.85e-04 | 2819.8ms/step | 21.3 samples/s
233
+ wandb: WARNING Tried to log to step 6190 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
234
+ Step 6200/50462 | Loss: 2.8627 | LR: 4.85e-04 | 2817.7ms/step | 21.3 samples/s
235
+ wandb: WARNING Tried to log to step 6200 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
236
+ Step 6210/50462 | Loss: 2.8903 | LR: 4.85e-04 | 2820.1ms/step | 21.3 samples/s
237
+ wandb: WARNING Tried to log to step 6210 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
238
+ Step 6220/50462 | Loss: 2.8524 | LR: 4.85e-04 | 2819.5ms/step | 21.3 samples/s
239
+ wandb: WARNING Tried to log to step 6220 that is less than the current step 6221. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
240
+ Step 6230/50462 | Loss: 2.8157 | LR: 4.85e-04 | 2821.0ms/step | 21.3 samples/s
241
+ Step 6240/50462 | Loss: 2.8295 | LR: 4.85e-04 | 2822.1ms/step | 21.3 samples/s
242
+ Step 6250/50462 | Loss: 2.8252 | LR: 4.85e-04 | 2819.6ms/step | 21.3 samples/s
243
+ Validation at step 6250: Loss = 2.8645, Perplexity = 17.54
244
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-6250
245
+ Step 6260/50462 | Loss: 2.8655 | LR: 4.85e-04 | 4744.2ms/step | 12.6 samples/s
246
+ Step 6270/50462 | Loss: 2.8385 | LR: 4.85e-04 | 2825.2ms/step | 21.2 samples/s
247
+ Step 6280/50462 | Loss: 2.8255 | LR: 4.85e-04 | 2823.4ms/step | 21.3 samples/s
248
+ Step 6290/50462 | Loss: 2.8550 | LR: 4.85e-04 | 2824.9ms/step | 21.2 samples/s
249
+ Step 6300/50462 | Loss: 2.8523 | LR: 4.85e-04 | 2830.5ms/step | 21.2 samples/s
250
+ Step 6310/50462 | Loss: 2.8538 | LR: 4.85e-04 | 2821.9ms/step | 21.3 samples/s
251
+ Step 6320/50462 | Loss: 2.8421 | LR: 4.85e-04 | 2827.6ms/step | 21.2 samples/s
252
+ Step 6330/50462 | Loss: 2.8555 | LR: 4.85e-04 | 2826.2ms/step | 21.2 samples/s
253
+ Step 6340/50462 | Loss: 2.8568 | LR: 4.85e-04 | 2827.0ms/step | 21.2 samples/s
254
+ Step 6350/50462 | Loss: 2.8298 | LR: 4.85e-04 | 2824.4ms/step | 21.2 samples/s
255
+ Step 6360/50462 | Loss: 2.8044 | LR: 4.84e-04 | 2826.0ms/step | 21.2 samples/s
256
+ Step 6370/50462 | Loss: 2.8636 | LR: 4.84e-04 | 2825.6ms/step | 21.2 samples/s
257
+ Step 6380/50462 | Loss: 2.8133 | LR: 4.84e-04 | 2824.3ms/step | 21.2 samples/s
258
+ Step 6390/50462 | Loss: 2.8303 | LR: 4.84e-04 | 2826.0ms/step | 21.2 samples/s
259
+ Step 6400/50462 | Loss: 2.7921 | LR: 4.84e-04 | 2819.7ms/step | 21.3 samples/s
260
+ Step 6410/50462 | Loss: 2.8089 | LR: 4.84e-04 | 2820.7ms/step | 21.3 samples/s
261
+ Step 6420/50462 | Loss: 2.8516 | LR: 4.84e-04 | 2825.8ms/step | 21.2 samples/s
262
+ Step 6430/50462 | Loss: 2.8520 | LR: 4.84e-04 | 2829.3ms/step | 21.2 samples/s
263
+ Step 6440/50462 | Loss: 2.8771 | LR: 4.84e-04 | 2825.5ms/step | 21.2 samples/s
264
+ Step 6450/50462 | Loss: 2.8404 | LR: 4.84e-04 | 2826.3ms/step | 21.2 samples/s
265
+ Step 6460/50462 | Loss: 2.8609 | LR: 4.84e-04 | 2825.8ms/step | 21.2 samples/s
266
+ Step 6470/50462 | Loss: 2.8627 | LR: 4.84e-04 | 2827.6ms/step | 21.2 samples/s
267
+ Step 6480/50462 | Loss: 2.8227 | LR: 4.84e-04 | 2828.5ms/step | 21.2 samples/s
268
+ Step 6490/50462 | Loss: 2.8173 | LR: 4.84e-04 | 2832.5ms/step | 21.2 samples/s
269
+ Step 6500/50462 | Loss: 2.8416 | LR: 4.84e-04 | 2830.2ms/step | 21.2 samples/s
270
+ Validation at step 6500: Loss = 2.8544, Perplexity = 17.36
271
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-6500
272
+ Step 6510/50462 | Loss: 2.8339 | LR: 4.84e-04 | 4724.2ms/step | 12.7 samples/s
273
+ Step 6520/50462 | Loss: 2.8390 | LR: 4.84e-04 | 2821.9ms/step | 21.3 samples/s
274
+ Step 6530/50462 | Loss: 2.8509 | LR: 4.84e-04 | 2821.2ms/step | 21.3 samples/s
275
+ Step 6540/50462 | Loss: 2.8420 | LR: 4.83e-04 | 2823.8ms/step | 21.2 samples/s
276
+ Step 6550/50462 | Loss: 2.8535 | LR: 4.83e-04 | 2827.1ms/step | 21.2 samples/s
277
+ Step 6560/50462 | Loss: 2.8307 | LR: 4.83e-04 | 2826.6ms/step | 21.2 samples/s
278
+ Step 6570/50462 | Loss: 2.8257 | LR: 4.83e-04 | 2828.4ms/step | 21.2 samples/s
279
+ Step 6580/50462 | Loss: 2.8226 | LR: 4.83e-04 | 2827.2ms/step | 21.2 samples/s
280
+ Step 6590/50462 | Loss: 2.8594 | LR: 4.83e-04 | 2827.3ms/step | 21.2 samples/s
281
+ Step 6600/50462 | Loss: 2.8827 | LR: 4.83e-04 | 2825.0ms/step | 21.2 samples/s
282
+ Step 6610/50462 | Loss: 2.8160 | LR: 4.83e-04 | 2826.7ms/step | 21.2 samples/s
283
+ Step 6620/50462 | Loss: 2.8175 | LR: 4.83e-04 | 2825.4ms/step | 21.2 samples/s
284
+ Step 6630/50462 | Loss: 2.8634 | LR: 4.83e-04 | 2828.3ms/step | 21.2 samples/s
285
+ Step 6640/50462 | Loss: 2.8553 | LR: 4.83e-04 | 2830.2ms/step | 21.2 samples/s
286
+ Step 6650/50462 | Loss: 2.8787 | LR: 4.83e-04 | 2831.6ms/step | 21.2 samples/s
287
+ Step 6660/50462 | Loss: 2.8487 | LR: 4.83e-04 | 2832.4ms/step | 21.2 samples/s
288
+ Step 6670/50462 | Loss: 2.8219 | LR: 4.83e-04 | 2833.4ms/step | 21.2 samples/s
289
+ Step 6680/50462 | Loss: 2.8446 | LR: 4.83e-04 | 2835.7ms/step | 21.2 samples/s
290
+ Step 6690/50462 | Loss: 2.8274 | LR: 4.83e-04 | 2837.0ms/step | 21.1 samples/s
291
+ Step 6700/50462 | Loss: 2.8233 | LR: 4.83e-04 | 2822.0ms/step | 21.3 samples/s
292
+ Step 6710/50462 | Loss: 2.7692 | LR: 4.82e-04 | 2826.6ms/step | 21.2 samples/s
293
+ Step 6720/50462 | Loss: 2.8327 | LR: 4.82e-04 | 2829.6ms/step | 21.2 samples/s
294
+ Step 6730/50462 | Loss: 2.8745 | LR: 4.82e-04 | 2827.9ms/step | 21.2 samples/s
295
+ Step 6740/50462 | Loss: 2.8562 | LR: 4.82e-04 | 2828.4ms/step | 21.2 samples/s
296
+ Step 6750/50462 | Loss: 2.8172 | LR: 4.82e-04 | 2830.9ms/step | 21.2 samples/s
297
+ Validation at step 6750: Loss = 2.8447, Perplexity = 17.20
298
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-6750
299
+ Step 6760/50462 | Loss: 2.7999 | LR: 4.82e-04 | 4742.2ms/step | 12.7 samples/s
300
+ Step 6770/50462 | Loss: 2.8325 | LR: 4.82e-04 | 2826.8ms/step | 21.2 samples/s
301
+ Step 6780/50462 | Loss: 2.8265 | LR: 4.82e-04 | 2827.0ms/step | 21.2 samples/s
302
+ Step 6790/50462 | Loss: 2.8427 | LR: 4.82e-04 | 2826.8ms/step | 21.2 samples/s
303
+ Step 6800/50462 | Loss: 2.8308 | LR: 4.82e-04 | 2831.5ms/step | 21.2 samples/s
304
+ Step 6810/50462 | Loss: 2.8202 | LR: 4.82e-04 | 2825.0ms/step | 21.2 samples/s
305
+ Step 6820/50462 | Loss: 2.8387 | LR: 4.82e-04 | 2829.4ms/step | 21.2 samples/s
306
+ Step 6830/50462 | Loss: 2.8369 | LR: 4.82e-04 | 2829.2ms/step | 21.2 samples/s
307
+ Step 6840/50462 | Loss: 2.8424 | LR: 4.82e-04 | 2827.2ms/step | 21.2 samples/s
308
+ Step 6850/50462 | Loss: 2.8462 | LR: 4.82e-04 | 2827.3ms/step | 21.2 samples/s
309
+ Step 6860/50462 | Loss: 2.8360 | LR: 4.82e-04 | 2827.5ms/step | 21.2 samples/s
310
+ Step 6870/50462 | Loss: 2.8156 | LR: 4.82e-04 | 2823.5ms/step | 21.3 samples/s
311
+ Step 6880/50462 | Loss: 2.8322 | LR: 4.81e-04 | 2827.1ms/step | 21.2 samples/s
312
+ Step 6890/50462 | Loss: 2.8454 | LR: 4.81e-04 | 2824.9ms/step | 21.2 samples/s
313
+ Step 6900/50462 | Loss: 2.8269 | LR: 4.81e-04 | 2824.2ms/step | 21.2 samples/s
314
+ Step 6910/50462 | Loss: 2.8452 | LR: 4.81e-04 | 2829.2ms/step | 21.2 samples/s
315
+ Step 6920/50462 | Loss: 2.8197 | LR: 4.81e-04 | 2826.6ms/step | 21.2 samples/s
316
+ Step 6930/50462 | Loss: 2.8001 | LR: 4.81e-04 | 2827.0ms/step | 21.2 samples/s
317
+ Step 6940/50462 | Loss: 2.8079 | LR: 4.81e-04 | 2827.8ms/step | 21.2 samples/s
318
+ Step 6950/50462 | Loss: 2.8084 | LR: 4.81e-04 | 2828.2ms/step | 21.2 samples/s
319
+ Step 6960/50462 | Loss: 2.7995 | LR: 4.81e-04 | 2827.1ms/step | 21.2 samples/s
320
+ Step 6970/50462 | Loss: 2.8182 | LR: 4.81e-04 | 2828.7ms/step | 21.2 samples/s
321
+ Step 6980/50462 | Loss: 2.8333 | LR: 4.81e-04 | 2825.2ms/step | 21.2 samples/s
322
+ Step 6990/50462 | Loss: 2.8061 | LR: 4.81e-04 | 2823.9ms/step | 21.2 samples/s
323
+ Step 7000/50462 | Loss: 2.8077 | LR: 4.81e-04 | 2826.0ms/step | 21.2 samples/s
324
+ Validation at step 7000: Loss = 2.8372, Perplexity = 17.07
325
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-7000
326
+ Step 7010/50462 | Loss: 2.8285 | LR: 4.81e-04 | 4713.6ms/step | 12.7 samples/s
327
+ Step 7020/50462 | Loss: 2.8495 | LR: 4.81e-04 | 2824.9ms/step | 21.2 samples/s
328
+ Step 7030/50462 | Loss: 2.8164 | LR: 4.81e-04 | 2828.3ms/step | 21.2 samples/s
329
+ Step 7040/50462 | Loss: 2.8421 | LR: 4.81e-04 | 2831.6ms/step | 21.2 samples/s
330
+ Step 7050/50462 | Loss: 2.8165 | LR: 4.80e-04 | 2831.3ms/step | 21.2 samples/s
331
+ Step 7060/50462 | Loss: 2.8048 | LR: 4.80e-04 | 2831.8ms/step | 21.2 samples/s
332
+ Step 7070/50462 | Loss: 2.8053 | LR: 4.80e-04 | 2830.2ms/step | 21.2 samples/s
333
+ Step 7080/50462 | Loss: 2.8174 | LR: 4.80e-04 | 2829.2ms/step | 21.2 samples/s
334
+ Step 7090/50462 | Loss: 2.8121 | LR: 4.80e-04 | 2829.8ms/step | 21.2 samples/s
335
+ Step 7100/50462 | Loss: 2.8201 | LR: 4.80e-04 | 2831.6ms/step | 21.2 samples/s
336
+ Step 7110/50462 | Loss: 2.7884 | LR: 4.80e-04 | 2828.6ms/step | 21.2 samples/s
337
+ Step 7120/50462 | Loss: 2.8225 | LR: 4.80e-04 | 2827.6ms/step | 21.2 samples/s
338
+ Step 7130/50462 | Loss: 2.8209 | LR: 4.80e-04 | 2829.3ms/step | 21.2 samples/s
339
+ Step 7140/50462 | Loss: 2.8234 | LR: 4.80e-04 | 2837.5ms/step | 21.1 samples/s
340
+ Step 7150/50462 | Loss: 2.8532 | LR: 4.80e-04 | 2838.6ms/step | 21.1 samples/s
341
+ Step 7160/50462 | Loss: 2.8421 | LR: 4.80e-04 | 2835.8ms/step | 21.2 samples/s
342
+ Step 7170/50462 | Loss: 2.8162 | LR: 4.80e-04 | 2836.7ms/step | 21.2 samples/s
343
+ Step 7180/50462 | Loss: 2.8106 | LR: 4.80e-04 | 2834.7ms/step | 21.2 samples/s
344
+ Step 7190/50462 | Loss: 2.8063 | LR: 4.80e-04 | 2843.9ms/step | 21.1 samples/s
345
+ Step 7200/50462 | Loss: 2.8102 | LR: 4.80e-04 | 2832.8ms/step | 21.2 samples/s
346
+ Step 7210/50462 | Loss: 2.8297 | LR: 4.79e-04 | 2832.2ms/step | 21.2 samples/s
347
+ Step 7220/50462 | Loss: 2.8047 | LR: 4.79e-04 | 2833.6ms/step | 21.2 samples/s
348
+ Step 7230/50462 | Loss: 2.7892 | LR: 4.79e-04 | 2829.4ms/step | 21.2 samples/s
349
+ Step 7240/50462 | Loss: 2.8157 | LR: 4.79e-04 | 2832.2ms/step | 21.2 samples/s
350
+ Step 7250/50462 | Loss: 2.8359 | LR: 4.79e-04 | 2835.0ms/step | 21.2 samples/s
351
+ Validation at step 7250: Loss = 2.8303, Perplexity = 16.95
352
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-7250
353
+ Step 7260/50462 | Loss: 2.8316 | LR: 4.79e-04 | 4728.0ms/step | 12.7 samples/s
354
+ Step 7270/50462 | Loss: 2.8153 | LR: 4.79e-04 | 2829.0ms/step | 21.2 samples/s
355
+ Step 7280/50462 | Loss: 2.8245 | LR: 4.79e-04 | 2831.0ms/step | 21.2 samples/s
356
+ Step 7290/50462 | Loss: 2.8046 | LR: 4.79e-04 | 2832.9ms/step | 21.2 samples/s
357
+ Step 7300/50462 | Loss: 2.8398 | LR: 4.79e-04 | 2836.7ms/step | 21.2 samples/s
358
+ Step 7310/50462 | Loss: 2.8421 | LR: 4.79e-04 | 2832.1ms/step | 21.2 samples/s
359
+ Step 7320/50462 | Loss: 2.8127 | LR: 4.79e-04 | 2832.3ms/step | 21.2 samples/s
360
+ Step 7330/50462 | Loss: 2.8090 | LR: 4.79e-04 | 2832.4ms/step | 21.2 samples/s
361
+ Step 7340/50462 | Loss: 2.8316 | LR: 4.79e-04 | 2831.0ms/step | 21.2 samples/s
362
+ Step 7350/50462 | Loss: 2.8172 | LR: 4.79e-04 | 2834.3ms/step | 21.2 samples/s
363
+ Step 7360/50462 | Loss: 2.7956 | LR: 4.79e-04 | 2831.1ms/step | 21.2 samples/s
364
+ Step 7370/50462 | Loss: 2.8436 | LR: 4.78e-04 | 2830.1ms/step | 21.2 samples/s
365
+ Step 7380/50462 | Loss: 2.7903 | LR: 4.78e-04 | 2833.2ms/step | 21.2 samples/s
366
+ Step 7390/50462 | Loss: 2.7718 | LR: 4.78e-04 | 2832.8ms/step | 21.2 samples/s
367
+ Step 7400/50462 | Loss: 2.8318 | LR: 4.78e-04 | 2832.7ms/step | 21.2 samples/s
368
+ Step 7410/50462 | Loss: 2.7899 | LR: 4.78e-04 | 2832.4ms/step | 21.2 samples/s
369
+ Step 7420/50462 | Loss: 2.8049 | LR: 4.78e-04 | 2832.6ms/step | 21.2 samples/s
370
+ Step 7430/50462 | Loss: 2.8237 | LR: 4.78e-04 | 2832.6ms/step | 21.2 samples/s
371
+ Step 7440/50462 | Loss: 2.7649 | LR: 4.78e-04 | 2832.7ms/step | 21.2 samples/s
372
+ Step 7450/50462 | Loss: 2.8128 | LR: 4.78e-04 | 2830.2ms/step | 21.2 samples/s
373
+ Step 7460/50462 | Loss: 2.7993 | LR: 4.78e-04 | 2831.4ms/step | 21.2 samples/s
374
+ Step 7470/50462 | Loss: 2.8039 | LR: 4.78e-04 | 2828.4ms/step | 21.2 samples/s
375
+ Step 7480/50462 | Loss: 2.8226 | LR: 4.78e-04 | 2834.6ms/step | 21.2 samples/s
376
+ Step 7490/50462 | Loss: 2.8183 | LR: 4.78e-04 | 2833.3ms/step | 21.2 samples/s
377
+ Step 7500/50462 | Loss: 2.8260 | LR: 4.78e-04 | 2835.8ms/step | 21.2 samples/s
378
+ Validation at step 7500: Loss = 2.8226, Perplexity = 16.82
379
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-7500
380
+ Step 7510/50462 | Loss: 2.7984 | LR: 4.78e-04 | 4804.0ms/step | 12.5 samples/s
381
+ Step 7520/50462 | Loss: 2.8030 | LR: 4.77e-04 | 2832.1ms/step | 21.2 samples/s
382
+ Step 7530/50462 | Loss: 2.8142 | LR: 4.77e-04 | 2833.8ms/step | 21.2 samples/s
383
+ Step 7540/50462 | Loss: 2.8141 | LR: 4.77e-04 | 2829.7ms/step | 21.2 samples/s
384
+ Step 7550/50462 | Loss: 2.7960 | LR: 4.77e-04 | 2832.5ms/step | 21.2 samples/s
385
+ Step 7560/50462 | Loss: 2.8297 | LR: 4.77e-04 | 2831.0ms/step | 21.2 samples/s
386
+ Step 7570/50462 | Loss: 2.7583 | LR: 4.77e-04 | 2834.3ms/step | 21.2 samples/s
387
+ Step 7580/50462 | Loss: 2.8149 | LR: 4.77e-04 | 2839.6ms/step | 21.1 samples/s
388
+ Step 7590/50462 | Loss: 2.8050 | LR: 4.77e-04 | 2834.0ms/step | 21.2 samples/s
389
+ Step 7600/50462 | Loss: 2.7879 | LR: 4.77e-04 | 2834.0ms/step | 21.2 samples/s
390
+ Step 7610/50462 | Loss: 2.7765 | LR: 4.77e-04 | 2832.4ms/step | 21.2 samples/s
391
+ Step 7620/50462 | Loss: 2.8211 | LR: 4.77e-04 | 2831.4ms/step | 21.2 samples/s
392
+ Step 7630/50462 | Loss: 2.7879 | LR: 4.77e-04 | 2834.5ms/step | 21.2 samples/s
393
+ Step 7640/50462 | Loss: 2.8146 | LR: 4.77e-04 | 2831.1ms/step | 21.2 samples/s
394
+ Step 7650/50462 | Loss: 2.7887 | LR: 4.77e-04 | 2826.6ms/step | 21.2 samples/s
395
+ Step 7660/50462 | Loss: 2.7751 | LR: 4.77e-04 | 2832.6ms/step | 21.2 samples/s
396
+ Step 7670/50462 | Loss: 2.7773 | LR: 4.76e-04 | 2826.7ms/step | 21.2 samples/s
397
+ Step 7680/50462 | Loss: 2.8116 | LR: 4.76e-04 | 2831.2ms/step | 21.2 samples/s
398
+ Step 7690/50462 | Loss: 2.7783 | LR: 4.76e-04 | 2833.0ms/step | 21.2 samples/s
399
+ Step 7700/50462 | Loss: 2.7857 | LR: 4.76e-04 | 2829.2ms/step | 21.2 samples/s
400
+ Step 7710/50462 | Loss: 2.8029 | LR: 4.76e-04 | 2835.2ms/step | 21.2 samples/s
401
+ Step 7720/50462 | Loss: 2.7885 | LR: 4.76e-04 | 2832.6ms/step | 21.2 samples/s
402
+ Step 7730/50462 | Loss: 2.8014 | LR: 4.76e-04 | 2830.9ms/step | 21.2 samples/s
403
+ Step 7740/50462 | Loss: 2.8146 | LR: 4.76e-04 | 2831.2ms/step | 21.2 samples/s
404
+ Step 7750/50462 | Loss: 2.8174 | LR: 4.76e-04 | 2829.3ms/step | 21.2 samples/s
405
+ Validation at step 7750: Loss = 2.8131, Perplexity = 16.66
406
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-7750
407
+ Step 7760/50462 | Loss: 2.7993 | LR: 4.76e-04 | 4732.3ms/step | 12.7 samples/s
408
+ Step 7770/50462 | Loss: 2.7832 | LR: 4.76e-04 | 2832.5ms/step | 21.2 samples/s
409
+ Step 7780/50462 | Loss: 2.7779 | LR: 4.76e-04 | 2832.1ms/step | 21.2 samples/s
410
+ Step 7790/50462 | Loss: 2.7838 | LR: 4.76e-04 | 2834.5ms/step | 21.2 samples/s
411
+ Step 7800/50462 | Loss: 2.7822 | LR: 4.76e-04 | 2836.6ms/step | 21.2 samples/s
412
+ Step 7810/50462 | Loss: 2.7876 | LR: 4.76e-04 | 2830.9ms/step | 21.2 samples/s
413
+ Step 7820/50462 | Loss: 2.8113 | LR: 4.75e-04 | 2834.8ms/step | 21.2 samples/s
414
+ Step 7830/50462 | Loss: 2.8010 | LR: 4.75e-04 | 2832.0ms/step | 21.2 samples/s
415
+ Step 7840/50462 | Loss: 2.8010 | LR: 4.75e-04 | 2832.6ms/step | 21.2 samples/s
416
+ Step 7850/50462 | Loss: 2.8107 | LR: 4.75e-04 | 2835.3ms/step | 21.2 samples/s
417
+ Step 7860/50462 | Loss: 2.8114 | LR: 4.75e-04 | 2829.9ms/step | 21.2 samples/s
418
+ Step 7870/50462 | Loss: 2.7901 | LR: 4.75e-04 | 2827.4ms/step | 21.2 samples/s
419
+ Step 7880/50462 | Loss: 2.8174 | LR: 4.75e-04 | 2839.2ms/step | 21.1 samples/s
420
+ Step 7890/50462 | Loss: 2.7936 | LR: 4.75e-04 | 2829.6ms/step | 21.2 samples/s
421
+ Step 7900/50462 | Loss: 2.7810 | LR: 4.75e-04 | 2847.7ms/step | 21.1 samples/s
422
+ Step 7910/50462 | Loss: 2.7693 | LR: 4.75e-04 | 2837.5ms/step | 21.1 samples/s
423
+ Step 7920/50462 | Loss: 2.8108 | LR: 4.75e-04 | 2831.0ms/step | 21.2 samples/s
424
+ Step 7930/50462 | Loss: 2.7878 | LR: 4.75e-04 | 2835.8ms/step | 21.2 samples/s
425
+ Step 7940/50462 | Loss: 2.7996 | LR: 4.75e-04 | 2835.8ms/step | 21.2 samples/s
426
+ Step 7950/50462 | Loss: 2.7874 | LR: 4.75e-04 | 2832.5ms/step | 21.2 samples/s
427
+ Step 7960/50462 | Loss: 2.7938 | LR: 4.74e-04 | 2833.2ms/step | 21.2 samples/s
428
+ Step 7970/50462 | Loss: 2.7931 | LR: 4.74e-04 | 2832.0ms/step | 21.2 samples/s
429
+ Step 7980/50462 | Loss: 2.7587 | LR: 4.74e-04 | 2831.3ms/step | 21.2 samples/s
430
+ Step 7990/50462 | Loss: 2.7533 | LR: 4.74e-04 | 2828.2ms/step | 21.2 samples/s
431
+ Step 8000/50462 | Loss: 2.7796 | LR: 4.74e-04 | 2829.2ms/step | 21.2 samples/s
432
+ Validation at step 8000: Loss = 2.8065, Perplexity = 16.55
433
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-8000
434
+ Step 8010/50462 | Loss: 2.8012 | LR: 4.74e-04 | 4733.1ms/step | 12.7 samples/s
435
+ Step 8020/50462 | Loss: 2.7603 | LR: 4.74e-04 | 2824.5ms/step | 21.2 samples/s
436
+ Step 8030/50462 | Loss: 2.7895 | LR: 4.74e-04 | 2826.0ms/step | 21.2 samples/s
437
+ Step 8040/50462 | Loss: 2.7949 | LR: 4.74e-04 | 2826.9ms/step | 21.2 samples/s
438
+ Step 8050/50462 | Loss: 2.8022 | LR: 4.74e-04 | 2830.8ms/step | 21.2 samples/s
439
+ Step 8060/50462 | Loss: 2.8108 | LR: 4.74e-04 | 2828.6ms/step | 21.2 samples/s
440
+ Step 8070/50462 | Loss: 2.7895 | LR: 4.74e-04 | 2831.8ms/step | 21.2 samples/s
441
+ Step 8080/50462 | Loss: 2.8107 | LR: 4.74e-04 | 2829.3ms/step | 21.2 samples/s
442
+ Step 8090/50462 | Loss: 2.8135 | LR: 4.74e-04 | 2828.4ms/step | 21.2 samples/s
443
+ Step 8100/50462 | Loss: 2.7970 | LR: 4.74e-04 | 2828.4ms/step | 21.2 samples/s
444
+ Step 8110/50462 | Loss: 2.7817 | LR: 4.73e-04 | 2826.3ms/step | 21.2 samples/s
445
+ Step 8120/50462 | Loss: 2.7818 | LR: 4.73e-04 | 2826.8ms/step | 21.2 samples/s
446
+ Step 8130/50462 | Loss: 2.7866 | LR: 4.73e-04 | 2829.5ms/step | 21.2 samples/s
447
+ Step 8140/50462 | Loss: 2.7823 | LR: 4.73e-04 | 2827.8ms/step | 21.2 samples/s
448
+ Step 8150/50462 | Loss: 2.7947 | LR: 4.73e-04 | 2824.0ms/step | 21.2 samples/s
449
+ Step 8160/50462 | Loss: 2.8141 | LR: 4.73e-04 | 2827.9ms/step | 21.2 samples/s
450
+ Step 8170/50462 | Loss: 2.7743 | LR: 4.73e-04 | 2828.2ms/step | 21.2 samples/s
451
+ Step 8180/50462 | Loss: 2.7726 | LR: 4.73e-04 | 2827.4ms/step | 21.2 samples/s
452
+ Step 8190/50462 | Loss: 2.7846 | LR: 4.73e-04 | 2839.0ms/step | 21.1 samples/s
453
+ Step 8200/50462 | Loss: 2.7729 | LR: 4.73e-04 | 2824.9ms/step | 21.2 samples/s
454
+ Step 8210/50462 | Loss: 2.8169 | LR: 4.73e-04 | 2828.4ms/step | 21.2 samples/s
455
+ Step 8220/50462 | Loss: 2.7852 | LR: 4.73e-04 | 2839.1ms/step | 21.1 samples/s
456
+ Step 8230/50462 | Loss: 2.7760 | LR: 4.73e-04 | 2829.6ms/step | 21.2 samples/s
457
+ Step 8240/50462 | Loss: 2.7593 | LR: 4.73e-04 | 2828.5ms/step | 21.2 samples/s
458
+ Step 8250/50462 | Loss: 2.7997 | LR: 4.72e-04 | 2828.1ms/step | 21.2 samples/s
459
+ Validation at step 8250: Loss = 2.7999, Perplexity = 16.44
460
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-8250
461
+ Step 8260/50462 | Loss: 2.7523 | LR: 4.72e-04 | 4773.8ms/step | 12.6 samples/s
462
+ Step 8270/50462 | Loss: 2.7759 | LR: 4.72e-04 | 2824.9ms/step | 21.2 samples/s
463
+ Step 8280/50462 | Loss: 2.7909 | LR: 4.72e-04 | 2828.9ms/step | 21.2 samples/s
464
+ Step 8290/50462 | Loss: 2.7802 | LR: 4.72e-04 | 2827.2ms/step | 21.2 samples/s
465
+ Step 8300/50462 | Loss: 2.7975 | LR: 4.72e-04 | 2824.6ms/step | 21.2 samples/s
466
+ Step 8310/50462 | Loss: 2.7950 | LR: 4.72e-04 | 2829.5ms/step | 21.2 samples/s
467
+ Step 8320/50462 | Loss: 2.7723 | LR: 4.72e-04 | 2827.7ms/step | 21.2 samples/s
468
+ Step 8330/50462 | Loss: 2.7928 | LR: 4.72e-04 | 2823.9ms/step | 21.2 samples/s
469
+ Step 8340/50462 | Loss: 2.7639 | LR: 4.72e-04 | 2846.1ms/step | 21.1 samples/s
470
+ Step 8350/50462 | Loss: 2.7707 | LR: 4.72e-04 | 2831.0ms/step | 21.2 samples/s
471
+ Step 8360/50462 | Loss: 2.7990 | LR: 4.72e-04 | 2829.4ms/step | 21.2 samples/s
472
+ Step 8370/50462 | Loss: 2.8025 | LR: 4.72e-04 | 2831.3ms/step | 21.2 samples/s
473
+ Step 8380/50462 | Loss: 2.8028 | LR: 4.71e-04 | 2832.1ms/step | 21.2 samples/s
474
+ Step 8390/50462 | Loss: 2.7690 | LR: 4.71e-04 | 2828.4ms/step | 21.2 samples/s
475
+ Step 8400/50462 | Loss: 2.7571 | LR: 4.71e-04 | 2827.9ms/step | 21.2 samples/s
476
+ Step 8410/50462 | Loss: 2.7676 | LR: 4.71e-04 | 2830.2ms/step | 21.2 samples/s
477
+ Step 8420/50462 | Loss: 2.7708 | LR: 4.71e-04 | 2827.5ms/step | 21.2 samples/s
478
+ Step 8430/50462 | Loss: 2.7216 | LR: 4.71e-04 | 2825.5ms/step | 21.2 samples/s
479
+ Step 8440/50462 | Loss: 2.7693 | LR: 4.71e-04 | 2826.9ms/step | 21.2 samples/s
480
+ Step 8450/50462 | Loss: 2.7839 | LR: 4.71e-04 | 2826.9ms/step | 21.2 samples/s
481
+ Step 8460/50462 | Loss: 2.7829 | LR: 4.71e-04 | 2834.4ms/step | 21.2 samples/s
482
+ Step 8470/50462 | Loss: 2.8129 | LR: 4.71e-04 | 2828.9ms/step | 21.2 samples/s
483
+ Step 8480/50462 | Loss: 2.8214 | LR: 4.71e-04 | 2827.1ms/step | 21.2 samples/s
484
+ Step 8490/50462 | Loss: 2.7758 | LR: 4.71e-04 | 2825.1ms/step | 21.2 samples/s
485
+ Step 8500/50462 | Loss: 2.7891 | LR: 4.71e-04 | 2831.6ms/step | 21.2 samples/s
486
+ Validation at step 8500: Loss = 2.7920, Perplexity = 16.31
487
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-8500
488
+ Step 8510/50462 | Loss: 2.8123 | LR: 4.71e-04 | 4720.9ms/step | 12.7 samples/s
489
+ Step 8520/50462 | Loss: 2.7670 | LR: 4.70e-04 | 2828.4ms/step | 21.2 samples/s
490
+ Step 8530/50462 | Loss: 2.7797 | LR: 4.70e-04 | 2828.3ms/step | 21.2 samples/s
491
+ Step 8540/50462 | Loss: 2.7798 | LR: 4.70e-04 | 2827.7ms/step | 21.2 samples/s
492
+ Step 8550/50462 | Loss: 2.8018 | LR: 4.70e-04 | 2841.3ms/step | 21.1 samples/s
493
+ Step 8560/50462 | Loss: 2.7387 | LR: 4.70e-04 | 2828.4ms/step | 21.2 samples/s
494
+ Step 8570/50462 | Loss: 2.8006 | LR: 4.70e-04 | 2827.3ms/step | 21.2 samples/s
495
+ Step 8580/50462 | Loss: 2.7432 | LR: 4.70e-04 | 2829.6ms/step | 21.2 samples/s
496
+ Step 8590/50462 | Loss: 2.7690 | LR: 4.70e-04 | 2836.0ms/step | 21.2 samples/s
497
+ Step 8600/50462 | Loss: 2.7692 | LR: 4.70e-04 | 2831.0ms/step | 21.2 samples/s
498
+ Step 8610/50462 | Loss: 2.7645 | LR: 4.70e-04 | 2830.1ms/step | 21.2 samples/s
499
+ Step 8620/50462 | Loss: 2.7955 | LR: 4.70e-04 | 2831.2ms/step | 21.2 samples/s
500
+ Step 8630/50462 | Loss: 2.7715 | LR: 4.70e-04 | 2830.8ms/step | 21.2 samples/s
501
+ Step 8640/50462 | Loss: 2.7557 | LR: 4.70e-04 | 2827.9ms/step | 21.2 samples/s
502
+ Step 8650/50462 | Loss: 2.7873 | LR: 4.69e-04 | 2827.3ms/step | 21.2 samples/s
503
+ Step 8660/50462 | Loss: 2.7826 | LR: 4.69e-04 | 2827.2ms/step | 21.2 samples/s
504
+ Step 8670/50462 | Loss: 2.7481 | LR: 4.69e-04 | 2828.3ms/step | 21.2 samples/s
505
+ Step 8680/50462 | Loss: 2.7883 | LR: 4.69e-04 | 2828.5ms/step | 21.2 samples/s
506
+ Step 8690/50462 | Loss: 2.7859 | LR: 4.69e-04 | 2827.6ms/step | 21.2 samples/s
507
+ Step 8700/50462 | Loss: 2.7838 | LR: 4.69e-04 | 2824.2ms/step | 21.2 samples/s
508
+ Step 8710/50462 | Loss: 2.7647 | LR: 4.69e-04 | 2826.2ms/step | 21.2 samples/s
509
+ Step 8720/50462 | Loss: 2.7783 | LR: 4.69e-04 | 2821.9ms/step | 21.3 samples/s
510
+ Step 8730/50462 | Loss: 2.7925 | LR: 4.69e-04 | 2818.3ms/step | 21.3 samples/s
511
+ Step 8740/50462 | Loss: 2.7903 | LR: 4.69e-04 | 2819.6ms/step | 21.3 samples/s
512
+ Step 8750/50462 | Loss: 2.7691 | LR: 4.69e-04 | 2819.6ms/step | 21.3 samples/s
513
+ Validation at step 8750: Loss = 2.7840, Perplexity = 16.18
514
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-8750
515
+ Step 8760/50462 | Loss: 2.7788 | LR: 4.69e-04 | 4684.9ms/step | 12.8 samples/s
epfl_final/train_logs/p2_1p2b_dense_seed72_resume8750.log ADDED
@@ -0,0 +1,270 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ W0904 04:19:28.650000 143 site-packages/torch/distributed/run.py:851]
2
+ W0904 04:19:28.650000 143 site-packages/torch/distributed/run.py:851] *****************************************
3
+ W0904 04:19:28.650000 143 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
4
+ W0904 04:19:28.650000 143 site-packages/torch/distributed/run.py:851] *****************************************
5
+ Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216
6
+ Running config:
7
+ {
8
+ "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
9
+ "model_dtype": "bfloat16",
10
+ "init_from_scratch": true,
11
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
12
+ "val_path": null,
13
+ "max_seq_length": 8192,
14
+ "num_workers": 8,
15
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216",
16
+ "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-8750",
17
+ "force_new_run": false,
18
+ "num_epochs": 1,
19
+ "max_steps": 50462,
20
+ "per_device_batch_size": 1,
21
+ "gradient_accumulation_steps": 15,
22
+ "learning_rate": 0.0005,
23
+ "weight_decay": 0.01,
24
+ "warmup_steps": 750,
25
+ "lr_scheduler": "cosine",
26
+ "max_grad_norm": 1.0,
27
+ "save_steps": 250,
28
+ "eval_steps": 250,
29
+ "eval_batches": 50,
30
+ "logging_steps": 10,
31
+ "seed": 72,
32
+ "verbose": true,
33
+ "checkpoint_activations": false,
34
+ "resume_micro_step_override": null,
35
+ "strategy": "ddp",
36
+ "mixed_precision": "bf16",
37
+ "wandb_enabled": true,
38
+ "wandb_project": "stochastic_sparsity",
39
+ "wandb_name": "1.2b dense 8k seed72 4gpu",
40
+ "wandb_tags": [
41
+ "qwen3",
42
+ "1.2b",
43
+ "8k",
44
+ "p2",
45
+ "dense",
46
+ "seed72"
47
+ ],
48
+ "hidden_size": 1536,
49
+ "intermediate_size": 6144,
50
+ "num_attention_heads": 12,
51
+ "num_key_value_heads": 4,
52
+ "num_hidden_layers": 28,
53
+ "head_dim": null,
54
+ "attention_dropout": null,
55
+ "rope_train_position_scale_set": null,
56
+ "attention_type": "sparse_stochastic",
57
+ "sparse_block_size": 128,
58
+ "sparse_num_random_blocks": 31,
59
+ "sparse_mask_pool_size": 1,
60
+ "sparse_val_dense_compare": false,
61
+ "sparse_layer_indices": null,
62
+ "sparse_mask_cache_dir": null,
63
+ "sparse_mask_seed": 0,
64
+ "sparse_mixed_nrb_set": null,
65
+ "dense_mode": true,
66
+ "dense_step_intervals": null
67
+ }
68
+ Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
69
+ Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-8750
70
+ Checkpoint activations: FalseCheckpoint activations: False
71
+ Checkpoint activations: False
72
+
73
+ Checkpoint activations: False
74
+ dense_mode=True: sparse attention layers will use standard causal SDPA.
75
+
76
+
77
+ ================================================================================
78
+ MODEL ARCHITECTURE
79
+ ================================================================================
80
+ Model config:
81
+ vocab_size: 151936
82
+ hidden_size: 1536
83
+ num_hidden_layers: 28
84
+ num_attention_heads: 12
85
+ num_key_value_heads: 4
86
+ head_dim: 128
87
+ intermediate_size: 6144
88
+ max_position_embeddings: 32768
89
+ rope_theta: 1000000
90
+ rope_train_position_scale_set: None
91
+ attention_type: sparse_stochastic
92
+ sparse_block_size: 128
93
+ sparse_num_random_blocks: 31
94
+ sparse_mask_pool_size: 1
95
+ sparse_max_seq_length: 8192
96
+
97
+ Parameter count:
98
+ Total parameters: 1,202,352,640
99
+ Trainable parameters: 1,202,352,640
100
+ Non-trainable parameters: 0
101
+ Model size (BF16): 2.40 GB
102
+
103
+ --------------------------------------------------------------------------------
104
+ LAYER-BY-LAYER SUMMARY
105
+ --------------------------------------------------------------------------------
106
+ ================================================================================================================================================================
107
+ Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
108
+ ================================================================================================================================================================
109
+ Qwen3ForCausalLM [1, 8192] -- -- True
110
+ ├─Qwen3Model: 1-1 -- [1, 8192, 1536] -- True
111
+ │ └─Embedding: 2-1 [1, 8192] [1, 8192, 1536] 233,373,696 True
112
+ │ └─ModuleList: 2-2 -- -- -- True
113
+ │ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
114
+ │ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
115
+ │ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
116
+ │ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
117
+ │ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
118
+ │ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
119
+ │ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
120
+ │ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
121
+ │ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
122
+ │ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
123
+ │ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
124
+ │ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
125
+ │ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
126
+ │ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
127
+ │ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
128
+ │ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
129
+ │ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
130
+ │ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
131
+ │ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
132
+ │ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
133
+ │ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
134
+ │ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
135
+ │ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
136
+ │ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
137
+ │ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
138
+ │ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
139
+ │ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
140
+ │ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
141
+ │ └─Qwen3RMSNorm: 2-3 [1, 8192, 1536] [1, 8192, 1536] 1,536 True
142
+ ├─Linear: 1-2 [1, 8192, 1536] [1, 8192, 151936] 233,373,696 True
143
+ ================================================================================================================================================================
144
+ Total params: 1,435,726,336
145
+ Trainable params: 1,435,726,336
146
+ Non-trainable params: 0
147
+ Total mult-adds (Units.GIGABYTES): 1.44
148
+ ================================================================================================================================================================
149
+ Input size (MB): 0.07
150
+ Forward/backward pass size (MB): 26218.59
151
+ Params size (MB): 2871.45
152
+ Estimated Total Size (MB): 29090.11
153
+ ================================================================================================================================================================
154
+ NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
155
+ ================================================================================
156
+
157
+ Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
158
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
159
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
160
+ Using pre-tokenized dataset (no runtime tokenization)
161
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
162
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
163
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
164
+
165
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
166
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
167
+ Dataset split: 3027764 train, 159356 validation
168
+ Training examples (packed, train split): 3027764
169
+ Validation examples (packed, val split): 159356
170
+ Per-device batch size: 1
171
+ World size (GPUs): 4
172
+ Gradient accumulation steps: 15
173
+ Effective batch size (seqs/step): 60
174
+ Sequence length: 8192
175
+ Tokens per optimizer step: 491,520
176
+ Micro-batches per epoch (per rank): 756941
177
+ Optimizer steps per epoch: 50462
178
+ Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
179
+ Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-8750...
180
+ Preserving checkpoint LR state for exact resume: current=[0.0004687250170709919], schedule_base=[0.0005]
181
+ Resumed from checkpoint at step 8750 (epoch 0, micro_step 131250)
182
+ Resuming training from step 8750
183
+ Resuming wandb run 7tfq6nwm
184
+ wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
185
+ wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
186
+ wandb: setting up run 7tfq6nwm
187
+ wandb: Tracking run with wandb version 0.26.1
188
+ wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260904_042057-7tfq6nwm
189
+ wandb: Run `wandb offline` to turn off syncing.
190
+ wandb: Resuming run 1.2b dense 8k seed72 4gpu
191
+ wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
192
+ wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/7tfq6nwm
193
+ Starting training...
194
+ Validation at step 8750: Loss = 2.7840, Perplexity = 16.18
195
+ wandb: WARNING Tried to log to step 8750 that is less than the current step 8751. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
196
+ Switching attention mode for step 8751: dense (dense step interval)
197
+ Step 8760/50462 | Loss: 2.7789 | LR: 4.69e-04 | 30187.4ms/step | 2.0 samples/s
198
+ Step 8770/50462 | Loss: 2.7657 | LR: 4.69e-04 | 2809.5ms/step | 21.4 samples/s
199
+ Step 8780/50462 | Loss: 2.8037 | LR: 4.68e-04 | 2815.7ms/step | 21.3 samples/s
200
+ Step 8790/50462 | Loss: 2.7473 | LR: 4.68e-04 | 2816.4ms/step | 21.3 samples/s
201
+ Step 8800/50462 | Loss: 2.7436 | LR: 4.68e-04 | 2819.2ms/step | 21.3 samples/s
202
+ Step 8810/50462 | Loss: 2.7757 | LR: 4.68e-04 | 2817.1ms/step | 21.3 samples/s
203
+ Step 8820/50462 | Loss: 2.7557 | LR: 4.68e-04 | 2815.9ms/step | 21.3 samples/s
204
+ Step 8830/50462 | Loss: 2.7731 | LR: 4.68e-04 | 2818.4ms/step | 21.3 samples/s
205
+ Step 8840/50462 | Loss: 2.7812 | LR: 4.68e-04 | 2818.2ms/step | 21.3 samples/s
206
+ Step 8850/50462 | Loss: 2.7587 | LR: 4.68e-04 | 2816.7ms/step | 21.3 samples/s
207
+ Step 8860/50462 | Loss: 2.7233 | LR: 4.68e-04 | 2818.0ms/step | 21.3 samples/s
208
+ Step 8870/50462 | Loss: 2.7967 | LR: 4.68e-04 | 2817.9ms/step | 21.3 samples/s
209
+ Step 8880/50462 | Loss: 2.7658 | LR: 4.68e-04 | 2821.9ms/step | 21.3 samples/s
210
+ Step 8890/50462 | Loss: 2.7572 | LR: 4.68e-04 | 2817.4ms/step | 21.3 samples/s
211
+ Step 8900/50462 | Loss: 2.7699 | LR: 4.68e-04 | 2823.1ms/step | 21.3 samples/s
212
+ Step 8910/50462 | Loss: 2.7876 | LR: 4.67e-04 | 2820.2ms/step | 21.3 samples/s
213
+ Step 8920/50462 | Loss: 2.7481 | LR: 4.67e-04 | 2820.0ms/step | 21.3 samples/s
214
+ Step 8930/50462 | Loss: 2.7610 | LR: 4.67e-04 | 2820.0ms/step | 21.3 samples/s
215
+ Step 8940/50462 | Loss: 2.7630 | LR: 4.67e-04 | 2820.9ms/step | 21.3 samples/s
216
+ Step 8950/50462 | Loss: 2.7806 | LR: 4.67e-04 | 2818.2ms/step | 21.3 samples/s
217
+ Step 8960/50462 | Loss: 2.7853 | LR: 4.67e-04 | 2814.6ms/step | 21.3 samples/s
218
+ Step 8970/50462 | Loss: 2.8039 | LR: 4.67e-04 | 2817.6ms/step | 21.3 samples/s
219
+ Step 8980/50462 | Loss: 2.7285 | LR: 4.67e-04 | 2818.9ms/step | 21.3 samples/s
220
+ Step 8990/50462 | Loss: 2.7869 | LR: 4.67e-04 | 2816.8ms/step | 21.3 samples/s
221
+ Step 9000/50462 | Loss: 2.7825 | LR: 4.67e-04 | 2814.1ms/step | 21.3 samples/s
222
+ Validation at step 9000: Loss = 2.7793, Perplexity = 16.11
223
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-9000
224
+ Step 9010/50462 | Loss: 2.7555 | LR: 4.67e-04 | 4736.9ms/step | 12.7 samples/s
225
+ Step 9020/50462 | Loss: 2.7954 | LR: 4.67e-04 | 2818.7ms/step | 21.3 samples/s
226
+ Step 9030/50462 | Loss: 2.7633 | LR: 4.67e-04 | 2816.4ms/step | 21.3 samples/s
227
+ Step 9040/50462 | Loss: 2.7525 | LR: 4.66e-04 | 2823.7ms/step | 21.2 samples/s
228
+ Step 9050/50462 | Loss: 2.7646 | LR: 4.66e-04 | 2819.6ms/step | 21.3 samples/s
229
+ Step 9060/50462 | Loss: 2.7677 | LR: 4.66e-04 | 2819.7ms/step | 21.3 samples/s
230
+ Step 9070/50462 | Loss: 2.7845 | LR: 4.66e-04 | 2820.8ms/step | 21.3 samples/s
231
+ Step 9080/50462 | Loss: 2.7606 | LR: 4.66e-04 | 2820.3ms/step | 21.3 samples/s
232
+ Step 9090/50462 | Loss: 2.7926 | LR: 4.66e-04 | 2821.5ms/step | 21.3 samples/s
233
+ Step 9100/50462 | Loss: 2.7167 | LR: 4.66e-04 | 2822.7ms/step | 21.3 samples/s
234
+ Step 9110/50462 | Loss: 2.7777 | LR: 4.66e-04 | 2822.6ms/step | 21.3 samples/s
235
+ Step 9120/50462 | Loss: 2.7821 | LR: 4.66e-04 | 2820.8ms/step | 21.3 samples/s
236
+ Step 9130/50462 | Loss: 2.7547 | LR: 4.66e-04 | 2822.1ms/step | 21.3 samples/s
237
+ Step 9140/50462 | Loss: 2.7391 | LR: 4.66e-04 | 2823.8ms/step | 21.2 samples/s
238
+ Step 9150/50462 | Loss: 2.7436 | LR: 4.66e-04 | 2822.2ms/step | 21.3 samples/s
239
+ Step 9160/50462 | Loss: 2.7022 | LR: 4.66e-04 | 2817.9ms/step | 21.3 samples/s
240
+ Step 9170/50462 | Loss: 2.7746 | LR: 4.65e-04 | 2820.7ms/step | 21.3 samples/s
241
+ Step 9180/50462 | Loss: 2.7842 | LR: 4.65e-04 | 2820.9ms/step | 21.3 samples/s
242
+ Step 9190/50462 | Loss: 2.7826 | LR: 4.65e-04 | 2819.4ms/step | 21.3 samples/s
243
+ Step 9200/50462 | Loss: 2.7867 | LR: 4.65e-04 | 2817.8ms/step | 21.3 samples/s
244
+ Step 9210/50462 | Loss: 2.7495 | LR: 4.65e-04 | 2818.7ms/step | 21.3 samples/s
245
+ Step 9220/50462 | Loss: 2.7309 | LR: 4.65e-04 | 2819.3ms/step | 21.3 samples/s
246
+ Step 9230/50462 | Loss: 2.7350 | LR: 4.65e-04 | 2819.2ms/step | 21.3 samples/s
247
+ Step 9240/50462 | Loss: 2.7455 | LR: 4.65e-04 | 2819.6ms/step | 21.3 samples/s
248
+ Step 9250/50462 | Loss: 2.7629 | LR: 4.65e-04 | 2823.3ms/step | 21.3 samples/s
249
+ Validation at step 9250: Loss = 2.7746, Perplexity = 16.03
250
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed72/run_20260903_094216/checkpoint-9250
251
+ Step 9260/50462 | Loss: 2.7622 | LR: 4.65e-04 | 4726.7ms/step | 12.7 samples/s
252
+ Step 9270/50462 | Loss: 2.7572 | LR: 4.65e-04 | 2820.4ms/step | 21.3 samples/s
253
+ Step 9280/50462 | Loss: 2.7671 | LR: 4.65e-04 | 2820.7ms/step | 21.3 samples/s
254
+ Step 9290/50462 | Loss: 2.7625 | LR: 4.64e-04 | 2822.5ms/step | 21.3 samples/s
255
+ Step 9300/50462 | Loss: 2.7504 | LR: 4.64e-04 | 2820.5ms/step | 21.3 samples/s
256
+ Step 9310/50462 | Loss: 2.7484 | LR: 4.64e-04 | 2818.5ms/step | 21.3 samples/s
257
+ Step 9320/50462 | Loss: 2.7604 | LR: 4.64e-04 | 2824.8ms/step | 21.2 samples/s
258
+ Step 9330/50462 | Loss: 2.7670 | LR: 4.64e-04 | 2823.9ms/step | 21.2 samples/s
259
+ Step 9340/50462 | Loss: 2.7511 | LR: 4.64e-04 | 2822.7ms/step | 21.3 samples/s
260
+ Step 9350/50462 | Loss: 2.7556 | LR: 4.64e-04 | 2824.2ms/step | 21.2 samples/s
261
+ Step 9360/50462 | Loss: 2.7818 | LR: 4.64e-04 | 2821.5ms/step | 21.3 samples/s
262
+ Step 9370/50462 | Loss: 2.7846 | LR: 4.64e-04 | 2825.5ms/step | 21.2 samples/s
263
+ Step 9380/50462 | Loss: 2.7812 | LR: 4.64e-04 | 2821.9ms/step | 21.3 samples/s
264
+ Step 9390/50462 | Loss: 2.7779 | LR: 4.64e-04 | 2824.5ms/step | 21.2 samples/s
265
+ Step 9400/50462 | Loss: 2.7706 | LR: 4.64e-04 | 2820.9ms/step | 21.3 samples/s
266
+ Step 9410/50462 | Loss: 2.7315 | LR: 4.63e-04 | 2821.5ms/step | 21.3 samples/s
267
+ Step 9420/50462 | Loss: 2.7492 | LR: 4.63e-04 | 2821.3ms/step | 21.3 samples/s
268
+ Step 9430/50462 | Loss: 2.7498 | LR: 4.63e-04 | 2823.2ms/step | 21.3 samples/s
269
+ Step 9440/50462 | Loss: 2.7252 | LR: 4.63e-04 | 2821.7ms/step | 21.3 samples/s
270
+ Step 9450/50462 | Loss: 2.7576 | LR: 4.63e-04 | 2822.1ms/step | 21.3 samples/s
epfl_final/train_logs/p2_1p2b_mixed_seed72_4gpu_resume45250_live.log ADDED
@@ -0,0 +1,762 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ W0903 03:29:56.506000 387 site-packages/torch/distributed/run.py:851]
2
+ W0903 03:29:56.506000 387 site-packages/torch/distributed/run.py:851] *****************************************
3
+ W0903 03:29:56.506000 387 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
4
+ W0903 03:29:56.506000 387 site-packages/torch/distributed/run.py:851] *****************************************
5
+ Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551
6
+ Running config:
7
+ {
8
+ "model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
9
+ "model_dtype": "bfloat16",
10
+ "init_from_scratch": true,
11
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
12
+ "val_path": null,
13
+ "max_seq_length": 8192,
14
+ "num_workers": 8,
15
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551",
16
+ "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-45250",
17
+ "force_new_run": false,
18
+ "num_epochs": 1,
19
+ "max_steps": 50462,
20
+ "per_device_batch_size": 1,
21
+ "gradient_accumulation_steps": 15,
22
+ "learning_rate": 0.0005,
23
+ "weight_decay": 0.01,
24
+ "warmup_steps": 750,
25
+ "lr_scheduler": "cosine",
26
+ "max_grad_norm": 1.0,
27
+ "save_steps": 250,
28
+ "eval_steps": 250,
29
+ "eval_batches": 50,
30
+ "logging_steps": 10,
31
+ "seed": 72,
32
+ "verbose": true,
33
+ "checkpoint_activations": true,
34
+ "resume_micro_step_override": null,
35
+ "strategy": "ddp",
36
+ "mixed_precision": "bf16",
37
+ "wandb_enabled": false,
38
+ "wandb_project": "stochastic_sparsity",
39
+ "wandb_name": "1.2b mixed 8k seed72 4gpu",
40
+ "wandb_tags": [
41
+ "qwen3",
42
+ "1.2b",
43
+ "8k",
44
+ "p2",
45
+ "mixed",
46
+ "seed72"
47
+ ],
48
+ "hidden_size": 1536,
49
+ "intermediate_size": 6144,
50
+ "num_attention_heads": 12,
51
+ "num_key_value_heads": 4,
52
+ "num_hidden_layers": null,
53
+ "head_dim": null,
54
+ "attention_dropout": null,
55
+ "rope_train_position_scale_set": null,
56
+ "attention_type": "sparse_stochastic",
57
+ "sparse_block_size": 128,
58
+ "sparse_num_random_blocks": 35,
59
+ "sparse_mask_pool_size": 10000,
60
+ "sparse_val_dense_compare": false,
61
+ "sparse_layer_indices": null,
62
+ "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
63
+ "sparse_mask_seed": 72,
64
+ "sparse_mixed_nrb_set": [
65
+ 63,
66
+ 47,
67
+ 31,
68
+ 21,
69
+ 11
70
+ ],
71
+ "dense_mode": false,
72
+ "dense_step_intervals": null
73
+ }
74
+ Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
75
+ Checkpoint activations: TrueCheckpoint activations: True
76
+
77
+ Checkpoint activations: True
78
+ Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-45250
79
+ Checkpoint activations: True
80
+
81
+
82
+ ================================================================================
83
+ MODEL ARCHITECTURE
84
+ ================================================================================
85
+ Model config:
86
+ vocab_size: 151936
87
+ hidden_size: 1536
88
+ num_hidden_layers: 28
89
+ num_attention_heads: 12
90
+ num_key_value_heads: 4
91
+ head_dim: 128
92
+ intermediate_size: 6144
93
+ max_position_embeddings: 32768
94
+ rope_theta: 1000000
95
+ rope_train_position_scale_set: None
96
+ attention_type: sparse_stochastic
97
+ sparse_block_size: 128
98
+ sparse_num_random_blocks: 35
99
+ sparse_mask_pool_size: 10000
100
+ sparse_max_seq_length: 8192
101
+
102
+ Parameter count:
103
+ Total parameters: 1,202,352,640
104
+ Trainable parameters: 1,202,352,640
105
+ Non-trainable parameters: 0
106
+ Model size (BF16): 2.40 GB
107
+
108
+ --------------------------------------------------------------------------------
109
+ LAYER-BY-LAYER SUMMARY
110
+ --------------------------------------------------------------------------------
111
+ /mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
112
+
113
+ SOLUTION: Use torch.compile(flex_attention)(...)
114
+
115
+ If you want to debug your score_mod/mask_mod, you can set:
116
+ torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
117
+
118
+ This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
119
+ _warn_once(
120
+ ================================================================================================================================================================
121
+ Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
122
+ ================================================================================================================================================================
123
+ Qwen3ForCausalLM [1, 8192] -- -- True
124
+ ├─Qwen3Model: 1-1 -- [1, 8192, 1536] -- True
125
+ │ └─Embedding: 2-1 [1, 8192] [1, 8192, 1536] 233,373,696 True
126
+ │ └─ModuleList: 2-2 -- -- -- True
127
+ │ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
128
+ │ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
129
+ │ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
130
+ │ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
131
+ │ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
132
+ │ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
133
+ │ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
134
+ │ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
135
+ │ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
136
+ │ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
137
+ │ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
138
+ │ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
139
+ │ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
140
+ │ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
141
+ │ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
142
+ │ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
143
+ │ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
144
+ │ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
145
+ │ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
146
+ │ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
147
+ │ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
148
+ │ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
149
+ │ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
150
+ │ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
151
+ │ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
152
+ │ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
153
+ │ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
154
+ │ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
155
+ │ └─Qwen3RMSNorm: 2-3 [1, 8192, 1536] [1, 8192, 1536] 1,536 True
156
+ ├─Linear: 1-2 [1, 8192, 1536] [1, 8192, 151936] 233,373,696 True
157
+ ================================================================================================================================================================
158
+ Total params: 1,435,726,336
159
+ Trainable params: 1,435,726,336
160
+ Non-trainable params: 0
161
+ Total mult-adds (Units.GIGABYTES): 1.44
162
+ ================================================================================================================================================================
163
+ Input size (MB): 0.07
164
+ Forward/backward pass size (MB): 26218.59
165
+ Params size (MB): 2871.45
166
+ Estimated Total Size (MB): 29090.11
167
+ ================================================================================================================================================================
168
+ NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
169
+ ================================================================================
170
+
171
+ Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
172
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
173
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
174
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
175
+ Using pre-tokenized dataset (no runtime tokenization)
176
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
177
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
178
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
179
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
180
+ Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
181
+ Dataset split: 3027764 train, 159356 validation
182
+ Training examples (packed, train split): 3027764
183
+ Validation examples (packed, val split): 159356
184
+ Per-device batch size: 1
185
+ World size (GPUs): 4
186
+ Gradient accumulation steps: 15
187
+ Effective batch size (seqs/step): 60
188
+ Sequence length: 8192
189
+ Tokens per optimizer step: 491,520
190
+ Micro-batches per epoch (per rank): 756941
191
+ Optimizer steps per epoch: 50462
192
+ Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
193
+ Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-45250...
194
+ Preserving checkpoint LR state for exact resume: current=[1.3438976469445447e-05], schedule_base=[0.0005]
195
+ Resumed from checkpoint at step 45250 (epoch 0, micro_step 646245)
196
+ Resuming training from step 45250
197
+ Starting training...
198
+ Validation at step 45250: Loss = 2.5530, Perplexity = 12.85
199
+ Switching attention mode for step 45251: sparse_stochastic (configured sparse attention)
200
+ Step 45260/50462 | Loss: 2.5328 | LR: 1.34e-05 | 94628.7ms/step | 0.6 samples/s
201
+ Step 45270/50462 | Loss: 2.5561 | LR: 1.33e-05 | 3982.1ms/step | 15.1 samples/s
202
+ Step 45280/50462 | Loss: 2.5512 | LR: 1.33e-05 | 3982.9ms/step | 15.1 samples/s
203
+ Step 45290/50462 | Loss: 2.5371 | LR: 1.32e-05 | 3979.4ms/step | 15.1 samples/s
204
+ Step 45300/50462 | Loss: 2.5093 | LR: 1.32e-05 | 3975.9ms/step | 15.1 samples/s
205
+ Step 45310/50462 | Loss: 2.5429 | LR: 1.31e-05 | 3983.9ms/step | 15.1 samples/s
206
+ Step 45320/50462 | Loss: 2.5242 | LR: 1.31e-05 | 3999.4ms/step | 15.0 samples/s
207
+ Step 45330/50462 | Loss: 2.5294 | LR: 1.30e-05 | 3979.3ms/step | 15.1 samples/s
208
+ Step 45340/50462 | Loss: 2.4970 | LR: 1.30e-05 | 3991.1ms/step | 15.0 samples/s
209
+ Step 45350/50462 | Loss: 2.5339 | LR: 1.29e-05 | 3978.7ms/step | 15.1 samples/s
210
+ Step 45360/50462 | Loss: 2.5427 | LR: 1.29e-05 | 3978.1ms/step | 15.1 samples/s
211
+ Step 45370/50462 | Loss: 2.5101 | LR: 1.28e-05 | 3987.0ms/step | 15.0 samples/s
212
+ Step 45380/50462 | Loss: 2.5453 | LR: 1.28e-05 | 3983.8ms/step | 15.1 samples/s
213
+ Step 45390/50462 | Loss: 2.5397 | LR: 1.27e-05 | 3989.6ms/step | 15.0 samples/s
214
+ Step 45400/50462 | Loss: 2.5295 | LR: 1.27e-05 | 3992.6ms/step | 15.0 samples/s
215
+ Step 45410/50462 | Loss: 2.5173 | LR: 1.26e-05 | 3995.9ms/step | 15.0 samples/s
216
+ Step 45420/50462 | Loss: 2.5306 | LR: 1.26e-05 | 3989.1ms/step | 15.0 samples/s
217
+ Step 45430/50462 | Loss: 2.4967 | LR: 1.25e-05 | 3998.2ms/step | 15.0 samples/s
218
+ Step 45440/50462 | Loss: 2.5309 | LR: 1.25e-05 | 3989.2ms/step | 15.0 samples/s
219
+ Step 45450/50462 | Loss: 2.5348 | LR: 1.24e-05 | 3989.2ms/step | 15.0 samples/s
220
+ Step 45460/50462 | Loss: 2.5291 | LR: 1.24e-05 | 3996.2ms/step | 15.0 samples/s
221
+ Step 45470/50462 | Loss: 2.5342 | LR: 1.23e-05 | 3988.4ms/step | 15.0 samples/s
222
+ Step 45480/50462 | Loss: 2.4890 | LR: 1.23e-05 | 3998.6ms/step | 15.0 samples/s
223
+ Step 45490/50462 | Loss: 2.5537 | LR: 1.22e-05 | 3985.3ms/step | 15.1 samples/s
224
+ Step 45500/50462 | Loss: 2.5304 | LR: 1.22e-05 | 3983.3ms/step | 15.1 samples/s
225
+ Validation at step 45500: Loss = 2.5520, Perplexity = 12.83
226
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-45500
227
+ Step 45510/50462 | Loss: 2.5291 | LR: 1.21e-05 | 6139.4ms/step | 9.8 samples/s
228
+ Step 45520/50462 | Loss: 2.5337 | LR: 1.21e-05 | 3998.9ms/step | 15.0 samples/s
229
+ Step 45530/50462 | Loss: 2.5579 | LR: 1.20e-05 | 4001.5ms/step | 15.0 samples/s
230
+ Step 45540/50462 | Loss: 2.5287 | LR: 1.20e-05 | 3993.2ms/step | 15.0 samples/s
231
+ Step 45550/50462 | Loss: 2.5385 | LR: 1.19e-05 | 3991.4ms/step | 15.0 samples/s
232
+ Step 45560/50462 | Loss: 2.5004 | LR: 1.19e-05 | 3988.9ms/step | 15.0 samples/s
233
+ Step 45570/50462 | Loss: 2.5196 | LR: 1.19e-05 | 3981.4ms/step | 15.1 samples/s
234
+ Step 45580/50462 | Loss: 2.5689 | LR: 1.18e-05 | 4000.3ms/step | 15.0 samples/s
235
+ Step 45590/50462 | Loss: 2.5002 | LR: 1.18e-05 | 3974.6ms/step | 15.1 samples/s
236
+ Step 45600/50462 | Loss: 2.5023 | LR: 1.17e-05 | 3978.7ms/step | 15.1 samples/s
237
+ Step 45610/50462 | Loss: 2.5439 | LR: 1.17e-05 | 3987.7ms/step | 15.0 samples/s
238
+ Step 45620/50462 | Loss: 2.4975 | LR: 1.16e-05 | 3986.3ms/step | 15.1 samples/s
239
+ Step 45630/50462 | Loss: 2.5424 | LR: 1.16e-05 | 3986.3ms/step | 15.1 samples/s
240
+ Step 45640/50462 | Loss: 2.5163 | LR: 1.15e-05 | 3991.3ms/step | 15.0 samples/s
241
+ Step 45650/50462 | Loss: 2.5144 | LR: 1.15e-05 | 3999.9ms/step | 15.0 samples/s
242
+ Step 45660/50462 | Loss: 2.5418 | LR: 1.14e-05 | 3985.2ms/step | 15.1 samples/s
243
+ Step 45670/50462 | Loss: 2.5415 | LR: 1.14e-05 | 3988.2ms/step | 15.0 samples/s
244
+ Step 45680/50462 | Loss: 2.5156 | LR: 1.13e-05 | 3985.2ms/step | 15.1 samples/s
245
+ Step 45690/50462 | Loss: 2.5190 | LR: 1.13e-05 | 3987.4ms/step | 15.0 samples/s
246
+ Step 45700/50462 | Loss: 2.5187 | LR: 1.12e-05 | 3990.8ms/step | 15.0 samples/s
247
+ Step 45710/50462 | Loss: 2.5569 | LR: 1.12e-05 | 3999.2ms/step | 15.0 samples/s
248
+ Step 45720/50462 | Loss: 2.5619 | LR: 1.11e-05 | 3983.5ms/step | 15.1 samples/s
249
+ Step 45730/50462 | Loss: 2.5248 | LR: 1.11e-05 | 3992.0ms/step | 15.0 samples/s
250
+ Step 45740/50462 | Loss: 2.5169 | LR: 1.10e-05 | 3973.7ms/step | 15.1 samples/s
251
+ Step 45750/50462 | Loss: 2.5267 | LR: 1.10e-05 | 3982.7ms/step | 15.1 samples/s
252
+ Validation at step 45750: Loss = 2.5527, Perplexity = 12.84
253
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-45750
254
+ Step 45760/50462 | Loss: 2.5087 | LR: 1.10e-05 | 5846.6ms/step | 10.3 samples/s
255
+ Step 45770/50462 | Loss: 2.5539 | LR: 1.09e-05 | 3989.0ms/step | 15.0 samples/s
256
+ Step 45780/50462 | Loss: 2.5169 | LR: 1.09e-05 | 4002.7ms/step | 15.0 samples/s
257
+ Step 45790/50462 | Loss: 2.5447 | LR: 1.08e-05 | 3985.6ms/step | 15.1 samples/s
258
+ Step 45800/50462 | Loss: 2.5200 | LR: 1.08e-05 | 3994.1ms/step | 15.0 samples/s
259
+ Step 45810/50462 | Loss: 2.5257 | LR: 1.07e-05 | 3990.0ms/step | 15.0 samples/s
260
+ Step 45820/50462 | Loss: 2.5364 | LR: 1.07e-05 | 3986.6ms/step | 15.1 samples/s
261
+ Step 45830/50462 | Loss: 2.4773 | LR: 1.06e-05 | 3997.4ms/step | 15.0 samples/s
262
+ Step 45840/50462 | Loss: 2.5160 | LR: 1.06e-05 | 3988.7ms/step | 15.0 samples/s
263
+ Step 45850/50462 | Loss: 2.5484 | LR: 1.05e-05 | 4006.1ms/step | 15.0 samples/s
264
+ Step 45860/50462 | Loss: 2.5463 | LR: 1.05e-05 | 3988.4ms/step | 15.0 samples/s
265
+ Step 45870/50462 | Loss: 2.5172 | LR: 1.05e-05 | 3988.8ms/step | 15.0 samples/s
266
+ Step 45880/50462 | Loss: 2.5355 | LR: 1.04e-05 | 3992.5ms/step | 15.0 samples/s
267
+ Step 45890/50462 | Loss: 2.5184 | LR: 1.04e-05 | 3984.6ms/step | 15.1 samples/s
268
+ Step 45900/50462 | Loss: 2.5346 | LR: 1.03e-05 | 3988.3ms/step | 15.0 samples/s
269
+ Step 45910/50462 | Loss: 2.5506 | LR: 1.03e-05 | 3982.0ms/step | 15.1 samples/s
270
+ Step 45920/50462 | Loss: 2.5315 | LR: 1.02e-05 | 3981.2ms/step | 15.1 samples/s
271
+ Step 45930/50462 | Loss: 2.4682 | LR: 1.02e-05 | 3998.4ms/step | 15.0 samples/s
272
+ Step 45940/50462 | Loss: 2.5201 | LR: 1.01e-05 | 3978.0ms/step | 15.1 samples/s
273
+ Step 45950/50462 | Loss: 2.5544 | LR: 1.01e-05 | 3993.6ms/step | 15.0 samples/s
274
+ Step 45960/50462 | Loss: 2.5339 | LR: 1.01e-05 | 3986.7ms/step | 15.1 samples/s
275
+ Step 45970/50462 | Loss: 2.5195 | LR: 1.00e-05 | 3983.9ms/step | 15.1 samples/s
276
+ Step 45980/50462 | Loss: 2.5625 | LR: 9.96e-06 | 3989.5ms/step | 15.0 samples/s
277
+ Step 45990/50462 | Loss: 2.5369 | LR: 9.92e-06 | 3998.2ms/step | 15.0 samples/s
278
+ Step 46000/50462 | Loss: 2.5041 | LR: 9.87e-06 | 3988.3ms/step | 15.0 samples/s
279
+ Validation at step 46000: Loss = 2.5516, Perplexity = 12.83
280
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-46000
281
+ Step 46010/50462 | Loss: 2.5184 | LR: 9.83e-06 | 5853.6ms/step | 10.3 samples/s
282
+ Step 46020/50462 | Loss: 2.5421 | LR: 9.79e-06 | 3980.1ms/step | 15.1 samples/s
283
+ Step 46030/50462 | Loss: 2.5225 | LR: 9.74e-06 | 3988.7ms/step | 15.0 samples/s
284
+ Step 46040/50462 | Loss: 2.5365 | LR: 9.70e-06 | 3986.4ms/step | 15.1 samples/s
285
+ Step 46050/50462 | Loss: 2.5737 | LR: 9.65e-06 | 3990.8ms/step | 15.0 samples/s
286
+ Step 46060/50462 | Loss: 2.5962 | LR: 9.61e-06 | 3995.0ms/step | 15.0 samples/s
287
+ Step 46070/50462 | Loss: 2.5060 | LR: 9.57e-06 | 3990.6ms/step | 15.0 samples/s
288
+ Step 46080/50462 | Loss: 2.5109 | LR: 9.52e-06 | 3981.4ms/step | 15.1 samples/s
289
+ Step 46090/50462 | Loss: 2.5484 | LR: 9.48e-06 | 3981.6ms/step | 15.1 samples/s
290
+ Step 46100/50462 | Loss: 2.5509 | LR: 9.44e-06 | 3984.8ms/step | 15.1 samples/s
291
+ Step 46110/50462 | Loss: 2.5519 | LR: 9.40e-06 | 3976.8ms/step | 15.1 samples/s
292
+ Step 46120/50462 | Loss: 2.5037 | LR: 9.35e-06 | 3968.5ms/step | 15.1 samples/s
293
+ Step 46130/50462 | Loss: 2.5374 | LR: 9.31e-06 | 3988.0ms/step | 15.0 samples/s
294
+ Step 46140/50462 | Loss: 2.5174 | LR: 9.27e-06 | 3981.4ms/step | 15.1 samples/s
295
+ Step 46150/50462 | Loss: 2.5040 | LR: 9.22e-06 | 3983.9ms/step | 15.1 samples/s
296
+ Step 46160/50462 | Loss: 2.5247 | LR: 9.18e-06 | 3997.4ms/step | 15.0 samples/s
297
+ Step 46170/50462 | Loss: 2.5329 | LR: 9.14e-06 | 3981.6ms/step | 15.1 samples/s
298
+ Step 46180/50462 | Loss: 2.5477 | LR: 9.10e-06 | 3992.0ms/step | 15.0 samples/s
299
+ Step 46190/50462 | Loss: 2.5032 | LR: 9.06e-06 | 3984.2ms/step | 15.1 samples/s
300
+ Step 46200/50462 | Loss: 2.5087 | LR: 9.01e-06 | 3993.6ms/step | 15.0 samples/s
301
+ Step 46210/50462 | Loss: 2.5498 | LR: 8.97e-06 | 3979.9ms/step | 15.1 samples/s
302
+ Step 46220/50462 | Loss: 2.5496 | LR: 8.93e-06 | 3993.9ms/step | 15.0 samples/s
303
+ Step 46230/50462 | Loss: 2.4956 | LR: 8.89e-06 | 3994.2ms/step | 15.0 samples/s
304
+ Step 46240/50462 | Loss: 2.5040 | LR: 8.85e-06 | 3990.2ms/step | 15.0 samples/s
305
+ Step 46250/50462 | Loss: 2.5175 | LR: 8.80e-06 | 3988.1ms/step | 15.0 samples/s
306
+ Validation at step 46250: Loss = 2.5520, Perplexity = 12.83
307
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-46250
308
+ Step 46260/50462 | Loss: 2.5501 | LR: 8.76e-06 | 5933.6ms/step | 10.1 samples/s
309
+ Step 46270/50462 | Loss: 2.5066 | LR: 8.72e-06 | 3980.0ms/step | 15.1 samples/s
310
+ Step 46280/50462 | Loss: 2.5293 | LR: 8.68e-06 | 3976.2ms/step | 15.1 samples/s
311
+ Step 46290/50462 | Loss: 2.5470 | LR: 8.64e-06 | 3983.8ms/step | 15.1 samples/s
312
+ Step 46300/50462 | Loss: 2.5454 | LR: 8.60e-06 | 3984.7ms/step | 15.1 samples/s
313
+ Step 46310/50462 | Loss: 2.5464 | LR: 8.56e-06 | 3981.5ms/step | 15.1 samples/s
314
+ Step 46320/50462 | Loss: 2.5423 | LR: 8.52e-06 | 3992.0ms/step | 15.0 samples/s
315
+ Step 46330/50462 | Loss: 2.5197 | LR: 8.47e-06 | 3992.8ms/step | 15.0 samples/s
316
+ Step 46340/50462 | Loss: 2.5390 | LR: 8.43e-06 | 3986.5ms/step | 15.1 samples/s
317
+ Step 46350/50462 | Loss: 2.5550 | LR: 8.39e-06 | 3982.2ms/step | 15.1 samples/s
318
+ Step 46360/50462 | Loss: 2.5484 | LR: 8.35e-06 | 3990.4ms/step | 15.0 samples/s
319
+ Step 46370/50462 | Loss: 2.5134 | LR: 8.31e-06 | 3991.0ms/step | 15.0 samples/s
320
+ Step 46380/50462 | Loss: 2.4913 | LR: 8.27e-06 | 3980.5ms/step | 15.1 samples/s
321
+ Step 46390/50462 | Loss: 2.5455 | LR: 8.23e-06 | 3981.7ms/step | 15.1 samples/s
322
+ Step 46400/50462 | Loss: 2.5518 | LR: 8.19e-06 | 3981.1ms/step | 15.1 samples/s
323
+ Step 46410/50462 | Loss: 2.5421 | LR: 8.15e-06 | 3977.7ms/step | 15.1 samples/s
324
+ Step 46420/50462 | Loss: 2.5063 | LR: 8.11e-06 | 3991.5ms/step | 15.0 samples/s
325
+ Step 46430/50462 | Loss: 2.5393 | LR: 8.07e-06 | 3981.1ms/step | 15.1 samples/s
326
+ Step 46440/50462 | Loss: 2.4993 | LR: 8.03e-06 | 3984.3ms/step | 15.1 samples/s
327
+ Step 46450/50462 | Loss: 2.5420 | LR: 7.99e-06 | 3992.0ms/step | 15.0 samples/s
328
+ Step 46460/50462 | Loss: 2.5491 | LR: 7.95e-06 | 3991.3ms/step | 15.0 samples/s
329
+ Step 46470/50462 | Loss: 2.5225 | LR: 7.91e-06 | 3996.5ms/step | 15.0 samples/s
330
+ Step 46480/50462 | Loss: 2.5436 | LR: 7.87e-06 | 3981.4ms/step | 15.1 samples/s
331
+ Step 46490/50462 | Loss: 2.4904 | LR: 7.83e-06 | 3985.3ms/step | 15.1 samples/s
332
+ Step 46500/50462 | Loss: 2.5163 | LR: 7.80e-06 | 3984.0ms/step | 15.1 samples/s
333
+ Validation at step 46500: Loss = 2.5517, Perplexity = 12.83
334
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-46500
335
+ Step 46510/50462 | Loss: 2.5490 | LR: 7.76e-06 | 5891.4ms/step | 10.2 samples/s
336
+ Step 46520/50462 | Loss: 2.5347 | LR: 7.72e-06 | 3984.7ms/step | 15.1 samples/s
337
+ Step 46530/50462 | Loss: 2.5299 | LR: 7.68e-06 | 3990.1ms/step | 15.0 samples/s
338
+ Step 46540/50462 | Loss: 2.5593 | LR: 7.64e-06 | 3989.3ms/step | 15.0 samples/s
339
+ Step 46550/50462 | Loss: 2.5568 | LR: 7.60e-06 | 3999.5ms/step | 15.0 samples/s
340
+ Step 46560/50462 | Loss: 2.5322 | LR: 7.56e-06 | 3985.1ms/step | 15.1 samples/s
341
+ Step 46570/50462 | Loss: 2.5736 | LR: 7.52e-06 | 3976.7ms/step | 15.1 samples/s
342
+ Step 46580/50462 | Loss: 2.5154 | LR: 7.49e-06 | 3981.2ms/step | 15.1 samples/s
343
+ Step 46590/50462 | Loss: 2.5483 | LR: 7.45e-06 | 3984.0ms/step | 15.1 samples/s
344
+ Step 46600/50462 | Loss: 2.5330 | LR: 7.41e-06 | 3976.3ms/step | 15.1 samples/s
345
+ Step 46610/50462 | Loss: 2.5206 | LR: 7.37e-06 | 3994.0ms/step | 15.0 samples/s
346
+ Step 46620/50462 | Loss: 2.5397 | LR: 7.33e-06 | 3992.8ms/step | 15.0 samples/s
347
+ Step 46630/50462 | Loss: 2.5250 | LR: 7.29e-06 | 3987.6ms/step | 15.0 samples/s
348
+ Step 46640/50462 | Loss: 2.5004 | LR: 7.26e-06 | 3979.9ms/step | 15.1 samples/s
349
+ Step 46650/50462 | Loss: 2.5253 | LR: 7.22e-06 | 3986.6ms/step | 15.1 samples/s
350
+ Step 46660/50462 | Loss: 2.5353 | LR: 7.18e-06 | 3981.2ms/step | 15.1 samples/s
351
+ Step 46670/50462 | Loss: 2.5213 | LR: 7.14e-06 | 3981.3ms/step | 15.1 samples/s
352
+ Step 46680/50462 | Loss: 2.5192 | LR: 7.11e-06 | 3989.6ms/step | 15.0 samples/s
353
+ Step 46690/50462 | Loss: 2.5424 | LR: 7.07e-06 | 3976.4ms/step | 15.1 samples/s
354
+ Step 46700/50462 | Loss: 2.5311 | LR: 7.03e-06 | 3981.0ms/step | 15.1 samples/s
355
+ Step 46710/50462 | Loss: 2.5512 | LR: 6.99e-06 | 3996.0ms/step | 15.0 samples/s
356
+ Step 46720/50462 | Loss: 2.5359 | LR: 6.96e-06 | 3993.0ms/step | 15.0 samples/s
357
+ Step 46730/50462 | Loss: 2.5458 | LR: 6.92e-06 | 3986.4ms/step | 15.1 samples/s
358
+ Step 46740/50462 | Loss: 2.5374 | LR: 6.88e-06 | 3999.5ms/step | 15.0 samples/s
359
+ Step 46750/50462 | Loss: 2.5320 | LR: 6.85e-06 | 3983.0ms/step | 15.1 samples/s
360
+ Validation at step 46750: Loss = 2.5523, Perplexity = 12.84
361
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-46750
362
+ Step 46760/50462 | Loss: 2.5146 | LR: 6.81e-06 | 5897.7ms/step | 10.2 samples/s
363
+ Step 46770/50462 | Loss: 2.5144 | LR: 6.77e-06 | 4010.2ms/step | 15.0 samples/s
364
+ Step 46780/50462 | Loss: 2.5329 | LR: 6.74e-06 | 3993.8ms/step | 15.0 samples/s
365
+ Step 46790/50462 | Loss: 2.4752 | LR: 6.70e-06 | 3990.5ms/step | 15.0 samples/s
366
+ Step 46800/50462 | Loss: 2.4960 | LR: 6.66e-06 | 3996.9ms/step | 15.0 samples/s
367
+ Step 46810/50462 | Loss: 2.5092 | LR: 6.63e-06 | 4000.8ms/step | 15.0 samples/s
368
+ Step 46820/50462 | Loss: 2.5498 | LR: 6.59e-06 | 4003.5ms/step | 15.0 samples/s
369
+ Step 46830/50462 | Loss: 2.5543 | LR: 6.56e-06 | 4014.9ms/step | 14.9 samples/s
370
+ Step 46840/50462 | Loss: 2.4882 | LR: 6.52e-06 | 3995.8ms/step | 15.0 samples/s
371
+ Step 46850/50462 | Loss: 2.5522 | LR: 6.48e-06 | 4023.3ms/step | 14.9 samples/s
372
+ Step 46860/50462 | Loss: 2.5146 | LR: 6.45e-06 | 4009.9ms/step | 15.0 samples/s
373
+ Step 46870/50462 | Loss: 2.5303 | LR: 6.41e-06 | 4014.8ms/step | 14.9 samples/s
374
+ Step 46880/50462 | Loss: 2.5104 | LR: 6.38e-06 | 3999.6ms/step | 15.0 samples/s
375
+ Step 46890/50462 | Loss: 2.5354 | LR: 6.34e-06 | 4008.1ms/step | 15.0 samples/s
376
+ Step 46900/50462 | Loss: 2.5347 | LR: 6.31e-06 | 3997.4ms/step | 15.0 samples/s
377
+ Step 46910/50462 | Loss: 2.5578 | LR: 6.27e-06 | 4009.0ms/step | 15.0 samples/s
378
+ Step 46920/50462 | Loss: 2.5231 | LR: 6.24e-06 | 4013.6ms/step | 14.9 samples/s
379
+ Step 46930/50462 | Loss: 2.5427 | LR: 6.20e-06 | 4011.2ms/step | 15.0 samples/s
380
+ Step 46940/50462 | Loss: 2.4928 | LR: 6.17e-06 | 3993.5ms/step | 15.0 samples/s
381
+ Step 46950/50462 | Loss: 2.5366 | LR: 6.13e-06 | 4013.1ms/step | 15.0 samples/s
382
+ Step 46960/50462 | Loss: 2.5286 | LR: 6.10e-06 | 3992.4ms/step | 15.0 samples/s
383
+ Step 46970/50462 | Loss: 2.5103 | LR: 6.06e-06 | 3989.4ms/step | 15.0 samples/s
384
+ Step 46980/50462 | Loss: 2.5090 | LR: 6.03e-06 | 3999.4ms/step | 15.0 samples/s
385
+ Step 46990/50462 | Loss: 2.5403 | LR: 5.99e-06 | 3994.2ms/step | 15.0 samples/s
386
+ Step 47000/50462 | Loss: 2.5575 | LR: 5.96e-06 | 4002.4ms/step | 15.0 samples/s
387
+ Validation at step 47000: Loss = 2.5522, Perplexity = 12.83
388
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-47000
389
+ Step 47010/50462 | Loss: 2.5457 | LR: 5.93e-06 | 6045.0ms/step | 9.9 samples/s
390
+ Step 47020/50462 | Loss: 2.5316 | LR: 5.89e-06 | 4006.3ms/step | 15.0 samples/s
391
+ Step 47030/50462 | Loss: 2.5480 | LR: 5.86e-06 | 4007.8ms/step | 15.0 samples/s
392
+ Step 47040/50462 | Loss: 2.5387 | LR: 5.82e-06 | 4005.5ms/step | 15.0 samples/s
393
+ Step 47050/50462 | Loss: 2.5654 | LR: 5.79e-06 | 4003.2ms/step | 15.0 samples/s
394
+ Step 47060/50462 | Loss: 2.5402 | LR: 5.76e-06 | 3990.7ms/step | 15.0 samples/s
395
+ Step 47070/50462 | Loss: 2.5222 | LR: 5.72e-06 | 3998.9ms/step | 15.0 samples/s
396
+ Step 47080/50462 | Loss: 2.5438 | LR: 5.69e-06 | 4011.2ms/step | 15.0 samples/s
397
+ Step 47090/50462 | Loss: 2.5501 | LR: 5.65e-06 | 4002.0ms/step | 15.0 samples/s
398
+ Step 47100/50462 | Loss: 2.5311 | LR: 5.62e-06 | 4000.5ms/step | 15.0 samples/s
399
+ Step 47110/50462 | Loss: 2.5445 | LR: 5.59e-06 | 4000.4ms/step | 15.0 samples/s
400
+ Step 47120/50462 | Loss: 2.5647 | LR: 5.56e-06 | 3999.8ms/step | 15.0 samples/s
401
+ Step 47130/50462 | Loss: 2.5583 | LR: 5.52e-06 | 3999.0ms/step | 15.0 samples/s
402
+ Step 47140/50462 | Loss: 2.5408 | LR: 5.49e-06 | 3994.7ms/step | 15.0 samples/s
403
+ Step 47150/50462 | Loss: 2.5695 | LR: 5.46e-06 | 3986.7ms/step | 15.1 samples/s
404
+ Step 47160/50462 | Loss: 2.5259 | LR: 5.42e-06 | 4002.4ms/step | 15.0 samples/s
405
+ Step 47170/50462 | Loss: 2.5598 | LR: 5.39e-06 | 3985.4ms/step | 15.1 samples/s
406
+ Step 47180/50462 | Loss: 2.5276 | LR: 5.36e-06 | 3987.0ms/step | 15.0 samples/s
407
+ Step 47190/50462 | Loss: 2.5317 | LR: 5.33e-06 | 3998.6ms/step | 15.0 samples/s
408
+ Step 47200/50462 | Loss: 2.5365 | LR: 5.29e-06 | 3991.2ms/step | 15.0 samples/s
409
+ Step 47210/50462 | Loss: 2.5590 | LR: 5.26e-06 | 3997.4ms/step | 15.0 samples/s
410
+ Step 47220/50462 | Loss: 2.5282 | LR: 5.23e-06 | 4002.0ms/step | 15.0 samples/s
411
+ Step 47230/50462 | Loss: 2.5610 | LR: 5.20e-06 | 3999.0ms/step | 15.0 samples/s
412
+ Step 47240/50462 | Loss: 2.5196 | LR: 5.16e-06 | 4000.0ms/step | 15.0 samples/s
413
+ Step 47250/50462 | Loss: 2.5365 | LR: 5.13e-06 | 3997.6ms/step | 15.0 samples/s
414
+ Validation at step 47250: Loss = 2.5521, Perplexity = 12.83
415
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-47250
416
+ Step 47260/50462 | Loss: 2.5526 | LR: 5.10e-06 | 5915.4ms/step | 10.1 samples/s
417
+ Step 47270/50462 | Loss: 2.5474 | LR: 5.07e-06 | 3996.5ms/step | 15.0 samples/s
418
+ Step 47280/50462 | Loss: 2.5568 | LR: 5.04e-06 | 3993.8ms/step | 15.0 samples/s
419
+ Step 47290/50462 | Loss: 2.5616 | LR: 5.01e-06 | 3991.0ms/step | 15.0 samples/s
420
+ Step 47300/50462 | Loss: 2.5336 | LR: 4.97e-06 | 3982.4ms/step | 15.1 samples/s
421
+ Step 47310/50462 | Loss: 2.5134 | LR: 4.94e-06 | 3986.8ms/step | 15.0 samples/s
422
+ Step 47320/50462 | Loss: 2.5314 | LR: 4.91e-06 | 4000.9ms/step | 15.0 samples/s
423
+ Step 47330/50462 | Loss: 2.5025 | LR: 4.88e-06 | 3984.5ms/step | 15.1 samples/s
424
+ Step 47340/50462 | Loss: 2.5189 | LR: 4.85e-06 | 3989.9ms/step | 15.0 samples/s
425
+ Step 47350/50462 | Loss: 2.5460 | LR: 4.82e-06 | 3996.2ms/step | 15.0 samples/s
426
+ Step 47360/50462 | Loss: 2.5184 | LR: 4.79e-06 | 3990.8ms/step | 15.0 samples/s
427
+ Step 47370/50462 | Loss: 2.5250 | LR: 4.76e-06 | 3991.3ms/step | 15.0 samples/s
428
+ Step 47380/50462 | Loss: 2.5333 | LR: 4.73e-06 | 3983.8ms/step | 15.1 samples/s
429
+ Step 47390/50462 | Loss: 2.5374 | LR: 4.70e-06 | 3995.7ms/step | 15.0 samples/s
430
+ Step 47400/50462 | Loss: 2.5201 | LR: 4.67e-06 | 4002.8ms/step | 15.0 samples/s
431
+ Step 47410/50462 | Loss: 2.5591 | LR: 4.64e-06 | 3990.0ms/step | 15.0 samples/s
432
+ Step 47420/50462 | Loss: 2.5590 | LR: 4.61e-06 | 3989.4ms/step | 15.0 samples/s
433
+ Step 47430/50462 | Loss: 2.5180 | LR: 4.58e-06 | 3983.9ms/step | 15.1 samples/s
434
+ Step 47440/50462 | Loss: 2.5359 | LR: 4.55e-06 | 3991.6ms/step | 15.0 samples/s
435
+ Step 47450/50462 | Loss: 2.5261 | LR: 4.52e-06 | 3987.6ms/step | 15.0 samples/s
436
+ Step 47460/50462 | Loss: 2.5128 | LR: 4.49e-06 | 3993.9ms/step | 15.0 samples/s
437
+ Step 47470/50462 | Loss: 2.5372 | LR: 4.46e-06 | 3979.1ms/step | 15.1 samples/s
438
+ Step 47480/50462 | Loss: 2.4950 | LR: 4.43e-06 | 3993.7ms/step | 15.0 samples/s
439
+ Step 47490/50462 | Loss: 2.5165 | LR: 4.40e-06 | 3998.6ms/step | 15.0 samples/s
440
+ Step 47500/50462 | Loss: 2.5530 | LR: 4.37e-06 | 3986.4ms/step | 15.1 samples/s
441
+ Validation at step 47500: Loss = 2.5528, Perplexity = 12.84
442
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-47500
443
+ Step 47510/50462 | Loss: 2.5468 | LR: 4.34e-06 | 5893.6ms/step | 10.2 samples/s
444
+ Step 47520/50462 | Loss: 2.5100 | LR: 4.31e-06 | 3991.7ms/step | 15.0 samples/s
445
+ Step 47530/50462 | Loss: 2.5641 | LR: 4.28e-06 | 4001.9ms/step | 15.0 samples/s
446
+ Step 47540/50462 | Loss: 2.5329 | LR: 4.25e-06 | 3995.0ms/step | 15.0 samples/s
447
+ Step 47550/50462 | Loss: 2.5410 | LR: 4.22e-06 | 3995.5ms/step | 15.0 samples/s
448
+ Step 47560/50462 | Loss: 2.5332 | LR: 4.19e-06 | 3992.1ms/step | 15.0 samples/s
449
+ Step 47570/50462 | Loss: 2.5226 | LR: 4.16e-06 | 4000.7ms/step | 15.0 samples/s
450
+ Step 47580/50462 | Loss: 2.5617 | LR: 4.13e-06 | 3988.8ms/step | 15.0 samples/s
451
+ Step 47590/50462 | Loss: 2.5187 | LR: 4.11e-06 | 3999.7ms/step | 15.0 samples/s
452
+ Step 47600/50462 | Loss: 2.5246 | LR: 4.08e-06 | 4006.7ms/step | 15.0 samples/s
453
+ Step 47610/50462 | Loss: 2.5295 | LR: 4.05e-06 | 3997.9ms/step | 15.0 samples/s
454
+ Step 47620/50462 | Loss: 2.5100 | LR: 4.02e-06 | 3995.6ms/step | 15.0 samples/s
455
+ Step 47630/50462 | Loss: 2.4924 | LR: 3.99e-06 | 4001.5ms/step | 15.0 samples/s
456
+ Step 47640/50462 | Loss: 2.5264 | LR: 3.97e-06 | 4023.1ms/step | 14.9 samples/s
457
+ Step 47650/50462 | Loss: 2.5560 | LR: 3.94e-06 | 4019.0ms/step | 14.9 samples/s
458
+ Step 47660/50462 | Loss: 2.5319 | LR: 3.91e-06 | 4000.2ms/step | 15.0 samples/s
459
+ Step 47670/50462 | Loss: 2.5049 | LR: 3.88e-06 | 4002.1ms/step | 15.0 samples/s
460
+ Step 47680/50462 | Loss: 2.5210 | LR: 3.85e-06 | 4002.1ms/step | 15.0 samples/s
461
+ Step 47690/50462 | Loss: 2.5076 | LR: 3.83e-06 | 4000.7ms/step | 15.0 samples/s
462
+ Step 47700/50462 | Loss: 2.5479 | LR: 3.80e-06 | 3995.8ms/step | 15.0 samples/s
463
+ Step 47710/50462 | Loss: 2.5457 | LR: 3.77e-06 | 3995.2ms/step | 15.0 samples/s
464
+ Step 47720/50462 | Loss: 2.5083 | LR: 3.74e-06 | 3987.3ms/step | 15.0 samples/s
465
+ Step 47730/50462 | Loss: 2.5401 | LR: 3.72e-06 | 3983.6ms/step | 15.1 samples/s
466
+ Step 47740/50462 | Loss: 2.5419 | LR: 3.69e-06 | 3997.0ms/step | 15.0 samples/s
467
+ Step 47750/50462 | Loss: 2.5209 | LR: 3.66e-06 | 4012.8ms/step | 15.0 samples/s
468
+ Validation at step 47750: Loss = 2.5518, Perplexity = 12.83
469
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-47750
470
+ Step 47760/50462 | Loss: 2.5490 | LR: 3.64e-06 | 6204.8ms/step | 9.7 samples/s
471
+ Step 47770/50462 | Loss: 2.5226 | LR: 3.61e-06 | 3999.4ms/step | 15.0 samples/s
472
+ Step 47780/50462 | Loss: 2.5444 | LR: 3.58e-06 | 4008.2ms/step | 15.0 samples/s
473
+ Step 47790/50462 | Loss: 2.5145 | LR: 3.56e-06 | 3995.9ms/step | 15.0 samples/s
474
+ Step 47800/50462 | Loss: 2.5684 | LR: 3.53e-06 | 4006.3ms/step | 15.0 samples/s
475
+ Step 47810/50462 | Loss: 2.4875 | LR: 3.50e-06 | 4008.1ms/step | 15.0 samples/s
476
+ Step 47820/50462 | Loss: 2.5147 | LR: 3.48e-06 | 4008.3ms/step | 15.0 samples/s
477
+ Step 47830/50462 | Loss: 2.5179 | LR: 3.45e-06 | 4000.3ms/step | 15.0 samples/s
478
+ Step 47840/50462 | Loss: 2.5144 | LR: 3.42e-06 | 4002.4ms/step | 15.0 samples/s
479
+ Step 47850/50462 | Loss: 2.5329 | LR: 3.40e-06 | 3990.5ms/step | 15.0 samples/s
480
+ Step 47860/50462 | Loss: 2.5348 | LR: 3.37e-06 | 4003.4ms/step | 15.0 samples/s
481
+ Step 47870/50462 | Loss: 2.5310 | LR: 3.35e-06 | 3997.5ms/step | 15.0 samples/s
482
+ Step 47880/50462 | Loss: 2.5469 | LR: 3.32e-06 | 4013.1ms/step | 15.0 samples/s
483
+ Step 47890/50462 | Loss: 2.5187 | LR: 3.30e-06 | 4003.7ms/step | 15.0 samples/s
484
+ Step 47900/50462 | Loss: 2.5376 | LR: 3.27e-06 | 3990.5ms/step | 15.0 samples/s
485
+ Step 47910/50462 | Loss: 2.5155 | LR: 3.24e-06 | 4000.4ms/step | 15.0 samples/s
486
+ Step 47920/50462 | Loss: 2.5657 | LR: 3.22e-06 | 3996.6ms/step | 15.0 samples/s
487
+ Step 47930/50462 | Loss: 2.5217 | LR: 3.19e-06 | 3998.4ms/step | 15.0 samples/s
488
+ Step 47940/50462 | Loss: 2.5504 | LR: 3.17e-06 | 3999.7ms/step | 15.0 samples/s
489
+ Step 47950/50462 | Loss: 2.5455 | LR: 3.14e-06 | 4016.0ms/step | 14.9 samples/s
490
+ Step 47960/50462 | Loss: 2.5267 | LR: 3.12e-06 | 4008.0ms/step | 15.0 samples/s
491
+ Step 47970/50462 | Loss: 2.5240 | LR: 3.09e-06 | 4002.6ms/step | 15.0 samples/s
492
+ Step 47980/50462 | Loss: 2.5380 | LR: 3.07e-06 | 3987.3ms/step | 15.0 samples/s
493
+ Step 47990/50462 | Loss: 2.5487 | LR: 3.04e-06 | 4006.9ms/step | 15.0 samples/s
494
+ Step 48000/50462 | Loss: 2.5889 | LR: 3.02e-06 | 4002.2ms/step | 15.0 samples/s
495
+ Validation at step 48000: Loss = 2.5522, Perplexity = 12.83
496
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-48000
497
+ Step 48010/50462 | Loss: 2.5012 | LR: 3.00e-06 | 5944.2ms/step | 10.1 samples/s
498
+ Step 48020/50462 | Loss: 2.5266 | LR: 2.97e-06 | 4002.4ms/step | 15.0 samples/s
499
+ Step 48030/50462 | Loss: 2.5327 | LR: 2.95e-06 | 3993.5ms/step | 15.0 samples/s
500
+ Step 48040/50462 | Loss: 2.5305 | LR: 2.92e-06 | 3993.3ms/step | 15.0 samples/s
501
+ Step 48050/50462 | Loss: 2.5289 | LR: 2.90e-06 | 4002.7ms/step | 15.0 samples/s
502
+ Step 48060/50462 | Loss: 2.5416 | LR: 2.87e-06 | 4002.1ms/step | 15.0 samples/s
503
+ Step 48070/50462 | Loss: 2.5362 | LR: 2.85e-06 | 3996.1ms/step | 15.0 samples/s
504
+ Step 48080/50462 | Loss: 2.5117 | LR: 2.83e-06 | 3996.1ms/step | 15.0 samples/s
505
+ Step 48090/50462 | Loss: 2.5466 | LR: 2.80e-06 | 4010.9ms/step | 15.0 samples/s
506
+ Step 48100/50462 | Loss: 2.5216 | LR: 2.78e-06 | 3993.1ms/step | 15.0 samples/s
507
+ Step 48110/50462 | Loss: 2.5543 | LR: 2.76e-06 | 3983.8ms/step | 15.1 samples/s
508
+ Step 48120/50462 | Loss: 2.5462 | LR: 2.73e-06 | 3999.9ms/step | 15.0 samples/s
509
+ Step 48130/50462 | Loss: 2.5347 | LR: 2.71e-06 | 4005.7ms/step | 15.0 samples/s
510
+ Step 48140/50462 | Loss: 2.5658 | LR: 2.69e-06 | 3983.4ms/step | 15.1 samples/s
511
+ Step 48150/50462 | Loss: 2.4975 | LR: 2.66e-06 | 3994.5ms/step | 15.0 samples/s
512
+ Step 48160/50462 | Loss: 2.5237 | LR: 2.64e-06 | 3988.2ms/step | 15.0 samples/s
513
+ Step 48170/50462 | Loss: 2.5381 | LR: 2.62e-06 | 3984.7ms/step | 15.1 samples/s
514
+ Step 48180/50462 | Loss: 2.5326 | LR: 2.60e-06 | 4004.7ms/step | 15.0 samples/s
515
+ Step 48190/50462 | Loss: 2.5345 | LR: 2.57e-06 | 3990.5ms/step | 15.0 samples/s
516
+ Step 48200/50462 | Loss: 2.5308 | LR: 2.55e-06 | 3987.2ms/step | 15.0 samples/s
517
+ Step 48210/50462 | Loss: 2.5168 | LR: 2.53e-06 | 3989.5ms/step | 15.0 samples/s
518
+ Step 48220/50462 | Loss: 2.5176 | LR: 2.51e-06 | 3996.3ms/step | 15.0 samples/s
519
+ Step 48230/50462 | Loss: 2.5588 | LR: 2.48e-06 | 3977.4ms/step | 15.1 samples/s
520
+ Step 48240/50462 | Loss: 2.5647 | LR: 2.46e-06 | 3991.5ms/step | 15.0 samples/s
521
+ Step 48250/50462 | Loss: 2.5086 | LR: 2.44e-06 | 3994.0ms/step | 15.0 samples/s
522
+ Validation at step 48250: Loss = 2.5525, Perplexity = 12.84
523
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-48250
524
+ Step 48260/50462 | Loss: 2.5331 | LR: 2.42e-06 | 5960.0ms/step | 10.1 samples/s
525
+ Step 48270/50462 | Loss: 2.4732 | LR: 2.39e-06 | 3986.9ms/step | 15.0 samples/s
526
+ Step 48280/50462 | Loss: 2.5215 | LR: 2.37e-06 | 3994.6ms/step | 15.0 samples/s
527
+ Step 48290/50462 | Loss: 2.5274 | LR: 2.35e-06 | 3990.3ms/step | 15.0 samples/s
528
+ Step 48300/50462 | Loss: 2.5485 | LR: 2.33e-06 | 3985.7ms/step | 15.1 samples/s
529
+ Step 48310/50462 | Loss: 2.5554 | LR: 2.31e-06 | 3997.3ms/step | 15.0 samples/s
530
+ Step 48320/50462 | Loss: 2.5106 | LR: 2.29e-06 | 3995.2ms/step | 15.0 samples/s
531
+ Step 48330/50462 | Loss: 2.5089 | LR: 2.27e-06 | 3994.7ms/step | 15.0 samples/s
532
+ Step 48340/50462 | Loss: 2.5552 | LR: 2.24e-06 | 3993.1ms/step | 15.0 samples/s
533
+ Step 48350/50462 | Loss: 2.5351 | LR: 2.22e-06 | 3989.7ms/step | 15.0 samples/s
534
+ Step 48360/50462 | Loss: 2.5170 | LR: 2.20e-06 | 3987.0ms/step | 15.0 samples/s
535
+ Step 48370/50462 | Loss: 2.5060 | LR: 2.18e-06 | 4011.7ms/step | 15.0 samples/s
536
+ Step 48380/50462 | Loss: 2.5049 | LR: 2.16e-06 | 4001.4ms/step | 15.0 samples/s
537
+ Step 48390/50462 | Loss: 2.5331 | LR: 2.14e-06 | 3989.6ms/step | 15.0 samples/s
538
+ Step 48400/50462 | Loss: 2.5390 | LR: 2.12e-06 | 3986.2ms/step | 15.1 samples/s
539
+ Step 48410/50462 | Loss: 2.5165 | LR: 2.10e-06 | 3974.3ms/step | 15.1 samples/s
540
+ Step 48420/50462 | Loss: 2.4957 | LR: 2.08e-06 | 3983.1ms/step | 15.1 samples/s
541
+ Step 48430/50462 | Loss: 2.5690 | LR: 2.06e-06 | 3990.6ms/step | 15.0 samples/s
542
+ Step 48440/50462 | Loss: 2.5504 | LR: 2.04e-06 | 4000.6ms/step | 15.0 samples/s
543
+ Step 48450/50462 | Loss: 2.5232 | LR: 2.02e-06 | 3993.0ms/step | 15.0 samples/s
544
+ Step 48460/50462 | Loss: 2.5535 | LR: 2.00e-06 | 3983.8ms/step | 15.1 samples/s
545
+ Step 48470/50462 | Loss: 2.5371 | LR: 1.98e-06 | 3983.4ms/step | 15.1 samples/s
546
+ Step 48480/50462 | Loss: 2.5504 | LR: 1.96e-06 | 3985.8ms/step | 15.1 samples/s
547
+ Step 48490/50462 | Loss: 2.5117 | LR: 1.94e-06 | 3994.9ms/step | 15.0 samples/s
548
+ Step 48500/50462 | Loss: 2.5395 | LR: 1.92e-06 | 3981.4ms/step | 15.1 samples/s
549
+ Validation at step 48500: Loss = 2.5515, Perplexity = 12.83
550
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-48500
551
+ Step 48510/50462 | Loss: 2.5360 | LR: 1.90e-06 | 5942.2ms/step | 10.1 samples/s
552
+ Step 48520/50462 | Loss: 2.5201 | LR: 1.88e-06 | 3998.7ms/step | 15.0 samples/s
553
+ Step 48530/50462 | Loss: 2.5084 | LR: 1.86e-06 | 3987.1ms/step | 15.0 samples/s
554
+ Step 48540/50462 | Loss: 2.5533 | LR: 1.84e-06 | 3999.9ms/step | 15.0 samples/s
555
+ Step 48550/50462 | Loss: 2.5326 | LR: 1.82e-06 | 4002.6ms/step | 15.0 samples/s
556
+ Step 48560/50462 | Loss: 2.5250 | LR: 1.80e-06 | 3987.7ms/step | 15.0 samples/s
557
+ Step 48570/50462 | Loss: 2.5659 | LR: 1.78e-06 | 3989.3ms/step | 15.0 samples/s
558
+ Step 48580/50462 | Loss: 2.5533 | LR: 1.77e-06 | 3998.2ms/step | 15.0 samples/s
559
+ Step 48590/50462 | Loss: 2.5435 | LR: 1.75e-06 | 3994.6ms/step | 15.0 samples/s
560
+ Step 48600/50462 | Loss: 2.5202 | LR: 1.73e-06 | 3995.4ms/step | 15.0 samples/s
561
+ Step 48610/50462 | Loss: 2.5214 | LR: 1.71e-06 | 3984.0ms/step | 15.1 samples/s
562
+ Step 48620/50462 | Loss: 2.5455 | LR: 1.69e-06 | 3974.4ms/step | 15.1 samples/s
563
+ Step 48630/50462 | Loss: 2.5240 | LR: 1.67e-06 | 3994.6ms/step | 15.0 samples/s
564
+ Step 48640/50462 | Loss: 2.5052 | LR: 1.66e-06 | 3995.5ms/step | 15.0 samples/s
565
+ Step 48650/50462 | Loss: 2.5539 | LR: 1.64e-06 | 4000.9ms/step | 15.0 samples/s
566
+ Step 48660/50462 | Loss: 2.5381 | LR: 1.62e-06 | 3996.1ms/step | 15.0 samples/s
567
+ Step 48670/50462 | Loss: 2.5144 | LR: 1.60e-06 | 3995.9ms/step | 15.0 samples/s
568
+ Step 48680/50462 | Loss: 2.5076 | LR: 1.58e-06 | 3993.5ms/step | 15.0 samples/s
569
+ Step 48690/50462 | Loss: 2.5404 | LR: 1.57e-06 | 4008.5ms/step | 15.0 samples/s
570
+ Step 48700/50462 | Loss: 2.5094 | LR: 1.55e-06 | 3992.2ms/step | 15.0 samples/s
571
+ Step 48710/50462 | Loss: 2.5630 | LR: 1.53e-06 | 3997.5ms/step | 15.0 samples/s
572
+ Step 48720/50462 | Loss: 2.5024 | LR: 1.51e-06 | 3997.2ms/step | 15.0 samples/s
573
+ Step 48730/50462 | Loss: 2.5215 | LR: 1.50e-06 | 3986.1ms/step | 15.1 samples/s
574
+ Step 48740/50462 | Loss: 2.4820 | LR: 1.48e-06 | 3989.6ms/step | 15.0 samples/s
575
+ Step 48750/50462 | Loss: 2.5206 | LR: 1.46e-06 | 3987.8ms/step | 15.0 samples/s
576
+ Validation at step 48750: Loss = 2.5526, Perplexity = 12.84
577
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-48750
578
+ Step 48760/50462 | Loss: 2.5113 | LR: 1.44e-06 | 5937.9ms/step | 10.1 samples/s
579
+ Step 48770/50462 | Loss: 2.5727 | LR: 1.43e-06 | 3991.8ms/step | 15.0 samples/s
580
+ Step 48780/50462 | Loss: 2.5205 | LR: 1.41e-06 | 3997.4ms/step | 15.0 samples/s
581
+ Step 48790/50462 | Loss: 2.5620 | LR: 1.39e-06 | 3996.0ms/step | 15.0 samples/s
582
+ Step 48800/50462 | Loss: 2.5212 | LR: 1.38e-06 | 3995.0ms/step | 15.0 samples/s
583
+ Step 48810/50462 | Loss: 2.5463 | LR: 1.36e-06 | 3997.0ms/step | 15.0 samples/s
584
+ Step 48820/50462 | Loss: 2.5311 | LR: 1.34e-06 | 4000.4ms/step | 15.0 samples/s
585
+ Step 48830/50462 | Loss: 2.5524 | LR: 1.33e-06 | 3999.1ms/step | 15.0 samples/s
586
+ Step 48840/50462 | Loss: 2.5567 | LR: 1.31e-06 | 4001.4ms/step | 15.0 samples/s
587
+ Step 48850/50462 | Loss: 2.5388 | LR: 1.30e-06 | 3997.6ms/step | 15.0 samples/s
588
+ Step 48860/50462 | Loss: 2.5289 | LR: 1.28e-06 | 3990.4ms/step | 15.0 samples/s
589
+ Step 48870/50462 | Loss: 2.5390 | LR: 1.26e-06 | 4000.0ms/step | 15.0 samples/s
590
+ Step 48880/50462 | Loss: 2.5509 | LR: 1.25e-06 | 4014.9ms/step | 14.9 samples/s
591
+ Step 48890/50462 | Loss: 2.5258 | LR: 1.23e-06 | 4011.9ms/step | 15.0 samples/s
592
+ Step 48900/50462 | Loss: 2.5320 | LR: 1.22e-06 | 3996.6ms/step | 15.0 samples/s
593
+ Step 48910/50462 | Loss: 2.5324 | LR: 1.20e-06 | 4002.3ms/step | 15.0 samples/s
594
+ Step 48920/50462 | Loss: 2.5437 | LR: 1.19e-06 | 4000.9ms/step | 15.0 samples/s
595
+ Step 48930/50462 | Loss: 2.5682 | LR: 1.17e-06 | 4004.8ms/step | 15.0 samples/s
596
+ Step 48940/50462 | Loss: 2.5276 | LR: 1.16e-06 | 4014.0ms/step | 14.9 samples/s
597
+ Step 48950/50462 | Loss: 2.5537 | LR: 1.14e-06 | 4018.0ms/step | 14.9 samples/s
598
+ Step 48960/50462 | Loss: 2.5182 | LR: 1.13e-06 | 4012.9ms/step | 15.0 samples/s
599
+ Step 48970/50462 | Loss: 2.5432 | LR: 1.11e-06 | 4014.5ms/step | 14.9 samples/s
600
+ Step 48980/50462 | Loss: 2.5229 | LR: 1.10e-06 | 4009.9ms/step | 15.0 samples/s
601
+ Step 48990/50462 | Loss: 2.5402 | LR: 1.08e-06 | 4007.3ms/step | 15.0 samples/s
602
+ Step 49000/50462 | Loss: 2.5578 | LR: 1.07e-06 | 4006.7ms/step | 15.0 samples/s
603
+ Validation at step 49000: Loss = 2.5524, Perplexity = 12.84
604
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-49000
605
+ Step 49010/50462 | Loss: 2.5187 | LR: 1.05e-06 | 5951.2ms/step | 10.1 samples/s
606
+ Step 49020/50462 | Loss: 2.5547 | LR: 1.04e-06 | 4013.7ms/step | 14.9 samples/s
607
+ Step 49030/50462 | Loss: 2.5473 | LR: 1.02e-06 | 4010.1ms/step | 15.0 samples/s
608
+ Step 49040/50462 | Loss: 2.5368 | LR: 1.01e-06 | 4009.5ms/step | 15.0 samples/s
609
+ Step 49050/50462 | Loss: 2.5611 | LR: 9.95e-07 | 4006.2ms/step | 15.0 samples/s
610
+ Step 49060/50462 | Loss: 2.5145 | LR: 9.81e-07 | 4006.7ms/step | 15.0 samples/s
611
+ Step 49070/50462 | Loss: 2.5091 | LR: 9.67e-07 | 4012.9ms/step | 15.0 samples/s
612
+ Step 49080/50462 | Loss: 2.5615 | LR: 9.53e-07 | 4022.4ms/step | 14.9 samples/s
613
+ Step 49090/50462 | Loss: 2.5518 | LR: 9.39e-07 | 4012.5ms/step | 15.0 samples/s
614
+ Step 49100/50462 | Loss: 2.5379 | LR: 9.25e-07 | 4007.5ms/step | 15.0 samples/s
615
+ Step 49110/50462 | Loss: 2.4802 | LR: 9.12e-07 | 3995.9ms/step | 15.0 samples/s
616
+ Step 49120/50462 | Loss: 2.5717 | LR: 8.99e-07 | 4002.9ms/step | 15.0 samples/s
617
+ Step 49130/50462 | Loss: 2.5384 | LR: 8.85e-07 | 4004.4ms/step | 15.0 samples/s
618
+ Step 49140/50462 | Loss: 2.5558 | LR: 8.72e-07 | 4012.5ms/step | 15.0 samples/s
619
+ Step 49150/50462 | Loss: 2.5023 | LR: 8.59e-07 | 4004.1ms/step | 15.0 samples/s
620
+ Step 49160/50462 | Loss: 2.5293 | LR: 8.46e-07 | 4010.8ms/step | 15.0 samples/s
621
+ Step 49170/50462 | Loss: 2.5600 | LR: 8.33e-07 | 4008.2ms/step | 15.0 samples/s
622
+ Step 49180/50462 | Loss: 2.5364 | LR: 8.20e-07 | 4020.0ms/step | 14.9 samples/s
623
+ Step 49190/50462 | Loss: 2.4869 | LR: 8.07e-07 | 4014.8ms/step | 14.9 samples/s
624
+ Step 49200/50462 | Loss: 2.5397 | LR: 7.95e-07 | 4003.1ms/step | 15.0 samples/s
625
+ Step 49210/50462 | Loss: 2.5740 | LR: 7.82e-07 | 3998.2ms/step | 15.0 samples/s
626
+ Step 49220/50462 | Loss: 2.5228 | LR: 7.70e-07 | 4003.1ms/step | 15.0 samples/s
627
+ Step 49230/50462 | Loss: 2.5302 | LR: 7.57e-07 | 4003.5ms/step | 15.0 samples/s
628
+ Step 49240/50462 | Loss: 2.5029 | LR: 7.45e-07 | 4013.4ms/step | 14.9 samples/s
629
+ Step 49250/50462 | Loss: 2.5419 | LR: 7.33e-07 | 3993.8ms/step | 15.0 samples/s
630
+ Validation at step 49250: Loss = 2.5514, Perplexity = 12.83
631
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-49250
632
+ Step 49260/50462 | Loss: 2.5592 | LR: 7.21e-07 | 5949.1ms/step | 10.1 samples/s
633
+ Step 49270/50462 | Loss: 2.5393 | LR: 7.09e-07 | 3999.2ms/step | 15.0 samples/s
634
+ Step 49280/50462 | Loss: 2.5224 | LR: 6.97e-07 | 4011.8ms/step | 15.0 samples/s
635
+ Step 49290/50462 | Loss: 2.5446 | LR: 6.85e-07 | 4006.3ms/step | 15.0 samples/s
636
+ Step 49300/50462 | Loss: 2.5395 | LR: 6.74e-07 | 3994.2ms/step | 15.0 samples/s
637
+ Step 49310/50462 | Loss: 2.5449 | LR: 6.62e-07 | 3997.1ms/step | 15.0 samples/s
638
+ Step 49320/50462 | Loss: 2.5460 | LR: 6.51e-07 | 4003.8ms/step | 15.0 samples/s
639
+ Step 49330/50462 | Loss: 2.5446 | LR: 6.39e-07 | 4006.0ms/step | 15.0 samples/s
640
+ Step 49340/50462 | Loss: 2.5285 | LR: 6.28e-07 | 4003.8ms/step | 15.0 samples/s
641
+ Step 49350/50462 | Loss: 2.5330 | LR: 6.17e-07 | 4008.9ms/step | 15.0 samples/s
642
+ Step 49360/50462 | Loss: 2.5305 | LR: 6.06e-07 | 4004.0ms/step | 15.0 samples/s
643
+ Step 49370/50462 | Loss: 2.5007 | LR: 5.95e-07 | 4002.1ms/step | 15.0 samples/s
644
+ Step 49380/50462 | Loss: 2.5318 | LR: 5.84e-07 | 3995.2ms/step | 15.0 samples/s
645
+ Step 49390/50462 | Loss: 2.5406 | LR: 5.73e-07 | 4007.1ms/step | 15.0 samples/s
646
+ Step 49400/50462 | Loss: 2.5271 | LR: 5.63e-07 | 4002.5ms/step | 15.0 samples/s
647
+ Step 49410/50462 | Loss: 2.5384 | LR: 5.52e-07 | 4011.3ms/step | 15.0 samples/s
648
+ Step 49420/50462 | Loss: 2.5077 | LR: 5.42e-07 | 3998.0ms/step | 15.0 samples/s
649
+ Step 49430/50462 | Loss: 2.4982 | LR: 5.31e-07 | 4010.4ms/step | 15.0 samples/s
650
+ Step 49440/50462 | Loss: 2.5401 | LR: 5.21e-07 | 4005.7ms/step | 15.0 samples/s
651
+ Step 49450/50462 | Loss: 2.4945 | LR: 5.11e-07 | 4013.4ms/step | 14.9 samples/s
652
+ Step 49460/50462 | Loss: 2.5220 | LR: 5.01e-07 | 4007.1ms/step | 15.0 samples/s
653
+ Step 49470/50462 | Loss: 2.5546 | LR: 4.91e-07 | 4007.1ms/step | 15.0 samples/s
654
+ Step 49480/50462 | Loss: 2.5401 | LR: 4.81e-07 | 4011.8ms/step | 15.0 samples/s
655
+ Step 49490/50462 | Loss: 2.5310 | LR: 4.72e-07 | 4007.4ms/step | 15.0 samples/s
656
+ Step 49500/50462 | Loss: 2.5566 | LR: 4.62e-07 | 4017.5ms/step | 14.9 samples/s
657
+ Validation at step 49500: Loss = 2.5521, Perplexity = 12.83
658
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-49500
659
+ Step 49510/50462 | Loss: 2.5082 | LR: 4.52e-07 | 5923.1ms/step | 10.1 samples/s
660
+ Step 49520/50462 | Loss: 2.5663 | LR: 4.43e-07 | 4005.5ms/step | 15.0 samples/s
661
+ Step 49530/50462 | Loss: 2.5148 | LR: 4.34e-07 | 4001.0ms/step | 15.0 samples/s
662
+ Step 49540/50462 | Loss: 2.5421 | LR: 4.24e-07 | 4007.9ms/step | 15.0 samples/s
663
+ Step 49550/50462 | Loss: 2.5200 | LR: 4.15e-07 | 4004.0ms/step | 15.0 samples/s
664
+ Step 49560/50462 | Loss: 2.5572 | LR: 4.06e-07 | 4015.3ms/step | 14.9 samples/s
665
+ Step 49570/50462 | Loss: 2.5033 | LR: 3.97e-07 | 4001.4ms/step | 15.0 samples/s
666
+ Step 49580/50462 | Loss: 2.5353 | LR: 3.88e-07 | 4009.0ms/step | 15.0 samples/s
667
+ Step 49590/50462 | Loss: 2.5294 | LR: 3.79e-07 | 4002.1ms/step | 15.0 samples/s
668
+ Step 49600/50462 | Loss: 2.5471 | LR: 3.71e-07 | 4004.1ms/step | 15.0 samples/s
669
+ Step 49610/50462 | Loss: 2.5330 | LR: 3.62e-07 | 4008.5ms/step | 15.0 samples/s
670
+ Step 49620/50462 | Loss: 2.5376 | LR: 3.54e-07 | 4006.7ms/step | 15.0 samples/s
671
+ Step 49630/50462 | Loss: 2.5351 | LR: 3.45e-07 | 4012.5ms/step | 15.0 samples/s
672
+ Step 49640/50462 | Loss: 2.5524 | LR: 3.37e-07 | 4012.6ms/step | 15.0 samples/s
673
+ Step 49650/50462 | Loss: 2.5128 | LR: 3.29e-07 | 3996.8ms/step | 15.0 samples/s
674
+ Step 49660/50462 | Loss: 2.5240 | LR: 3.21e-07 | 4009.8ms/step | 15.0 samples/s
675
+ Step 49670/50462 | Loss: 2.5579 | LR: 3.13e-07 | 4010.6ms/step | 15.0 samples/s
676
+ Step 49680/50462 | Loss: 2.5405 | LR: 3.05e-07 | 4000.0ms/step | 15.0 samples/s
677
+ Step 49690/50462 | Loss: 2.5474 | LR: 2.97e-07 | 4008.4ms/step | 15.0 samples/s
678
+ Step 49700/50462 | Loss: 2.5267 | LR: 2.90e-07 | 4001.0ms/step | 15.0 samples/s
679
+ Step 49710/50462 | Loss: 2.5286 | LR: 2.82e-07 | 4003.6ms/step | 15.0 samples/s
680
+ Step 49720/50462 | Loss: 2.5228 | LR: 2.75e-07 | 4008.2ms/step | 15.0 samples/s
681
+ Step 49730/50462 | Loss: 2.5330 | LR: 2.67e-07 | 4011.4ms/step | 15.0 samples/s
682
+ Step 49740/50462 | Loss: 2.5524 | LR: 2.60e-07 | 4004.1ms/step | 15.0 samples/s
683
+ Step 49750/50462 | Loss: 2.5274 | LR: 2.53e-07 | 4014.6ms/step | 14.9 samples/s
684
+ Validation at step 49750: Loss = 2.5522, Perplexity = 12.84
685
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-49750
686
+ Step 49760/50462 | Loss: 2.5174 | LR: 2.46e-07 | 5959.3ms/step | 10.1 samples/s
687
+ Step 49770/50462 | Loss: 2.5142 | LR: 2.39e-07 | 4015.2ms/step | 14.9 samples/s
688
+ Step 49780/50462 | Loss: 2.4925 | LR: 2.32e-07 | 3997.6ms/step | 15.0 samples/s
689
+ Step 49790/50462 | Loss: 2.5381 | LR: 2.25e-07 | 4004.4ms/step | 15.0 samples/s
690
+ Step 49800/50462 | Loss: 2.5445 | LR: 2.19e-07 | 4002.1ms/step | 15.0 samples/s
691
+ Step 49810/50462 | Loss: 2.5415 | LR: 2.12e-07 | 3998.4ms/step | 15.0 samples/s
692
+ Step 49820/50462 | Loss: 2.5400 | LR: 2.06e-07 | 4003.0ms/step | 15.0 samples/s
693
+ Step 49830/50462 | Loss: 2.5502 | LR: 1.99e-07 | 4012.5ms/step | 15.0 samples/s
694
+ Step 49840/50462 | Loss: 2.5008 | LR: 1.93e-07 | 4010.8ms/step | 15.0 samples/s
695
+ Step 49850/50462 | Loss: 2.5092 | LR: 1.87e-07 | 3997.4ms/step | 15.0 samples/s
696
+ Step 49860/50462 | Loss: 2.5410 | LR: 1.81e-07 | 3997.1ms/step | 15.0 samples/s
697
+ Step 49870/50462 | Loss: 2.5257 | LR: 1.75e-07 | 4001.6ms/step | 15.0 samples/s
698
+ Step 49880/50462 | Loss: 2.5176 | LR: 1.69e-07 | 3996.1ms/step | 15.0 samples/s
699
+ Step 49890/50462 | Loss: 2.5128 | LR: 1.63e-07 | 3998.9ms/step | 15.0 samples/s
700
+ Step 49900/50462 | Loss: 2.5631 | LR: 1.58e-07 | 4014.1ms/step | 14.9 samples/s
701
+ Step 49910/50462 | Loss: 2.5404 | LR: 1.52e-07 | 4006.7ms/step | 15.0 samples/s
702
+ Step 49920/50462 | Loss: 2.5574 | LR: 1.47e-07 | 4000.8ms/step | 15.0 samples/s
703
+ Step 49930/50462 | Loss: 2.5108 | LR: 1.41e-07 | 3992.1ms/step | 15.0 samples/s
704
+ Step 49940/50462 | Loss: 2.4691 | LR: 1.36e-07 | 4005.8ms/step | 15.0 samples/s
705
+ Step 49950/50462 | Loss: 2.5467 | LR: 1.31e-07 | 4008.4ms/step | 15.0 samples/s
706
+ Step 49960/50462 | Loss: 2.5321 | LR: 1.26e-07 | 4008.8ms/step | 15.0 samples/s
707
+ Step 49970/50462 | Loss: 2.5465 | LR: 1.21e-07 | 3997.2ms/step | 15.0 samples/s
708
+ Step 49980/50462 | Loss: 2.5368 | LR: 1.16e-07 | 3998.9ms/step | 15.0 samples/s
709
+ Step 49990/50462 | Loss: 2.5749 | LR: 1.11e-07 | 4005.1ms/step | 15.0 samples/s
710
+ Step 50000/50462 | Loss: 2.5255 | LR: 1.07e-07 | 4010.5ms/step | 15.0 samples/s
711
+ Validation at step 50000: Loss = 2.5521, Perplexity = 12.83
712
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-50000
713
+ Step 50010/50462 | Loss: 2.5491 | LR: 1.02e-07 | 5929.7ms/step | 10.1 samples/s
714
+ Step 50020/50462 | Loss: 2.5360 | LR: 9.75e-08 | 4003.2ms/step | 15.0 samples/s
715
+ Step 50030/50462 | Loss: 2.5297 | LR: 9.32e-08 | 3997.3ms/step | 15.0 samples/s
716
+ Step 50040/50462 | Loss: 2.5319 | LR: 8.89e-08 | 4016.6ms/step | 14.9 samples/s
717
+ Step 50050/50462 | Loss: 2.5042 | LR: 8.47e-08 | 4000.9ms/step | 15.0 samples/s
718
+ Step 50060/50462 | Loss: 2.5170 | LR: 8.07e-08 | 4001.5ms/step | 15.0 samples/s
719
+ Step 50070/50462 | Loss: 2.5394 | LR: 7.67e-08 | 4017.4ms/step | 14.9 samples/s
720
+ Step 50080/50462 | Loss: 2.5006 | LR: 7.28e-08 | 4009.6ms/step | 15.0 samples/s
721
+ Step 50090/50462 | Loss: 2.4954 | LR: 6.91e-08 | 3986.3ms/step | 15.1 samples/s
722
+ Step 50100/50462 | Loss: 2.5347 | LR: 6.54e-08 | 4008.0ms/step | 15.0 samples/s
723
+ Step 50110/50462 | Loss: 2.5528 | LR: 6.19e-08 | 3998.6ms/step | 15.0 samples/s
724
+ Step 50120/50462 | Loss: 2.5639 | LR: 5.84e-08 | 4004.6ms/step | 15.0 samples/s
725
+ Step 50130/50462 | Loss: 2.5250 | LR: 5.50e-08 | 3999.3ms/step | 15.0 samples/s
726
+ Step 50140/50462 | Loss: 2.5484 | LR: 5.18e-08 | 3985.0ms/step | 15.1 samples/s
727
+ Step 50150/50462 | Loss: 2.5591 | LR: 4.86e-08 | 3991.5ms/step | 15.0 samples/s
728
+ Step 50160/50462 | Loss: 2.4908 | LR: 4.55e-08 | 4003.5ms/step | 15.0 samples/s
729
+ Step 50170/50462 | Loss: 2.4990 | LR: 4.26e-08 | 4004.2ms/step | 15.0 samples/s
730
+ Step 50180/50462 | Loss: 2.5454 | LR: 3.97e-08 | 3997.2ms/step | 15.0 samples/s
731
+ Step 50190/50462 | Loss: 2.5747 | LR: 3.69e-08 | 4001.4ms/step | 15.0 samples/s
732
+ Step 50200/50462 | Loss: 2.5187 | LR: 3.43e-08 | 4004.5ms/step | 15.0 samples/s
733
+ Step 50210/50462 | Loss: 2.5689 | LR: 3.17e-08 | 3991.4ms/step | 15.0 samples/s
734
+ Step 50220/50462 | Loss: 2.5416 | LR: 2.92e-08 | 4008.3ms/step | 15.0 samples/s
735
+ Step 50230/50462 | Loss: 2.5340 | LR: 2.69e-08 | 3979.2ms/step | 15.1 samples/s
736
+ Step 50240/50462 | Loss: 2.5250 | LR: 2.46e-08 | 4003.1ms/step | 15.0 samples/s
737
+ Step 50250/50462 | Loss: 2.5153 | LR: 2.24e-08 | 4008.2ms/step | 15.0 samples/s
738
+ Validation at step 50250: Loss = 2.5524, Perplexity = 12.84
739
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-50250
740
+ Step 50260/50462 | Loss: 2.5258 | LR: 2.04e-08 | 5933.2ms/step | 10.1 samples/s
741
+ Step 50270/50462 | Loss: 2.5478 | LR: 1.84e-08 | 3986.9ms/step | 15.0 samples/s
742
+ Step 50280/50462 | Loss: 2.5632 | LR: 1.65e-08 | 4005.4ms/step | 15.0 samples/s
743
+ Step 50290/50462 | Loss: 2.5076 | LR: 1.48e-08 | 3998.4ms/step | 15.0 samples/s
744
+ Step 50300/50462 | Loss: 2.5241 | LR: 1.31e-08 | 4006.1ms/step | 15.0 samples/s
745
+ Step 50310/50462 | Loss: 2.5025 | LR: 1.15e-08 | 3989.3ms/step | 15.0 samples/s
746
+ Step 50320/50462 | Loss: 2.5299 | LR: 1.01e-08 | 3997.7ms/step | 15.0 samples/s
747
+ Step 50330/50462 | Loss: 2.5175 | LR: 8.70e-09 | 3988.0ms/step | 15.0 samples/s
748
+ Step 50340/50462 | Loss: 2.5031 | LR: 7.43e-09 | 3986.0ms/step | 15.1 samples/s
749
+ Step 50350/50462 | Loss: 2.5019 | LR: 6.26e-09 | 4002.9ms/step | 15.0 samples/s
750
+ Step 50360/50462 | Loss: 2.5602 | LR: 5.19e-09 | 4008.9ms/step | 15.0 samples/s
751
+ Step 50370/50462 | Loss: 2.5583 | LR: 4.23e-09 | 4000.1ms/step | 15.0 samples/s
752
+ Step 50380/50462 | Loss: 2.5750 | LR: 3.36e-09 | 4001.4ms/step | 15.0 samples/s
753
+ Step 50390/50462 | Loss: 2.5512 | LR: 2.59e-09 | 4000.6ms/step | 15.0 samples/s
754
+ Step 50400/50462 | Loss: 2.5111 | LR: 1.92e-09 | 4000.5ms/step | 15.0 samples/s
755
+ Step 50410/50462 | Loss: 2.5527 | LR: 1.35e-09 | 3991.8ms/step | 15.0 samples/s
756
+ Step 50420/50462 | Loss: 2.5309 | LR: 8.81e-10 | 3988.1ms/step | 15.0 samples/s
757
+ Step 50430/50462 | Loss: 2.5423 | LR: 5.11e-10 | 4011.3ms/step | 15.0 samples/s
758
+ Step 50440/50462 | Loss: 2.5589 | LR: 2.42e-10 | 4002.4ms/step | 15.0 samples/s
759
+ Step 50450/50462 | Loss: 2.5344 | LR: 7.19e-11 | 4006.2ms/step | 15.0 samples/s
760
+ Step 50460/50462 | Loss: 2.5888 | LR: 2.00e-12 | 4006.7ms/step | 15.0 samples/s
761
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-50462
762
+ Training complete!
epfl_final/train_logs/p2_1p2b_mixed_seed72_h100_resume35000.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/train_logs/p4_1p2b_retrieval_40gb.log ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ safe_1p2b_retrieval 0
2
+ ALL_DONE
epfl_final/train_logs/p4_1p2b_retrieval_40gb_retry.log ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ safe_1p2b_retrieval 0
2
+ ALL_DONE
epfl_final/train_logs/p4_3b_mixed_resume_partial.log ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ partial_3b_mixed_jobs 4
2
+ START 0 3b_mixed
3
+ START 1 3b_mixed
4
+ START 2 3b_mixed
5
+ START 3 3b_mixed
6
+ DONE 3 3b_mixed 0
7
+ DONE 1 3b_mixed 0
8
+ DONE 0 3b_mixed 0
9
+ DONE 2 3b_mixed 0
10
+ ALL_DONE
epfl_final/train_logs/p4_40gb_safe_remaining.log ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ safe_remaining 0
2
+ ALL_DONE
epfl_final/train_logs/p4_40gb_safe_remaining_retry.log ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ safe_1p2b_retrieval 8
2
+ START 0 1p2b_dense
3
+ DONE 0 1p2b_dense 0
4
+ START 0 1p2b_dense
5
+ DONE 0 1p2b_dense 0
6
+ START 0 1p2b_dense
7
+ DONE 0 1p2b_dense 0
8
+ START 0 1p2b_dense
9
+ DONE 0 1p2b_dense 0
10
+ START 0 1p2b_mixed
11
+ DONE 0 1p2b_mixed 0
12
+ START 0 1p2b_mixed
13
+ DONE 0 1p2b_mixed 0
14
+ START 0 1p2b_mixed
epfl_final/train_logs/p4_a10080_recovery_latest.log ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ START_RECOVERY
2
+ large_jobs 17
3
+ START 0 1p2b_mixed_seed70
4
+ START 1 1p2b_dense
5
+ START 2 1p2b_dense
6
+ START 3 1p2b_dense
epfl_final/train_logs/p4_a10080_resume_latest.log ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ large_jobs 17
2
+ START 0 1p2b_mixed_seed70
3
+ START 1 1p2b_dense
4
+ START 2 1p2b_dense
5
+ START 3 1p2b_dense
6
+ DONE 2 1p2b_dense 0
7
+ START 2 1p2b_dense
8
+ DONE 3 1p2b_dense 0
9
+ START 3 1p2b_mixed
10
+ DONE 1 1p2b_dense 0
11
+ START 1 1p2b_mixed
12
+ DONE 0 1p2b_mixed_seed70 0
13
+ START 0 1p2b_mixed
epfl_final/train_logs/p4_large_resume_mixed_after_restart.log ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ large_jobs 1
2
+ START 0 1p2b_mixed_seed70
epfl_final/train_logs/p4_large_retry_after_health.log ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ large_jobs 8
2
+ START 0 3b_dense
3
+ START 1 3b_dense
4
+ START 2 3b_dense
5
+ START 3 3b_dense
6
+ DONE 2 3b_dense 0
7
+ START 2 3b_mixed
8
+ DONE 0 3b_dense 0
9
+ START 0 3b_mixed
10
+ DONE 1 3b_dense 0
11
+ START 1 3b_mixed
12
+ DONE 3 3b_dense 0
13
+ START 3 3b_mixed
epfl_final/train_logs/p4_safe16_b.log ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ SAFE16_B 6
2
+ START 0 1p2b_dense_seed70_16384_pg19_ood_windows
3
+ START 1 1p2b_mixed_seed70_16384_fineweb_val_seed42
4
+ START 2 1p2b_mixed_seed70_16384_arxiv_ood_windows
5
+ START 3 1p2b_dense_seed71_16384_pg19_ood_windows
6
+ START 4 1p2b_mixed_seed71_16384_fineweb_val_seed42
7
+ TIMEOUT_SKIP 0 1p2b_dense_seed70_16384_pg19_ood_windows
8
+ TIMEOUT_SKIP 1 1p2b_mixed_seed70_16384_fineweb_val_seed42
9
+ TIMEOUT_SKIP 2 1p2b_mixed_seed70_16384_arxiv_ood_windows
10
+ TIMEOUT_SKIP 3 1p2b_dense_seed71_16384_pg19_ood_windows
11
+ TIMEOUT_SKIP 4 1p2b_mixed_seed71_16384_fineweb_val_seed42
12
+ START 0 1p2b_mixed_seed71_16384_arxiv_ood_windows
13
+ TIMEOUT_SKIP 0 1p2b_mixed_seed71_16384_arxiv_ood_windows
14
+ ALL_DONE
epfl_final/train_logs/queue_seed70_mixed_repaired_after_seed72.log ADDED
@@ -0,0 +1 @@
 
 
1
+ find: ‘/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70_repaired’: No such file or directory
epfl_final/train_logs/queue_seed70_mixed_repaired_after_seed72.pid ADDED
@@ -0,0 +1 @@
 
 
1
+ 115
epfl_final/train_logs/retry_crashed_40gb.log ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ safe_1p2b_retrieval 8
2
+ START 0 1p2b_dense
3
+ START 1 1p2b_dense
4
+ START 2 1p2b_dense
5
+ START 3 1p2b_dense
6
+ START 4 1p2b_mixed
7
+ DONE 0 1p2b_dense 0
8
+ START 0 1p2b_mixed
9
+ DONE 1 1p2b_dense 0
10
+ DONE 2 1p2b_dense 0
11
+ START 1 1p2b_mixed
12
+ START 2 1p2b_mixed
13
+ DONE 4 1p2b_mixed 0
14
+ DONE 3 1p2b_dense 0
15
+ DONE 0 1p2b_mixed 0
16
+ DONE 2 1p2b_mixed 0
17
+ DONE 1 1p2b_mixed 0
18
+ ALL_DONE
epfl_final/train_logs/temp_dense_downstream_a_20260906.log ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ JOBS 24 DONE 0
2
+ START 1 365m_dense_s42_L30720_passkey_tau0.85
3
+ START 2 365m_dense_s42_L30720_niah_multikey_tau0.85
4
+ START 3 365m_dense_s42_L30720_niah_multivalue_tau0.85
5
+ START 4 365m_dense_s42_L30720_vt_tau0.85
6
+ DONE 1 365m_dense_s42_L30720_passkey_tau0.85 0 True
7
+ DONE 2 365m_dense_s42_L30720_niah_multikey_tau0.85 0 True
8
+ DONE 3 365m_dense_s42_L30720_niah_multivalue_tau0.85 0 True
9
+ DONE 4 365m_dense_s42_L30720_vt_tau0.85 0 True
10
+ START 1 1p2b_dense_s42_L30720_niah_multikey_tau0.85
11
+ START 2 1p2b_dense_s42_L30720_niah_multivalue_tau0.85
12
+ START 3 1p2b_dense_s42_L30720_vt_tau0.85
13
+ START 4 1p2b_mixed_s42_L8192_passkey_tau0.95
14
+ DONE 4 1p2b_mixed_s42_L8192_passkey_tau0.95 0 True
15
+ START 4 1p2b_mixed_s42_L8192_niah_multivalue_tau0.95
16
+ DONE 4 1p2b_mixed_s42_L8192_niah_multivalue_tau0.95 0 True
17
+ START 4 1p2b_mixed_s42_L30720_niah_multivalue_tau0.8
18
+ DONE 1 1p2b_dense_s42_L30720_niah_multikey_tau0.85 0 True
19
+ DONE 3 1p2b_dense_s42_L30720_vt_tau0.85 0 True
20
+ DONE 2 1p2b_dense_s42_L30720_niah_multivalue_tau0.85 0 True
21
+ DONE 4 1p2b_mixed_s42_L30720_niah_multivalue_tau0.8 0 True
22
+ COMPLETE 23 / 24
epfl_final/train_logs/temp_dense_downstream_b_20260906.log ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ JOBS 24 DONE 0
2
+ START 0 365m_mixed_s42_L8192_passkey_tau0.95
3
+ START 1 365m_mixed_s42_L8192_niah_multikey_tau0.95
4
+ START 2 365m_mixed_s42_L8192_niah_multivalue_tau0.95
5
+ START 3 365m_mixed_s42_L8192_vt_tau0.95
6
+ START 4 365m_mixed_s42_L30720_passkey_tau0.8
7
+ DONE 0 365m_mixed_s42_L8192_passkey_tau0.95 0 True
8
+ DONE 1 365m_mixed_s42_L8192_niah_multikey_tau0.95 0 True
9
+ DONE 3 365m_mixed_s42_L8192_vt_tau0.95 0 True
10
+ START 0 365m_mixed_s42_L30720_niah_multikey_tau0.8
11
+ START 1 365m_mixed_s42_L30720_niah_multivalue_tau0.8
12
+ START 3 365m_mixed_s42_L30720_vt_tau0.8
13
+ DONE 2 365m_mixed_s42_L8192_niah_multivalue_tau0.95 0 True
14
+ START 2 1p2b_dense_s42_L30720_passkey_tau0.85
15
+ DONE 4 365m_mixed_s42_L30720_passkey_tau0.8 0 True
16
+ START 4 1p2b_mixed_s42_L8192_niah_multikey_tau0.95
17
+ DONE 0 365m_mixed_s42_L30720_niah_multikey_tau0.8 0 True
18
+ DONE 1 365m_mixed_s42_L30720_niah_multivalue_tau0.8 0 True
19
+ DONE 3 365m_mixed_s42_L30720_vt_tau0.8 0 True
20
+ START 0 1p2b_mixed_s42_L8192_vt_tau0.95
21
+ START 1 1p2b_mixed_s42_L30720_passkey_tau0.8
22
+ START 3 1p2b_mixed_s42_L30720_niah_multikey_tau0.8
23
+ DONE 4 1p2b_mixed_s42_L8192_niah_multikey_tau0.95 0 True
24
+ START 4 1p2b_mixed_s42_L30720_vt_tau0.8
25
+ DONE 0 1p2b_mixed_s42_L8192_vt_tau0.95 0 True
26
+ DONE 2 1p2b_dense_s42_L30720_passkey_tau0.85 0 True
27
+ DONE 1 1p2b_mixed_s42_L30720_passkey_tau0.8 0 True
28
+ DONE 3 1p2b_mixed_s42_L30720_niah_multikey_tau0.8 0 True
29
+ DONE 4 1p2b_mixed_s42_L30720_vt_tau0.8 0 True
30
+ COMPLETE 24 / 24
epfl_final/train_logs/temp_protocol_40a_20260906.log ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ JOBS 36 DONE 0
2
+ START 0 365m_dense_s42_L8192_dense
3
+ START 1 365m_dense_s42_L8192_topk_tight
4
+ START 2 365m_dense_s42_L8192_random_full
5
+ START 3 365m_dense_s42_L16384_dense
6
+ START 4 365m_dense_s42_L16384_topk_tight
7
+ DONE 0 365m_dense_s42_L8192_dense 0 True
8
+ START 0 365m_dense_s42_L16384_random_full
9
+ DONE 3 365m_dense_s42_L16384_dense 0 True
10
+ START 3 365m_dense_s42_L24576_dense
11
+ DONE 2 365m_dense_s42_L8192_random_full 0 True
12
+ START 2 365m_dense_s42_L24576_topk_tight
13
+ DONE 1 365m_dense_s42_L8192_topk_tight 0 True
14
+ START 1 365m_dense_s42_L24576_random_full
15
+ DONE 4 365m_dense_s42_L16384_topk_tight 0 True
16
+ START 4 365m_dense_s42_L30720_dense
17
+ DONE 3 365m_dense_s42_L24576_dense 0 True
18
+ START 3 365m_dense_s42_L30720_topk_tight
19
+ DONE 0 365m_dense_s42_L16384_random_full 0 True
20
+ START 0 365m_dense_s42_L30720_random_full
21
+ DONE 4 365m_dense_s42_L30720_dense 0 True
22
+ START 4 365m_mixed_s42_L8192_dense
23
+ DONE 2 365m_dense_s42_L24576_topk_tight 0 True
24
+ START 2 365m_mixed_s42_L8192_topk_tight
25
+ DONE 1 365m_dense_s42_L24576_random_full 0 True
26
+ DONE 3 365m_dense_s42_L30720_topk_tight 0 True
27
+ START 1 365m_mixed_s42_L8192_random_full
28
+ START 3 365m_mixed_s42_L16384_dense
29
+ DONE 4 365m_mixed_s42_L8192_dense 0 True
30
+ START 4 365m_mixed_s42_L16384_topk_tight
31
+ DONE 2 365m_mixed_s42_L8192_topk_tight 0 True
32
+ START 2 365m_mixed_s42_L16384_random_full
33
+ DONE 3 365m_mixed_s42_L16384_dense 0 True
34
+ DONE 4 365m_mixed_s42_L16384_topk_tight 0 True
35
+ START 3 365m_mixed_s42_L24576_dense
36
+ START 4 365m_mixed_s42_L24576_topk_tight
37
+ DONE 1 365m_mixed_s42_L8192_random_full 0 True
38
+ START 1 365m_mixed_s42_L24576_random_full
39
+ DONE 2 365m_mixed_s42_L16384_random_full 0 True
40
+ START 2 365m_mixed_s42_L30720_dense
41
+ DONE 0 365m_dense_s42_L30720_random_full 0 True
42
+ DONE 3 365m_mixed_s42_L24576_dense 0 True
43
+ START 0 365m_mixed_s42_L30720_topk_tight
44
+ START 3 365m_mixed_s42_L30720_random_full
45
+ DONE 4 365m_mixed_s42_L24576_topk_tight 0 True
46
+ START 4 1p2b_dense_s42_L8192_dense
47
+ DONE 2 365m_mixed_s42_L30720_dense 0 True
48
+ START 2 1p2b_dense_s42_L8192_topk_tight
49
+ DONE 1 365m_mixed_s42_L24576_random_full 0 True
50
+ START 1 1p2b_dense_s42_L8192_random_full
51
+ DONE 0 365m_mixed_s42_L30720_topk_tight 0 True
52
+ START 0 1p2b_dense_s42_L16384_dense
53
+ DONE 4 1p2b_dense_s42_L8192_dense 0 True
54
+ START 4 1p2b_dense_s42_L16384_topk_tight
55
+ DONE 3 365m_mixed_s42_L30720_random_full 0 True
56
+ START 3 1p2b_dense_s42_L16384_random_full
57
+ DONE 1 1p2b_dense_s42_L8192_random_full 0 True
58
+ START 1 1p2b_mixed_s42_L8192_dense
59
+ DONE 2 1p2b_dense_s42_L8192_topk_tight 0 True
60
+ DONE 0 1p2b_dense_s42_L16384_dense 0 True
61
+ START 0 1p2b_mixed_s42_L8192_topk_tight
62
+ START 2 1p2b_mixed_s42_L8192_random_full
63
+ DONE 4 1p2b_dense_s42_L16384_topk_tight 0 True
64
+ START 4 1p2b_mixed_s42_L16384_dense
65
+ DONE 0 1p2b_mixed_s42_L8192_topk_tight 0 True
66
+ START 0 1p2b_mixed_s42_L16384_topk_tight
67
+ DONE 1 1p2b_mixed_s42_L8192_dense 0 True
68
+ START 1 1p2b_mixed_s42_L16384_random_full
69
+ DONE 2 1p2b_mixed_s42_L8192_random_full 0 True
70
+ DONE 3 1p2b_dense_s42_L16384_random_full 0 True
71
+ DONE 4 1p2b_mixed_s42_L16384_dense 0 True
72
+ DONE 0 1p2b_mixed_s42_L16384_topk_tight 0 True
73
+ DONE 1 1p2b_mixed_s42_L16384_random_full 0 True
74
+ COMPLETE 36 / 36
epfl_final/train_logs/temp_protocol_40b_20260906.log ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ JOBS 24 DONE 0
2
+ START 0 365m_dense_s42_L8192_topk_full
3
+ START 1 365m_dense_s42_L8192_random_tight
4
+ START 2 365m_dense_s42_L16384_topk_full
5
+ START 3 365m_dense_s42_L16384_random_tight
6
+ START 4 365m_dense_s42_L24576_topk_full
7
+ DONE 1 365m_dense_s42_L8192_random_tight 0 True
8
+ DONE 3 365m_dense_s42_L16384_random_tight 0 True
9
+ START 1 365m_dense_s42_L24576_random_tight
10
+ START 3 365m_dense_s42_L30720_topk_full
11
+ DONE 0 365m_dense_s42_L8192_topk_full 0 True
12
+ START 0 365m_dense_s42_L30720_random_tight
13
+ DONE 1 365m_dense_s42_L24576_random_tight 0 True
14
+ START 1 365m_mixed_s42_L8192_topk_full
15
+ DONE 4 365m_dense_s42_L24576_topk_full 0 True
16
+ START 4 365m_mixed_s42_L8192_random_tight
17
+ DONE 2 365m_dense_s42_L16384_topk_full 0 True
18
+ START 2 365m_mixed_s42_L16384_topk_full
19
+ DONE 0 365m_dense_s42_L30720_random_tight 0 True
20
+ START 0 365m_mixed_s42_L16384_random_tight
21
+ DONE 1 365m_mixed_s42_L8192_topk_full 0 True
22
+ DONE 4 365m_mixed_s42_L8192_random_tight 0 True
23
+ START 1 365m_mixed_s42_L24576_topk_full
24
+ START 4 365m_mixed_s42_L24576_random_tight
25
+ DONE 3 365m_dense_s42_L30720_topk_full 0 True
26
+ START 3 365m_mixed_s42_L30720_topk_full
27
+ DONE 2 365m_mixed_s42_L16384_topk_full 0 True
28
+ DONE 0 365m_mixed_s42_L16384_random_tight 0 True
29
+ START 0 365m_mixed_s42_L30720_random_tight
30
+ START 2 1p2b_dense_s42_L8192_topk_full
31
+ DONE 4 365m_mixed_s42_L24576_random_tight 0 True
32
+ START 4 1p2b_dense_s42_L8192_random_tight
33
+ DONE 1 365m_mixed_s42_L24576_topk_full 0 True
34
+ START 1 1p2b_dense_s42_L16384_topk_full
35
+ DONE 3 365m_mixed_s42_L30720_topk_full 0 True
36
+ START 3 1p2b_dense_s42_L16384_random_tight
37
+ DONE 0 365m_mixed_s42_L30720_random_tight 0 True
38
+ START 0 1p2b_mixed_s42_L8192_topk_full
39
+ DONE 4 1p2b_dense_s42_L8192_random_tight 0 True
40
+ START 4 1p2b_mixed_s42_L8192_random_tight
41
+ DONE 3 1p2b_dense_s42_L16384_random_tight 0 True
42
+ START 3 1p2b_mixed_s42_L16384_topk_full
43
+ DONE 2 1p2b_dense_s42_L8192_topk_full 0 True
44
+ START 2 1p2b_mixed_s42_L16384_random_tight
45
+ DONE 0 1p2b_mixed_s42_L8192_topk_full 0 True
46
+ DONE 1 1p2b_dense_s42_L16384_topk_full 0 True
47
+ DONE 4 1p2b_mixed_s42_L8192_random_tight 0 True
48
+ DONE 2 1p2b_mixed_s42_L16384_random_tight 0 True
49
+ DONE 3 1p2b_mixed_s42_L16384_topk_full 0 True
50
+ COMPLETE 24 / 24
epfl_final/train_logs/temp_protocol_80_20260906.log ADDED
@@ -0,0 +1,120 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ JOBS 60 DONE 0
2
+ START 0 1p2b_dense_s42_L24576_dense
3
+ START 1 1p2b_dense_s42_L24576_topk_full
4
+ START 2 1p2b_dense_s42_L24576_topk_tight
5
+ START 3 1p2b_dense_s42_L24576_random_full
6
+ DONE 0 1p2b_dense_s42_L24576_dense 0 True
7
+ START 0 1p2b_dense_s42_L24576_random_tight
8
+ DONE 2 1p2b_dense_s42_L24576_topk_tight 0 True
9
+ START 2 1p2b_dense_s42_L30720_dense
10
+ DONE 1 1p2b_dense_s42_L24576_topk_full 0 True
11
+ START 1 1p2b_dense_s42_L30720_topk_full
12
+ DONE 3 1p2b_dense_s42_L24576_random_full 0 True
13
+ START 3 1p2b_dense_s42_L30720_topk_tight
14
+ DONE 0 1p2b_dense_s42_L24576_random_tight 0 True
15
+ START 0 1p2b_dense_s42_L30720_random_full
16
+ DONE 2 1p2b_dense_s42_L30720_dense 0 True
17
+ START 2 1p2b_dense_s42_L30720_random_tight
18
+ DONE 3 1p2b_dense_s42_L30720_topk_tight 0 True
19
+ START 3 1p2b_mixed_s42_L24576_dense
20
+ DONE 1 1p2b_dense_s42_L30720_topk_full 0 True
21
+ START 1 1p2b_mixed_s42_L24576_topk_full
22
+ DONE 2 1p2b_dense_s42_L30720_random_tight 0 True
23
+ START 2 1p2b_mixed_s42_L24576_topk_tight
24
+ DONE 3 1p2b_mixed_s42_L24576_dense 0 True
25
+ START 3 1p2b_mixed_s42_L24576_random_full
26
+ DONE 2 1p2b_mixed_s42_L24576_topk_tight 0 True
27
+ START 2 1p2b_mixed_s42_L24576_random_tight
28
+ DONE 0 1p2b_dense_s42_L30720_random_full 0 True
29
+ START 0 1p2b_mixed_s42_L30720_dense
30
+ DONE 1 1p2b_mixed_s42_L24576_topk_full 0 True
31
+ START 1 1p2b_mixed_s42_L30720_topk_full
32
+ DONE 2 1p2b_mixed_s42_L24576_random_tight 0 True
33
+ START 2 1p2b_mixed_s42_L30720_topk_tight
34
+ DONE 3 1p2b_mixed_s42_L24576_random_full 0 True
35
+ START 3 1p2b_mixed_s42_L30720_random_full
36
+ DONE 0 1p2b_mixed_s42_L30720_dense 0 True
37
+ START 0 1p2b_mixed_s42_L30720_random_tight
38
+ DONE 1 1p2b_mixed_s42_L30720_topk_full 0 True
39
+ START 1 3b_dense_L8192_dense
40
+ DONE 2 1p2b_mixed_s42_L30720_topk_tight 0 True
41
+ START 2 3b_dense_L8192_topk_full
42
+ DONE 0 1p2b_mixed_s42_L30720_random_tight 0 True
43
+ START 0 3b_dense_L8192_topk_tight
44
+ DONE 3 1p2b_mixed_s42_L30720_random_full 0 True
45
+ START 3 3b_dense_L8192_random_full
46
+ DONE 1 3b_dense_L8192_dense 0 True
47
+ START 1 3b_dense_L8192_random_tight
48
+ DONE 2 3b_dense_L8192_topk_full 0 True
49
+ START 2 3b_dense_L16384_dense
50
+ DONE 0 3b_dense_L8192_topk_tight 0 True
51
+ START 0 3b_dense_L16384_topk_full
52
+ DONE 3 3b_dense_L8192_random_full 0 True
53
+ START 3 3b_dense_L16384_topk_tight
54
+ DONE 1 3b_dense_L8192_random_tight 0 True
55
+ START 1 3b_dense_L16384_random_full
56
+ DONE 2 3b_dense_L16384_dense 0 True
57
+ START 2 3b_dense_L16384_random_tight
58
+ DONE 0 3b_dense_L16384_topk_full 0 True
59
+ DONE 3 3b_dense_L16384_topk_tight 0 True
60
+ START 0 3b_dense_L24576_dense
61
+ START 3 3b_dense_L24576_topk_full
62
+ DONE 1 3b_dense_L16384_random_full 0 True
63
+ START 1 3b_dense_L24576_topk_tight
64
+ DONE 2 3b_dense_L16384_random_tight 0 True
65
+ START 2 3b_dense_L24576_random_full
66
+ DONE 0 3b_dense_L24576_dense 0 True
67
+ START 0 3b_dense_L24576_random_tight
68
+ DONE 3 3b_dense_L24576_topk_full 0 True
69
+ DONE 1 3b_dense_L24576_topk_tight 0 True
70
+ START 1 3b_dense_L30720_dense
71
+ START 3 3b_dense_L30720_topk_full
72
+ DONE 0 3b_dense_L24576_random_tight 0 True
73
+ START 0 3b_dense_L30720_topk_tight
74
+ DONE 2 3b_dense_L24576_random_full 0 True
75
+ START 2 3b_dense_L30720_random_full
76
+ DONE 1 3b_dense_L30720_dense 0 True
77
+ START 1 3b_dense_L30720_random_tight
78
+ DONE 0 3b_dense_L30720_topk_tight 0 True
79
+ START 0 3b_mixed_L8192_dense
80
+ DONE 3 3b_dense_L30720_topk_full 0 True
81
+ START 3 3b_mixed_L8192_topk_full
82
+ DONE 1 3b_dense_L30720_random_tight 0 True
83
+ START 1 3b_mixed_L8192_topk_tight
84
+ DONE 0 3b_mixed_L8192_dense 0 True
85
+ START 0 3b_mixed_L8192_random_full
86
+ DONE 3 3b_mixed_L8192_topk_full 0 True
87
+ START 3 3b_mixed_L8192_random_tight
88
+ DONE 2 3b_dense_L30720_random_full 0 True
89
+ START 2 3b_mixed_L16384_dense
90
+ DONE 1 3b_mixed_L8192_topk_tight 0 True
91
+ START 1 3b_mixed_L16384_topk_full
92
+ DONE 3 3b_mixed_L8192_random_tight 0 True
93
+ START 3 3b_mixed_L16384_topk_tight
94
+ DONE 0 3b_mixed_L8192_random_full 0 True
95
+ START 0 3b_mixed_L16384_random_full
96
+ DONE 2 3b_mixed_L16384_dense 0 True
97
+ START 2 3b_mixed_L24576_dense
98
+ DONE 1 3b_mixed_L16384_topk_full 0 True
99
+ START 1 3b_mixed_L24576_topk_full
100
+ DONE 3 3b_mixed_L16384_topk_tight 0 True
101
+ START 3 3b_mixed_L24576_topk_tight
102
+ DONE 0 3b_mixed_L16384_random_full 0 True
103
+ START 0 3b_mixed_L24576_random_full
104
+ DONE 2 3b_mixed_L24576_dense 0 True
105
+ START 2 3b_mixed_L24576_random_tight
106
+ DONE 3 3b_mixed_L24576_topk_tight 0 True
107
+ START 3 3b_mixed_L30720_dense
108
+ DONE 1 3b_mixed_L24576_topk_full 0 True
109
+ START 1 3b_mixed_L30720_topk_full
110
+ DONE 2 3b_mixed_L24576_random_tight 0 True
111
+ START 2 3b_mixed_L30720_topk_tight
112
+ DONE 0 3b_mixed_L24576_random_full 0 True
113
+ START 0 3b_mixed_L30720_random_full
114
+ DONE 3 3b_mixed_L30720_dense 0 True
115
+ START 3 3b_mixed_L30720_random_tight
116
+ DONE 1 3b_mixed_L30720_topk_full 0 True
117
+ DONE 2 3b_mixed_L30720_topk_tight 0 True
118
+ DONE 3 3b_mixed_L30720_random_tight 0 True
119
+ DONE 0 3b_mixed_L30720_random_full 0 True
120
+ COMPLETE 60 / 60