Hugging Face's logo

rubentium
/
sparse-checkpoints

Model card Files
xet
 Community
rubentium commited on
Commit
a6d1a04
·
verified ·
1 Parent(s): 83d5197

EPFL ALiBi and training provenance 151-200 of 1184

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +6 -0
  2. epfl_final/wandb/run-20260430_154759-yujhhjfq/files/config.yaml +154 -0
  3. epfl_final/wandb/run-20260505_213713-4i6l0dd0/files/output.log +697 -0
  4. epfl_final/wandb/run-20260505_213713-4i6l0dd0/files/requirements.txt +97 -0
  5. epfl_final/wandb/run-20260505_213713-4i6l0dd0/files/wandb-metadata.json +36 -0
  6. epfl_final/wandb/run-20260505_213713-4i6l0dd0/logs/debug-internal.log +0 -0
  7. epfl_final/wandb/run-20260505_213713-4i6l0dd0/logs/debug.log +19 -0
  8. epfl_final/wandb/run-20260505_213713-4i6l0dd0/run-4i6l0dd0.wandb +3 -0
  9. epfl_final/wandb/run-20260506_173734-nmzybrc7/files/output.log +496 -0
  10. epfl_final/wandb/run-20260506_173734-nmzybrc7/files/requirements.txt +97 -0
  11. epfl_final/wandb/run-20260506_173734-nmzybrc7/files/wandb-metadata.json +36 -0
  12. epfl_final/wandb/run-20260506_173734-nmzybrc7/logs/debug-internal.log +0 -0
  13. epfl_final/wandb/run-20260506_173734-nmzybrc7/logs/debug.log +19 -0
  14. epfl_final/wandb/run-20260506_173734-nmzybrc7/run-nmzybrc7.wandb +3 -0
  15. epfl_final/wandb/run-20260506_200450-z06f2580/files/output.log +1045 -0
  16. epfl_final/wandb/run-20260506_200450-z06f2580/files/requirements.txt +97 -0
  17. epfl_final/wandb/run-20260506_200450-z06f2580/files/wandb-metadata.json +36 -0
  18. epfl_final/wandb/run-20260506_200450-z06f2580/logs/debug-internal.log +0 -0
  19. epfl_final/wandb/run-20260506_200450-z06f2580/logs/debug.log +19 -0
  20. epfl_final/wandb/run-20260506_200450-z06f2580/run-z06f2580.wandb +3 -0
  21. epfl_final/wandb/run-20260509_201725-v6634orv/files/config.yaml +134 -0
  22. epfl_final/wandb/run-20260509_201725-v6634orv/files/output.log +0 -0
  23. epfl_final/wandb/run-20260509_201725-v6634orv/files/requirements.txt +97 -0
  24. epfl_final/wandb/run-20260509_201725-v6634orv/files/wandb-metadata.json +36 -0
  25. epfl_final/wandb/run-20260509_201725-v6634orv/files/wandb-summary.json +1 -0
  26. epfl_final/wandb/run-20260509_201725-v6634orv/logs/debug-internal.log +0 -0
  27. epfl_final/wandb/run-20260509_201725-v6634orv/logs/debug.log +24 -0
  28. epfl_final/wandb/run-20260509_201725-v6634orv/run-v6634orv.wandb +3 -0
  29. epfl_final/wandb/run-20260511_155411-fii64tue/files/config.yaml +133 -0
  30. epfl_final/wandb/run-20260511_155411-fii64tue/files/output.log +467 -0
  31. epfl_final/wandb/run-20260511_155411-fii64tue/files/requirements.txt +97 -0
  32. epfl_final/wandb/run-20260511_155411-fii64tue/files/wandb-metadata.json +36 -0
  33. epfl_final/wandb/run-20260511_155411-fii64tue/files/wandb-summary.json +1 -0
  34. epfl_final/wandb/run-20260511_155411-fii64tue/logs/debug-internal.log +0 -0
  35. epfl_final/wandb/run-20260511_155411-fii64tue/logs/debug.log +21 -0
  36. epfl_final/wandb/run-20260511_155411-fii64tue/run-fii64tue.wandb +3 -0
  37. epfl_final/wandb/run-20260511_201933-jzi5hfrk/files/config.yaml +134 -0
  38. epfl_final/wandb/run-20260511_201933-jzi5hfrk/files/output.log +0 -0
  39. epfl_final/wandb/run-20260511_201933-jzi5hfrk/files/requirements.txt +97 -0
  40. epfl_final/wandb/run-20260511_201933-jzi5hfrk/files/wandb-metadata.json +36 -0
  41. epfl_final/wandb/run-20260511_201933-jzi5hfrk/files/wandb-summary.json +1 -0
  42. epfl_final/wandb/run-20260511_201933-jzi5hfrk/logs/debug-internal.log +0 -0
  43. epfl_final/wandb/run-20260511_201933-jzi5hfrk/logs/debug.log +24 -0
  44. epfl_final/wandb/run-20260511_201933-jzi5hfrk/run-jzi5hfrk.wandb +3 -0
  45. epfl_final/wandb/run-20260511_215604-377k1je4/files/output.log +147 -0
  46. epfl_final/wandb/run-20260511_215604-377k1je4/files/requirements.txt +97 -0
  47. epfl_final/wandb/run-20260511_215604-377k1je4/files/wandb-metadata.json +36 -0
  48. epfl_final/wandb/run-20260511_215604-377k1je4/files/wandb-summary.json +1 -0
  49. epfl_final/wandb/run-20260511_215604-377k1je4/logs/debug-internal.log +22 -0
  50. epfl_final/wandb/run-20260511_215604-377k1je4/logs/debug.log +21 -0
.gitattributes CHANGED
@@ -211,3 +211,9 @@ epfl_final/wandb/run-20260430_153400-3fhhow4v/run-3fhhow4v.wandb filter=lfs diff
211
  epfl_final/wandb/run-20260430_154427-zytgzzs7/run-zytgzzs7.wandb filter=lfs diff=lfs merge=lfs -text
212
  epfl_final/wandb/run-20260430_154531-8xz0mg7d/run-8xz0mg7d.wandb filter=lfs diff=lfs merge=lfs -text
213
  epfl_final/wandb/run-20260430_154759-yujhhjfq/run-yujhhjfq.wandb filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
211
  epfl_final/wandb/run-20260430_154427-zytgzzs7/run-zytgzzs7.wandb filter=lfs diff=lfs merge=lfs -text
212
  epfl_final/wandb/run-20260430_154531-8xz0mg7d/run-8xz0mg7d.wandb filter=lfs diff=lfs merge=lfs -text
213
  epfl_final/wandb/run-20260430_154759-yujhhjfq/run-yujhhjfq.wandb filter=lfs diff=lfs merge=lfs -text
214
+ epfl_final/wandb/run-20260505_213713-4i6l0dd0/run-4i6l0dd0.wandb filter=lfs diff=lfs merge=lfs -text
215
+ epfl_final/wandb/run-20260506_173734-nmzybrc7/run-nmzybrc7.wandb filter=lfs diff=lfs merge=lfs -text
216
+ epfl_final/wandb/run-20260506_200450-z06f2580/run-z06f2580.wandb filter=lfs diff=lfs merge=lfs -text
217
+ epfl_final/wandb/run-20260509_201725-v6634orv/run-v6634orv.wandb filter=lfs diff=lfs merge=lfs -text
218
+ epfl_final/wandb/run-20260511_155411-fii64tue/run-fii64tue.wandb filter=lfs diff=lfs merge=lfs -text
219
+ epfl_final/wandb/run-20260511_201933-jzi5hfrk/run-jzi5hfrk.wandb filter=lfs diff=lfs merge=lfs -text
epfl_final/wandb/run-20260430_154759-yujhhjfq/files/config.yaml ADDED
@@ -0,0 +1,154 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ _wandb:
2
+ value:
3
+ cli_version: 0.26.1
4
+ e:
5
+ wlkifov2nkv7u22tti0lvhg8pz6y8y20:
6
+ codePath: SparseAtt/train.py
7
+ codePathLocal: SparseAtt/train.py
8
+ cpu_count: 96
9
+ cpu_count_logical: 192
10
+ cudaVersion: "13.0"
11
+ disk:
12
+ /:
13
+ total: "9374931091456"
14
+ used: "6049983488000"
15
+ email: ruben.navasardyan@mail.utoronto.ca
16
+ executable: /usr/bin/python3
17
+ gpu: NVIDIA H100 80GB HBM3
18
+ gpu_count: 1
19
+ gpu_nvidia:
20
+ - architecture: Hopper
21
+ cudaCores: 16896
22
+ memoryTotal: "85520809984"
23
+ name: NVIDIA H100 80GB HBM3
24
+ uuid: GPU-410cba62-f098-3454-c960-ebc55bd081af
25
+ host: h100-workspace-7-0-1
26
+ memory:
27
+ total: "1623005847552"
28
+ os: Linux-6.8.0-59-generic-x86_64-with-glibc2.35
29
+ program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
30
+ python: CPython 3.10.12
31
+ root: /mloscratch/homes/navasard/sparse
32
+ startedAt: "2026-04-30T15:47:59.246695Z"
33
+ writerId: wlkifov2nkv7u22tti0lvhg8pz6y8y20
34
+ m: []
35
+ python_version: 3.10.12
36
+ t:
37
+ "1":
38
+ - 1
39
+ - 49
40
+ - 51
41
+ "2":
42
+ - 1
43
+ - 11
44
+ - 49
45
+ - 51
46
+ - 71
47
+ "3":
48
+ - 15
49
+ - 16
50
+ - 61
51
+ "4": 3.10.12
52
+ "5": 0.26.1
53
+ "12": 0.26.1
54
+ "13": linux-x86_64
55
+ attention_type:
56
+ value: sparse_stochastic
57
+ checkpoint_activations:
58
+ value: false
59
+ eval_batches:
60
+ value: 500
61
+ eval_steps:
62
+ value: 250
63
+ gradient_accumulation_steps:
64
+ value: 8
65
+ init_from_scratch:
66
+ value: true
67
+ learning_rate:
68
+ value: 0.0005
69
+ logging_steps:
70
+ value: 10
71
+ lr_scheduler:
72
+ value: cosine
73
+ max_grad_norm:
74
+ value: 1
75
+ max_seq_length:
76
+ value: 3840
77
+ max_steps:
78
+ value: 10005
79
+ mixed_precision:
80
+ value: bf16
81
+ model_dtype:
82
+ value: bfloat16
83
+ model_path:
84
+ value: /mloscratch/homes/navasard/qwen3_06b_base
85
+ num_epochs:
86
+ value: 1
87
+ num_workers:
88
+ value: 8
89
+ output_dir:
90
+ value: ./checkpoints/run_20260430_154756
91
+ per_device_batch_size:
92
+ value: 8
93
+ resume_from_checkpoint:
94
+ value: null
95
+ save_steps:
96
+ value: 2500
97
+ seed:
98
+ value: 42
99
+ sparse_block_size:
100
+ value: 128
101
+ sparse_layer_indices:
102
+ value:
103
+ - 4
104
+ - 5
105
+ - 6
106
+ - 7
107
+ - 8
108
+ - 9
109
+ - 10
110
+ - 11
111
+ - 12
112
+ - 13
113
+ - 14
114
+ - 15
115
+ - 16
116
+ - 17
117
+ - 18
118
+ - 19
119
+ - 20
120
+ - 21
121
+ - 22
122
+ - 23
123
+ - 24
124
+ - 25
125
+ - 26
126
+ - 27
127
+ sparse_mask_pool_size:
128
+ value: 1000
129
+ sparse_num_random_blocks:
130
+ value: 9
131
+ sparse_val_dense_compare:
132
+ value: true
133
+ strategy:
134
+ value: ddp
135
+ train_path:
136
+ value: /mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized
137
+ val_path:
138
+ value: /mloscratch/homes/navasard/data/slimpajama_packed_val/tokenized
139
+ verbose:
140
+ value: true
141
+ wandb_enabled:
142
+ value: true
143
+ wandb_name:
144
+ value: null
145
+ wandb_project:
146
+ value: stochastic_sparsity
147
+ wandb_tags:
148
+ value:
149
+ - qwen3
150
+ - 0.6b
151
+ warmup_steps:
152
+ value: 100
153
+ weight_decay:
154
+ value: 0.01
epfl_final/wandb/run-20260505_213713-4i6l0dd0/files/output.log ADDED
@@ -0,0 +1,697 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ ================================================================================
4
+ MODEL ARCHITECTURE
5
+ ================================================================================
6
+ Model config:
7
+ vocab_size: 151936
8
+ hidden_size: 1024
9
+ num_hidden_layers: 28
10
+ num_attention_heads: 16
11
+ num_key_value_heads: 8
12
+ head_dim: 128
13
+ intermediate_size: 3072
14
+ max_position_embeddings: 32768
15
+ rope_theta: 1000000
16
+ attention_type: dense
17
+
18
+ Parameter count:
19
+ Total parameters: 596,049,920
20
+ Trainable parameters: 596,049,920
21
+ Non-trainable parameters: 0
22
+ Model size (BF16): 1.19 GB
23
+
24
+ --------------------------------------------------------------------------------
25
+ LAYER-BY-LAYER SUMMARY
26
+ --------------------------------------------------------------------------------
27
+ ===========================================================================================================================================================
28
+ Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
29
+ ===========================================================================================================================================================
30
+ Qwen3ForCausalLM [3, 8192] -- -- True
31
+ ├─Qwen3Model: 1-1 -- [3, 8192, 1024] -- True
32
+ │ └─Embedding: 2-1 [3, 8192] [3, 8192, 1024] 155,582,464 True
33
+ │ └─ModuleList: 2-2 -- -- -- True
34
+ │ │ └─Qwen3DecoderLayer: 3-1 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
35
+ │ │ └─Qwen3DecoderLayer: 3-2 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
36
+ │ │ └─Qwen3DecoderLayer: 3-3 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
37
+ │ │ └─Qwen3DecoderLayer: 3-4 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
38
+ │ │ └─Qwen3DecoderLayer: 3-5 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
39
+ │ │ └─Qwen3DecoderLayer: 3-6 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
40
+ │ │ └─Qwen3DecoderLayer: 3-7 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
41
+ │ │ └─Qwen3DecoderLayer: 3-8 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
42
+ │ │ └─Qwen3DecoderLayer: 3-9 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
43
+ │ │ └─Qwen3DecoderLayer: 3-10 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
44
+ │ │ └─Qwen3DecoderLayer: 3-11 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
45
+ │ │ └─Qwen3DecoderLayer: 3-12 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
46
+ │ │ └─Qwen3DecoderLayer: 3-13 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
47
+ │ │ └─Qwen3DecoderLayer: 3-14 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
48
+ │ │ └─Qwen3DecoderLayer: 3-15 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
49
+ │ │ └─Qwen3DecoderLayer: 3-16 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
50
+ │ │ └─Qwen3DecoderLayer: 3-17 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
51
+ │ │ └─Qwen3DecoderLayer: 3-18 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
52
+ │ │ └─Qwen3DecoderLayer: 3-19 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
53
+ │ │ └─Qwen3DecoderLayer: 3-20 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
54
+ │ │ └─Qwen3DecoderLayer: 3-21 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
55
+ │ │ └─Qwen3DecoderLayer: 3-22 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
56
+ │ │ └��Qwen3DecoderLayer: 3-23 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
57
+ │ │ └─Qwen3DecoderLayer: 3-24 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
58
+ │ │ └─Qwen3DecoderLayer: 3-25 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
59
+ │ │ └─Qwen3DecoderLayer: 3-26 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
60
+ │ │ └─Qwen3DecoderLayer: 3-27 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
61
+ │ │ └─Qwen3DecoderLayer: 3-28 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
62
+ │ └─Qwen3RMSNorm: 2-3 [3, 8192, 1024] [3, 8192, 1024] 1,024 True
63
+ ├─Linear: 1-2 [3, 8192, 1024] [3, 8192, 151936] 155,582,464 True
64
+ ===========================================================================================================================================================
65
+ Total params: 751,632,384
66
+ Trainable params: 751,632,384
67
+ Non-trainable params: 0
68
+ Total mult-adds (G): 2.25
69
+ ===========================================================================================================================================================
70
+ Input size (MB): 0.20
71
+ Forward/backward pass size (MB): 63052.97
72
+ Params size (MB): 1503.26
73
+ Estimated Total Size (MB): 64556.43
74
+ ===========================================================================================================================================================
75
+ ================================================================================
76
+
77
+ Loading dataset from /mloscratch/homes/navasard/data/the_pile_train/tokenized...
78
+ Using pre-tokenized dataset (no runtime tokenization)
79
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/the_pile_train/tokenized...
80
+ Loaded 5315146 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1328786)
81
+ Dataset split: 1063028 train, 265758 validation
82
+ Training examples (packed, train split): 1063028
83
+ Validation examples (packed, val split): 265758
84
+ Per-device batch size: 3
85
+ World size (GPUs): 1
86
+ Gradient accumulation steps: 8
87
+ Effective batch size (seqs/step): 24
88
+ Sequence length: 8192
89
+ Tokens per optimizer step: 196,608
90
+ Micro-batches per epoch (per rank): 354342
91
+ Optimizer steps per epoch: 44292
92
+ Total optimizer steps: 20005 (capped by max_steps; 0.45 epochs)
93
+ Starting training...
94
+ Step 10/20005 | Loss: 12.1031 | LR: 2.00e-05 | 11142.3ms/step | 2.2 samples/s
95
+ Step 20/20005 | Loss: 11.5938 | LR: 4.00e-05 | 5762.4ms/step | 4.2 samples/s
96
+ Step 30/20005 | Loss: 10.7006 | LR: 6.00e-05 | 5759.6ms/step | 4.2 samples/s
97
+ Step 40/20005 | Loss: 10.2108 | LR: 8.00e-05 | 5755.6ms/step | 4.2 samples/s
98
+ Step 50/20005 | Loss: 9.3895 | LR: 1.00e-04 | 5833.5ms/step | 4.1 samples/s
99
+ Step 60/20005 | Loss: 8.6356 | LR: 1.20e-04 | 5746.8ms/step | 4.2 samples/s
100
+ Step 70/20005 | Loss: 8.0401 | LR: 1.40e-04 | 5755.9ms/step | 4.2 samples/s
101
+ Step 80/20005 | Loss: 7.6302 | LR: 1.60e-04 | 5760.2ms/step | 4.2 samples/s
102
+ Step 90/20005 | Loss: 7.6538 | LR: 1.80e-04 | 5762.4ms/step | 4.2 samples/s
103
+ Step 100/20005 | Loss: 7.3930 | LR: 2.00e-04 | 5773.9ms/step | 4.2 samples/s
104
+ Step 110/20005 | Loss: 7.2834 | LR: 2.20e-04 | 5779.4ms/step | 4.2 samples/s
105
+ Step 120/20005 | Loss: 7.1788 | LR: 2.40e-04 | 5781.2ms/step | 4.2 samples/s
106
+ Step 130/20005 | Loss: 6.9958 | LR: 2.60e-04 | 5784.0ms/step | 4.1 samples/s
107
+ Step 140/20005 | Loss: 6.8817 | LR: 2.80e-04 | 5791.5ms/step | 4.1 samples/s
108
+ Step 150/20005 | Loss: 6.7182 | LR: 3.00e-04 | 5795.4ms/step | 4.1 samples/s
109
+ Step 160/20005 | Loss: 6.6074 | LR: 3.20e-04 | 5795.4ms/step | 4.1 samples/s
110
+ Step 170/20005 | Loss: 6.5873 | LR: 3.40e-04 | 5794.9ms/step | 4.1 samples/s
111
+ Step 180/20005 | Loss: 6.4443 | LR: 3.60e-04 | 5795.9ms/step | 4.1 samples/s
112
+ Step 190/20005 | Loss: 6.2919 | LR: 3.80e-04 | 5796.7ms/step | 4.1 samples/s
113
+ Step 200/20005 | Loss: 6.3783 | LR: 4.00e-04 | 5801.0ms/step | 4.1 samples/s
114
+ Step 210/20005 | Loss: 6.1791 | LR: 4.20e-04 | 5798.4ms/step | 4.1 samples/s
115
+ Step 220/20005 | Loss: 6.1359 | LR: 4.40e-04 | 5802.1ms/step | 4.1 samples/s
116
+ Step 230/20005 | Loss: 6.0648 | LR: 4.60e-04 | 5800.1ms/step | 4.1 samples/s
117
+ Step 240/20005 | Loss: 6.0271 | LR: 4.80e-04 | 5798.7ms/step | 4.1 samples/s
118
+ Step 250/20005 | Loss: 5.9168 | LR: 5.00e-04 | 5804.5ms/step | 4.1 samples/s
119
+ Validation at step 250: Loss = 5.9538, Perplexity = 385.21
120
+ Step 260/20005 | Loss: 5.8859 | LR: 5.00e-04 | 42798.9ms/step | 0.6 samples/s
121
+ Step 270/20005 | Loss: 5.8554 | LR: 5.00e-04 | 5801.8ms/step | 4.1 samples/s
122
+ Step 280/20005 | Loss: 5.8664 | LR: 5.00e-04 | 5802.3ms/step | 4.1 samples/s
123
+ Step 290/20005 | Loss: 5.7446 | LR: 5.00e-04 | 5812.5ms/step | 4.1 samples/s
124
+ Step 300/20005 | Loss: 5.7423 | LR: 5.00e-04 | 5804.7ms/step | 4.1 samples/s
125
+ Step 310/20005 | Loss: 5.6704 | LR: 5.00e-04 | 5804.3ms/step | 4.1 samples/s
126
+ Step 320/20005 | Loss: 5.5512 | LR: 5.00e-04 | 5809.0ms/step | 4.1 samples/s
127
+ Step 330/20005 | Loss: 5.5059 | LR: 5.00e-04 | 5804.2ms/step | 4.1 samples/s
128
+ Step 340/20005 | Loss: 5.3304 | LR: 5.00e-04 | 5804.9ms/step | 4.1 samples/s
129
+ Step 350/20005 | Loss: 5.3472 | LR: 5.00e-04 | 5811.4ms/step | 4.1 samples/s
130
+ Step 360/20005 | Loss: 5.4208 | LR: 5.00e-04 | 5806.3ms/step | 4.1 samples/s
131
+ Step 370/20005 | Loss: 5.1753 | LR: 5.00e-04 | 5805.3ms/step | 4.1 samples/s
132
+ Step 380/20005 | Loss: 5.2463 | LR: 5.00e-04 | 5810.9ms/step | 4.1 samples/s
133
+ Step 390/20005 | Loss: 5.2254 | LR: 5.00e-04 | 5806.4ms/step | 4.1 samples/s
134
+ Step 400/20005 | Loss: 5.1079 | LR: 5.00e-04 | 5808.4ms/step | 4.1 samples/s
135
+ Step 410/20005 | Loss: 5.1257 | LR: 5.00e-04 | 5806.7ms/step | 4.1 samples/s
136
+ Step 420/20005 | Loss: 4.9416 | LR: 5.00e-04 | 5807.1ms/step | 4.1 samples/s
137
+ Step 430/20005 | Loss: 5.0110 | LR: 5.00e-04 | 5807.5ms/step | 4.1 samples/s
138
+ Step 440/20005 | Loss: 4.8328 | LR: 5.00e-04 | 5805.8ms/step | 4.1 samples/s
139
+ Step 450/20005 | Loss: 4.8581 | LR: 5.00e-04 | 5808.4ms/step | 4.1 samples/s
140
+ Step 460/20005 | Loss: 4.8352 | LR: 5.00e-04 | 5810.2ms/step | 4.1 samples/s
141
+ Step 470/20005 | Loss: 4.7408 | LR: 5.00e-04 | 5812.1ms/step | 4.1 samples/s
142
+ Step 480/20005 | Loss: 4.6448 | LR: 5.00e-04 | 5895.1ms/step | 4.1 samples/s
143
+ Step 490/20005 | Loss: 4.6115 | LR: 5.00e-04 | 5810.4ms/step | 4.1 samples/s
144
+ Step 500/20005 | Loss: 4.4683 | LR: 5.00e-04 | 5814.6ms/step | 4.1 samples/s
145
+ Validation at step 500: Loss = 4.5455, Perplexity = 94.20
146
+ Step 510/20005 | Loss: 4.6236 | LR: 5.00e-04 | 41002.8ms/step | 0.6 samples/s
147
+ Step 520/20005 | Loss: 4.5253 | LR: 5.00e-04 | 5809.2ms/step | 4.1 samples/s
148
+ Step 530/20005 | Loss: 4.4373 | LR: 5.00e-04 | 5809.7ms/step | 4.1 samples/s
149
+ Step 540/20005 | Loss: 4.4541 | LR: 5.00e-04 | 5809.8ms/step | 4.1 samples/s
150
+ Step 550/20005 | Loss: 4.3815 | LR: 5.00e-04 | 5810.9ms/step | 4.1 samples/s
151
+ Step 560/20005 | Loss: 4.3735 | LR: 5.00e-04 | 5808.3ms/step | 4.1 samples/s
152
+ Step 570/20005 | Loss: 4.2842 | LR: 5.00e-04 | 5809.5ms/step | 4.1 samples/s
153
+ Step 580/20005 | Loss: 4.2415 | LR: 5.00e-04 | 5807.3ms/step | 4.1 samples/s
154
+ Step 590/20005 | Loss: 4.2345 | LR: 5.00e-04 | 5806.0ms/step | 4.1 samples/s
155
+ Step 600/20005 | Loss: 4.1789 | LR: 5.00e-04 | 5811.5ms/step | 4.1 samples/s
156
+ Step 610/20005 | Loss: 4.1109 | LR: 5.00e-04 | 5811.9ms/step | 4.1 samples/s
157
+ Step 620/20005 | Loss: 4.0885 | LR: 5.00e-04 | 5807.8ms/step | 4.1 samples/s
158
+ Step 630/20005 | Loss: 4.0650 | LR: 5.00e-04 | 5809.2ms/step | 4.1 samples/s
159
+ Step 640/20005 | Loss: 4.1462 | LR: 5.00e-04 | 5811.1ms/step | 4.1 samples/s
160
+ Step 650/20005 | Loss: 4.1200 | LR: 4.99e-04 | 5813.0ms/step | 4.1 samples/s
161
+ Step 660/20005 | Loss: 4.0761 | LR: 4.99e-04 | 5808.2ms/step | 4.1 samples/s
162
+ Step 670/20005 | Loss: 3.9784 | LR: 4.99e-04 | 5811.1ms/step | 4.1 samples/s
163
+ Step 680/20005 | Loss: 3.9679 | LR: 4.99e-04 | 5809.9ms/step | 4.1 samples/s
164
+ Step 690/20005 | Loss: 3.8426 | LR: 4.99e-04 | 5808.7ms/step | 4.1 samples/s
165
+ Step 700/20005 | Loss: 3.9139 | LR: 4.99e-04 | 5810.7ms/step | 4.1 samples/s
166
+ Step 710/20005 | Loss: 3.9618 | LR: 4.99e-04 | 5810.8ms/step | 4.1 samples/s
167
+ Step 720/20005 | Loss: 3.9317 | LR: 4.99e-04 | 5809.6ms/step | 4.1 samples/s
168
+ Step 730/20005 | Loss: 3.8755 | LR: 4.99e-04 | 5806.4ms/step | 4.1 samples/s
169
+ Step 740/20005 | Loss: 3.8398 | LR: 4.99e-04 | 5811.6ms/step | 4.1 samples/s
170
+ Step 750/20005 | Loss: 3.8248 | LR: 4.99e-04 | 5812.4ms/step | 4.1 samples/s
171
+ Validation at step 750: Loss = 3.8531, Perplexity = 47.14
172
+ Step 760/20005 | Loss: 3.8301 | LR: 4.99e-04 | 41011.9ms/step | 0.6 samples/s
173
+ Step 770/20005 | Loss: 3.8163 | LR: 4.99e-04 | 5811.6ms/step | 4.1 samples/s
174
+ Step 780/20005 | Loss: 3.8503 | LR: 4.99e-04 | 5814.1ms/step | 4.1 samples/s
175
+ Step 790/20005 | Loss: 3.7563 | LR: 4.99e-04 | 5811.4ms/step | 4.1 samples/s
176
+ Step 800/20005 | Loss: 3.7060 | LR: 4.99e-04 | 5808.4ms/step | 4.1 samples/s
177
+ Step 810/20005 | Loss: 3.7502 | LR: 4.99e-04 | 5809.3ms/step | 4.1 samples/s
178
+ Step 820/20005 | Loss: 3.7263 | LR: 4.99e-04 | 5811.2ms/step | 4.1 samples/s
179
+ Step 830/20005 | Loss: 3.7480 | LR: 4.99e-04 | 5811.2ms/step | 4.1 samples/s
180
+ Step 840/20005 | Loss: 3.7300 | LR: 4.99e-04 | 5812.0ms/step | 4.1 samples/s
181
+ Step 850/20005 | Loss: 3.7008 | LR: 4.99e-04 | 5810.6ms/step | 4.1 samples/s
182
+ Step 860/20005 | Loss: 3.7313 | LR: 4.99e-04 | 5809.4ms/step | 4.1 samples/s
183
+ Step 870/20005 | Loss: 3.6913 | LR: 4.99e-04 | 5812.1ms/step | 4.1 samples/s
184
+ Step 880/20005 | Loss: 3.6455 | LR: 4.99e-04 | 5812.5ms/step | 4.1 samples/s
185
+ Step 890/20005 | Loss: 3.5717 | LR: 4.99e-04 | 5810.7ms/step | 4.1 samples/s
186
+ Step 900/20005 | Loss: 3.6340 | LR: 4.99e-04 | 5809.6ms/step | 4.1 samples/s
187
+ Step 910/20005 | Loss: 3.6133 | LR: 4.99e-04 | 5809.2ms/step | 4.1 samples/s
188
+ Step 920/20005 | Loss: 3.5735 | LR: 4.99e-04 | 5814.5ms/step | 4.1 samples/s
189
+ Step 930/20005 | Loss: 3.5416 | LR: 4.99e-04 | 5812.5ms/step | 4.1 samples/s
190
+ Step 940/20005 | Loss: 3.6553 | LR: 4.98e-04 | 5817.0ms/step | 4.1 samples/s
191
+ Step 950/20005 | Loss: 3.5010 | LR: 4.98e-04 | 5853.4ms/step | 4.1 samples/s
192
+ Step 960/20005 | Loss: 3.5733 | LR: 4.98e-04 | 5814.6ms/step | 4.1 samples/s
193
+ Step 970/20005 | Loss: 3.4923 | LR: 4.98e-04 | 5811.7ms/step | 4.1 samples/s
194
+ Step 980/20005 | Loss: 3.5255 | LR: 4.98e-04 | 5813.2ms/step | 4.1 samples/s
195
+ Step 990/20005 | Loss: 3.5597 | LR: 4.98e-04 | 5811.9ms/step | 4.1 samples/s
196
+ Step 1000/20005 | Loss: 3.4905 | LR: 4.98e-04 | 5815.3ms/step | 4.1 samples/s
197
+ Validation at step 1000: Loss = 3.5304, Perplexity = 34.14
198
+ Step 1010/20005 | Loss: 3.5151 | LR: 4.98e-04 | 41015.6ms/step | 0.6 samples/s
199
+ Step 1020/20005 | Loss: 3.4903 | LR: 4.98e-04 | 5812.5ms/step | 4.1 samples/s
200
+ Step 1030/20005 | Loss: 3.5029 | LR: 4.98e-04 | 5812.6ms/step | 4.1 samples/s
201
+ Step 1040/20005 | Loss: 3.5418 | LR: 4.98e-04 | 5811.2ms/step | 4.1 samples/s
202
+ Step 1050/20005 | Loss: 3.3820 | LR: 4.98e-04 | 5811.0ms/step | 4.1 samples/s
203
+ Step 1060/20005 | Loss: 3.4867 | LR: 4.98e-04 | 5817.4ms/step | 4.1 samples/s
204
+ Step 1070/20005 | Loss: 3.4437 | LR: 4.98e-04 | 5814.5ms/step | 4.1 samples/s
205
+ Step 1080/20005 | Loss: 3.5662 | LR: 4.98e-04 | 5814.2ms/step | 4.1 samples/s
206
+ Step 1090/20005 | Loss: 3.4792 | LR: 4.98e-04 | 5813.3ms/step | 4.1 samples/s
207
+ Step 1100/20005 | Loss: 3.3973 | LR: 4.98e-04 | 5816.3ms/step | 4.1 samples/s
208
+ Step 1110/20005 | Loss: 3.4931 | LR: 4.98e-04 | 5818.0ms/step | 4.1 samples/s
209
+ Step 1120/20005 | Loss: 3.5327 | LR: 4.98e-04 | 5815.7ms/step | 4.1 samples/s
210
+ Step 1130/20005 | Loss: 3.4647 | LR: 4.98e-04 | 5817.0ms/step | 4.1 samples/s
211
+ Step 1140/20005 | Loss: 3.4436 | LR: 4.98e-04 | 5814.5ms/step | 4.1 samples/s
212
+ Step 1150/20005 | Loss: 3.4381 | LR: 4.97e-04 | 5819.8ms/step | 4.1 samples/s
213
+ Step 1160/20005 | Loss: 3.4283 | LR: 4.97e-04 | 5816.5ms/step | 4.1 samples/s
214
+ Step 1170/20005 | Loss: 3.4346 | LR: 4.97e-04 | 5816.3ms/step | 4.1 samples/s
215
+ Step 1180/20005 | Loss: 3.4342 | LR: 4.97e-04 | 5812.3ms/step | 4.1 samples/s
216
+ Step 1190/20005 | Loss: 3.4301 | LR: 4.97e-04 | 5817.1ms/step | 4.1 samples/s
217
+ Step 1200/20005 | Loss: 3.3507 | LR: 4.97e-04 | 5818.0ms/step | 4.1 samples/s
218
+ Step 1210/20005 | Loss: 3.2779 | LR: 4.97e-04 | 5810.1ms/step | 4.1 samples/s
219
+ Step 1220/20005 | Loss: 3.4393 | LR: 4.97e-04 | 5815.8ms/step | 4.1 samples/s
220
+ Step 1230/20005 | Loss: 3.3387 | LR: 4.97e-04 | 5814.9ms/step | 4.1 samples/s
221
+ Step 1240/20005 | Loss: 3.4264 | LR: 4.97e-04 | 5815.1ms/step | 4.1 samples/s
222
+ Step 1250/20005 | Loss: 3.2624 | LR: 4.97e-04 | 5814.9ms/step | 4.1 samples/s
223
+ Validation at step 1250: Loss = 3.3519, Perplexity = 28.56
224
+ Step 1260/20005 | Loss: 3.3179 | LR: 4.97e-04 | 41016.2ms/step | 0.6 samples/s
225
+ Step 1270/20005 | Loss: 3.3501 | LR: 4.97e-04 | 5815.5ms/step | 4.1 samples/s
226
+ Step 1280/20005 | Loss: 3.2788 | LR: 4.97e-04 | 5815.0ms/step | 4.1 samples/s
227
+ Step 1290/20005 | Loss: 3.3351 | LR: 4.97e-04 | 5813.6ms/step | 4.1 samples/s
228
+ Step 1300/20005 | Loss: 3.2674 | LR: 4.97e-04 | 5811.5ms/step | 4.1 samples/s
229
+ Step 1310/20005 | Loss: 3.2681 | LR: 4.96e-04 | 5809.9ms/step | 4.1 samples/s
230
+ Step 1320/20005 | Loss: 3.2306 | LR: 4.96e-04 | 5813.6ms/step | 4.1 samples/s
231
+ Step 1330/20005 | Loss: 3.3539 | LR: 4.96e-04 | 5815.9ms/step | 4.1 samples/s
232
+ Step 1340/20005 | Loss: 3.3054 | LR: 4.96e-04 | 5813.6ms/step | 4.1 samples/s
233
+ Step 1350/20005 | Loss: 3.2983 | LR: 4.96e-04 | 5810.8ms/step | 4.1 samples/s
234
+ Step 1360/20005 | Loss: 3.2526 | LR: 4.96e-04 | 5810.6ms/step | 4.1 samples/s
235
+ Step 1370/20005 | Loss: 3.2730 | LR: 4.96e-04 | 5812.6ms/step | 4.1 samples/s
236
+ Step 1380/20005 | Loss: 3.2460 | LR: 4.96e-04 | 5811.8ms/step | 4.1 samples/s
237
+ Step 1390/20005 | Loss: 3.2202 | LR: 4.96e-04 | 5813.2ms/step | 4.1 samples/s
238
+ Step 1400/20005 | Loss: 3.2724 | LR: 4.96e-04 | 5810.7ms/step | 4.1 samples/s
239
+ Step 1410/20005 | Loss: 3.2776 | LR: 4.96e-04 | 5810.2ms/step | 4.1 samples/s
240
+ Step 1420/20005 | Loss: 3.2775 | LR: 4.96e-04 | 5850.5ms/step | 4.1 samples/s
241
+ Step 1430/20005 | Loss: 3.2651 | LR: 4.96e-04 | 5811.9ms/step | 4.1 samples/s
242
+ Step 1440/20005 | Loss: 3.2635 | LR: 4.96e-04 | 5810.5ms/step | 4.1 samples/s
243
+ Step 1450/20005 | Loss: 3.3174 | LR: 4.95e-04 | 5813.9ms/step | 4.1 samples/s
244
+ Step 1460/20005 | Loss: 3.2486 | LR: 4.95e-04 | 5811.5ms/step | 4.1 samples/s
245
+ Step 1470/20005 | Loss: 3.2349 | LR: 4.95e-04 | 5812.7ms/step | 4.1 samples/s
246
+ Step 1480/20005 | Loss: 3.2286 | LR: 4.95e-04 | 5812.2ms/step | 4.1 samples/s
247
+ Step 1490/20005 | Loss: 3.2052 | LR: 4.95e-04 | 5812.6ms/step | 4.1 samples/s
248
+ Step 1500/20005 | Loss: 3.2613 | LR: 4.95e-04 | 5809.3ms/step | 4.1 samples/s
249
+ Validation at step 1500: Loss = 3.2403, Perplexity = 25.54
250
+ Step 1510/20005 | Loss: 3.1793 | LR: 4.95e-04 | 41013.4ms/step | 0.6 samples/s
251
+ Step 1520/20005 | Loss: 3.2195 | LR: 4.95e-04 | 5809.3ms/step | 4.1 samples/s
252
+ Step 1530/20005 | Loss: 3.2168 | LR: 4.95e-04 | 5812.3ms/step | 4.1 samples/s
253
+ Step 1540/20005 | Loss: 3.1593 | LR: 4.95e-04 | 5812.5ms/step | 4.1 samples/s
254
+ Step 1550/20005 | Loss: 3.2935 | LR: 4.95e-04 | 5811.9ms/step | 4.1 samples/s
255
+ Step 1560/20005 | Loss: 3.2095 | LR: 4.95e-04 | 5810.0ms/step | 4.1 samples/s
256
+ Step 1570/20005 | Loss: 3.2053 | LR: 4.95e-04 | 5810.8ms/step | 4.1 samples/s
257
+ Step 1580/20005 | Loss: 3.2232 | LR: 4.94e-04 | 5809.2ms/step | 4.1 samples/s
258
+ Step 1590/20005 | Loss: 3.1531 | LR: 4.94e-04 | 5807.2ms/step | 4.1 samples/s
259
+ Step 1600/20005 | Loss: 3.2274 | LR: 4.94e-04 | 5807.0ms/step | 4.1 samples/s
260
+ Step 1610/20005 | Loss: 3.1333 | LR: 4.94e-04 | 5811.1ms/step | 4.1 samples/s
261
+ Step 1620/20005 | Loss: 3.1941 | LR: 4.94e-04 | 5814.2ms/step | 4.1 samples/s
262
+ Step 1630/20005 | Loss: 3.1469 | LR: 4.94e-04 | 5810.6ms/step | 4.1 samples/s
263
+ Step 1640/20005 | Loss: 3.2688 | LR: 4.94e-04 | 5811.7ms/step | 4.1 samples/s
264
+ Step 1650/20005 | Loss: 3.2282 | LR: 4.94e-04 | 5812.1ms/step | 4.1 samples/s
265
+ Step 1660/20005 | Loss: 3.1980 | LR: 4.94e-04 | 5809.6ms/step | 4.1 samples/s
266
+ Step 1670/20005 | Loss: 3.0894 | LR: 4.94e-04 | 5812.7ms/step | 4.1 samples/s
267
+ Step 1680/20005 | Loss: 3.2300 | LR: 4.94e-04 | 5809.7ms/step | 4.1 samples/s
268
+ Step 1690/20005 | Loss: 3.0874 | LR: 4.93e-04 | 5809.4ms/step | 4.1 samples/s
269
+ Step 1700/20005 | Loss: 3.1658 | LR: 4.93e-04 | 5812.6ms/step | 4.1 samples/s
270
+ Step 1710/20005 | Loss: 3.0258 | LR: 4.93e-04 | 5813.4ms/step | 4.1 samples/s
271
+ Step 1720/20005 | Loss: 3.1192 | LR: 4.93e-04 | 5808.8ms/step | 4.1 samples/s
272
+ Step 1730/20005 | Loss: 3.1511 | LR: 4.93e-04 | 5813.5ms/step | 4.1 samples/s
273
+ Step 1740/20005 | Loss: 3.1087 | LR: 4.93e-04 | 5809.8ms/step | 4.1 samples/s
274
+ Step 1750/20005 | Loss: 3.1846 | LR: 4.93e-04 | 5811.9ms/step | 4.1 samples/s
275
+ Validation at step 1750: Loss = 3.1317, Perplexity = 22.91
276
+ Step 1760/20005 | Loss: 3.1319 | LR: 4.93e-04 | 41012.5ms/step | 0.6 samples/s
277
+ Step 1770/20005 | Loss: 3.1531 | LR: 4.93e-04 | 5811.7ms/step | 4.1 samples/s
278
+ Step 1780/20005 | Loss: 3.1205 | LR: 4.93e-04 | 5807.7ms/step | 4.1 samples/s
279
+ Step 1790/20005 | Loss: 3.0976 | LR: 4.93e-04 | 5808.5ms/step | 4.1 samples/s
280
+ Step 1800/20005 | Loss: 3.0564 | LR: 4.92e-04 | 5813.9ms/step | 4.1 samples/s
281
+ Step 1810/20005 | Loss: 3.2074 | LR: 4.92e-04 | 5807.3ms/step | 4.1 samples/s
282
+ Step 1820/20005 | Loss: 3.0795 | LR: 4.92e-04 | 5810.8ms/step | 4.1 samples/s
283
+ Step 1830/20005 | Loss: 3.1276 | LR: 4.92e-04 | 5808.1ms/step | 4.1 samples/s
284
+ Step 1840/20005 | Loss: 3.0406 | LR: 4.92e-04 | 5808.5ms/step | 4.1 samples/s
285
+ Step 1850/20005 | Loss: 3.0512 | LR: 4.92e-04 | 5809.4ms/step | 4.1 samples/s
286
+ Step 1860/20005 | Loss: 3.0493 | LR: 4.92e-04 | 5811.5ms/step | 4.1 samples/s
287
+ Step 1870/20005 | Loss: 3.0489 | LR: 4.92e-04 | 5811.9ms/step | 4.1 samples/s
288
+ Step 1880/20005 | Loss: 3.0611 | LR: 4.92e-04 | 5850.7ms/step | 4.1 samples/s
289
+ Step 1890/20005 | Loss: 3.0266 | LR: 4.92e-04 | 5812.1ms/step | 4.1 samples/s
290
+ Step 1900/20005 | Loss: 3.1198 | LR: 4.91e-04 | 5810.0ms/step | 4.1 samples/s
291
+ Step 1910/20005 | Loss: 3.0534 | LR: 4.91e-04 | 5808.8ms/step | 4.1 samples/s
292
+ Step 1920/20005 | Loss: 3.0460 | LR: 4.91e-04 | 5813.3ms/step | 4.1 samples/s
293
+ Step 1930/20005 | Loss: 3.1177 | LR: 4.91e-04 | 5813.9ms/step | 4.1 samples/s
294
+ Step 1940/20005 | Loss: 2.9266 | LR: 4.91e-04 | 5815.1ms/step | 4.1 samples/s
295
+ Step 1950/20005 | Loss: 3.1093 | LR: 4.91e-04 | 5816.1ms/step | 4.1 samples/s
296
+ Step 1960/20005 | Loss: 3.0315 | LR: 4.91e-04 | 5816.2ms/step | 4.1 samples/s
297
+ Step 1970/20005 | Loss: 3.0259 | LR: 4.91e-04 | 5811.9ms/step | 4.1 samples/s
298
+ Step 1980/20005 | Loss: 3.0024 | LR: 4.91e-04 | 5815.3ms/step | 4.1 samples/s
299
+ Step 1990/20005 | Loss: 3.0007 | LR: 4.90e-04 | 5813.6ms/step | 4.1 samples/s
300
+ Step 2000/20005 | Loss: 3.0923 | LR: 4.90e-04 | 5806.6ms/step | 4.1 samples/s
301
+ Validation at step 2000: Loss = 3.0505, Perplexity = 21.13
302
+ Saved checkpoint to checkpoints/run_20260505_213703/checkpoint-2000
303
+ Step 2010/20005 | Loss: 3.1051 | LR: 4.90e-04 | 41536.6ms/step | 0.6 samples/s
304
+ Step 2020/20005 | Loss: 3.0532 | LR: 4.90e-04 | 5808.2ms/step | 4.1 samples/s
305
+ Step 2030/20005 | Loss: 3.0733 | LR: 4.90e-04 | 5809.3ms/step | 4.1 samples/s
306
+ Step 2040/20005 | Loss: 3.0438 | LR: 4.90e-04 | 5811.1ms/step | 4.1 samples/s
307
+ Step 2050/20005 | Loss: 2.9897 | LR: 4.90e-04 | 5811.6ms/step | 4.1 samples/s
308
+ Step 2060/20005 | Loss: 3.0117 | LR: 4.90e-04 | 5810.2ms/step | 4.1 samples/s
309
+ Step 2070/20005 | Loss: 3.0327 | LR: 4.90e-04 | 5806.7ms/step | 4.1 samples/s
310
+ Step 2080/20005 | Loss: 3.0403 | LR: 4.89e-04 | 5808.7ms/step | 4.1 samples/s
311
+ Step 2090/20005 | Loss: 3.0105 | LR: 4.89e-04 | 5809.7ms/step | 4.1 samples/s
312
+ Step 2100/20005 | Loss: 2.9851 | LR: 4.89e-04 | 5810.8ms/step | 4.1 samples/s
313
+ Step 2110/20005 | Loss: 3.0197 | LR: 4.89e-04 | 5811.2ms/step | 4.1 samples/s
314
+ Step 2120/20005 | Loss: 3.0110 | LR: 4.89e-04 | 5809.9ms/step | 4.1 samples/s
315
+ Step 2130/20005 | Loss: 3.1198 | LR: 4.89e-04 | 5810.6ms/step | 4.1 samples/s
316
+ Step 2140/20005 | Loss: 2.9169 | LR: 4.89e-04 | 5812.6ms/step | 4.1 samples/s
317
+ Step 2150/20005 | Loss: 3.0773 | LR: 4.89e-04 | 5807.6ms/step | 4.1 samples/s
318
+ Step 2160/20005 | Loss: 3.0327 | LR: 4.89e-04 | 5810.0ms/step | 4.1 samples/s
319
+ Step 2170/20005 | Loss: 2.9799 | LR: 4.88e-04 | 5812.6ms/step | 4.1 samples/s
320
+ Step 2180/20005 | Loss: 2.9976 | LR: 4.88e-04 | 5811.7ms/step | 4.1 samples/s
321
+ Step 2190/20005 | Loss: 2.9744 | LR: 4.88e-04 | 5811.1ms/step | 4.1 samples/s
322
+ Step 2200/20005 | Loss: 2.9789 | LR: 4.88e-04 | 5811.2ms/step | 4.1 samples/s
323
+ Step 2210/20005 | Loss: 2.9896 | LR: 4.88e-04 | 5811.0ms/step | 4.1 samples/s
324
+ Step 2220/20005 | Loss: 2.9993 | LR: 4.88e-04 | 5807.6ms/step | 4.1 samples/s
325
+ Step 2230/20005 | Loss: 2.9487 | LR: 4.88e-04 | 5808.8ms/step | 4.1 samples/s
326
+ Step 2240/20005 | Loss: 2.9469 | LR: 4.88e-04 | 5808.0ms/step | 4.1 samples/s
327
+ Step 2250/20005 | Loss: 3.0251 | LR: 4.87e-04 | 5810.9ms/step | 4.1 samples/s
328
+ Validation at step 2250: Loss = 2.9884, Perplexity = 19.85
329
+ Step 2260/20005 | Loss: 2.9684 | LR: 4.87e-04 | 41016.1ms/step | 0.6 samples/s
330
+ Step 2270/20005 | Loss: 2.9679 | LR: 4.87e-04 | 5812.1ms/step | 4.1 samples/s
331
+ Step 2280/20005 | Loss: 3.0049 | LR: 4.87e-04 | 5806.3ms/step | 4.1 samples/s
332
+ Step 2290/20005 | Loss: 2.9259 | LR: 4.87e-04 | 5809.3ms/step | 4.1 samples/s
333
+ Step 2300/20005 | Loss: 2.9914 | LR: 4.87e-04 | 5812.6ms/step | 4.1 samples/s
334
+ Step 2310/20005 | Loss: 2.9534 | LR: 4.87e-04 | 5811.4ms/step | 4.1 samples/s
335
+ Step 2320/20005 | Loss: 2.8371 | LR: 4.87e-04 | 5812.3ms/step | 4.1 samples/s
336
+ Step 2330/20005 | Loss: 2.9542 | LR: 4.86e-04 | 5812.0ms/step | 4.1 samples/s
337
+ Step 2340/20005 | Loss: 2.9293 | LR: 4.86e-04 | 5815.9ms/step | 4.1 samples/s
338
+ Step 2350/20005 | Loss: 2.9422 | LR: 4.86e-04 | 5851.9ms/step | 4.1 samples/s
339
+ Step 2360/20005 | Loss: 2.9272 | LR: 4.86e-04 | 5810.0ms/step | 4.1 samples/s
340
+ Step 2370/20005 | Loss: 2.9814 | LR: 4.86e-04 | 5809.9ms/step | 4.1 samples/s
341
+ Step 2380/20005 | Loss: 2.9808 | LR: 4.86e-04 | 5814.8ms/step | 4.1 samples/s
342
+ Step 2390/20005 | Loss: 2.9888 | LR: 4.86e-04 | 5810.9ms/step | 4.1 samples/s
343
+ Step 2400/20005 | Loss: 3.0021 | LR: 4.86e-04 | 5811.6ms/step | 4.1 samples/s
344
+ Step 2410/20005 | Loss: 2.8874 | LR: 4.85e-04 | 5810.4ms/step | 4.1 samples/s
345
+ Step 2420/20005 | Loss: 2.9374 | LR: 4.85e-04 | 5812.3ms/step | 4.1 samples/s
346
+ Step 2430/20005 | Loss: 2.9527 | LR: 4.85e-04 | 5811.4ms/step | 4.1 samples/s
347
+ Step 2440/20005 | Loss: 3.0686 | LR: 4.85e-04 | 5808.7ms/step | 4.1 samples/s
348
+ Step 2450/20005 | Loss: 2.9517 | LR: 4.85e-04 | 5808.1ms/step | 4.1 samples/s
349
+ Step 2460/20005 | Loss: 2.8594 | LR: 4.85e-04 | 5812.2ms/step | 4.1 samples/s
350
+ Step 2470/20005 | Loss: 2.9005 | LR: 4.85e-04 | 5812.7ms/step | 4.1 samples/s
351
+ Step 2480/20005 | Loss: 2.9289 | LR: 4.84e-04 | 5810.1ms/step | 4.1 samples/s
352
+ Step 2490/20005 | Loss: 2.9550 | LR: 4.84e-04 | 5810.8ms/step | 4.1 samples/s
353
+ Step 2500/20005 | Loss: 2.9537 | LR: 4.84e-04 | 5813.4ms/step | 4.1 samples/s
354
+ Validation at step 2500: Loss = 2.9390, Perplexity = 18.90
355
+ Step 2510/20005 | Loss: 2.9456 | LR: 4.84e-04 | 41005.2ms/step | 0.6 samples/s
356
+ Step 2520/20005 | Loss: 2.8767 | LR: 4.84e-04 | 5811.5ms/step | 4.1 samples/s
357
+ Step 2530/20005 | Loss: 2.9016 | LR: 4.84e-04 | 5810.8ms/step | 4.1 samples/s
358
+ Step 2540/20005 | Loss: 2.9625 | LR: 4.84e-04 | 5813.0ms/step | 4.1 samples/s
359
+ Step 2550/20005 | Loss: 2.9278 | LR: 4.83e-04 | 5812.2ms/step | 4.1 samples/s
360
+ Step 2560/20005 | Loss: 2.9166 | LR: 4.83e-04 | 5814.2ms/step | 4.1 samples/s
361
+ Step 2570/20005 | Loss: 2.9357 | LR: 4.83e-04 | 5810.3ms/step | 4.1 samples/s
362
+ Step 2580/20005 | Loss: 2.8754 | LR: 4.83e-04 | 5811.7ms/step | 4.1 samples/s
363
+ Step 2590/20005 | Loss: 2.9905 | LR: 4.83e-04 | 5811.0ms/step | 4.1 samples/s
364
+ Step 2600/20005 | Loss: 2.8788 | LR: 4.83e-04 | 5813.9ms/step | 4.1 samples/s
365
+ Step 2610/20005 | Loss: 2.8794 | LR: 4.83e-04 | 5813.5ms/step | 4.1 samples/s
366
+ Step 2620/20005 | Loss: 2.9479 | LR: 4.82e-04 | 5813.5ms/step | 4.1 samples/s
367
+ Step 2630/20005 | Loss: 2.8816 | LR: 4.82e-04 | 5812.1ms/step | 4.1 samples/s
368
+ Step 2640/20005 | Loss: 2.8476 | LR: 4.82e-04 | 5809.6ms/step | 4.1 samples/s
369
+ Step 2650/20005 | Loss: 2.9432 | LR: 4.82e-04 | 5812.1ms/step | 4.1 samples/s
370
+ Step 2660/20005 | Loss: 2.9021 | LR: 4.82e-04 | 5811.8ms/step | 4.1 samples/s
371
+ Step 2670/20005 | Loss: 2.8588 | LR: 4.82e-04 | 5810.0ms/step | 4.1 samples/s
372
+ Step 2680/20005 | Loss: 2.8981 | LR: 4.82e-04 | 5808.5ms/step | 4.1 samples/s
373
+ Step 2690/20005 | Loss: 2.8678 | LR: 4.81e-04 | 5809.9ms/step | 4.1 samples/s
374
+ Step 2700/20005 | Loss: 2.8914 | LR: 4.81e-04 | 5814.9ms/step | 4.1 samples/s
375
+ Step 2710/20005 | Loss: 2.8227 | LR: 4.81e-04 | 5807.5ms/step | 4.1 samples/s
376
+ Step 2720/20005 | Loss: 2.9196 | LR: 4.81e-04 | 5811.1ms/step | 4.1 samples/s
377
+ Step 2730/20005 | Loss: 2.9920 | LR: 4.81e-04 | 5812.7ms/step | 4.1 samples/s
378
+ Step 2740/20005 | Loss: 2.8563 | LR: 4.81e-04 | 5813.5ms/step | 4.1 samples/s
379
+ Step 2750/20005 | Loss: 2.8787 | LR: 4.81e-04 | 5815.5ms/step | 4.1 samples/s
380
+ Validation at step 2750: Loss = 2.8935, Perplexity = 18.06
381
+ Step 2760/20005 | Loss: 2.8689 | LR: 4.80e-04 | 41009.7ms/step | 0.6 samples/s
382
+ Step 2770/20005 | Loss: 2.9807 | LR: 4.80e-04 | 5806.9ms/step | 4.1 samples/s
383
+ Step 2780/20005 | Loss: 2.8822 | LR: 4.80e-04 | 5808.1ms/step | 4.1 samples/s
384
+ Step 2790/20005 | Loss: 2.9279 | LR: 4.80e-04 | 5809.2ms/step | 4.1 samples/s
385
+ Step 2800/20005 | Loss: 2.8770 | LR: 4.80e-04 | 5812.1ms/step | 4.1 samples/s
386
+ Step 2810/20005 | Loss: 2.9119 | LR: 4.80e-04 | 5810.5ms/step | 4.1 samples/s
387
+ Step 2820/20005 | Loss: 2.9154 | LR: 4.79e-04 | 5848.9ms/step | 4.1 samples/s
388
+ Step 2830/20005 | Loss: 2.8084 | LR: 4.79e-04 | 5809.7ms/step | 4.1 samples/s
389
+ Step 2840/20005 | Loss: 2.8542 | LR: 4.79e-04 | 5813.8ms/step | 4.1 samples/s
390
+ Step 2850/20005 | Loss: 2.8210 | LR: 4.79e-04 | 5813.2ms/step | 4.1 samples/s
391
+ Step 2860/20005 | Loss: 2.7724 | LR: 4.79e-04 | 5810.5ms/step | 4.1 samples/s
392
+ Step 2870/20005 | Loss: 2.9941 | LR: 4.79e-04 | 5811.4ms/step | 4.1 samples/s
393
+ Step 2880/20005 | Loss: 2.8269 | LR: 4.78e-04 | 5814.1ms/step | 4.1 samples/s
394
+ Step 2890/20005 | Loss: 2.7923 | LR: 4.78e-04 | 5810.9ms/step | 4.1 samples/s
395
+ Step 2900/20005 | Loss: 2.8665 | LR: 4.78e-04 | 5813.9ms/step | 4.1 samples/s
396
+ Step 2910/20005 | Loss: 2.8135 | LR: 4.78e-04 | 5814.6ms/step | 4.1 samples/s
397
+ Step 2920/20005 | Loss: 2.8151 | LR: 4.78e-04 | 5811.0ms/step | 4.1 samples/s
398
+ Step 2930/20005 | Loss: 2.8713 | LR: 4.78e-04 | 5810.7ms/step | 4.1 samples/s
399
+ Step 2940/20005 | Loss: 2.8756 | LR: 4.77e-04 | 5809.7ms/step | 4.1 samples/s
400
+ Step 2950/20005 | Loss: 2.7846 | LR: 4.77e-04 | 5807.4ms/step | 4.1 samples/s
401
+ Step 2960/20005 | Loss: 2.8644 | LR: 4.77e-04 | 5814.4ms/step | 4.1 samples/s
402
+ Step 2970/20005 | Loss: 2.8643 | LR: 4.77e-04 | 5815.4ms/step | 4.1 samples/s
403
+ Step 2980/20005 | Loss: 2.8627 | LR: 4.77e-04 | 5809.6ms/step | 4.1 samples/s
404
+ Step 2990/20005 | Loss: 2.8380 | LR: 4.77e-04 | 5810.2ms/step | 4.1 samples/s
405
+ Step 3000/20005 | Loss: 2.7958 | LR: 4.76e-04 | 5809.1ms/step | 4.1 samples/s
406
+ Validation at step 3000: Loss = 2.8582, Perplexity = 17.43
407
+ Step 3010/20005 | Loss: 2.8251 | LR: 4.76e-04 | 41013.0ms/step | 0.6 samples/s
408
+ Step 3020/20005 | Loss: 2.8032 | LR: 4.76e-04 | 5812.7ms/step | 4.1 samples/s
409
+ Step 3030/20005 | Loss: 2.8149 | LR: 4.76e-04 | 5813.1ms/step | 4.1 samples/s
410
+ Step 3040/20005 | Loss: 2.8608 | LR: 4.76e-04 | 5812.5ms/step | 4.1 samples/s
411
+ Step 3050/20005 | Loss: 2.9347 | LR: 4.76e-04 | 5813.7ms/step | 4.1 samples/s
412
+ Step 3060/20005 | Loss: 2.7499 | LR: 4.75e-04 | 5808.6ms/step | 4.1 samples/s
413
+ Step 3070/20005 | Loss: 2.8834 | LR: 4.75e-04 | 5810.4ms/step | 4.1 samples/s
414
+ Step 3080/20005 | Loss: 2.7995 | LR: 4.75e-04 | 5810.8ms/step | 4.1 samples/s
415
+ Step 3090/20005 | Loss: 2.8934 | LR: 4.75e-04 | 5808.0ms/step | 4.1 samples/s
416
+ Step 3100/20005 | Loss: 2.8546 | LR: 4.75e-04 | 5811.2ms/step | 4.1 samples/s
417
+ Step 3110/20005 | Loss: 2.8050 | LR: 4.75e-04 | 5811.5ms/step | 4.1 samples/s
418
+ Step 3120/20005 | Loss: 2.8322 | LR: 4.74e-04 | 5813.5ms/step | 4.1 samples/s
419
+ Step 3130/20005 | Loss: 2.8530 | LR: 4.74e-04 | 5815.4ms/step | 4.1 samples/s
420
+ Step 3140/20005 | Loss: 2.8598 | LR: 4.74e-04 | 5813.1ms/step | 4.1 samples/s
421
+ Step 3150/20005 | Loss: 2.8326 | LR: 4.74e-04 | 5811.0ms/step | 4.1 samples/s
422
+ Step 3160/20005 | Loss: 2.9055 | LR: 4.74e-04 | 5810.0ms/step | 4.1 samples/s
423
+ Step 3170/20005 | Loss: 2.8030 | LR: 4.74e-04 | 5810.1ms/step | 4.1 samples/s
424
+ Step 3180/20005 | Loss: 2.7387 | LR: 4.73e-04 | 5812.2ms/step | 4.1 samples/s
425
+ Step 3190/20005 | Loss: 2.8134 | LR: 4.73e-04 | 5813.8ms/step | 4.1 samples/s
426
+ Step 3200/20005 | Loss: 2.8793 | LR: 4.73e-04 | 5806.4ms/step | 4.1 samples/s
427
+ Step 3210/20005 | Loss: 2.7051 | LR: 4.73e-04 | 5810.1ms/step | 4.1 samples/s
428
+ Step 3220/20005 | Loss: 2.8230 | LR: 4.73e-04 | 5811.8ms/step | 4.1 samples/s
429
+ Step 3230/20005 | Loss: 2.8299 | LR: 4.72e-04 | 5809.6ms/step | 4.1 samples/s
430
+ Step 3240/20005 | Loss: 2.7705 | LR: 4.72e-04 | 5811.2ms/step | 4.1 samples/s
431
+ Step 3250/20005 | Loss: 2.7539 | LR: 4.72e-04 | 5811.3ms/step | 4.1 samples/s
432
+ Validation at step 3250: Loss = 2.8184, Perplexity = 16.75
433
+ Step 3260/20005 | Loss: 2.7569 | LR: 4.72e-04 | 41014.4ms/step | 0.6 samples/s
434
+ Step 3270/20005 | Loss: 2.7931 | LR: 4.72e-04 | 5812.8ms/step | 4.1 samples/s
435
+ Step 3280/20005 | Loss: 2.7479 | LR: 4.72e-04 | 5808.8ms/step | 4.1 samples/s
436
+ Step 3290/20005 | Loss: 2.8138 | LR: 4.71e-04 | 5852.5ms/step | 4.1 samples/s
437
+ Step 3300/20005 | Loss: 2.8090 | LR: 4.71e-04 | 5811.7ms/step | 4.1 samples/s
438
+ Step 3310/20005 | Loss: 2.8093 | LR: 4.71e-04 | 5812.1ms/step | 4.1 samples/s
439
+ Step 3320/20005 | Loss: 2.7947 | LR: 4.71e-04 | 5811.3ms/step | 4.1 samples/s
440
+ Step 3330/20005 | Loss: 2.8460 | LR: 4.71e-04 | 5811.9ms/step | 4.1 samples/s
441
+ Step 3340/20005 | Loss: 2.8323 | LR: 4.70e-04 | 5811.2ms/step | 4.1 samples/s
442
+ Step 3350/20005 | Loss: 2.7699 | LR: 4.70e-04 | 5811.0ms/step | 4.1 samples/s
443
+ Step 3360/20005 | Loss: 2.7592 | LR: 4.70e-04 | 5809.3ms/step | 4.1 samples/s
444
+ Step 3370/20005 | Loss: 2.8019 | LR: 4.70e-04 | 5809.2ms/step | 4.1 samples/s
445
+ Step 3380/20005 | Loss: 2.7589 | LR: 4.70e-04 | 5807.5ms/step | 4.1 samples/s
446
+ Step 3390/20005 | Loss: 2.7989 | LR: 4.69e-04 | 5812.2ms/step | 4.1 samples/s
447
+ Step 3400/20005 | Loss: 2.7220 | LR: 4.69e-04 | 5812.0ms/step | 4.1 samples/s
448
+ Step 3410/20005 | Loss: 2.7442 | LR: 4.69e-04 | 5811.4ms/step | 4.1 samples/s
449
+ Step 3420/20005 | Loss: 2.7948 | LR: 4.69e-04 | 5810.2ms/step | 4.1 samples/s
450
+ Step 3430/20005 | Loss: 2.8496 | LR: 4.69e-04 | 5806.6ms/step | 4.1 samples/s
451
+ Step 3440/20005 | Loss: 2.7826 | LR: 4.69e-04 | 5812.1ms/step | 4.1 samples/s
452
+ Step 3450/20005 | Loss: 2.8063 | LR: 4.68e-04 | 5816.1ms/step | 4.1 samples/s
453
+ Step 3460/20005 | Loss: 2.8033 | LR: 4.68e-04 | 5810.6ms/step | 4.1 samples/s
454
+ Step 3470/20005 | Loss: 2.8292 | LR: 4.68e-04 | 5814.1ms/step | 4.1 samples/s
455
+ Step 3480/20005 | Loss: 2.7888 | LR: 4.68e-04 | 5813.4ms/step | 4.1 samples/s
456
+ Step 3490/20005 | Loss: 2.7871 | LR: 4.68e-04 | 5811.9ms/step | 4.1 samples/s
457
+ Step 3500/20005 | Loss: 2.6815 | LR: 4.67e-04 | 5811.8ms/step | 4.1 samples/s
458
+ Validation at step 3500: Loss = 2.7876, Perplexity = 16.24
459
+ Step 3510/20005 | Loss: 2.8247 | LR: 4.67e-04 | 41016.7ms/step | 0.6 samples/s
460
+ Step 3520/20005 | Loss: 2.7604 | LR: 4.67e-04 | 5814.3ms/step | 4.1 samples/s
461
+ Step 3530/20005 | Loss: 2.7122 | LR: 4.67e-04 | 5814.1ms/step | 4.1 samples/s
462
+ Step 3540/20005 | Loss: 2.8006 | LR: 4.67e-04 | 5813.3ms/step | 4.1 samples/s
463
+ Step 3550/20005 | Loss: 2.7926 | LR: 4.66e-04 | 5816.1ms/step | 4.1 samples/s
464
+ Step 3560/20005 | Loss: 2.7180 | LR: 4.66e-04 | 5814.9ms/step | 4.1 samples/s
465
+ Step 3570/20005 | Loss: 2.7342 | LR: 4.66e-04 | 5812.5ms/step | 4.1 samples/s
466
+ Step 3580/20005 | Loss: 2.6993 | LR: 4.66e-04 | 5814.6ms/step | 4.1 samples/s
467
+ Step 3590/20005 | Loss: 2.7827 | LR: 4.66e-04 | 5815.2ms/step | 4.1 samples/s
468
+ Step 3600/20005 | Loss: 2.8069 | LR: 4.65e-04 | 5811.5ms/step | 4.1 samples/s
469
+ Step 3610/20005 | Loss: 2.8327 | LR: 4.65e-04 | 5811.1ms/step | 4.1 samples/s
470
+ Step 3620/20005 | Loss: 2.7596 | LR: 4.65e-04 | 5813.9ms/step | 4.1 samples/s
471
+ Step 3630/20005 | Loss: 2.8664 | LR: 4.65e-04 | 5812.6ms/step | 4.1 samples/s
472
+ Step 3640/20005 | Loss: 2.7676 | LR: 4.65e-04 | 5814.5ms/step | 4.1 samples/s
473
+ Step 3650/20005 | Loss: 2.8229 | LR: 4.64e-04 | 5816.2ms/step | 4.1 samples/s
474
+ Step 3660/20005 | Loss: 2.7831 | LR: 4.64e-04 | 5814.0ms/step | 4.1 samples/s
475
+ Step 3670/20005 | Loss: 2.7453 | LR: 4.64e-04 | 5817.0ms/step | 4.1 samples/s
476
+ Step 3680/20005 | Loss: 2.7262 | LR: 4.64e-04 | 5815.7ms/step | 4.1 samples/s
477
+ Step 3690/20005 | Loss: 2.7442 | LR: 4.64e-04 | 5816.1ms/step | 4.1 samples/s
478
+ Step 3700/20005 | Loss: 2.8463 | LR: 4.63e-04 | 5814.2ms/step | 4.1 samples/s
479
+ Step 3710/20005 | Loss: 2.7687 | LR: 4.63e-04 | 5812.7ms/step | 4.1 samples/s
480
+ Step 3720/20005 | Loss: 2.7136 | LR: 4.63e-04 | 5811.2ms/step | 4.1 samples/s
481
+ Step 3730/20005 | Loss: 2.7107 | LR: 4.63e-04 | 5811.4ms/step | 4.1 samples/s
482
+ Step 3740/20005 | Loss: 2.8167 | LR: 4.62e-04 | 5809.5ms/step | 4.1 samples/s
483
+ Step 3750/20005 | Loss: 2.7129 | LR: 4.62e-04 | 5811.4ms/step | 4.1 samples/s
484
+ Validation at step 3750: Loss = 2.7602, Perplexity = 15.80
485
+ Step 3760/20005 | Loss: 2.7660 | LR: 4.62e-04 | 41052.6ms/step | 0.6 samples/s
486
+ Step 3770/20005 | Loss: 2.7921 | LR: 4.62e-04 | 5812.6ms/step | 4.1 samples/s
487
+ Step 3780/20005 | Loss: 2.7641 | LR: 4.62e-04 | 5810.6ms/step | 4.1 samples/s
488
+ Step 3790/20005 | Loss: 2.8571 | LR: 4.61e-04 | 5809.0ms/step | 4.1 samples/s
489
+ Step 3800/20005 | Loss: 2.6938 | LR: 4.61e-04 | 5812.3ms/step | 4.1 samples/s
490
+ Step 3810/20005 | Loss: 2.8074 | LR: 4.61e-04 | 5811.3ms/step | 4.1 samples/s
491
+ Step 3820/20005 | Loss: 2.6964 | LR: 4.61e-04 | 5814.2ms/step | 4.1 samples/s
492
+ Step 3830/20005 | Loss: 2.7067 | LR: 4.61e-04 | 5814.1ms/step | 4.1 samples/s
493
+ Step 3840/20005 | Loss: 2.7791 | LR: 4.60e-04 | 5810.5ms/step | 4.1 samples/s
494
+ Step 3850/20005 | Loss: 2.7951 | LR: 4.60e-04 | 5811.3ms/step | 4.1 samples/s
495
+ Step 3860/20005 | Loss: 2.7405 | LR: 4.60e-04 | 5811.6ms/step | 4.1 samples/s
496
+ Step 3870/20005 | Loss: 2.6990 | LR: 4.60e-04 | 5812.4ms/step | 4.1 samples/s
497
+ Step 3880/20005 | Loss: 2.8132 | LR: 4.59e-04 | 5808.5ms/step | 4.1 samples/s
498
+ Step 3890/20005 | Loss: 2.6765 | LR: 4.59e-04 | 5810.6ms/step | 4.1 samples/s
499
+ Step 3900/20005 | Loss: 2.7411 | LR: 4.59e-04 | 5811.3ms/step | 4.1 samples/s
500
+ Step 3910/20005 | Loss: 2.6058 | LR: 4.59e-04 | 5813.8ms/step | 4.1 samples/s
501
+ Step 3920/20005 | Loss: 2.6630 | LR: 4.59e-04 | 5808.9ms/step | 4.1 samples/s
502
+ Step 3930/20005 | Loss: 2.7304 | LR: 4.58e-04 | 5813.4ms/step | 4.1 samples/s
503
+ Step 3940/20005 | Loss: 2.7411 | LR: 4.58e-04 | 5812.3ms/step | 4.1 samples/s
504
+ Step 3950/20005 | Loss: 2.7543 | LR: 4.58e-04 | 5813.6ms/step | 4.1 samples/s
505
+ Step 3960/20005 | Loss: 2.7778 | LR: 4.58e-04 | 5814.2ms/step | 4.1 samples/s
506
+ Step 3970/20005 | Loss: 2.6364 | LR: 4.58e-04 | 5809.1ms/step | 4.1 samples/s
507
+ Step 3980/20005 | Loss: 2.6053 | LR: 4.57e-04 | 5814.2ms/step | 4.1 samples/s
508
+ Step 3990/20005 | Loss: 2.7217 | LR: 4.57e-04 | 5812.1ms/step | 4.1 samples/s
509
+ Step 4000/20005 | Loss: 2.7271 | LR: 4.57e-04 | 5811.6ms/step | 4.1 samples/s
510
+ Validation at step 4000: Loss = 2.7345, Perplexity = 15.40
511
+ Saved checkpoint to checkpoints/run_20260505_213703/checkpoint-4000
512
+ Step 4010/20005 | Loss: 2.7555 | LR: 4.57e-04 | 41507.3ms/step | 0.6 samples/s
513
+ Step 4020/20005 | Loss: 2.6734 | LR: 4.56e-04 | 5809.1ms/step | 4.1 samples/s
514
+ Step 4030/20005 | Loss: 2.7163 | LR: 4.56e-04 | 5810.2ms/step | 4.1 samples/s
515
+ Step 4040/20005 | Loss: 2.7805 | LR: 4.56e-04 | 5811.9ms/step | 4.1 samples/s
516
+ Step 4050/20005 | Loss: 2.7664 | LR: 4.56e-04 | 5813.9ms/step | 4.1 samples/s
517
+ Step 4060/20005 | Loss: 2.7627 | LR: 4.55e-04 | 5813.0ms/step | 4.1 samples/s
518
+ Step 4070/20005 | Loss: 2.7325 | LR: 4.55e-04 | 5811.6ms/step | 4.1 samples/s
519
+ Step 4080/20005 | Loss: 2.7629 | LR: 4.55e-04 | 5810.2ms/step | 4.1 samples/s
520
+ Step 4090/20005 | Loss: 2.6950 | LR: 4.55e-04 | 5811.0ms/step | 4.1 samples/s
521
+ Step 4100/20005 | Loss: 2.6856 | LR: 4.55e-04 | 5810.7ms/step | 4.1 samples/s
522
+ Step 4110/20005 | Loss: 2.6639 | LR: 4.54e-04 | 5811.4ms/step | 4.1 samples/s
523
+ Step 4120/20005 | Loss: 2.6826 | LR: 4.54e-04 | 5810.0ms/step | 4.1 samples/s
524
+ Step 4130/20005 | Loss: 2.6829 | LR: 4.54e-04 | 5811.1ms/step | 4.1 samples/s
525
+ Step 4140/20005 | Loss: 2.8031 | LR: 4.54e-04 | 5813.0ms/step | 4.1 samples/s
526
+ Step 4150/20005 | Loss: 2.7295 | LR: 4.53e-04 | 5812.2ms/step | 4.1 samples/s
527
+ Step 4160/20005 | Loss: 2.6696 | LR: 4.53e-04 | 5810.6ms/step | 4.1 samples/s
528
+ Step 4170/20005 | Loss: 2.7318 | LR: 4.53e-04 | 5808.6ms/step | 4.1 samples/s
529
+ Step 4180/20005 | Loss: 2.6314 | LR: 4.53e-04 | 5812.7ms/step | 4.1 samples/s
530
+ Step 4190/20005 | Loss: 2.7562 | LR: 4.53e-04 | 5813.4ms/step | 4.1 samples/s
531
+ Step 4200/20005 | Loss: 2.6794 | LR: 4.52e-04 | 5813.0ms/step | 4.1 samples/s
532
+ Step 4210/20005 | Loss: 2.7494 | LR: 4.52e-04 | 5808.0ms/step | 4.1 samples/s
533
+ Step 4220/20005 | Loss: 2.7343 | LR: 4.52e-04 | 5813.0ms/step | 4.1 samples/s
534
+ Step 4230/20005 | Loss: 2.6236 | LR: 4.52e-04 | 5849.9ms/step | 4.1 samples/s
535
+ Step 4240/20005 | Loss: 2.7105 | LR: 4.51e-04 | 5814.3ms/step | 4.1 samples/s
536
+ Step 4250/20005 | Loss: 2.7933 | LR: 4.51e-04 | 5813.6ms/step | 4.1 samples/s
537
+ Validation at step 4250: Loss = 2.7202, Perplexity = 15.18
538
+ Step 4260/20005 | Loss: 2.6575 | LR: 4.51e-04 | 41016.6ms/step | 0.6 samples/s
539
+ Step 4270/20005 | Loss: 2.7397 | LR: 4.51e-04 | 5814.2ms/step | 4.1 samples/s
540
+ Step 4280/20005 | Loss: 2.6631 | LR: 4.50e-04 | 5813.3ms/step | 4.1 samples/s
541
+ Step 4290/20005 | Loss: 2.7004 | LR: 4.50e-04 | 5813.0ms/step | 4.1 samples/s
542
+ Step 4300/20005 | Loss: 2.6326 | LR: 4.50e-04 | 5813.4ms/step | 4.1 samples/s
543
+ Step 4310/20005 | Loss: 2.7162 | LR: 4.50e-04 | 5808.9ms/step | 4.1 samples/s
544
+ Step 4320/20005 | Loss: 2.6519 | LR: 4.49e-04 | 5814.1ms/step | 4.1 samples/s
545
+ Step 4330/20005 | Loss: 2.6678 | LR: 4.49e-04 | 5814.6ms/step | 4.1 samples/s
546
+ Step 4340/20005 | Loss: 2.7463 | LR: 4.49e-04 | 5815.0ms/step | 4.1 samples/s
547
+ Step 4350/20005 | Loss: 2.6902 | LR: 4.49e-04 | 5813.8ms/step | 4.1 samples/s
548
+ Step 4360/20005 | Loss: 2.5958 | LR: 4.48e-04 | 5811.7ms/step | 4.1 samples/s
549
+ Step 4370/20005 | Loss: 2.7132 | LR: 4.48e-04 | 5816.5ms/step | 4.1 samples/s
550
+ Step 4380/20005 | Loss: 2.7344 | LR: 4.48e-04 | 5814.4ms/step | 4.1 samples/s
551
+ Step 4390/20005 | Loss: 2.7231 | LR: 4.48e-04 | 5815.6ms/step | 4.1 samples/s
552
+ Step 4400/20005 | Loss: 2.7515 | LR: 4.48e-04 | 5812.8ms/step | 4.1 samples/s
553
+ Step 4410/20005 | Loss: 2.6891 | LR: 4.47e-04 | 5813.4ms/step | 4.1 samples/s
554
+ Step 4420/20005 | Loss: 2.6536 | LR: 4.47e-04 | 5817.2ms/step | 4.1 samples/s
555
+ Step 4430/20005 | Loss: 2.7288 | LR: 4.47e-04 | 5816.9ms/step | 4.1 samples/s
556
+ Step 4440/20005 | Loss: 2.6529 | LR: 4.47e-04 | 5815.6ms/step | 4.1 samples/s
557
+ Step 4450/20005 | Loss: 2.6640 | LR: 4.46e-04 | 5817.7ms/step | 4.1 samples/s
558
+ Step 4460/20005 | Loss: 2.6356 | LR: 4.46e-04 | 5816.6ms/step | 4.1 samples/s
559
+ Step 4470/20005 | Loss: 2.7730 | LR: 4.46e-04 | 5819.1ms/step | 4.1 samples/s
560
+ Step 4480/20005 | Loss: 2.6717 | LR: 4.46e-04 | 5814.2ms/step | 4.1 samples/s
561
+ Step 4490/20005 | Loss: 2.7667 | LR: 4.45e-04 | 5815.7ms/step | 4.1 samples/s
562
+ Step 4500/20005 | Loss: 2.7165 | LR: 4.45e-04 | 5818.7ms/step | 4.1 samples/s
563
+ Validation at step 4500: Loss = 2.6988, Perplexity = 14.86
564
+ Step 4510/20005 | Loss: 2.6900 | LR: 4.45e-04 | 41019.2ms/step | 0.6 samples/s
565
+ Step 4520/20005 | Loss: 2.6917 | LR: 4.45e-04 | 5816.4ms/step | 4.1 samples/s
566
+ Step 4530/20005 | Loss: 2.7396 | LR: 4.44e-04 | 5819.9ms/step | 4.1 samples/s
567
+ Step 4540/20005 | Loss: 2.7525 | LR: 4.44e-04 | 5818.3ms/step | 4.1 samples/s
568
+ Step 4550/20005 | Loss: 2.7379 | LR: 4.44e-04 | 5811.0ms/step | 4.1 samples/s
569
+ Step 4560/20005 | Loss: 2.6798 | LR: 4.44e-04 | 5818.6ms/step | 4.1 samples/s
570
+ Step 4570/20005 | Loss: 2.7027 | LR: 4.43e-04 | 5815.8ms/step | 4.1 samples/s
571
+ Step 4580/20005 | Loss: 2.7053 | LR: 4.43e-04 | 5821.0ms/step | 4.1 samples/s
572
+ Step 4590/20005 | Loss: 2.6662 | LR: 4.43e-04 | 5814.0ms/step | 4.1 samples/s
573
+ Step 4600/20005 | Loss: 2.6223 | LR: 4.43e-04 | 5813.6ms/step | 4.1 samples/s
574
+ Step 4610/20005 | Loss: 2.6855 | LR: 4.42e-04 | 5815.7ms/step | 4.1 samples/s
575
+ Step 4620/20005 | Loss: 2.7027 | LR: 4.42e-04 | 5813.3ms/step | 4.1 samples/s
576
+ Step 4630/20005 | Loss: 2.6082 | LR: 4.42e-04 | 5814.4ms/step | 4.1 samples/s
577
+ Step 4640/20005 | Loss: 2.5936 | LR: 4.42e-04 | 5813.5ms/step | 4.1 samples/s
578
+ Step 4650/20005 | Loss: 2.7066 | LR: 4.41e-04 | 5815.6ms/step | 4.1 samples/s
579
+ Step 4660/20005 | Loss: 2.5791 | LR: 4.41e-04 | 5814.2ms/step | 4.1 samples/s
580
+ Step 4670/20005 | Loss: 2.7396 | LR: 4.41e-04 | 5813.3ms/step | 4.1 samples/s
581
+ Step 4680/20005 | Loss: 2.7135 | LR: 4.40e-04 | 5817.6ms/step | 4.1 samples/s
582
+ Step 4690/20005 | Loss: 2.6128 | LR: 4.40e-04 | 5814.2ms/step | 4.1 samples/s
583
+ Step 4700/20005 | Loss: 2.7533 | LR: 4.40e-04 | 5812.2ms/step | 4.1 samples/s
584
+ Step 4710/20005 | Loss: 2.6620 | LR: 4.40e-04 | 5859.2ms/step | 4.1 samples/s
585
+ Step 4720/20005 | Loss: 2.7244 | LR: 4.39e-04 | 5807.4ms/step | 4.1 samples/s
586
+ Step 4730/20005 | Loss: 2.6293 | LR: 4.39e-04 | 5813.4ms/step | 4.1 samples/s
587
+ Step 4740/20005 | Loss: 2.6827 | LR: 4.39e-04 | 5814.7ms/step | 4.1 samples/s
588
+ Step 4750/20005 | Loss: 2.6336 | LR: 4.39e-04 | 5809.6ms/step | 4.1 samples/s
589
+ Validation at step 4750: Loss = 2.6747, Perplexity = 14.51
590
+ Step 4760/20005 | Loss: 2.6474 | LR: 4.38e-04 | 41016.1ms/step | 0.6 samples/s
591
+ Step 4770/20005 | Loss: 2.6701 | LR: 4.38e-04 | 5816.5ms/step | 4.1 samples/s
592
+ Step 4780/20005 | Loss: 2.6674 | LR: 4.38e-04 | 5815.8ms/step | 4.1 samples/s
593
+ Step 4790/20005 | Loss: 2.6144 | LR: 4.38e-04 | 5814.3ms/step | 4.1 samples/s
594
+ Step 4800/20005 | Loss: 2.6652 | LR: 4.37e-04 | 5817.0ms/step | 4.1 samples/s
595
+ Step 4810/20005 | Loss: 2.6978 | LR: 4.37e-04 | 5814.9ms/step | 4.1 samples/s
596
+ Step 4820/20005 | Loss: 2.6463 | LR: 4.37e-04 | 5812.2ms/step | 4.1 samples/s
597
+ Step 4830/20005 | Loss: 2.6040 | LR: 4.37e-04 | 5811.6ms/step | 4.1 samples/s
598
+ Step 4840/20005 | Loss: 2.7247 | LR: 4.36e-04 | 5808.2ms/step | 4.1 samples/s
599
+ Step 4850/20005 | Loss: 2.7645 | LR: 4.36e-04 | 5810.2ms/step | 4.1 samples/s
600
+ Step 4860/20005 | Loss: 2.7207 | LR: 4.36e-04 | 5808.3ms/step | 4.1 samples/s
601
+ Step 4870/20005 | Loss: 2.6404 | LR: 4.36e-04 | 5811.9ms/step | 4.1 samples/s
602
+ Step 4880/20005 | Loss: 2.6303 | LR: 4.35e-04 | 5810.4ms/step | 4.1 samples/s
603
+ Step 4890/20005 | Loss: 2.6022 | LR: 4.35e-04 | 5809.7ms/step | 4.1 samples/s
604
+ Step 4900/20005 | Loss: 2.7073 | LR: 4.35e-04 | 5812.4ms/step | 4.1 samples/s
605
+ Step 4910/20005 | Loss: 2.7051 | LR: 4.34e-04 | 5813.6ms/step | 4.1 samples/s
606
+ Step 4920/20005 | Loss: 2.6350 | LR: 4.34e-04 | 5809.9ms/step | 4.1 samples/s
607
+ Step 4930/20005 | Loss: 2.6252 | LR: 4.34e-04 | 5808.5ms/step | 4.1 samples/s
608
+ Step 4940/20005 | Loss: 2.6657 | LR: 4.34e-04 | 5807.3ms/step | 4.1 samples/s
609
+ Step 4950/20005 | Loss: 2.6188 | LR: 4.33e-04 | 5813.4ms/step | 4.1 samples/s
610
+ Step 4960/20005 | Loss: 2.6007 | LR: 4.33e-04 | 5808.2ms/step | 4.1 samples/s
611
+ Step 4970/20005 | Loss: 2.6472 | LR: 4.33e-04 | 5812.3ms/step | 4.1 samples/s
612
+ Step 4980/20005 | Loss: 2.7392 | LR: 4.33e-04 | 5814.4ms/step | 4.1 samples/s
613
+ Step 4990/20005 | Loss: 2.7054 | LR: 4.32e-04 | 5812.3ms/step | 4.1 samples/s
614
+ Step 5000/20005 | Loss: 2.7248 | LR: 4.32e-04 | 5813.0ms/step | 4.1 samples/s
615
+ Validation at step 5000: Loss = 2.6573, Perplexity = 14.26
616
+ Step 5010/20005 | Loss: 2.5971 | LR: 4.32e-04 | 41013.6ms/step | 0.6 samples/s
617
+ Step 5020/20005 | Loss: 2.6208 | LR: 4.31e-04 | 5811.4ms/step | 4.1 samples/s
618
+ Step 5030/20005 | Loss: 2.5785 | LR: 4.31e-04 | 5810.9ms/step | 4.1 samples/s
619
+ Step 5040/20005 | Loss: 2.5314 | LR: 4.31e-04 | 5813.2ms/step | 4.1 samples/s
620
+ Step 5050/20005 | Loss: 2.6525 | LR: 4.31e-04 | 5813.6ms/step | 4.1 samples/s
621
+ Step 5060/20005 | Loss: 2.5788 | LR: 4.30e-04 | 5811.7ms/step | 4.1 samples/s
622
+ Step 5070/20005 | Loss: 2.5588 | LR: 4.30e-04 | 5824.7ms/step | 4.1 samples/s
623
+ Step 5080/20005 | Loss: 2.6807 | LR: 4.30e-04 | 5824.9ms/step | 4.1 samples/s
624
+ Step 5090/20005 | Loss: 2.6606 | LR: 4.30e-04 | 5815.9ms/step | 4.1 samples/s
625
+ Step 5100/20005 | Loss: 2.6776 | LR: 4.29e-04 | 5816.3ms/step | 4.1 samples/s
626
+ Step 5110/20005 | Loss: 2.7371 | LR: 4.29e-04 | 5817.9ms/step | 4.1 samples/s
627
+ Step 5120/20005 | Loss: 2.5616 | LR: 4.29e-04 | 5816.7ms/step | 4.1 samples/s
628
+ Step 5130/20005 | Loss: 2.6053 | LR: 4.28e-04 | 5817.7ms/step | 4.1 samples/s
629
+ Step 5140/20005 | Loss: 2.6342 | LR: 4.28e-04 | 5817.7ms/step | 4.1 samples/s
630
+ Step 5150/20005 | Loss: 2.6366 | LR: 4.28e-04 | 5821.4ms/step | 4.1 samples/s
631
+ Step 5160/20005 | Loss: 2.7361 | LR: 4.28e-04 | 5816.6ms/step | 4.1 samples/s
632
+ Step 5170/20005 | Loss: 2.6602 | LR: 4.27e-04 | 5802.7ms/step | 4.1 samples/s
633
+ Step 5180/20005 | Loss: 2.6408 | LR: 4.27e-04 | 5851.3ms/step | 4.1 samples/s
634
+ Step 5190/20005 | Loss: 2.6586 | LR: 4.27e-04 | 5797.9ms/step | 4.1 samples/s
635
+ Step 5200/20005 | Loss: 2.6770 | LR: 4.26e-04 | 5802.0ms/step | 4.1 samples/s
636
+ Step 5210/20005 | Loss: 2.6575 | LR: 4.26e-04 | 5801.8ms/step | 4.1 samples/s
637
+ Step 5220/20005 | Loss: 2.5600 | LR: 4.26e-04 | 5804.0ms/step | 4.1 samples/s
638
+ Step 5230/20005 | Loss: 2.6149 | LR: 4.26e-04 | 5805.9ms/step | 4.1 samples/s
639
+ Step 5240/20005 | Loss: 2.7000 | LR: 4.25e-04 | 5804.2ms/step | 4.1 samples/s
640
+ Step 5250/20005 | Loss: 2.6361 | LR: 4.25e-04 | 5803.6ms/step | 4.1 samples/s
641
+ Validation at step 5250: Loss = 2.6406, Perplexity = 14.02
642
+ Step 5260/20005 | Loss: 2.6441 | LR: 4.25e-04 | 40969.9ms/step | 0.6 samples/s
643
+ Step 5270/20005 | Loss: 2.6648 | LR: 4.24e-04 | 5801.4ms/step | 4.1 samples/s
644
+ Step 5280/20005 | Loss: 2.6067 | LR: 4.24e-04 | 5801.4ms/step | 4.1 samples/s
645
+ Step 5290/20005 | Loss: 2.6228 | LR: 4.24e-04 | 5802.2ms/step | 4.1 samples/s
646
+ Step 5300/20005 | Loss: 2.5862 | LR: 4.24e-04 | 5803.7ms/step | 4.1 samples/s
647
+ Step 5310/20005 | Loss: 2.6006 | LR: 4.23e-04 | 5807.4ms/step | 4.1 samples/s
648
+ Step 5320/20005 | Loss: 2.6101 | LR: 4.23e-04 | 5801.4ms/step | 4.1 samples/s
649
+ Step 5330/20005 | Loss: 2.5875 | LR: 4.23e-04 | 5801.2ms/step | 4.1 samples/s
650
+ Step 5340/20005 | Loss: 2.5546 | LR: 4.22e-04 | 5800.4ms/step | 4.1 samples/s
651
+ Step 5350/20005 | Loss: 2.6454 | LR: 4.22e-04 | 5804.8ms/step | 4.1 samples/s
652
+ Step 5360/20005 | Loss: 2.6255 | LR: 4.22e-04 | 5804.3ms/step | 4.1 samples/s
653
+ Step 5370/20005 | Loss: 2.6495 | LR: 4.22e-04 | 5807.9ms/step | 4.1 samples/s
654
+ Step 5380/20005 | Loss: 2.5398 | LR: 4.21e-04 | 5802.9ms/step | 4.1 samples/s
655
+ Step 5390/20005 | Loss: 2.6720 | LR: 4.21e-04 | 5811.0ms/step | 4.1 samples/s
656
+ Step 5400/20005 | Loss: 2.6251 | LR: 4.21e-04 | 5817.1ms/step | 4.1 samples/s
657
+ Step 5410/20005 | Loss: 2.6015 | LR: 4.20e-04 | 5823.6ms/step | 4.1 samples/s
658
+ Step 5420/20005 | Loss: 2.5848 | LR: 4.20e-04 | 5819.4ms/step | 4.1 samples/s
659
+ Step 5430/20005 | Loss: 2.5988 | LR: 4.20e-04 | 5820.6ms/step | 4.1 samples/s
660
+ Step 5440/20005 | Loss: 2.6136 | LR: 4.20e-04 | 5823.8ms/step | 4.1 samples/s
661
+ Step 5450/20005 | Loss: 2.5693 | LR: 4.19e-04 | 5824.3ms/step | 4.1 samples/s
662
+ Step 5460/20005 | Loss: 2.5766 | LR: 4.19e-04 | 5824.2ms/step | 4.1 samples/s
663
+ Step 5470/20005 | Loss: 2.6115 | LR: 4.19e-04 | 5825.7ms/step | 4.1 samples/s
664
+ Step 5480/20005 | Loss: 2.5542 | LR: 4.18e-04 | 5823.7ms/step | 4.1 samples/s
665
+ Step 5490/20005 | Loss: 2.6211 | LR: 4.18e-04 | 5823.0ms/step | 4.1 samples/s
666
+ Step 5500/20005 | Loss: 2.6421 | LR: 4.18e-04 | 5821.3ms/step | 4.1 samples/s
667
+ Validation at step 5500: Loss = 2.6272, Perplexity = 13.83
668
+ Step 5510/20005 | Loss: 2.6057 | LR: 4.18e-04 | 40989.2ms/step | 0.6 samples/s
669
+ Step 5520/20005 | Loss: 2.6298 | LR: 4.17e-04 | 5806.8ms/step | 4.1 samples/s
670
+ Step 5530/20005 | Loss: 2.5617 | LR: 4.17e-04 | 5817.7ms/step | 4.1 samples/s
671
+ Step 5540/20005 | Loss: 2.5625 | LR: 4.17e-04 | 5811.9ms/step | 4.1 samples/s
672
+ Step 5550/20005 | Loss: 2.5544 | LR: 4.16e-04 | 5802.5ms/step | 4.1 samples/s
673
+ Step 5560/20005 | Loss: 2.5817 | LR: 4.16e-04 | 5805.2ms/step | 4.1 samples/s
674
+ Step 5570/20005 | Loss: 2.7236 | LR: 4.16e-04 | 5803.1ms/step | 4.1 samples/s
675
+ Step 5580/20005 | Loss: 2.6420 | LR: 4.15e-04 | 5808.7ms/step | 4.1 samples/s
676
+ Step 5590/20005 | Loss: 2.6013 | LR: 4.15e-04 | 5802.6ms/step | 4.1 samples/s
677
+ Step 5600/20005 | Loss: 2.6410 | LR: 4.15e-04 | 5808.7ms/step | 4.1 samples/s
678
+ Step 5610/20005 | Loss: 2.5924 | LR: 4.15e-04 | 5805.3ms/step | 4.1 samples/s
679
+ Step 5620/20005 | Loss: 2.6147 | LR: 4.14e-04 | 5802.3ms/step | 4.1 samples/s
680
+ Step 5630/20005 | Loss: 2.6338 | LR: 4.14e-04 | 5804.2ms/step | 4.1 samples/s
681
+ Step 5640/20005 | Loss: 2.6364 | LR: 4.14e-04 | 5802.2ms/step | 4.1 samples/s
682
+ Step 5650/20005 | Loss: 2.6607 | LR: 4.13e-04 | 5843.6ms/step | 4.1 samples/s
683
+ Step 5660/20005 | Loss: 2.5265 | LR: 4.13e-04 | 5805.7ms/step | 4.1 samples/s
684
+ Step 5670/20005 | Loss: 2.6036 | LR: 4.13e-04 | 5814.1ms/step | 4.1 samples/s
685
+ Step 5680/20005 | Loss: 2.5456 | LR: 4.12e-04 | 5811.3ms/step | 4.1 samples/s
686
+ Step 5690/20005 | Loss: 2.5657 | LR: 4.12e-04 | 5812.6ms/step | 4.1 samples/s
687
+ Step 5700/20005 | Loss: 2.6663 | LR: 4.12e-04 | 5815.5ms/step | 4.1 samples/s
688
+ Step 5710/20005 | Loss: 2.6079 | LR: 4.12e-04 | 5810.8ms/step | 4.1 samples/s
689
+ Step 5720/20005 | Loss: 2.6251 | LR: 4.11e-04 | 5811.5ms/step | 4.1 samples/s
690
+ Step 5730/20005 | Loss: 2.6698 | LR: 4.11e-04 | 5811.0ms/step | 4.1 samples/s
691
+ Step 5740/20005 | Loss: 2.5895 | LR: 4.11e-04 | 5814.5ms/step | 4.1 samples/s
692
+ Step 5750/20005 | Loss: 2.6105 | LR: 4.10e-04 | 5810.2ms/step | 4.1 samples/s
693
+ Validation at step 5750: Loss = 2.6144, Perplexity = 13.66
694
+ Step 5760/20005 | Loss: 2.5906 | LR: 4.10e-04 | 41010.3ms/step | 0.6 samples/s
695
+ Step 5770/20005 | Loss: 2.6256 | LR: 4.10e-04 | 5810.6ms/step | 4.1 samples/s
696
+ Step 5780/20005 | Loss: 2.6565 | LR: 4.09e-04 | 5811.5ms/step | 4.1 samples/s
697
+ Step 5790/20005 | Loss: 2.6120 | LR: 4.09e-04 | 5810.7ms/step | 4.1 samples/s
epfl_final/wandb/run-20260505_213713-4i6l0dd0/files/requirements.txt ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pytz==2026.2
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.6.3
8
+ tzdata==2026.2
9
+ typing_extensions==4.15.0
10
+ triton==3.6.0
11
+ tqdm==4.67.3
12
+ torchinfo==1.8.0
13
+ sympy==1.14.0
14
+ smmap==5.0.3
15
+ six==1.17.0
16
+ shellingham==1.5.4
17
+ safetensors==0.7.0
18
+ regex==2026.4.4
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.4.1
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.2.6
38
+ networkx==3.4.2
39
+ mdurl==0.1.2
40
+ MarkupSafe==3.0.3
41
+ idna==3.13
42
+ hf-xet==1.4.3
43
+ h11==0.16.0
44
+ fsspec==2026.2.0
45
+ frozenlist==1.8.0
46
+ filelock==3.29.0
47
+ dill==0.4.1
48
+ cuda-pathfinder==1.5.4
49
+ click==8.3.3
50
+ charset-normalizer==3.4.7
51
+ certifi==2026.4.22
52
+ attrs==26.1.0
53
+ async-timeout==5.0.1
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ typing-inspection==0.4.2
58
+ sentry-sdk==2.59.0
59
+ requests==2.33.1
60
+ python-dateutil==2.9.0.post0
61
+ pydantic_core==2.46.3
62
+ nvidia-cusparse==12.6.3.3
63
+ nvidia-cufft==12.0.0.61
64
+ nvidia-cudnn-cu13==9.19.0.56
65
+ multiprocess==0.70.19
66
+ multidict==6.7.1
67
+ markdown-it-py==4.0.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ exceptiongroup==1.3.1
72
+ cuda-bindings==13.2.0
73
+ aiosignal==1.4.0
74
+ yarl==1.23.0
75
+ rich==15.0.0
76
+ pydantic==2.13.3
77
+ pandas==2.3.3
78
+ nvidia-cusolver==12.0.4.66
79
+ GitPython==3.1.49
80
+ anyio==4.13.0
81
+ wandb==0.26.1
82
+ typer==0.25.1
83
+ httpx==0.28.1
84
+ aiohttp==3.13.5
85
+ torch==2.11.0
86
+ huggingface_hub==1.13.0
87
+ tokenizers==0.22.2
88
+ datasets==4.8.5
89
+ accelerate==1.13.0
90
+ transformers==5.8.0
91
+ PyGObject==3.42.1
92
+ dbus-python==1.2.18
93
+ distro==1.7.0
94
+ pip==22.0.2
95
+ setuptools==59.6.0
96
+ ssh-import-id==5.11
97
+ wheel==0.37.1
epfl_final/wandb/run-20260505_213713-4i6l0dd0/files/wandb-metadata.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.10.12",
4
+ "startedAt": "2026-05-05T21:37:13.817285Z",
5
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
6
+ "codePath": "SparseAtt/train.py",
7
+ "codePathLocal": "SparseAtt/train.py",
8
+ "email": "ruben.navasardyan@mail.utoronto.ca",
9
+ "root": "/mloscratch/homes/navasard/sparse",
10
+ "host": "h100-node-0-0",
11
+ "executable": "/usr/bin/python3",
12
+ "cpu_count": 96,
13
+ "cpu_count_logical": 192,
14
+ "gpu": "NVIDIA H100 80GB HBM3",
15
+ "gpu_count": 1,
16
+ "disk": {
17
+ "/": {
18
+ "total": "9374931091456",
19
+ "used": "7270149292032"
20
+ }
21
+ },
22
+ "memory": {
23
+ "total": "1623005847552"
24
+ },
25
+ "gpu_nvidia": [
26
+ {
27
+ "name": "NVIDIA H100 80GB HBM3",
28
+ "memoryTotal": "85520809984",
29
+ "cudaCores": 16896,
30
+ "architecture": "Hopper",
31
+ "uuid": "GPU-a0a4877f-b532-7a97-dd64-6d98c46f4daa"
32
+ }
33
+ ],
34
+ "cudaVersion": "13.0",
35
+ "writerId": "5yf5sr5qwvp8pz24wbq0vpdgy4vs7rmf"
36
+ }
epfl_final/wandb/run-20260505_213713-4i6l0dd0/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/wandb/run-20260505_213713-4i6l0dd0/logs/debug.log ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-05-05 21:37:13,835 INFO MainThread:4222 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-05-05 21:37:13,835 INFO MainThread:4222 [wandb_setup.py:_flush():81] Configure stats pid to 4222
3
+ 2026-05-05 21:37:13,836 INFO MainThread:4222 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-05-05 21:37:13,837 INFO MainThread:4222 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260505_213713-4i6l0dd0/logs/debug.log
5
+ 2026-05-05 21:37:13,837 INFO MainThread:4222 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260505_213713-4i6l0dd0/logs/debug-internal.log
6
+ 2026-05-05 21:37:13,838 INFO MainThread:4222 [wandb_init.py:init():850] calling init triggers
7
+ 2026-05-05 21:37:13,839 INFO MainThread:4222 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/the_pile_train/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': './checkpoints/run_20260505_213703', 'resume_from_checkpoint': None, 'num_epochs': 1, 'max_steps': 20005, 'per_device_batch_size': 3, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 250, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 2000, 'eval_steps': 250, 'eval_batches': 200, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': None, 'wandb_tags': ['qwen3', '0.6b'], 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 19, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27], '_wandb': {}}
9
+ 2026-05-05 21:37:13,839 INFO MainThread:4222 [wandb_init.py:init():898] starting backend
10
+ 2026-05-05 21:37:14,071 INFO MainThread:4222 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-05-05 21:37:14,383 INFO MainThread:4222 [wandb_init.py:init():918] backend started and connected
12
+ 2026-05-05 21:37:14,389 INFO MainThread:4222 [wandb_init.py:init():988] updated telemetry
13
+ 2026-05-05 21:37:14,390 INFO MainThread:4222 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-05-05 21:37:14,790 INFO MainThread:4222 [wandb_init.py:init():1056] starting run threads in backend
15
+ 2026-05-05 21:37:14,844 INFO MainThread:4222 [wandb_run.py:_console_start():2554] atexit reg
16
+ 2026-05-05 21:37:14,844 INFO MainThread:4222 [wandb_run.py:_redirect():2403] redirect: wrap_raw
17
+ 2026-05-05 21:37:14,844 INFO MainThread:4222 [wandb_run.py:_redirect():2472] Wrapping output streams.
18
+ 2026-05-05 21:37:14,844 INFO MainThread:4222 [wandb_run.py:_redirect():2495] Redirects installed.
19
+ 2026-05-05 21:37:14,846 INFO MainThread:4222 [wandb_init.py:init():1094] run started, returning control to user process
epfl_final/wandb/run-20260505_213713-4i6l0dd0/run-4i6l0dd0.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:67c95d8d63408b8e5ea9ca1bf4ec866fc834f039b103ce1ab555b1f54faf400b
3
+ size 6062080
epfl_final/wandb/run-20260506_173734-nmzybrc7/files/output.log ADDED
@@ -0,0 +1,496 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ ================================================================================
4
+ MODEL ARCHITECTURE
5
+ ================================================================================
6
+ Model config:
7
+ vocab_size: 151936
8
+ hidden_size: 1024
9
+ num_hidden_layers: 28
10
+ num_attention_heads: 16
11
+ num_key_value_heads: 8
12
+ head_dim: 128
13
+ intermediate_size: 3072
14
+ max_position_embeddings: 32768
15
+ rope_theta: 1000000
16
+ attention_type: sparse_stochastic
17
+ sparse_block_size: 128
18
+ sparse_num_random_blocks: 20
19
+ sparse_mask_pool_size: 1000
20
+ sparse_max_seq_length: 8192
21
+
22
+ Parameter count:
23
+ Total parameters: 596,049,920
24
+ Trainable parameters: 596,049,920
25
+ Non-trainable parameters: 0
26
+ Model size (BF16): 1.19 GB
27
+
28
+ --------------------------------------------------------------------------------
29
+ LAYER-BY-LAYER SUMMARY
30
+ --------------------------------------------------------------------------------
31
+ /home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
32
+
33
+ SOLUTION: Use torch.compile(flex_attention)(...)
34
+
35
+ If you want to debug your score_mod/mask_mod, you can set:
36
+ torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
37
+
38
+ This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
39
+ _warn_once(
40
+ ================================================================================================================================================================
41
+ Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
42
+ ================================================================================================================================================================
43
+ Qwen3ForCausalLM [3, 8192] -- -- True
44
+ ├─Qwen3Model: 1-1 -- [3, 8192, 1024] -- True
45
+ │ └─Embedding: 2-1 [3, 8192] [3, 8192, 1024] 155,582,464 True
46
+ │ └─ModuleList: 2-2 -- -- -- True
47
+ │ │ └─Qwen3DecoderLayer: 3-1 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
48
+ │ │ └─Qwen3DecoderLayer: 3-2 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
49
+ │ │ └─Qwen3DecoderLayer: 3-3 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
50
+ │ │ └─Qwen3DecoderLayer: 3-4 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
51
+ │ │ └─Qwen3DecoderLayer: 3-5 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
52
+ │ │ └─Qwen3DecoderLayer: 3-6 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
53
+ │ │ └─Qwen3DecoderLayer: 3-7 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
54
+ │ │ └─Qwen3DecoderLayer: 3-8 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
55
+ │ │ └─Qwen3DecoderLayer: 3-9 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
56
+ │ │ └─Qwen3DecoderLayer: 3-10 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
57
+ │ │ └─Qwen3DecoderLayer: 3-11 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
58
+ │ │ └─Qwen3DecoderLayer: 3-12 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
59
+ │ │ └─Qwen3DecoderLayer: 3-13 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
60
+ │ │ └─Qwen3DecoderLayer: 3-14 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
61
+ │ │ └─Qwen3DecoderLayer: 3-15 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
62
+ │ │ └─Qwen3DecoderLayer: 3-16 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
63
+ │ │ └─Qwen3DecoderLayer: 3-17 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
64
+ │ │ └─Qwen3DecoderLayer: 3-18 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
65
+ │ │ └─Qwen3DecoderLayer: 3-19 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
66
+ │ │ └─Qwen3DecoderLayer: 3-20 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
67
+ │ │ └─Qwen3DecoderLayer: 3-21 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
68
+ │ │ └─Qwen3DecoderLayer: 3-22 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
69
+ │ │ └─Qwen3DecoderLayer: 3-23 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
70
+ │ │ └─Qwen3DecoderLayer: 3-24 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
71
+ │ │ └─Qwen3DecoderLayer: 3-25 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
72
+ │ │ └─Qwen3DecoderLayer: 3-26 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
73
+ │ │ └─Qwen3DecoderLayer: 3-27 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
74
+ │ │ └─Qwen3DecoderLayer: 3-28 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
75
+ │ └─Qwen3RMSNorm: 2-3 [3, 8192, 1024] [3, 8192, 1024] 1,024 True
76
+ ├─Linear: 1-2 [3, 8192, 1024] [3, 8192, 151936] 155,582,464 True
77
+ ================================================================================================================================================================
78
+ Total params: 751,632,384
79
+ Trainable params: 751,632,384
80
+ Non-trainable params: 0
81
+ Total mult-adds (G): 2.25
82
+ ================================================================================================================================================================
83
+ Input size (MB): 0.20
84
+ Forward/backward pass size (MB): 63052.97
85
+ Params size (MB): 1503.26
86
+ Estimated Total Size (MB): 64556.43
87
+ ================================================================================================================================================================
88
+ ================================================================================
89
+
90
+ Loading dataset from /mloscratch/homes/navasard/data/the_pile_train/tokenized...
91
+ Using pre-tokenized dataset (no runtime tokenization)
92
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/the_pile_train/tokenized...
93
+ Loaded 5315146 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1328786)
94
+ Dataset split: 1063028 train, 265758 validation
95
+ Training examples (packed, train split): 1063028
96
+ Validation examples (packed, val split): 265758
97
+ Per-device batch size: 3
98
+ World size (GPUs): 1
99
+ Gradient accumulation steps: 8
100
+ Effective batch size (seqs/step): 24
101
+ Sequence length: 8192
102
+ Tokens per optimizer step: 196,608
103
+ Micro-batches per epoch (per rank): 354342
104
+ Optimizer steps per epoch: 44292
105
+ Total optimizer steps: 20005 (capped by max_steps; 0.45 epochs)
106
+ Starting training...
107
+ Step 10/20005 | Loss: 12.0953 | LR: 2.00e-05 | 9747.4ms/step | 2.5 samples/s
108
+ Step 20/20005 | Loss: 11.5112 | LR: 4.00e-05 | 2455.1ms/step | 9.8 samples/s
109
+ Step 30/20005 | Loss: 10.6615 | LR: 6.00e-05 | 2445.4ms/step | 9.8 samples/s
110
+ Step 40/20005 | Loss: 10.1898 | LR: 8.00e-05 | 2439.2ms/step | 9.8 samples/s
111
+ Step 50/20005 | Loss: 9.3633 | LR: 1.00e-04 | 2437.9ms/step | 9.8 samples/s
112
+ Step 60/20005 | Loss: 8.6099 | LR: 1.20e-04 | 2437.9ms/step | 9.8 samples/s
113
+ Step 70/20005 | Loss: 8.0160 | LR: 1.40e-04 | 2443.9ms/step | 9.8 samples/s
114
+ Step 80/20005 | Loss: 7.6071 | LR: 1.60e-04 | 2450.5ms/step | 9.8 samples/s
115
+ Step 90/20005 | Loss: 7.6196 | LR: 1.80e-04 | 2459.7ms/step | 9.8 samples/s
116
+ Step 100/20005 | Loss: 7.3453 | LR: 2.00e-04 | 2461.2ms/step | 9.8 samples/s
117
+ Step 110/20005 | Loss: 7.2536 | LR: 2.20e-04 | 2465.1ms/step | 9.7 samples/s
118
+ Step 120/20005 | Loss: 7.1495 | LR: 2.40e-04 | 2464.9ms/step | 9.7 samples/s
119
+ Step 130/20005 | Loss: 6.9748 | LR: 2.60e-04 | 2466.5ms/step | 9.7 samples/s
120
+ Step 140/20005 | Loss: 6.8463 | LR: 2.80e-04 | 2472.3ms/step | 9.7 samples/s
121
+ Step 150/20005 | Loss: 6.6894 | LR: 3.00e-04 | 2469.1ms/step | 9.7 samples/s
122
+ Step 160/20005 | Loss: 6.5733 | LR: 3.20e-04 | 2472.4ms/step | 9.7 samples/s
123
+ Step 170/20005 | Loss: 6.5434 | LR: 3.40e-04 | 2471.1ms/step | 9.7 samples/s
124
+ Step 180/20005 | Loss: 6.4437 | LR: 3.60e-04 | 2469.6ms/step | 9.7 samples/s
125
+ Step 190/20005 | Loss: 6.2488 | LR: 3.80e-04 | 2469.1ms/step | 9.7 samples/s
126
+ Step 200/20005 | Loss: 6.3373 | LR: 4.00e-04 | 2470.4ms/step | 9.7 samples/s
127
+ Step 210/20005 | Loss: 6.1415 | LR: 4.20e-04 | 2472.3ms/step | 9.7 samples/s
128
+ Step 220/20005 | Loss: 6.0566 | LR: 4.40e-04 | 2472.8ms/step | 9.7 samples/s
129
+ Step 230/20005 | Loss: 6.0284 | LR: 4.60e-04 | 2472.8ms/step | 9.7 samples/s
130
+ Step 240/20005 | Loss: 6.0281 | LR: 4.80e-04 | 2472.8ms/step | 9.7 samples/s
131
+ Step 250/20005 | Loss: 5.8878 | LR: 5.00e-04 | 2474.1ms/step | 9.7 samples/s
132
+ Validation at step 250: Loss = 5.9092, Perplexity = 368.41
133
+ Step 260/20005 | Loss: 5.8341 | LR: 5.00e-04 | 23166.1ms/step | 1.0 samples/s
134
+ Step 270/20005 | Loss: 5.7961 | LR: 5.00e-04 | 2467.7ms/step | 9.7 samples/s
135
+ Step 280/20005 | Loss: 5.8218 | LR: 5.00e-04 | 2469.0ms/step | 9.7 samples/s
136
+ Step 290/20005 | Loss: 5.7036 | LR: 5.00e-04 | 2469.6ms/step | 9.7 samples/s
137
+ Step 300/20005 | Loss: 5.6739 | LR: 5.00e-04 | 2469.6ms/step | 9.7 samples/s
138
+ Step 310/20005 | Loss: 5.6067 | LR: 5.00e-04 | 2470.2ms/step | 9.7 samples/s
139
+ Step 320/20005 | Loss: 5.5022 | LR: 5.00e-04 | 2472.2ms/step | 9.7 samples/s
140
+ Step 330/20005 | Loss: 5.4496 | LR: 5.00e-04 | 2469.2ms/step | 9.7 samples/s
141
+ Step 340/20005 | Loss: 5.2664 | LR: 5.00e-04 | 2470.2ms/step | 9.7 samples/s
142
+ Step 350/20005 | Loss: 5.2576 | LR: 5.00e-04 | 2469.4ms/step | 9.7 samples/s
143
+ Step 360/20005 | Loss: 5.3474 | LR: 5.00e-04 | 2468.0ms/step | 9.7 samples/s
144
+ Step 370/20005 | Loss: 5.0944 | LR: 5.00e-04 | 2468.5ms/step | 9.7 samples/s
145
+ Step 380/20005 | Loss: 5.1576 | LR: 5.00e-04 | 2473.3ms/step | 9.7 samples/s
146
+ Step 390/20005 | Loss: 5.1447 | LR: 5.00e-04 | 2473.1ms/step | 9.7 samples/s
147
+ Step 400/20005 | Loss: 5.0084 | LR: 5.00e-04 | 2475.2ms/step | 9.7 samples/s
148
+ Step 410/20005 | Loss: 5.0430 | LR: 5.00e-04 | 2472.0ms/step | 9.7 samples/s
149
+ Step 420/20005 | Loss: 4.8453 | LR: 5.00e-04 | 2470.4ms/step | 9.7 samples/s
150
+ Step 430/20005 | Loss: 4.9373 | LR: 5.00e-04 | 2472.6ms/step | 9.7 samples/s
151
+ Step 440/20005 | Loss: 4.7404 | LR: 5.00e-04 | 2473.2ms/step | 9.7 samples/s
152
+ Step 450/20005 | Loss: 4.8040 | LR: 5.00e-04 | 2470.3ms/step | 9.7 samples/s
153
+ Step 460/20005 | Loss: 4.8016 | LR: 5.00e-04 | 2468.5ms/step | 9.7 samples/s
154
+ Step 470/20005 | Loss: 4.7038 | LR: 5.00e-04 | 2470.9ms/step | 9.7 samples/s
155
+ Step 480/20005 | Loss: 4.6288 | LR: 5.00e-04 | 2470.2ms/step | 9.7 samples/s
156
+ Step 490/20005 | Loss: 4.6210 | LR: 5.00e-04 | 2470.4ms/step | 9.7 samples/s
157
+ Step 500/20005 | Loss: 4.5110 | LR: 5.00e-04 | 2470.2ms/step | 9.7 samples/s
158
+ Validation at step 500: Loss = 4.5643, Perplexity = 95.99
159
+ Step 510/20005 | Loss: 4.6094 | LR: 5.00e-04 | 20008.5ms/step | 1.2 samples/s
160
+ Step 520/20005 | Loss: 4.5267 | LR: 5.00e-04 | 2473.6ms/step | 9.7 samples/s
161
+ Step 530/20005 | Loss: 4.4512 | LR: 5.00e-04 | 2471.4ms/step | 9.7 samples/s
162
+ Step 540/20005 | Loss: 4.4637 | LR: 5.00e-04 | 2470.1ms/step | 9.7 samples/s
163
+ Step 550/20005 | Loss: 4.3918 | LR: 5.00e-04 | 2469.5ms/step | 9.7 samples/s
164
+ Step 560/20005 | Loss: 4.4136 | LR: 5.00e-04 | 2471.2ms/step | 9.7 samples/s
165
+ Step 570/20005 | Loss: 4.3111 | LR: 5.00e-04 | 2470.1ms/step | 9.7 samples/s
166
+ Step 580/20005 | Loss: 4.2604 | LR: 5.00e-04 | 2466.7ms/step | 9.7 samples/s
167
+ Step 590/20005 | Loss: 4.2515 | LR: 5.00e-04 | 2471.6ms/step | 9.7 samples/s
168
+ Step 600/20005 | Loss: 4.2258 | LR: 5.00e-04 | 2472.6ms/step | 9.7 samples/s
169
+ Step 610/20005 | Loss: 4.1722 | LR: 5.00e-04 | 2471.3ms/step | 9.7 samples/s
170
+ Step 620/20005 | Loss: 4.1427 | LR: 5.00e-04 | 2471.6ms/step | 9.7 samples/s
171
+ Step 630/20005 | Loss: 4.1107 | LR: 5.00e-04 | 2470.0ms/step | 9.7 samples/s
172
+ Step 640/20005 | Loss: 4.1992 | LR: 5.00e-04 | 2469.4ms/step | 9.7 samples/s
173
+ Step 650/20005 | Loss: 4.1674 | LR: 4.99e-04 | 2472.4ms/step | 9.7 samples/s
174
+ Step 660/20005 | Loss: 4.1263 | LR: 4.99e-04 | 2471.2ms/step | 9.7 samples/s
175
+ Step 670/20005 | Loss: 4.0386 | LR: 4.99e-04 | 2474.5ms/step | 9.7 samples/s
176
+ Step 680/20005 | Loss: 4.0233 | LR: 4.99e-04 | 2472.6ms/step | 9.7 samples/s
177
+ Step 690/20005 | Loss: 3.9074 | LR: 4.99e-04 | 2467.9ms/step | 9.7 samples/s
178
+ Step 700/20005 | Loss: 3.9711 | LR: 4.99e-04 | 2467.3ms/step | 9.7 samples/s
179
+ Step 710/20005 | Loss: 4.0219 | LR: 4.99e-04 | 2472.1ms/step | 9.7 samples/s
180
+ Step 720/20005 | Loss: 3.9824 | LR: 4.99e-04 | 2474.1ms/step | 9.7 samples/s
181
+ Step 730/20005 | Loss: 3.9286 | LR: 4.99e-04 | 2473.0ms/step | 9.7 samples/s
182
+ Step 740/20005 | Loss: 3.8975 | LR: 4.99e-04 | 2475.8ms/step | 9.7 samples/s
183
+ Step 750/20005 | Loss: 3.8868 | LR: 4.99e-04 | 2472.0ms/step | 9.7 samples/s
184
+ Validation at step 750: Loss = 3.9122, Perplexity = 50.01
185
+ Step 760/20005 | Loss: 3.9005 | LR: 4.99e-04 | 19998.9ms/step | 1.2 samples/s
186
+ Step 770/20005 | Loss: 3.8754 | LR: 4.99e-04 | 2471.5ms/step | 9.7 samples/s
187
+ Step 780/20005 | Loss: 3.9505 | LR: 4.99e-04 | 2470.8ms/step | 9.7 samples/s
188
+ Step 790/20005 | Loss: 3.8299 | LR: 4.99e-04 | 2472.9ms/step | 9.7 samples/s
189
+ Step 800/20005 | Loss: 3.7659 | LR: 4.99e-04 | 2471.3ms/step | 9.7 samples/s
190
+ Step 810/20005 | Loss: 3.8095 | LR: 4.99e-04 | 2469.0ms/step | 9.7 samples/s
191
+ Step 820/20005 | Loss: 3.8006 | LR: 4.99e-04 | 2470.9ms/step | 9.7 samples/s
192
+ Step 830/20005 | Loss: 3.8261 | LR: 4.99e-04 | 2472.9ms/step | 9.7 samples/s
193
+ Step 840/20005 | Loss: 3.8026 | LR: 4.99e-04 | 2469.8ms/step | 9.7 samples/s
194
+ Step 850/20005 | Loss: 3.7642 | LR: 4.99e-04 | 2472.0ms/step | 9.7 samples/s
195
+ Step 860/20005 | Loss: 3.7994 | LR: 4.99e-04 | 2472.7ms/step | 9.7 samples/s
196
+ Step 870/20005 | Loss: 3.7685 | LR: 4.99e-04 | 2473.4ms/step | 9.7 samples/s
197
+ Step 880/20005 | Loss: 3.7097 | LR: 4.99e-04 | 2470.1ms/step | 9.7 samples/s
198
+ Step 890/20005 | Loss: 3.6376 | LR: 4.99e-04 | 2468.6ms/step | 9.7 samples/s
199
+ Step 900/20005 | Loss: 3.7063 | LR: 4.99e-04 | 2474.3ms/step | 9.7 samples/s
200
+ Step 910/20005 | Loss: 3.6934 | LR: 4.99e-04 | 2471.7ms/step | 9.7 samples/s
201
+ Step 920/20005 | Loss: 3.6495 | LR: 4.99e-04 | 2470.7ms/step | 9.7 samples/s
202
+ Step 930/20005 | Loss: 3.5923 | LR: 4.99e-04 | 2471.9ms/step | 9.7 samples/s
203
+ Step 940/20005 | Loss: 3.7267 | LR: 4.98e-04 | 2470.2ms/step | 9.7 samples/s
204
+ Step 950/20005 | Loss: 3.5760 | LR: 4.98e-04 | 2468.5ms/step | 9.7 samples/s
205
+ Step 960/20005 | Loss: 3.6459 | LR: 4.98e-04 | 2469.6ms/step | 9.7 samples/s
206
+ Step 970/20005 | Loss: 3.5800 | LR: 4.98e-04 | 2471.9ms/step | 9.7 samples/s
207
+ Step 980/20005 | Loss: 3.5983 | LR: 4.98e-04 | 2475.0ms/step | 9.7 samples/s
208
+ Step 990/20005 | Loss: 3.6347 | LR: 4.98e-04 | 2474.8ms/step | 9.7 samples/s
209
+ Step 1000/20005 | Loss: 3.5700 | LR: 4.98e-04 | 2472.7ms/step | 9.7 samples/s
210
+ Validation at step 1000: Loss = 3.6054, Perplexity = 36.79
211
+ Step 1010/20005 | Loss: 3.5783 | LR: 4.98e-04 | 20042.8ms/step | 1.2 samples/s
212
+ Step 1020/20005 | Loss: 3.5671 | LR: 4.98e-04 | 2473.3ms/step | 9.7 samples/s
213
+ Step 1030/20005 | Loss: 3.5786 | LR: 4.98e-04 | 2470.0ms/step | 9.7 samples/s
214
+ Step 1040/20005 | Loss: 3.6036 | LR: 4.98e-04 | 2572.7ms/step | 9.3 samples/s
215
+ Step 1050/20005 | Loss: 3.4463 | LR: 4.98e-04 | 2481.5ms/step | 9.7 samples/s
216
+ Step 1060/20005 | Loss: 3.5629 | LR: 4.98e-04 | 2484.6ms/step | 9.7 samples/s
217
+ Step 1070/20005 | Loss: 3.5151 | LR: 4.98e-04 | 2482.3ms/step | 9.7 samples/s
218
+ Step 1080/20005 | Loss: 3.6346 | LR: 4.98e-04 | 2482.4ms/step | 9.7 samples/s
219
+ Step 1090/20005 | Loss: 3.5463 | LR: 4.98e-04 | 2483.7ms/step | 9.7 samples/s
220
+ Step 1100/20005 | Loss: 3.4785 | LR: 4.98e-04 | 2485.4ms/step | 9.7 samples/s
221
+ Step 1110/20005 | Loss: 3.5641 | LR: 4.98e-04 | 2482.7ms/step | 9.7 samples/s
222
+ Step 1120/20005 | Loss: 3.5929 | LR: 4.98e-04 | 2481.2ms/step | 9.7 samples/s
223
+ Step 1130/20005 | Loss: 3.5342 | LR: 4.98e-04 | 2480.7ms/step | 9.7 samples/s
224
+ Step 1140/20005 | Loss: 3.5100 | LR: 4.98e-04 | 2483.6ms/step | 9.7 samples/s
225
+ Step 1150/20005 | Loss: 3.5179 | LR: 4.97e-04 | 2488.2ms/step | 9.6 samples/s
226
+ Step 1160/20005 | Loss: 3.4899 | LR: 4.97e-04 | 2485.5ms/step | 9.7 samples/s
227
+ Step 1170/20005 | Loss: 3.5013 | LR: 4.97e-04 | 2483.8ms/step | 9.7 samples/s
228
+ Step 1180/20005 | Loss: 3.4978 | LR: 4.97e-04 | 2482.7ms/step | 9.7 samples/s
229
+ Step 1190/20005 | Loss: 3.5004 | LR: 4.97e-04 | 2483.5ms/step | 9.7 samples/s
230
+ Step 1200/20005 | Loss: 3.4190 | LR: 4.97e-04 | 2484.0ms/step | 9.7 samples/s
231
+ Step 1210/20005 | Loss: 3.3502 | LR: 4.97e-04 | 2484.1ms/step | 9.7 samples/s
232
+ Step 1220/20005 | Loss: 3.5036 | LR: 4.97e-04 | 2483.9ms/step | 9.7 samples/s
233
+ Step 1230/20005 | Loss: 3.4153 | LR: 4.97e-04 | 2481.5ms/step | 9.7 samples/s
234
+ Step 1240/20005 | Loss: 3.5046 | LR: 4.97e-04 | 2480.7ms/step | 9.7 samples/s
235
+ Step 1250/20005 | Loss: 3.3397 | LR: 4.97e-04 | 2482.6ms/step | 9.7 samples/s
236
+ Validation at step 1250: Loss = 3.4208, Perplexity = 30.60
237
+ Step 1260/20005 | Loss: 3.3878 | LR: 4.97e-04 | 20092.1ms/step | 1.2 samples/s
238
+ Step 1270/20005 | Loss: 3.4112 | LR: 4.97e-04 | 2480.6ms/step | 9.7 samples/s
239
+ Step 1280/20005 | Loss: 3.3534 | LR: 4.97e-04 | 2481.9ms/step | 9.7 samples/s
240
+ Step 1290/20005 | Loss: 3.4114 | LR: 4.97e-04 | 2482.4ms/step | 9.7 samples/s
241
+ Step 1300/20005 | Loss: 3.3480 | LR: 4.97e-04 | 2481.8ms/step | 9.7 samples/s
242
+ Step 1310/20005 | Loss: 3.3368 | LR: 4.96e-04 | 2481.3ms/step | 9.7 samples/s
243
+ Step 1320/20005 | Loss: 3.3049 | LR: 4.96e-04 | 2482.6ms/step | 9.7 samples/s
244
+ Step 1330/20005 | Loss: 3.4321 | LR: 4.96e-04 | 2480.4ms/step | 9.7 samples/s
245
+ Step 1340/20005 | Loss: 3.3693 | LR: 4.96e-04 | 2480.2ms/step | 9.7 samples/s
246
+ Step 1350/20005 | Loss: 3.3928 | LR: 4.96e-04 | 2482.5ms/step | 9.7 samples/s
247
+ Step 1360/20005 | Loss: 3.3521 | LR: 4.96e-04 | 2482.5ms/step | 9.7 samples/s
248
+ Step 1370/20005 | Loss: 3.3616 | LR: 4.96e-04 | 2481.4ms/step | 9.7 samples/s
249
+ Step 1380/20005 | Loss: 3.3294 | LR: 4.96e-04 | 2485.7ms/step | 9.7 samples/s
250
+ Step 1390/20005 | Loss: 3.2962 | LR: 4.96e-04 | 2482.7ms/step | 9.7 samples/s
251
+ Step 1400/20005 | Loss: 3.3390 | LR: 4.96e-04 | 2482.3ms/step | 9.7 samples/s
252
+ Step 1410/20005 | Loss: 3.3442 | LR: 4.96e-04 | 2478.7ms/step | 9.7 samples/s
253
+ Step 1420/20005 | Loss: 3.3456 | LR: 4.96e-04 | 2480.5ms/step | 9.7 samples/s
254
+ Step 1430/20005 | Loss: 3.3379 | LR: 4.96e-04 | 2482.0ms/step | 9.7 samples/s
255
+ Step 1440/20005 | Loss: 3.3419 | LR: 4.96e-04 | 2478.3ms/step | 9.7 samples/s
256
+ Step 1450/20005 | Loss: 3.3965 | LR: 4.95e-04 | 2480.7ms/step | 9.7 samples/s
257
+ Step 1460/20005 | Loss: 3.3223 | LR: 4.95e-04 | 2479.9ms/step | 9.7 samples/s
258
+ Step 1470/20005 | Loss: 3.3176 | LR: 4.95e-04 | 2479.8ms/step | 9.7 samples/s
259
+ Step 1480/20005 | Loss: 3.3046 | LR: 4.95e-04 | 2480.6ms/step | 9.7 samples/s
260
+ Step 1490/20005 | Loss: 3.2875 | LR: 4.95e-04 | 2480.2ms/step | 9.7 samples/s
261
+ Step 1500/20005 | Loss: 3.3622 | LR: 4.95e-04 | 2481.7ms/step | 9.7 samples/s
262
+ Validation at step 1500: Loss = 3.3429, Perplexity = 28.30
263
+ Step 1510/20005 | Loss: 3.2782 | LR: 4.95e-04 | 20084.7ms/step | 1.2 samples/s
264
+ Step 1520/20005 | Loss: 3.3071 | LR: 4.95e-04 | 2483.2ms/step | 9.7 samples/s
265
+ Step 1530/20005 | Loss: 3.3001 | LR: 4.95e-04 | 2481.1ms/step | 9.7 samples/s
266
+ Step 1540/20005 | Loss: 3.2394 | LR: 4.95e-04 | 2484.8ms/step | 9.7 samples/s
267
+ Step 1550/20005 | Loss: 3.3707 | LR: 4.95e-04 | 2484.1ms/step | 9.7 samples/s
268
+ Step 1560/20005 | Loss: 3.2867 | LR: 4.95e-04 | 2480.0ms/step | 9.7 samples/s
269
+ Step 1570/20005 | Loss: 3.2819 | LR: 4.95e-04 | 2481.8ms/step | 9.7 samples/s
270
+ Step 1580/20005 | Loss: 3.2988 | LR: 4.94e-04 | 2480.3ms/step | 9.7 samples/s
271
+ Step 1590/20005 | Loss: 3.2332 | LR: 4.94e-04 | 2480.0ms/step | 9.7 samples/s
272
+ Step 1600/20005 | Loss: 3.3096 | LR: 4.94e-04 | 2481.9ms/step | 9.7 samples/s
273
+ Step 1610/20005 | Loss: 3.2179 | LR: 4.94e-04 | 2484.5ms/step | 9.7 samples/s
274
+ Step 1620/20005 | Loss: 3.2699 | LR: 4.94e-04 | 2479.5ms/step | 9.7 samples/s
275
+ Step 1630/20005 | Loss: 3.2273 | LR: 4.94e-04 | 2482.2ms/step | 9.7 samples/s
276
+ Step 1640/20005 | Loss: 3.3416 | LR: 4.94e-04 | 2481.6ms/step | 9.7 samples/s
277
+ Step 1650/20005 | Loss: 3.3088 | LR: 4.94e-04 | 2483.9ms/step | 9.7 samples/s
278
+ Step 1660/20005 | Loss: 3.2767 | LR: 4.94e-04 | 2485.8ms/step | 9.7 samples/s
279
+ Step 1670/20005 | Loss: 3.1671 | LR: 4.94e-04 | 2485.2ms/step | 9.7 samples/s
280
+ Step 1680/20005 | Loss: 3.3102 | LR: 4.94e-04 | 2482.1ms/step | 9.7 samples/s
281
+ Step 1690/20005 | Loss: 3.1658 | LR: 4.93e-04 | 2480.9ms/step | 9.7 samples/s
282
+ Step 1700/20005 | Loss: 3.2453 | LR: 4.93e-04 | 2484.1ms/step | 9.7 samples/s
283
+ Step 1710/20005 | Loss: 3.1070 | LR: 4.93e-04 | 2483.1ms/step | 9.7 samples/s
284
+ Step 1720/20005 | Loss: 3.1972 | LR: 4.93e-04 | 2480.9ms/step | 9.7 samples/s
285
+ Step 1730/20005 | Loss: 3.2333 | LR: 4.93e-04 | 2480.2ms/step | 9.7 samples/s
286
+ Step 1740/20005 | Loss: 3.1906 | LR: 4.93e-04 | 2485.7ms/step | 9.7 samples/s
287
+ Step 1750/20005 | Loss: 3.2600 | LR: 4.93e-04 | 2484.0ms/step | 9.7 samples/s
288
+ Validation at step 1750: Loss = 3.2076, Perplexity = 24.72
289
+ Step 1760/20005 | Loss: 3.2116 | LR: 4.93e-04 | 20073.0ms/step | 1.2 samples/s
290
+ Step 1770/20005 | Loss: 3.2311 | LR: 4.93e-04 | 2482.8ms/step | 9.7 samples/s
291
+ Step 1780/20005 | Loss: 3.2018 | LR: 4.93e-04 | 2480.3ms/step | 9.7 samples/s
292
+ Step 1790/20005 | Loss: 3.1756 | LR: 4.93e-04 | 2480.4ms/step | 9.7 samples/s
293
+ Step 1800/20005 | Loss: 3.1392 | LR: 4.92e-04 | 2482.7ms/step | 9.7 samples/s
294
+ Step 1810/20005 | Loss: 3.2799 | LR: 4.92e-04 | 2482.7ms/step | 9.7 samples/s
295
+ Step 1820/20005 | Loss: 3.1527 | LR: 4.92e-04 | 2483.2ms/step | 9.7 samples/s
296
+ Step 1830/20005 | Loss: 3.2052 | LR: 4.92e-04 | 2582.8ms/step | 9.3 samples/s
297
+ Step 1840/20005 | Loss: 3.1211 | LR: 4.92e-04 | 2482.1ms/step | 9.7 samples/s
298
+ Step 1850/20005 | Loss: 3.1366 | LR: 4.92e-04 | 2483.4ms/step | 9.7 samples/s
299
+ Step 1860/20005 | Loss: 3.1339 | LR: 4.92e-04 | 2485.6ms/step | 9.7 samples/s
300
+ Step 1870/20005 | Loss: 3.1281 | LR: 4.92e-04 | 2481.2ms/step | 9.7 samples/s
301
+ Step 1880/20005 | Loss: 3.1423 | LR: 4.92e-04 | 2480.9ms/step | 9.7 samples/s
302
+ Step 1890/20005 | Loss: 3.1088 | LR: 4.92e-04 | 2484.9ms/step | 9.7 samples/s
303
+ Step 1900/20005 | Loss: 3.2010 | LR: 4.91e-04 | 2480.0ms/step | 9.7 samples/s
304
+ Step 1910/20005 | Loss: 3.1371 | LR: 4.91e-04 | 2481.1ms/step | 9.7 samples/s
305
+ Step 1920/20005 | Loss: 3.1164 | LR: 4.91e-04 | 2482.7ms/step | 9.7 samples/s
306
+ Step 1930/20005 | Loss: 3.1960 | LR: 4.91e-04 | 2482.0ms/step | 9.7 samples/s
307
+ Step 1940/20005 | Loss: 3.0022 | LR: 4.91e-04 | 2482.1ms/step | 9.7 samples/s
308
+ Step 1950/20005 | Loss: 3.1884 | LR: 4.91e-04 | 2484.2ms/step | 9.7 samples/s
309
+ Step 1960/20005 | Loss: 3.1131 | LR: 4.91e-04 | 2481.6ms/step | 9.7 samples/s
310
+ Step 1970/20005 | Loss: 3.1066 | LR: 4.91e-04 | 2480.0ms/step | 9.7 samples/s
311
+ Step 1980/20005 | Loss: 3.0815 | LR: 4.91e-04 | 2481.2ms/step | 9.7 samples/s
312
+ Step 1990/20005 | Loss: 3.0854 | LR: 4.90e-04 | 2485.2ms/step | 9.7 samples/s
313
+ Step 2000/20005 | Loss: 3.1739 | LR: 4.90e-04 | 2483.9ms/step | 9.7 samples/s
314
+ Validation at step 2000: Loss = 3.1330, Perplexity = 22.94
315
+ Saved checkpoint to checkpoints/run_20260506_171628/checkpoint-2000
316
+ Step 2010/20005 | Loss: 3.1899 | LR: 4.90e-04 | 20595.1ms/step | 1.2 samples/s
317
+ Step 2020/20005 | Loss: 3.1322 | LR: 4.90e-04 | 2483.0ms/step | 9.7 samples/s
318
+ Step 2030/20005 | Loss: 3.1557 | LR: 4.90e-04 | 2483.2ms/step | 9.7 samples/s
319
+ Step 2040/20005 | Loss: 3.1301 | LR: 4.90e-04 | 2481.5ms/step | 9.7 samples/s
320
+ Step 2050/20005 | Loss: 3.0754 | LR: 4.90e-04 | 2484.8ms/step | 9.7 samples/s
321
+ Step 2060/20005 | Loss: 3.0958 | LR: 4.90e-04 | 2482.1ms/step | 9.7 samples/s
322
+ Step 2070/20005 | Loss: 3.1173 | LR: 4.90e-04 | 2482.0ms/step | 9.7 samples/s
323
+ Step 2080/20005 | Loss: 3.1208 | LR: 4.89e-04 | 2479.9ms/step | 9.7 samples/s
324
+ Step 2090/20005 | Loss: 3.0896 | LR: 4.89e-04 | 2479.4ms/step | 9.7 samples/s
325
+ Step 2100/20005 | Loss: 3.0643 | LR: 4.89e-04 | 2480.4ms/step | 9.7 samples/s
326
+ Step 2110/20005 | Loss: 3.1077 | LR: 4.89e-04 | 2480.6ms/step | 9.7 samples/s
327
+ Step 2120/20005 | Loss: 3.0934 | LR: 4.89e-04 | 2481.7ms/step | 9.7 samples/s
328
+ Step 2130/20005 | Loss: 3.2026 | LR: 4.89e-04 | 2482.3ms/step | 9.7 samples/s
329
+ Step 2140/20005 | Loss: 2.9999 | LR: 4.89e-04 | 2481.5ms/step | 9.7 samples/s
330
+ Step 2150/20005 | Loss: 3.1597 | LR: 4.89e-04 | 2478.5ms/step | 9.7 samples/s
331
+ Step 2160/20005 | Loss: 3.1149 | LR: 4.89e-04 | 2479.0ms/step | 9.7 samples/s
332
+ Step 2170/20005 | Loss: 3.0599 | LR: 4.88e-04 | 2480.0ms/step | 9.7 samples/s
333
+ Step 2180/20005 | Loss: 3.0751 | LR: 4.88e-04 | 2478.7ms/step | 9.7 samples/s
334
+ Step 2190/20005 | Loss: 3.0508 | LR: 4.88e-04 | 2481.1ms/step | 9.7 samples/s
335
+ Step 2200/20005 | Loss: 3.0565 | LR: 4.88e-04 | 2482.9ms/step | 9.7 samples/s
336
+ Step 2210/20005 | Loss: 3.0679 | LR: 4.88e-04 | 2479.5ms/step | 9.7 samples/s
337
+ Step 2220/20005 | Loss: 3.0740 | LR: 4.88e-04 | 2480.0ms/step | 9.7 samples/s
338
+ Step 2230/20005 | Loss: 3.0267 | LR: 4.88e-04 | 2482.2ms/step | 9.7 samples/s
339
+ Step 2240/20005 | Loss: 3.0273 | LR: 4.88e-04 | 2480.1ms/step | 9.7 samples/s
340
+ Step 2250/20005 | Loss: 3.1010 | LR: 4.87e-04 | 2484.6ms/step | 9.7 samples/s
341
+ Validation at step 2250: Loss = 3.0659, Perplexity = 21.45
342
+ Step 2260/20005 | Loss: 3.0495 | LR: 4.87e-04 | 20211.6ms/step | 1.2 samples/s
343
+ Step 2270/20005 | Loss: 3.0487 | LR: 4.87e-04 | 2485.8ms/step | 9.7 samples/s
344
+ Step 2280/20005 | Loss: 3.0875 | LR: 4.87e-04 | 2485.9ms/step | 9.7 samples/s
345
+ Step 2290/20005 | Loss: 3.0081 | LR: 4.87e-04 | 2482.6ms/step | 9.7 samples/s
346
+ Step 2300/20005 | Loss: 3.0695 | LR: 4.87e-04 | 2485.7ms/step | 9.7 samples/s
347
+ Step 2310/20005 | Loss: 3.0367 | LR: 4.87e-04 | 2482.3ms/step | 9.7 samples/s
348
+ Step 2320/20005 | Loss: 2.9226 | LR: 4.87e-04 | 2481.8ms/step | 9.7 samples/s
349
+ Step 2330/20005 | Loss: 3.0387 | LR: 4.86e-04 | 2482.7ms/step | 9.7 samples/s
350
+ Step 2340/20005 | Loss: 3.0105 | LR: 4.86e-04 | 2484.5ms/step | 9.7 samples/s
351
+ Step 2350/20005 | Loss: 3.0209 | LR: 4.86e-04 | 2486.6ms/step | 9.7 samples/s
352
+ Step 2360/20005 | Loss: 3.0097 | LR: 4.86e-04 | 2490.4ms/step | 9.6 samples/s
353
+ Step 2370/20005 | Loss: 3.0632 | LR: 4.86e-04 | 2485.8ms/step | 9.7 samples/s
354
+ Step 2380/20005 | Loss: 3.0618 | LR: 4.86e-04 | 2485.3ms/step | 9.7 samples/s
355
+ Step 2390/20005 | Loss: 3.0674 | LR: 4.86e-04 | 2488.0ms/step | 9.6 samples/s
356
+ Step 2400/20005 | Loss: 3.0802 | LR: 4.86e-04 | 2485.4ms/step | 9.7 samples/s
357
+ Step 2410/20005 | Loss: 2.9627 | LR: 4.85e-04 | 2487.1ms/step | 9.6 samples/s
358
+ Step 2420/20005 | Loss: 3.0272 | LR: 4.85e-04 | 2485.9ms/step | 9.7 samples/s
359
+ Step 2430/20005 | Loss: 3.0372 | LR: 4.85e-04 | 2486.8ms/step | 9.7 samples/s
360
+ Step 2440/20005 | Loss: 3.1485 | LR: 4.85e-04 | 2483.2ms/step | 9.7 samples/s
361
+ Step 2450/20005 | Loss: 3.0323 | LR: 4.85e-04 | 2483.8ms/step | 9.7 samples/s
362
+ Step 2460/20005 | Loss: 2.9436 | LR: 4.85e-04 | 2484.7ms/step | 9.7 samples/s
363
+ Step 2470/20005 | Loss: 2.9838 | LR: 4.85e-04 | 2485.8ms/step | 9.7 samples/s
364
+ Step 2480/20005 | Loss: 3.0096 | LR: 4.84e-04 | 2485.2ms/step | 9.7 samples/s
365
+ Step 2490/20005 | Loss: 3.0263 | LR: 4.84e-04 | 2483.6ms/step | 9.7 samples/s
366
+ Step 2500/20005 | Loss: 3.0297 | LR: 4.84e-04 | 2486.2ms/step | 9.7 samples/s
367
+ Validation at step 2500: Loss = 3.0174, Perplexity = 20.44
368
+ Step 2510/20005 | Loss: 3.0233 | LR: 4.84e-04 | 20150.6ms/step | 1.2 samples/s
369
+ Step 2520/20005 | Loss: 2.9520 | LR: 4.84e-04 | 2486.3ms/step | 9.7 samples/s
370
+ Step 2530/20005 | Loss: 2.9792 | LR: 4.84e-04 | 2487.3ms/step | 9.6 samples/s
371
+ Step 2540/20005 | Loss: 3.0410 | LR: 4.84e-04 | 2481.8ms/step | 9.7 samples/s
372
+ Step 2550/20005 | Loss: 3.0117 | LR: 4.83e-04 | 2483.7ms/step | 9.7 samples/s
373
+ Step 2560/20005 | Loss: 2.9952 | LR: 4.83e-04 | 2482.3ms/step | 9.7 samples/s
374
+ Step 2570/20005 | Loss: 3.0182 | LR: 4.83e-04 | 2486.8ms/step | 9.7 samples/s
375
+ Step 2580/20005 | Loss: 2.9531 | LR: 4.83e-04 | 2484.5ms/step | 9.7 samples/s
376
+ Step 2590/20005 | Loss: 3.0715 | LR: 4.83e-04 | 2483.8ms/step | 9.7 samples/s
377
+ Step 2600/20005 | Loss: 2.9748 | LR: 4.83e-04 | 2486.1ms/step | 9.7 samples/s
378
+ Step 2610/20005 | Loss: 2.9719 | LR: 4.83e-04 | 2484.1ms/step | 9.7 samples/s
379
+ Step 2620/20005 | Loss: 3.0272 | LR: 4.82e-04 | 2641.3ms/step | 9.1 samples/s
380
+ Step 2630/20005 | Loss: 2.9623 | LR: 4.82e-04 | 2483.6ms/step | 9.7 samples/s
381
+ Step 2640/20005 | Loss: 2.9279 | LR: 4.82e-04 | 2487.8ms/step | 9.6 samples/s
382
+ Step 2650/20005 | Loss: 3.0301 | LR: 4.82e-04 | 2486.4ms/step | 9.7 samples/s
383
+ Step 2660/20005 | Loss: 2.9855 | LR: 4.82e-04 | 2485.6ms/step | 9.7 samples/s
384
+ Step 2670/20005 | Loss: 2.9391 | LR: 4.82e-04 | 2484.7ms/step | 9.7 samples/s
385
+ Step 2680/20005 | Loss: 2.9792 | LR: 4.82e-04 | 2491.4ms/step | 9.6 samples/s
386
+ Step 2690/20005 | Loss: 2.9522 | LR: 4.81e-04 | 2489.8ms/step | 9.6 samples/s
387
+ Step 2700/20005 | Loss: 2.9745 | LR: 4.81e-04 | 2487.5ms/step | 9.6 samples/s
388
+ Step 2710/20005 | Loss: 2.9118 | LR: 4.81e-04 | 2491.0ms/step | 9.6 samples/s
389
+ Step 2720/20005 | Loss: 2.9993 | LR: 4.81e-04 | 2478.8ms/step | 9.7 samples/s
390
+ Step 2730/20005 | Loss: 3.0726 | LR: 4.81e-04 | 2481.5ms/step | 9.7 samples/s
391
+ Step 2740/20005 | Loss: 2.9389 | LR: 4.81e-04 | 2486.7ms/step | 9.7 samples/s
392
+ Step 2750/20005 | Loss: 2.9551 | LR: 4.81e-04 | 2489.4ms/step | 9.6 samples/s
393
+ Validation at step 2750: Loss = 2.9700, Perplexity = 19.49
394
+ Step 2760/20005 | Loss: 2.9470 | LR: 4.80e-04 | 20153.4ms/step | 1.2 samples/s
395
+ Step 2770/20005 | Loss: 3.0599 | LR: 4.80e-04 | 2489.6ms/step | 9.6 samples/s
396
+ Step 2780/20005 | Loss: 2.9620 | LR: 4.80e-04 | 2484.6ms/step | 9.7 samples/s
397
+ Step 2790/20005 | Loss: 3.0053 | LR: 4.80e-04 | 2482.9ms/step | 9.7 samples/s
398
+ Step 2800/20005 | Loss: 2.9547 | LR: 4.80e-04 | 2484.7ms/step | 9.7 samples/s
399
+ Step 2810/20005 | Loss: 2.9998 | LR: 4.80e-04 | 2487.3ms/step | 9.6 samples/s
400
+ Step 2820/20005 | Loss: 3.0059 | LR: 4.79e-04 | 2488.8ms/step | 9.6 samples/s
401
+ Step 2830/20005 | Loss: 2.8953 | LR: 4.79e-04 | 2482.8ms/step | 9.7 samples/s
402
+ Step 2840/20005 | Loss: 2.9341 | LR: 4.79e-04 | 2487.6ms/step | 9.6 samples/s
403
+ Step 2850/20005 | Loss: 2.8997 | LR: 4.79e-04 | 2486.9ms/step | 9.7 samples/s
404
+ Step 2860/20005 | Loss: 2.8518 | LR: 4.79e-04 | 2486.5ms/step | 9.7 samples/s
405
+ Step 2870/20005 | Loss: 3.0818 | LR: 4.79e-04 | 2485.5ms/step | 9.7 samples/s
406
+ Step 2880/20005 | Loss: 2.9127 | LR: 4.78e-04 | 2482.8ms/step | 9.7 samples/s
407
+ Step 2890/20005 | Loss: 2.8781 | LR: 4.78e-04 | 2483.6ms/step | 9.7 samples/s
408
+ Step 2900/20005 | Loss: 2.9468 | LR: 4.78e-04 | 2484.3ms/step | 9.7 samples/s
409
+ Step 2910/20005 | Loss: 2.8944 | LR: 4.78e-04 | 2485.6ms/step | 9.7 samples/s
410
+ Step 2920/20005 | Loss: 2.9003 | LR: 4.78e-04 | 2487.8ms/step | 9.6 samples/s
411
+ Step 2930/20005 | Loss: 2.9508 | LR: 4.78e-04 | 2487.2ms/step | 9.6 samples/s
412
+ Step 2940/20005 | Loss: 2.9581 | LR: 4.77e-04 | 2484.5ms/step | 9.7 samples/s
413
+ Step 2950/20005 | Loss: 2.8649 | LR: 4.77e-04 | 2485.0ms/step | 9.7 samples/s
414
+ Step 2960/20005 | Loss: 2.9527 | LR: 4.77e-04 | 2488.1ms/step | 9.6 samples/s
415
+ Step 2970/20005 | Loss: 2.9560 | LR: 4.77e-04 | 2487.6ms/step | 9.6 samples/s
416
+ Step 2980/20005 | Loss: 2.9454 | LR: 4.77e-04 | 2485.0ms/step | 9.7 samples/s
417
+ Step 2990/20005 | Loss: 2.9172 | LR: 4.77e-04 | 2483.9ms/step | 9.7 samples/s
418
+ Step 3000/20005 | Loss: 2.8768 | LR: 4.76e-04 | 2483.7ms/step | 9.7 samples/s
419
+ Validation at step 3000: Loss = 2.9385, Perplexity = 18.89
420
+ Step 3010/20005 | Loss: 2.9027 | LR: 4.76e-04 | 20163.4ms/step | 1.2 samples/s
421
+ Step 3020/20005 | Loss: 2.8841 | LR: 4.76e-04 | 2490.1ms/step | 9.6 samples/s
422
+ Step 3030/20005 | Loss: 2.8952 | LR: 4.76e-04 | 2490.2ms/step | 9.6 samples/s
423
+ Step 3040/20005 | Loss: 2.9458 | LR: 4.76e-04 | 2486.3ms/step | 9.7 samples/s
424
+ Step 3050/20005 | Loss: 3.0163 | LR: 4.76e-04 | 2488.3ms/step | 9.6 samples/s
425
+ Step 3060/20005 | Loss: 2.8302 | LR: 4.75e-04 | 2483.8ms/step | 9.7 samples/s
426
+ Step 3070/20005 | Loss: 2.9645 | LR: 4.75e-04 | 2485.0ms/step | 9.7 samples/s
427
+ Step 3080/20005 | Loss: 2.8695 | LR: 4.75e-04 | 2486.8ms/step | 9.7 samples/s
428
+ Step 3090/20005 | Loss: 2.9766 | LR: 4.75e-04 | 2487.1ms/step | 9.6 samples/s
429
+ Step 3100/20005 | Loss: 2.9351 | LR: 4.75e-04 | 2488.3ms/step | 9.6 samples/s
430
+ Step 3110/20005 | Loss: 2.8867 | LR: 4.75e-04 | 2489.0ms/step | 9.6 samples/s
431
+ Step 3120/20005 | Loss: 2.9147 | LR: 4.74e-04 | 2486.7ms/step | 9.7 samples/s
432
+ Step 3130/20005 | Loss: 2.9249 | LR: 4.74e-04 | 2489.3ms/step | 9.6 samples/s
433
+ Step 3140/20005 | Loss: 2.9412 | LR: 4.74e-04 | 2487.1ms/step | 9.6 samples/s
434
+ Step 3150/20005 | Loss: 2.9122 | LR: 4.74e-04 | 2490.4ms/step | 9.6 samples/s
435
+ Step 3160/20005 | Loss: 2.9904 | LR: 4.74e-04 | 2489.2ms/step | 9.6 samples/s
436
+ Step 3170/20005 | Loss: 2.8860 | LR: 4.74e-04 | 2483.5ms/step | 9.7 samples/s
437
+ Step 3180/20005 | Loss: 2.8189 | LR: 4.73e-04 | 2489.9ms/step | 9.6 samples/s
438
+ Step 3190/20005 | Loss: 2.8932 | LR: 4.73e-04 | 2487.6ms/step | 9.6 samples/s
439
+ Step 3200/20005 | Loss: 2.9586 | LR: 4.73e-04 | 2482.2ms/step | 9.7 samples/s
440
+ Step 3210/20005 | Loss: 2.7848 | LR: 4.73e-04 | 2484.2ms/step | 9.7 samples/s
441
+ Step 3220/20005 | Loss: 2.8996 | LR: 4.73e-04 | 2485.9ms/step | 9.7 samples/s
442
+ Step 3230/20005 | Loss: 2.9051 | LR: 4.72e-04 | 2486.1ms/step | 9.7 samples/s
443
+ Step 3240/20005 | Loss: 2.8546 | LR: 4.72e-04 | 2484.9ms/step | 9.7 samples/s
444
+ Step 3250/20005 | Loss: 2.8317 | LR: 4.72e-04 | 2481.8ms/step | 9.7 samples/s
445
+ Validation at step 3250: Loss = 2.8979, Perplexity = 18.14
446
+ Step 3260/20005 | Loss: 2.8391 | LR: 4.72e-04 | 20153.4ms/step | 1.2 samples/s
447
+ Step 3270/20005 | Loss: 2.8688 | LR: 4.72e-04 | 2487.9ms/step | 9.6 samples/s
448
+ Step 3280/20005 | Loss: 2.8271 | LR: 4.72e-04 | 2486.2ms/step | 9.7 samples/s
449
+ Step 3290/20005 | Loss: 2.8906 | LR: 4.71e-04 | 2486.1ms/step | 9.7 samples/s
450
+ Step 3300/20005 | Loss: 2.8870 | LR: 4.71e-04 | 2486.2ms/step | 9.7 samples/s
451
+ Step 3310/20005 | Loss: 2.8872 | LR: 4.71e-04 | 2486.1ms/step | 9.7 samples/s
452
+ Step 3320/20005 | Loss: 2.8777 | LR: 4.71e-04 | 2485.0ms/step | 9.7 samples/s
453
+ Step 3330/20005 | Loss: 2.9292 | LR: 4.71e-04 | 2482.9ms/step | 9.7 samples/s
454
+ Step 3340/20005 | Loss: 2.9075 | LR: 4.70e-04 | 2486.3ms/step | 9.7 samples/s
455
+ Step 3350/20005 | Loss: 2.8418 | LR: 4.70e-04 | 2485.6ms/step | 9.7 samples/s
456
+ Step 3360/20005 | Loss: 2.8374 | LR: 4.70e-04 | 2485.8ms/step | 9.7 samples/s
457
+ Step 3370/20005 | Loss: 2.8806 | LR: 4.70e-04 | 2486.6ms/step | 9.7 samples/s
458
+ Step 3380/20005 | Loss: 2.8556 | LR: 4.70e-04 | 2486.1ms/step | 9.7 samples/s
459
+ Step 3390/20005 | Loss: 2.8850 | LR: 4.69e-04 | 2483.9ms/step | 9.7 samples/s
460
+ Step 3400/20005 | Loss: 2.7990 | LR: 4.69e-04 | 2487.9ms/step | 9.6 samples/s
461
+ Step 3410/20005 | Loss: 2.8183 | LR: 4.69e-04 | 2488.4ms/step | 9.6 samples/s
462
+ Step 3420/20005 | Loss: 2.8690 | LR: 4.69e-04 | 2585.4ms/step | 9.3 samples/s
463
+ Step 3430/20005 | Loss: 2.9224 | LR: 4.69e-04 | 2487.0ms/step | 9.7 samples/s
464
+ Step 3440/20005 | Loss: 2.8586 | LR: 4.69e-04 | 2490.0ms/step | 9.6 samples/s
465
+ Step 3450/20005 | Loss: 2.8856 | LR: 4.68e-04 | 2487.9ms/step | 9.6 samples/s
466
+ Step 3460/20005 | Loss: 2.8790 | LR: 4.68e-04 | 2486.1ms/step | 9.7 samples/s
467
+ Step 3470/20005 | Loss: 2.9077 | LR: 4.68e-04 | 2487.2ms/step | 9.6 samples/s
468
+ Step 3480/20005 | Loss: 2.8669 | LR: 4.68e-04 | 2487.2ms/step | 9.6 samples/s
469
+ Step 3490/20005 | Loss: 2.8640 | LR: 4.68e-04 | 2488.6ms/step | 9.6 samples/s
470
+ Step 3500/20005 | Loss: 2.7557 | LR: 4.67e-04 | 2491.2ms/step | 9.6 samples/s
471
+ Validation at step 3500: Loss = 2.8650, Perplexity = 17.55
472
+ Step 3510/20005 | Loss: 2.9039 | LR: 4.67e-04 | 20195.0ms/step | 1.2 samples/s
473
+ Step 3520/20005 | Loss: 2.8394 | LR: 4.67e-04 | 2491.3ms/step | 9.6 samples/s
474
+ Step 3530/20005 | Loss: 2.7960 | LR: 4.67e-04 | 2486.9ms/step | 9.7 samples/s
475
+ Step 3540/20005 | Loss: 2.8816 | LR: 4.67e-04 | 2486.8ms/step | 9.7 samples/s
476
+ Step 3550/20005 | Loss: 2.8719 | LR: 4.66e-04 | 2488.1ms/step | 9.6 samples/s
477
+ Step 3560/20005 | Loss: 2.7991 | LR: 4.66e-04 | 2488.5ms/step | 9.6 samples/s
478
+ Step 3570/20005 | Loss: 2.8148 | LR: 4.66e-04 | 2486.1ms/step | 9.7 samples/s
479
+ Step 3580/20005 | Loss: 2.7765 | LR: 4.66e-04 | 2488.7ms/step | 9.6 samples/s
480
+ Step 3590/20005 | Loss: 2.8594 | LR: 4.66e-04 | 2487.3ms/step | 9.6 samples/s
481
+ Step 3600/20005 | Loss: 2.8868 | LR: 4.65e-04 | 2487.5ms/step | 9.6 samples/s
482
+ Step 3610/20005 | Loss: 2.9076 | LR: 4.65e-04 | 2485.3ms/step | 9.7 samples/s
483
+ Step 3620/20005 | Loss: 2.8390 | LR: 4.65e-04 | 2490.2ms/step | 9.6 samples/s
484
+ Step 3630/20005 | Loss: 2.9418 | LR: 4.65e-04 | 2489.0ms/step | 9.6 samples/s
485
+ Step 3640/20005 | Loss: 2.8458 | LR: 4.65e-04 | 2488.1ms/step | 9.6 samples/s
486
+ Step 3650/20005 | Loss: 2.9035 | LR: 4.64e-04 | 2487.4ms/step | 9.6 samples/s
487
+ Step 3660/20005 | Loss: 2.8621 | LR: 4.64e-04 | 2486.7ms/step | 9.7 samples/s
488
+ Step 3670/20005 | Loss: 2.8240 | LR: 4.64e-04 | 2487.5ms/step | 9.6 samples/s
489
+ Step 3680/20005 | Loss: 2.7992 | LR: 4.64e-04 | 2486.5ms/step | 9.7 samples/s
490
+ Step 3690/20005 | Loss: 2.8206 | LR: 4.64e-04 | 2487.2ms/step | 9.6 samples/s
491
+ Step 3700/20005 | Loss: 2.9252 | LR: 4.63e-04 | 2488.6ms/step | 9.6 samples/s
492
+ Step 3710/20005 | Loss: 2.8458 | LR: 4.63e-04 | 2487.8ms/step | 9.6 samples/s
493
+ Step 3720/20005 | Loss: 2.7925 | LR: 4.63e-04 | 2486.8ms/step | 9.7 samples/s
494
+ Step 3730/20005 | Loss: 2.7895 | LR: 4.63e-04 | 2486.2ms/step | 9.7 samples/s
495
+ Step 3740/20005 | Loss: 2.8958 | LR: 4.62e-04 | 2486.5ms/step | 9.7 samples/s
496
+ Step 3750/20005 | Loss: 2.7859 | LR: 4.62e-04 | 2487.0ms/step | 9.7 samples/s
epfl_final/wandb/run-20260506_173734-nmzybrc7/files/requirements.txt ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pytz==2026.2
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.6.3
8
+ tzdata==2026.2
9
+ typing_extensions==4.15.0
10
+ triton==3.6.0
11
+ tqdm==4.67.3
12
+ torchinfo==1.8.0
13
+ sympy==1.14.0
14
+ smmap==5.0.3
15
+ six==1.17.0
16
+ shellingham==1.5.4
17
+ safetensors==0.7.0
18
+ regex==2026.4.4
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.4.1
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.2.6
38
+ networkx==3.4.2
39
+ mdurl==0.1.2
40
+ MarkupSafe==3.0.3
41
+ idna==3.13
42
+ hf-xet==1.5.0
43
+ h11==0.16.0
44
+ fsspec==2026.2.0
45
+ frozenlist==1.8.0
46
+ filelock==3.29.0
47
+ dill==0.4.1
48
+ cuda-pathfinder==1.5.4
49
+ click==8.3.3
50
+ charset-normalizer==3.4.7
51
+ certifi==2026.4.22
52
+ attrs==26.1.0
53
+ async-timeout==5.0.1
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ typing-inspection==0.4.2
58
+ sentry-sdk==2.59.0
59
+ requests==2.33.1
60
+ python-dateutil==2.9.0.post0
61
+ pydantic_core==2.46.4
62
+ nvidia-cusparse==12.6.3.3
63
+ nvidia-cufft==12.0.0.61
64
+ nvidia-cudnn-cu13==9.19.0.56
65
+ multiprocess==0.70.19
66
+ multidict==6.7.1
67
+ markdown-it-py==4.1.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ exceptiongroup==1.3.1
72
+ cuda-bindings==13.2.0
73
+ aiosignal==1.4.0
74
+ yarl==1.23.0
75
+ rich==15.0.0
76
+ pydantic==2.13.4
77
+ pandas==2.3.3
78
+ nvidia-cusolver==12.0.4.66
79
+ GitPython==3.1.50
80
+ anyio==4.13.0
81
+ wandb==0.26.1
82
+ typer==0.25.1
83
+ httpx==0.28.1
84
+ aiohttp==3.13.5
85
+ torch==2.11.0
86
+ huggingface_hub==1.14.0
87
+ tokenizers==0.22.2
88
+ datasets==4.8.5
89
+ accelerate==1.13.0
90
+ transformers==5.8.0
91
+ PyGObject==3.42.1
92
+ dbus-python==1.2.18
93
+ distro==1.7.0
94
+ pip==22.0.2
95
+ setuptools==59.6.0
96
+ ssh-import-id==5.11
97
+ wheel==0.37.1
epfl_final/wandb/run-20260506_173734-nmzybrc7/files/wandb-metadata.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.10.12",
4
+ "startedAt": "2026-05-06T17:37:34.812193Z",
5
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
6
+ "codePath": "SparseAtt/train.py",
7
+ "codePathLocal": "SparseAtt/train.py",
8
+ "email": "ruben.navasardyan@mail.utoronto.ca",
9
+ "root": "/mloscratch/homes/navasard/sparse",
10
+ "host": "h100-workspace-6-0-5",
11
+ "executable": "/usr/bin/python3",
12
+ "cpu_count": 96,
13
+ "cpu_count_logical": 192,
14
+ "gpu": "NVIDIA H100 80GB HBM3",
15
+ "gpu_count": 1,
16
+ "disk": {
17
+ "/": {
18
+ "total": "9374931091456",
19
+ "used": "6703238864896"
20
+ }
21
+ },
22
+ "memory": {
23
+ "total": "1623005741056"
24
+ },
25
+ "gpu_nvidia": [
26
+ {
27
+ "name": "NVIDIA H100 80GB HBM3",
28
+ "memoryTotal": "85520809984",
29
+ "cudaCores": 16896,
30
+ "architecture": "Hopper",
31
+ "uuid": "GPU-5cb9f8b7-4a13-4f85-d73e-6bcbebebefb8"
32
+ }
33
+ ],
34
+ "cudaVersion": "13.0",
35
+ "writerId": "r2idnc2nhy1o898ozhpgg24vyxnymq1c"
36
+ }
epfl_final/wandb/run-20260506_173734-nmzybrc7/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/wandb/run-20260506_173734-nmzybrc7/logs/debug.log ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-05-06 17:37:34,827 INFO MainThread:5753 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-05-06 17:37:34,827 INFO MainThread:5753 [wandb_setup.py:_flush():81] Configure stats pid to 5753
3
+ 2026-05-06 17:37:34,828 INFO MainThread:5753 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-05-06 17:37:34,829 INFO MainThread:5753 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260506_173734-nmzybrc7/logs/debug.log
5
+ 2026-05-06 17:37:34,829 INFO MainThread:5753 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260506_173734-nmzybrc7/logs/debug-internal.log
6
+ 2026-05-06 17:37:34,830 INFO MainThread:5753 [wandb_init.py:init():850] calling init triggers
7
+ 2026-05-06 17:37:34,830 INFO MainThread:5753 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/the_pile_train/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': './checkpoints/run_20260506_171628', 'resume_from_checkpoint': None, 'num_epochs': 1, 'max_steps': 20005, 'per_device_batch_size': 3, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 250, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 2000, 'eval_steps': 250, 'eval_batches': 200, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': None, 'wandb_tags': ['qwen3', '0.6b'], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 20, 'sparse_mask_pool_size': 1000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, '_wandb': {}}
9
+ 2026-05-06 17:37:34,831 INFO MainThread:5753 [wandb_init.py:init():898] starting backend
10
+ 2026-05-06 17:37:35,048 INFO MainThread:5753 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-05-06 17:37:35,985 INFO MainThread:5753 [wandb_init.py:init():918] backend started and connected
12
+ 2026-05-06 17:37:35,987 INFO MainThread:5753 [wandb_init.py:init():988] updated telemetry
13
+ 2026-05-06 17:37:35,988 INFO MainThread:5753 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-05-06 17:37:36,623 INFO MainThread:5753 [wandb_init.py:init():1056] starting run threads in backend
15
+ 2026-05-06 17:37:36,695 INFO MainThread:5753 [wandb_run.py:_console_start():2554] atexit reg
16
+ 2026-05-06 17:37:36,695 INFO MainThread:5753 [wandb_run.py:_redirect():2403] redirect: wrap_raw
17
+ 2026-05-06 17:37:36,696 INFO MainThread:5753 [wandb_run.py:_redirect():2472] Wrapping output streams.
18
+ 2026-05-06 17:37:36,697 INFO MainThread:5753 [wandb_run.py:_redirect():2495] Redirects installed.
19
+ 2026-05-06 17:37:36,699 INFO MainThread:5753 [wandb_init.py:init():1094] run started, returning control to user process
epfl_final/wandb/run-20260506_173734-nmzybrc7/run-nmzybrc7.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cbf3a13c8fd1c19cd8cab2e86b440c960baf0468fd5f5322ee96237f5f288bd2
3
+ size 1966080
epfl_final/wandb/run-20260506_200450-z06f2580/files/output.log ADDED
@@ -0,0 +1,1045 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ ================================================================================
4
+ MODEL ARCHITECTURE
5
+ ================================================================================
6
+ Model config:
7
+ vocab_size: 151936
8
+ hidden_size: 1024
9
+ num_hidden_layers: 28
10
+ num_attention_heads: 16
11
+ num_key_value_heads: 8
12
+ head_dim: 128
13
+ intermediate_size: 3072
14
+ max_position_embeddings: 32768
15
+ rope_theta: 1000000
16
+ attention_type: sparse_stochastic
17
+ sparse_block_size: 128
18
+ sparse_num_random_blocks: 31
19
+ sparse_mask_pool_size: 1000
20
+ sparse_max_seq_length: 8192
21
+
22
+ Parameter count:
23
+ Total parameters: 596,049,920
24
+ Trainable parameters: 596,049,920
25
+ Non-trainable parameters: 0
26
+ Model size (BF16): 1.19 GB
27
+
28
+ --------------------------------------------------------------------------------
29
+ LAYER-BY-LAYER SUMMARY
30
+ --------------------------------------------------------------------------------
31
+ /home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
32
+
33
+ SOLUTION: Use torch.compile(flex_attention)(...)
34
+
35
+ If you want to debug your score_mod/mask_mod, you can set:
36
+ torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
37
+
38
+ This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
39
+ _warn_once(
40
+ ================================================================================================================================================================
41
+ Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
42
+ ================================================================================================================================================================
43
+ Qwen3ForCausalLM [3, 8192] -- -- True
44
+ ├─Qwen3Model: 1-1 -- [3, 8192, 1024] -- True
45
+ │ └─Embedding: 2-1 [3, 8192] [3, 8192, 1024] 155,582,464 True
46
+ │ └─ModuleList: 2-2 -- -- -- True
47
+ │ │ └─Qwen3DecoderLayer: 3-1 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
48
+ │ │ └─Qwen3DecoderLayer: 3-2 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
49
+ │ │ └─Qwen3DecoderLayer: 3-3 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
50
+ │ │ └─Qwen3DecoderLayer: 3-4 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
51
+ │ │ └─Qwen3DecoderLayer: 3-5 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
52
+ │ │ └─Qwen3DecoderLayer: 3-6 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
53
+ │ │ └─Qwen3DecoderLayer: 3-7 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
54
+ │ │ └─Qwen3DecoderLayer: 3-8 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
55
+ │ │ └─Qwen3DecoderLayer: 3-9 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
56
+ │ │ └─Qwen3DecoderLayer: 3-10 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
57
+ │ │ └─Qwen3DecoderLayer: 3-11 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
58
+ │ │ └─Qwen3DecoderLayer: 3-12 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
59
+ │ │ └─Qwen3DecoderLayer: 3-13 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
60
+ │ │ └─Qwen3DecoderLayer: 3-14 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
61
+ │ │ └─Qwen3DecoderLayer: 3-15 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
62
+ │ │ └─Qwen3DecoderLayer: 3-16 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
63
+ │ │ └─Qwen3DecoderLayer: 3-17 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
64
+ │ │ └─Qwen3DecoderLayer: 3-18 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
65
+ │ │ └─Qwen3DecoderLayer: 3-19 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
66
+ │ │ └─Qwen3DecoderLayer: 3-20 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
67
+ │ │ └─Qwen3DecoderLayer: 3-21 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
68
+ │ │ └─Qwen3DecoderLayer: 3-22 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
69
+ │ │ └─Qwen3DecoderLayer: 3-23 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
70
+ │ │ └─Qwen3DecoderLayer: 3-24 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
71
+ │ │ └─Qwen3DecoderLayer: 3-25 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
72
+ │ │ └─Qwen3DecoderLayer: 3-26 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
73
+ │ │ └─Qwen3DecoderLayer: 3-27 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
74
+ │ │ └─Qwen3DecoderLayer: 3-28 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
75
+ │ └─Qwen3RMSNorm: 2-3 [3, 8192, 1024] [3, 8192, 1024] 1,024 True
76
+ ├─Linear: 1-2 [3, 8192, 1024] [3, 8192, 151936] 155,582,464 True
77
+ ================================================================================================================================================================
78
+ Total params: 751,632,384
79
+ Trainable params: 751,632,384
80
+ Non-trainable params: 0
81
+ Total mult-adds (G): 2.25
82
+ ================================================================================================================================================================
83
+ Input size (MB): 0.20
84
+ Forward/backward pass size (MB): 63052.97
85
+ Params size (MB): 1503.26
86
+ Estimated Total Size (MB): 64556.43
87
+ ================================================================================================================================================================
88
+ ================================================================================
89
+
90
+ Loading dataset from /mloscratch/homes/navasard/data/the_pile_train/tokenized...
91
+ Using pre-tokenized dataset (no runtime tokenization)
92
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/the_pile_train/tokenized...
93
+ Loaded 5315146 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1328786)
94
+ Dataset split: 1063028 train, 265758 validation
95
+ Training examples (packed, train split): 1063028
96
+ Validation examples (packed, val split): 265758
97
+ Per-device batch size: 3
98
+ World size (GPUs): 1
99
+ Gradient accumulation steps: 8
100
+ Effective batch size (seqs/step): 24
101
+ Sequence length: 8192
102
+ Tokens per optimizer step: 196,608
103
+ Micro-batches per epoch (per rank): 354342
104
+ Optimizer steps per epoch: 44292
105
+ Total optimizer steps: 20005 (capped by max_steps; 0.45 epochs)
106
+ Starting training...
107
+ Step 10/20005 | Loss: 12.0989 | LR: 2.00e-05 | 10410.8ms/step | 2.3 samples/s
108
+ Step 20/20005 | Loss: 11.5447 | LR: 4.00e-05 | 2730.6ms/step | 8.8 samples/s
109
+ Step 30/20005 | Loss: 10.6770 | LR: 6.00e-05 | 2728.6ms/step | 8.8 samples/s
110
+ Step 40/20005 | Loss: 10.1986 | LR: 8.00e-05 | 2726.3ms/step | 8.8 samples/s
111
+ Step 50/20005 | Loss: 9.3747 | LR: 1.00e-04 | 2718.2ms/step | 8.8 samples/s
112
+ Step 60/20005 | Loss: 8.6205 | LR: 1.20e-04 | 2719.9ms/step | 8.8 samples/s
113
+ Step 70/20005 | Loss: 8.0284 | LR: 1.40e-04 | 2724.0ms/step | 8.8 samples/s
114
+ Step 80/20005 | Loss: 7.6163 | LR: 1.60e-04 | 2728.9ms/step | 8.8 samples/s
115
+ Step 90/20005 | Loss: 7.6306 | LR: 1.80e-04 | 2736.0ms/step | 8.8 samples/s
116
+ Step 100/20005 | Loss: 7.3455 | LR: 2.00e-04 | 2738.1ms/step | 8.8 samples/s
117
+ Step 110/20005 | Loss: 7.2508 | LR: 2.20e-04 | 2746.7ms/step | 8.7 samples/s
118
+ Step 120/20005 | Loss: 7.1419 | LR: 2.40e-04 | 2746.8ms/step | 8.7 samples/s
119
+ Step 130/20005 | Loss: 6.9745 | LR: 2.60e-04 | 2748.9ms/step | 8.7 samples/s
120
+ Step 140/20005 | Loss: 6.8496 | LR: 2.80e-04 | 2748.7ms/step | 8.7 samples/s
121
+ Step 150/20005 | Loss: 6.6922 | LR: 3.00e-04 | 2748.5ms/step | 8.7 samples/s
122
+ Step 160/20005 | Loss: 6.5952 | LR: 3.20e-04 | 2750.2ms/step | 8.7 samples/s
123
+ Step 170/20005 | Loss: 6.5753 | LR: 3.40e-04 | 2747.4ms/step | 8.7 samples/s
124
+ Step 180/20005 | Loss: 6.4846 | LR: 3.60e-04 | 2749.7ms/step | 8.7 samples/s
125
+ Step 190/20005 | Loss: 6.2716 | LR: 3.80e-04 | 2750.8ms/step | 8.7 samples/s
126
+ Step 200/20005 | Loss: 6.3449 | LR: 4.00e-04 | 2753.6ms/step | 8.7 samples/s
127
+ Step 210/20005 | Loss: 6.1360 | LR: 4.20e-04 | 2751.2ms/step | 8.7 samples/s
128
+ Step 220/20005 | Loss: 6.1078 | LR: 4.40e-04 | 2752.5ms/step | 8.7 samples/s
129
+ Step 230/20005 | Loss: 6.0445 | LR: 4.60e-04 | 2752.1ms/step | 8.7 samples/s
130
+ Step 240/20005 | Loss: 6.0357 | LR: 4.80e-04 | 2754.3ms/step | 8.7 samples/s
131
+ Step 250/20005 | Loss: 5.9084 | LR: 5.00e-04 | 2751.2ms/step | 8.7 samples/s
132
+ Validation at step 250: Loss = 5.9291, Perplexity = 375.83
133
+ Step 260/20005 | Loss: 5.8606 | LR: 5.00e-04 | 25384.1ms/step | 0.9 samples/s
134
+ Step 270/20005 | Loss: 5.8475 | LR: 5.00e-04 | 2758.6ms/step | 8.7 samples/s
135
+ Step 280/20005 | Loss: 5.8624 | LR: 5.00e-04 | 2758.3ms/step | 8.7 samples/s
136
+ Step 290/20005 | Loss: 5.7226 | LR: 5.00e-04 | 2760.3ms/step | 8.7 samples/s
137
+ Step 300/20005 | Loss: 5.7034 | LR: 5.00e-04 | 2757.5ms/step | 8.7 samples/s
138
+ Step 310/20005 | Loss: 5.6323 | LR: 5.00e-04 | 2758.7ms/step | 8.7 samples/s
139
+ Step 320/20005 | Loss: 5.5215 | LR: 5.00e-04 | 2758.0ms/step | 8.7 samples/s
140
+ Step 330/20005 | Loss: 5.4634 | LR: 5.00e-04 | 2760.2ms/step | 8.7 samples/s
141
+ Step 340/20005 | Loss: 5.2741 | LR: 5.00e-04 | 2762.0ms/step | 8.7 samples/s
142
+ Step 350/20005 | Loss: 5.2777 | LR: 5.00e-04 | 2759.4ms/step | 8.7 samples/s
143
+ Step 360/20005 | Loss: 5.3587 | LR: 5.00e-04 | 2759.2ms/step | 8.7 samples/s
144
+ Step 370/20005 | Loss: 5.1093 | LR: 5.00e-04 | 2766.5ms/step | 8.7 samples/s
145
+ Step 380/20005 | Loss: 5.1675 | LR: 5.00e-04 | 2762.0ms/step | 8.7 samples/s
146
+ Step 390/20005 | Loss: 5.1510 | LR: 5.00e-04 | 2765.5ms/step | 8.7 samples/s
147
+ Step 400/20005 | Loss: 5.0181 | LR: 5.00e-04 | 2763.6ms/step | 8.7 samples/s
148
+ Step 410/20005 | Loss: 5.0407 | LR: 5.00e-04 | 2760.6ms/step | 8.7 samples/s
149
+ Step 420/20005 | Loss: 4.8463 | LR: 5.00e-04 | 2759.6ms/step | 8.7 samples/s
150
+ Step 430/20005 | Loss: 4.9312 | LR: 5.00e-04 | 2757.2ms/step | 8.7 samples/s
151
+ Step 440/20005 | Loss: 4.7367 | LR: 5.00e-04 | 2757.2ms/step | 8.7 samples/s
152
+ Step 450/20005 | Loss: 4.8000 | LR: 5.00e-04 | 2760.0ms/step | 8.7 samples/s
153
+ Step 460/20005 | Loss: 4.7938 | LR: 5.00e-04 | 2764.0ms/step | 8.7 samples/s
154
+ Step 470/20005 | Loss: 4.7001 | LR: 5.00e-04 | 2764.6ms/step | 8.7 samples/s
155
+ Step 480/20005 | Loss: 4.6015 | LR: 5.00e-04 | 2762.7ms/step | 8.7 samples/s
156
+ Step 490/20005 | Loss: 4.5671 | LR: 5.00e-04 | 2762.0ms/step | 8.7 samples/s
157
+ Step 500/20005 | Loss: 4.4572 | LR: 5.00e-04 | 2764.6ms/step | 8.7 samples/s
158
+ Validation at step 500: Loss = 4.5364, Perplexity = 93.35
159
+ Step 510/20005 | Loss: 4.5766 | LR: 5.00e-04 | 21967.2ms/step | 1.1 samples/s
160
+ Step 520/20005 | Loss: 4.4948 | LR: 5.00e-04 | 2759.1ms/step | 8.7 samples/s
161
+ Step 530/20005 | Loss: 4.4216 | LR: 5.00e-04 | 2761.1ms/step | 8.7 samples/s
162
+ Step 540/20005 | Loss: 4.4558 | LR: 5.00e-04 | 2761.3ms/step | 8.7 samples/s
163
+ Step 550/20005 | Loss: 4.3696 | LR: 5.00e-04 | 2761.9ms/step | 8.7 samples/s
164
+ Step 560/20005 | Loss: 4.3700 | LR: 5.00e-04 | 2760.0ms/step | 8.7 samples/s
165
+ Step 570/20005 | Loss: 4.2611 | LR: 5.00e-04 | 2759.5ms/step | 8.7 samples/s
166
+ Step 580/20005 | Loss: 4.2245 | LR: 5.00e-04 | 2760.0ms/step | 8.7 samples/s
167
+ Step 590/20005 | Loss: 4.2480 | LR: 5.00e-04 | 2760.9ms/step | 8.7 samples/s
168
+ Step 600/20005 | Loss: 4.2026 | LR: 5.00e-04 | 2760.8ms/step | 8.7 samples/s
169
+ Step 610/20005 | Loss: 4.1299 | LR: 5.00e-04 | 2759.2ms/step | 8.7 samples/s
170
+ Step 620/20005 | Loss: 4.1015 | LR: 5.00e-04 | 2763.8ms/step | 8.7 samples/s
171
+ Step 630/20005 | Loss: 4.0786 | LR: 5.00e-04 | 2763.7ms/step | 8.7 samples/s
172
+ Step 640/20005 | Loss: 4.1684 | LR: 5.00e-04 | 2765.4ms/step | 8.7 samples/s
173
+ Step 650/20005 | Loss: 4.1388 | LR: 4.99e-04 | 2762.8ms/step | 8.7 samples/s
174
+ Step 660/20005 | Loss: 4.1002 | LR: 4.99e-04 | 2765.0ms/step | 8.7 samples/s
175
+ Step 670/20005 | Loss: 3.9987 | LR: 4.99e-04 | 2766.6ms/step | 8.7 samples/s
176
+ Step 680/20005 | Loss: 3.9821 | LR: 4.99e-04 | 2765.3ms/step | 8.7 samples/s
177
+ Step 690/20005 | Loss: 3.8683 | LR: 4.99e-04 | 2763.5ms/step | 8.7 samples/s
178
+ Step 700/20005 | Loss: 3.9370 | LR: 4.99e-04 | 2758.8ms/step | 8.7 samples/s
179
+ Step 710/20005 | Loss: 3.9901 | LR: 4.99e-04 | 2760.0ms/step | 8.7 samples/s
180
+ Step 720/20005 | Loss: 3.9598 | LR: 4.99e-04 | 2762.6ms/step | 8.7 samples/s
181
+ Step 730/20005 | Loss: 3.9113 | LR: 4.99e-04 | 2759.9ms/step | 8.7 samples/s
182
+ Step 740/20005 | Loss: 3.8747 | LR: 4.99e-04 | 2764.2ms/step | 8.7 samples/s
183
+ Step 750/20005 | Loss: 3.8602 | LR: 4.99e-04 | 2759.8ms/step | 8.7 samples/s
184
+ Validation at step 750: Loss = 3.8865, Perplexity = 48.74
185
+ Step 760/20005 | Loss: 3.8667 | LR: 4.99e-04 | 21963.8ms/step | 1.1 samples/s
186
+ Step 770/20005 | Loss: 3.8535 | LR: 4.99e-04 | 2761.2ms/step | 8.7 samples/s
187
+ Step 780/20005 | Loss: 3.8783 | LR: 4.99e-04 | 2760.9ms/step | 8.7 samples/s
188
+ Step 790/20005 | Loss: 3.7873 | LR: 4.99e-04 | 2760.0ms/step | 8.7 samples/s
189
+ Step 800/20005 | Loss: 3.7243 | LR: 4.99e-04 | 2759.9ms/step | 8.7 samples/s
190
+ Step 810/20005 | Loss: 3.7751 | LR: 4.99e-04 | 2759.9ms/step | 8.7 samples/s
191
+ Step 820/20005 | Loss: 3.7464 | LR: 4.99e-04 | 2758.2ms/step | 8.7 samples/s
192
+ Step 830/20005 | Loss: 3.7759 | LR: 4.99e-04 | 2763.4ms/step | 8.7 samples/s
193
+ Step 840/20005 | Loss: 3.7637 | LR: 4.99e-04 | 2882.4ms/step | 8.3 samples/s
194
+ Step 850/20005 | Loss: 3.7363 | LR: 4.99e-04 | 2762.8ms/step | 8.7 samples/s
195
+ Step 860/20005 | Loss: 3.7785 | LR: 4.99e-04 | 2767.0ms/step | 8.7 samples/s
196
+ Step 870/20005 | Loss: 3.7385 | LR: 4.99e-04 | 2763.1ms/step | 8.7 samples/s
197
+ Step 880/20005 | Loss: 3.6628 | LR: 4.99e-04 | 2763.5ms/step | 8.7 samples/s
198
+ Step 890/20005 | Loss: 3.5887 | LR: 4.99e-04 | 2765.1ms/step | 8.7 samples/s
199
+ Step 900/20005 | Loss: 3.6553 | LR: 4.99e-04 | 2763.9ms/step | 8.7 samples/s
200
+ Step 910/20005 | Loss: 3.6443 | LR: 4.99e-04 | 2765.2ms/step | 8.7 samples/s
201
+ Step 920/20005 | Loss: 3.6155 | LR: 4.99e-04 | 2765.1ms/step | 8.7 samples/s
202
+ Step 930/20005 | Loss: 3.5568 | LR: 4.99e-04 | 2765.5ms/step | 8.7 samples/s
203
+ Step 940/20005 | Loss: 3.6848 | LR: 4.98e-04 | 2767.2ms/step | 8.7 samples/s
204
+ Step 950/20005 | Loss: 3.5370 | LR: 4.98e-04 | 2764.4ms/step | 8.7 samples/s
205
+ Step 960/20005 | Loss: 3.6071 | LR: 4.98e-04 | 2765.6ms/step | 8.7 samples/s
206
+ Step 970/20005 | Loss: 3.5247 | LR: 4.98e-04 | 2760.6ms/step | 8.7 samples/s
207
+ Step 980/20005 | Loss: 3.5528 | LR: 4.98e-04 | 2762.4ms/step | 8.7 samples/s
208
+ Step 990/20005 | Loss: 3.5869 | LR: 4.98e-04 | 2763.2ms/step | 8.7 samples/s
209
+ Step 1000/20005 | Loss: 3.5220 | LR: 4.98e-04 | 2765.2ms/step | 8.7 samples/s
210
+ Validation at step 1000: Loss = 3.5638, Perplexity = 35.30
211
+ Step 1010/20005 | Loss: 3.5427 | LR: 4.98e-04 | 21969.3ms/step | 1.1 samples/s
212
+ Step 1020/20005 | Loss: 3.5228 | LR: 4.98e-04 | 2764.6ms/step | 8.7 samples/s
213
+ Step 1030/20005 | Loss: 3.5350 | LR: 4.98e-04 | 2768.7ms/step | 8.7 samples/s
214
+ Step 1040/20005 | Loss: 3.5662 | LR: 4.98e-04 | 2765.9ms/step | 8.7 samples/s
215
+ Step 1050/20005 | Loss: 3.4038 | LR: 4.98e-04 | 2762.1ms/step | 8.7 samples/s
216
+ Step 1060/20005 | Loss: 3.5186 | LR: 4.98e-04 | 2770.1ms/step | 8.7 samples/s
217
+ Step 1070/20005 | Loss: 3.4663 | LR: 4.98e-04 | 2765.6ms/step | 8.7 samples/s
218
+ Step 1080/20005 | Loss: 3.5870 | LR: 4.98e-04 | 2764.3ms/step | 8.7 samples/s
219
+ Step 1090/20005 | Loss: 3.5031 | LR: 4.98e-04 | 2761.2ms/step | 8.7 samples/s
220
+ Step 1100/20005 | Loss: 3.4334 | LR: 4.98e-04 | 2761.7ms/step | 8.7 samples/s
221
+ Step 1110/20005 | Loss: 3.5186 | LR: 4.98e-04 | 2760.7ms/step | 8.7 samples/s
222
+ Step 1120/20005 | Loss: 3.5534 | LR: 4.98e-04 | 2761.2ms/step | 8.7 samples/s
223
+ Step 1130/20005 | Loss: 3.4956 | LR: 4.98e-04 | 2760.9ms/step | 8.7 samples/s
224
+ Step 1140/20005 | Loss: 3.4621 | LR: 4.98e-04 | 2760.2ms/step | 8.7 samples/s
225
+ Step 1150/20005 | Loss: 3.4725 | LR: 4.97e-04 | 2761.9ms/step | 8.7 samples/s
226
+ Step 1160/20005 | Loss: 3.4526 | LR: 4.97e-04 | 2766.1ms/step | 8.7 samples/s
227
+ Step 1170/20005 | Loss: 3.4601 | LR: 4.97e-04 | 2771.7ms/step | 8.7 samples/s
228
+ Step 1180/20005 | Loss: 3.4587 | LR: 4.97e-04 | 2770.3ms/step | 8.7 samples/s
229
+ Step 1190/20005 | Loss: 3.4606 | LR: 4.97e-04 | 2763.5ms/step | 8.7 samples/s
230
+ Step 1200/20005 | Loss: 3.3753 | LR: 4.97e-04 | 2764.2ms/step | 8.7 samples/s
231
+ Step 1210/20005 | Loss: 3.3043 | LR: 4.97e-04 | 2768.2ms/step | 8.7 samples/s
232
+ Step 1220/20005 | Loss: 3.4626 | LR: 4.97e-04 | 2768.1ms/step | 8.7 samples/s
233
+ Step 1230/20005 | Loss: 3.3684 | LR: 4.97e-04 | 2766.0ms/step | 8.7 samples/s
234
+ Step 1240/20005 | Loss: 3.4586 | LR: 4.97e-04 | 2764.4ms/step | 8.7 samples/s
235
+ Step 1250/20005 | Loss: 3.2903 | LR: 4.97e-04 | 2764.3ms/step | 8.7 samples/s
236
+ Validation at step 1250: Loss = 3.3759, Perplexity = 29.25
237
+ Step 1260/20005 | Loss: 3.3458 | LR: 4.97e-04 | 21969.4ms/step | 1.1 samples/s
238
+ Step 1270/20005 | Loss: 3.3751 | LR: 4.97e-04 | 2762.5ms/step | 8.7 samples/s
239
+ Step 1280/20005 | Loss: 3.3108 | LR: 4.97e-04 | 2766.7ms/step | 8.7 samples/s
240
+ Step 1290/20005 | Loss: 3.3678 | LR: 4.97e-04 | 2764.9ms/step | 8.7 samples/s
241
+ Step 1300/20005 | Loss: 3.2969 | LR: 4.97e-04 | 2766.5ms/step | 8.7 samples/s
242
+ Step 1310/20005 | Loss: 3.2972 | LR: 4.96e-04 | 2763.8ms/step | 8.7 samples/s
243
+ Step 1320/20005 | Loss: 3.2614 | LR: 4.96e-04 | 2766.0ms/step | 8.7 samples/s
244
+ Step 1330/20005 | Loss: 3.3838 | LR: 4.96e-04 | 2768.1ms/step | 8.7 samples/s
245
+ Step 1340/20005 | Loss: 3.3408 | LR: 4.96e-04 | 2765.0ms/step | 8.7 samples/s
246
+ Step 1350/20005 | Loss: 3.3330 | LR: 4.96e-04 | 2769.1ms/step | 8.7 samples/s
247
+ Step 1360/20005 | Loss: 3.2903 | LR: 4.96e-04 | 2766.2ms/step | 8.7 samples/s
248
+ Step 1370/20005 | Loss: 3.3151 | LR: 4.96e-04 | 2764.7ms/step | 8.7 samples/s
249
+ Step 1380/20005 | Loss: 3.2862 | LR: 4.96e-04 | 2762.6ms/step | 8.7 samples/s
250
+ Step 1390/20005 | Loss: 3.2497 | LR: 4.96e-04 | 2759.5ms/step | 8.7 samples/s
251
+ Step 1400/20005 | Loss: 3.3090 | LR: 4.96e-04 | 2762.3ms/step | 8.7 samples/s
252
+ Step 1410/20005 | Loss: 3.3080 | LR: 4.96e-04 | 2766.6ms/step | 8.7 samples/s
253
+ Step 1420/20005 | Loss: 3.3029 | LR: 4.96e-04 | 2761.4ms/step | 8.7 samples/s
254
+ Step 1430/20005 | Loss: 3.3020 | LR: 4.96e-04 | 2761.2ms/step | 8.7 samples/s
255
+ Step 1440/20005 | Loss: 3.2956 | LR: 4.96e-04 | 2770.4ms/step | 8.7 samples/s
256
+ Step 1450/20005 | Loss: 3.3466 | LR: 4.95e-04 | 2766.4ms/step | 8.7 samples/s
257
+ Step 1460/20005 | Loss: 3.2753 | LR: 4.95e-04 | 2762.0ms/step | 8.7 samples/s
258
+ Step 1470/20005 | Loss: 3.2611 | LR: 4.95e-04 | 2759.9ms/step | 8.7 samples/s
259
+ Step 1480/20005 | Loss: 3.2540 | LR: 4.95e-04 | 2760.5ms/step | 8.7 samples/s
260
+ Step 1490/20005 | Loss: 3.2429 | LR: 4.95e-04 | 2766.5ms/step | 8.7 samples/s
261
+ Step 1500/20005 | Loss: 3.2993 | LR: 4.95e-04 | 2761.7ms/step | 8.7 samples/s
262
+ Validation at step 1500: Loss = 3.2771, Perplexity = 26.50
263
+ Step 1510/20005 | Loss: 3.2156 | LR: 4.95e-04 | 21965.5ms/step | 1.1 samples/s
264
+ Step 1520/20005 | Loss: 3.2559 | LR: 4.95e-04 | 2766.6ms/step | 8.7 samples/s
265
+ Step 1530/20005 | Loss: 3.2496 | LR: 4.95e-04 | 2762.0ms/step | 8.7 samples/s
266
+ Step 1540/20005 | Loss: 3.1913 | LR: 4.95e-04 | 2761.2ms/step | 8.7 samples/s
267
+ Step 1550/20005 | Loss: 3.3294 | LR: 4.95e-04 | 2762.6ms/step | 8.7 samples/s
268
+ Step 1560/20005 | Loss: 3.2475 | LR: 4.95e-04 | 2789.0ms/step | 8.6 samples/s
269
+ Step 1570/20005 | Loss: 3.2430 | LR: 4.95e-04 | 2780.6ms/step | 8.6 samples/s
270
+ Step 1580/20005 | Loss: 3.2574 | LR: 4.94e-04 | 2770.3ms/step | 8.7 samples/s
271
+ Step 1590/20005 | Loss: 3.1857 | LR: 4.94e-04 | 2767.9ms/step | 8.7 samples/s
272
+ Step 1600/20005 | Loss: 3.2606 | LR: 4.94e-04 | 2763.0ms/step | 8.7 samples/s
273
+ Step 1610/20005 | Loss: 3.1693 | LR: 4.94e-04 | 2762.6ms/step | 8.7 samples/s
274
+ Step 1620/20005 | Loss: 3.2251 | LR: 4.94e-04 | 2761.9ms/step | 8.7 samples/s
275
+ Step 1630/20005 | Loss: 3.1801 | LR: 4.94e-04 | 2760.1ms/step | 8.7 samples/s
276
+ Step 1640/20005 | Loss: 3.2999 | LR: 4.94e-04 | 2759.9ms/step | 8.7 samples/s
277
+ Step 1650/20005 | Loss: 3.2658 | LR: 4.94e-04 | 2765.9ms/step | 8.7 samples/s
278
+ Step 1660/20005 | Loss: 3.2310 | LR: 4.94e-04 | 2760.0ms/step | 8.7 samples/s
279
+ Step 1670/20005 | Loss: 3.1202 | LR: 4.94e-04 | 2765.1ms/step | 8.7 samples/s
280
+ Step 1680/20005 | Loss: 3.2660 | LR: 4.94e-04 | 2767.8ms/step | 8.7 samples/s
281
+ Step 1690/20005 | Loss: 3.1254 | LR: 4.93e-04 | 2764.7ms/step | 8.7 samples/s
282
+ Step 1700/20005 | Loss: 3.2053 | LR: 4.93e-04 | 2866.8ms/step | 8.4 samples/s
283
+ Step 1710/20005 | Loss: 3.0661 | LR: 4.93e-04 | 2766.2ms/step | 8.7 samples/s
284
+ Step 1720/20005 | Loss: 3.1562 | LR: 4.93e-04 | 2763.9ms/step | 8.7 samples/s
285
+ Step 1730/20005 | Loss: 3.1911 | LR: 4.93e-04 | 2766.0ms/step | 8.7 samples/s
286
+ Step 1740/20005 | Loss: 3.1470 | LR: 4.93e-04 | 2762.8ms/step | 8.7 samples/s
287
+ Step 1750/20005 | Loss: 3.2204 | LR: 4.93e-04 | 2769.6ms/step | 8.7 samples/s
288
+ Validation at step 1750: Loss = 3.1644, Perplexity = 23.67
289
+ Step 1760/20005 | Loss: 3.1679 | LR: 4.93e-04 | 21967.9ms/step | 1.1 samples/s
290
+ Step 1770/20005 | Loss: 3.1877 | LR: 4.93e-04 | 2766.5ms/step | 8.7 samples/s
291
+ Step 1780/20005 | Loss: 3.1568 | LR: 4.93e-04 | 2761.7ms/step | 8.7 samples/s
292
+ Step 1790/20005 | Loss: 3.1350 | LR: 4.93e-04 | 2763.6ms/step | 8.7 samples/s
293
+ Step 1800/20005 | Loss: 3.0924 | LR: 4.92e-04 | 2766.1ms/step | 8.7 samples/s
294
+ Step 1810/20005 | Loss: 3.2371 | LR: 4.92e-04 | 2763.6ms/step | 8.7 samples/s
295
+ Step 1820/20005 | Loss: 3.1143 | LR: 4.92e-04 | 2765.1ms/step | 8.7 samples/s
296
+ Step 1830/20005 | Loss: 3.1666 | LR: 4.92e-04 | 2763.8ms/step | 8.7 samples/s
297
+ Step 1840/20005 | Loss: 3.0805 | LR: 4.92e-04 | 2762.7ms/step | 8.7 samples/s
298
+ Step 1850/20005 | Loss: 3.0941 | LR: 4.92e-04 | 2761.4ms/step | 8.7 samples/s
299
+ Step 1860/20005 | Loss: 3.0905 | LR: 4.92e-04 | 2761.4ms/step | 8.7 samples/s
300
+ Step 1870/20005 | Loss: 3.0872 | LR: 4.92e-04 | 2761.6ms/step | 8.7 samples/s
301
+ Step 1880/20005 | Loss: 3.0970 | LR: 4.92e-04 | 2763.1ms/step | 8.7 samples/s
302
+ Step 1890/20005 | Loss: 3.0578 | LR: 4.92e-04 | 2759.9ms/step | 8.7 samples/s
303
+ Step 1900/20005 | Loss: 3.1589 | LR: 4.91e-04 | 2762.7ms/step | 8.7 samples/s
304
+ Step 1910/20005 | Loss: 3.0947 | LR: 4.91e-04 | 2768.7ms/step | 8.7 samples/s
305
+ Step 1920/20005 | Loss: 3.0868 | LR: 4.91e-04 | 2766.5ms/step | 8.7 samples/s
306
+ Step 1930/20005 | Loss: 3.1592 | LR: 4.91e-04 | 2769.9ms/step | 8.7 samples/s
307
+ Step 1940/20005 | Loss: 2.9666 | LR: 4.91e-04 | 2765.5ms/step | 8.7 samples/s
308
+ Step 1950/20005 | Loss: 3.1476 | LR: 4.91e-04 | 2768.1ms/step | 8.7 samples/s
309
+ Step 1960/20005 | Loss: 3.0691 | LR: 4.91e-04 | 2762.9ms/step | 8.7 samples/s
310
+ Step 1970/20005 | Loss: 3.0647 | LR: 4.91e-04 | 2764.2ms/step | 8.7 samples/s
311
+ Step 1980/20005 | Loss: 3.0428 | LR: 4.91e-04 | 2766.0ms/step | 8.7 samples/s
312
+ Step 1990/20005 | Loss: 3.0410 | LR: 4.90e-04 | 2762.5ms/step | 8.7 samples/s
313
+ Step 2000/20005 | Loss: 3.1315 | LR: 4.90e-04 | 2760.8ms/step | 8.7 samples/s
314
+ Validation at step 2000: Loss = 3.0909, Perplexity = 22.00
315
+ Saved checkpoint to checkpoints/run_20260506_193527/checkpoint-2000
316
+ Step 2010/20005 | Loss: 3.1517 | LR: 4.90e-04 | 22555.4ms/step | 1.1 samples/s
317
+ Step 2020/20005 | Loss: 3.0912 | LR: 4.90e-04 | 2760.8ms/step | 8.7 samples/s
318
+ Step 2030/20005 | Loss: 3.1107 | LR: 4.90e-04 | 2768.6ms/step | 8.7 samples/s
319
+ Step 2040/20005 | Loss: 3.0892 | LR: 4.90e-04 | 2770.8ms/step | 8.7 samples/s
320
+ Step 2050/20005 | Loss: 3.0426 | LR: 4.90e-04 | 2764.5ms/step | 8.7 samples/s
321
+ Step 2060/20005 | Loss: 3.0659 | LR: 4.90e-04 | 2769.1ms/step | 8.7 samples/s
322
+ Step 2070/20005 | Loss: 3.0791 | LR: 4.90e-04 | 2761.1ms/step | 8.7 samples/s
323
+ Step 2080/20005 | Loss: 3.0813 | LR: 4.89e-04 | 2763.9ms/step | 8.7 samples/s
324
+ Step 2090/20005 | Loss: 3.0492 | LR: 4.89e-04 | 2765.0ms/step | 8.7 samples/s
325
+ Step 2100/20005 | Loss: 3.0255 | LR: 4.89e-04 | 2760.9ms/step | 8.7 samples/s
326
+ Step 2110/20005 | Loss: 3.0726 | LR: 4.89e-04 | 2760.3ms/step | 8.7 samples/s
327
+ Step 2120/20005 | Loss: 3.0578 | LR: 4.89e-04 | 2761.6ms/step | 8.7 samples/s
328
+ Step 2130/20005 | Loss: 3.1649 | LR: 4.89e-04 | 2762.0ms/step | 8.7 samples/s
329
+ Step 2140/20005 | Loss: 2.9619 | LR: 4.89e-04 | 2766.5ms/step | 8.7 samples/s
330
+ Step 2150/20005 | Loss: 3.1198 | LR: 4.89e-04 | 2761.9ms/step | 8.7 samples/s
331
+ Step 2160/20005 | Loss: 3.0763 | LR: 4.89e-04 | 2762.1ms/step | 8.7 samples/s
332
+ Step 2170/20005 | Loss: 3.0274 | LR: 4.88e-04 | 2762.1ms/step | 8.7 samples/s
333
+ Step 2180/20005 | Loss: 3.0355 | LR: 4.88e-04 | 2761.5ms/step | 8.7 samples/s
334
+ Step 2190/20005 | Loss: 3.0168 | LR: 4.88e-04 | 2760.2ms/step | 8.7 samples/s
335
+ Step 2200/20005 | Loss: 3.0221 | LR: 4.88e-04 | 2761.2ms/step | 8.7 samples/s
336
+ Step 2210/20005 | Loss: 3.0307 | LR: 4.88e-04 | 2761.1ms/step | 8.7 samples/s
337
+ Step 2220/20005 | Loss: 3.0357 | LR: 4.88e-04 | 2759.8ms/step | 8.7 samples/s
338
+ Step 2230/20005 | Loss: 2.9930 | LR: 4.88e-04 | 2765.7ms/step | 8.7 samples/s
339
+ Step 2240/20005 | Loss: 2.9925 | LR: 4.88e-04 | 2765.9ms/step | 8.7 samples/s
340
+ Step 2250/20005 | Loss: 3.0709 | LR: 4.87e-04 | 2761.2ms/step | 8.7 samples/s
341
+ Validation at step 2250: Loss = 3.0346, Perplexity = 20.79
342
+ Step 2260/20005 | Loss: 3.0123 | LR: 4.87e-04 | 21969.4ms/step | 1.1 samples/s
343
+ Step 2270/20005 | Loss: 3.0142 | LR: 4.87e-04 | 2765.3ms/step | 8.7 samples/s
344
+ Step 2280/20005 | Loss: 3.0513 | LR: 4.87e-04 | 2760.0ms/step | 8.7 samples/s
345
+ Step 2290/20005 | Loss: 2.9671 | LR: 4.87e-04 | 2766.9ms/step | 8.7 samples/s
346
+ Step 2300/20005 | Loss: 3.0316 | LR: 4.87e-04 | 2761.9ms/step | 8.7 samples/s
347
+ Step 2310/20005 | Loss: 2.9959 | LR: 4.87e-04 | 2765.3ms/step | 8.7 samples/s
348
+ Step 2320/20005 | Loss: 2.8850 | LR: 4.87e-04 | 2764.4ms/step | 8.7 samples/s
349
+ Step 2330/20005 | Loss: 3.0029 | LR: 4.86e-04 | 2763.6ms/step | 8.7 samples/s
350
+ Step 2340/20005 | Loss: 2.9750 | LR: 4.86e-04 | 2760.9ms/step | 8.7 samples/s
351
+ Step 2350/20005 | Loss: 2.9845 | LR: 4.86e-04 | 2776.0ms/step | 8.6 samples/s
352
+ Step 2360/20005 | Loss: 2.9698 | LR: 4.86e-04 | 2769.0ms/step | 8.7 samples/s
353
+ Step 2370/20005 | Loss: 3.0263 | LR: 4.86e-04 | 2769.0ms/step | 8.7 samples/s
354
+ Step 2380/20005 | Loss: 3.0247 | LR: 4.86e-04 | 2764.6ms/step | 8.7 samples/s
355
+ Step 2390/20005 | Loss: 3.0306 | LR: 4.86e-04 | 2769.2ms/step | 8.7 samples/s
356
+ Step 2400/20005 | Loss: 3.0468 | LR: 4.86e-04 | 2767.2ms/step | 8.7 samples/s
357
+ Step 2410/20005 | Loss: 2.9311 | LR: 4.85e-04 | 2763.1ms/step | 8.7 samples/s
358
+ Step 2420/20005 | Loss: 2.9804 | LR: 4.85e-04 | 2769.6ms/step | 8.7 samples/s
359
+ Step 2430/20005 | Loss: 2.9943 | LR: 4.85e-04 | 2763.9ms/step | 8.7 samples/s
360
+ Step 2440/20005 | Loss: 3.1119 | LR: 4.85e-04 | 2761.8ms/step | 8.7 samples/s
361
+ Step 2450/20005 | Loss: 2.9980 | LR: 4.85e-04 | 2770.0ms/step | 8.7 samples/s
362
+ Step 2460/20005 | Loss: 2.9099 | LR: 4.85e-04 | 2768.0ms/step | 8.7 samples/s
363
+ Step 2470/20005 | Loss: 2.9476 | LR: 4.85e-04 | 2764.8ms/step | 8.7 samples/s
364
+ Step 2480/20005 | Loss: 2.9730 | LR: 4.84e-04 | 2766.5ms/step | 8.7 samples/s
365
+ Step 2490/20005 | Loss: 2.9905 | LR: 4.84e-04 | 2763.6ms/step | 8.7 samples/s
366
+ Step 2500/20005 | Loss: 2.9910 | LR: 4.84e-04 | 2763.6ms/step | 8.7 samples/s
367
+ Validation at step 2500: Loss = 2.9803, Perplexity = 19.69
368
+ Step 2510/20005 | Loss: 2.9858 | LR: 4.84e-04 | 21960.4ms/step | 1.1 samples/s
369
+ Step 2520/20005 | Loss: 2.9145 | LR: 4.84e-04 | 2769.6ms/step | 8.7 samples/s
370
+ Step 2530/20005 | Loss: 2.9410 | LR: 4.84e-04 | 2767.0ms/step | 8.7 samples/s
371
+ Step 2540/20005 | Loss: 3.0019 | LR: 4.84e-04 | 2763.3ms/step | 8.7 samples/s
372
+ Step 2550/20005 | Loss: 2.9718 | LR: 4.83e-04 | 2864.3ms/step | 8.4 samples/s
373
+ Step 2560/20005 | Loss: 2.9669 | LR: 4.83e-04 | 2760.7ms/step | 8.7 samples/s
374
+ Step 2570/20005 | Loss: 2.9860 | LR: 4.83e-04 | 2763.3ms/step | 8.7 samples/s
375
+ Step 2580/20005 | Loss: 2.9181 | LR: 4.83e-04 | 2763.4ms/step | 8.7 samples/s
376
+ Step 2590/20005 | Loss: 3.0322 | LR: 4.83e-04 | 2763.0ms/step | 8.7 samples/s
377
+ Step 2600/20005 | Loss: 2.9227 | LR: 4.83e-04 | 2763.2ms/step | 8.7 samples/s
378
+ Step 2610/20005 | Loss: 2.9246 | LR: 4.83e-04 | 2760.5ms/step | 8.7 samples/s
379
+ Step 2620/20005 | Loss: 2.9878 | LR: 4.82e-04 | 2760.3ms/step | 8.7 samples/s
380
+ Step 2630/20005 | Loss: 2.9312 | LR: 4.82e-04 | 2762.5ms/step | 8.7 samples/s
381
+ Step 2640/20005 | Loss: 2.8957 | LR: 4.82e-04 | 2764.8ms/step | 8.7 samples/s
382
+ Step 2650/20005 | Loss: 2.9927 | LR: 4.82e-04 | 2767.0ms/step | 8.7 samples/s
383
+ Step 2660/20005 | Loss: 2.9486 | LR: 4.82e-04 | 2766.1ms/step | 8.7 samples/s
384
+ Step 2670/20005 | Loss: 2.9007 | LR: 4.82e-04 | 2763.3ms/step | 8.7 samples/s
385
+ Step 2680/20005 | Loss: 2.9492 | LR: 4.82e-04 | 2760.4ms/step | 8.7 samples/s
386
+ Step 2690/20005 | Loss: 2.9333 | LR: 4.81e-04 | 2761.1ms/step | 8.7 samples/s
387
+ Step 2700/20005 | Loss: 2.9460 | LR: 4.81e-04 | 2760.4ms/step | 8.7 samples/s
388
+ Step 2710/20005 | Loss: 2.8776 | LR: 4.81e-04 | 2761.4ms/step | 8.7 samples/s
389
+ Step 2720/20005 | Loss: 2.9657 | LR: 4.81e-04 | 2760.6ms/step | 8.7 samples/s
390
+ Step 2730/20005 | Loss: 3.0355 | LR: 4.81e-04 | 2762.1ms/step | 8.7 samples/s
391
+ Step 2740/20005 | Loss: 2.9029 | LR: 4.81e-04 | 2761.8ms/step | 8.7 samples/s
392
+ Step 2750/20005 | Loss: 2.9181 | LR: 4.81e-04 | 2765.1ms/step | 8.7 samples/s
393
+ Validation at step 2750: Loss = 2.9368, Perplexity = 18.85
394
+ Step 2760/20005 | Loss: 2.9134 | LR: 4.80e-04 | 21967.9ms/step | 1.1 samples/s
395
+ Step 2770/20005 | Loss: 3.0252 | LR: 4.80e-04 | 2763.5ms/step | 8.7 samples/s
396
+ Step 2780/20005 | Loss: 2.9300 | LR: 4.80e-04 | 2761.7ms/step | 8.7 samples/s
397
+ Step 2790/20005 | Loss: 2.9710 | LR: 4.80e-04 | 2760.1ms/step | 8.7 samples/s
398
+ Step 2800/20005 | Loss: 2.9183 | LR: 4.80e-04 | 2761.2ms/step | 8.7 samples/s
399
+ Step 2810/20005 | Loss: 2.9678 | LR: 4.80e-04 | 2759.7ms/step | 8.7 samples/s
400
+ Step 2820/20005 | Loss: 2.9757 | LR: 4.79e-04 | 2763.7ms/step | 8.7 samples/s
401
+ Step 2830/20005 | Loss: 2.8640 | LR: 4.79e-04 | 2763.4ms/step | 8.7 samples/s
402
+ Step 2840/20005 | Loss: 2.9046 | LR: 4.79e-04 | 2764.8ms/step | 8.7 samples/s
403
+ Step 2850/20005 | Loss: 2.8705 | LR: 4.79e-04 | 2763.1ms/step | 8.7 samples/s
404
+ Step 2860/20005 | Loss: 2.8204 | LR: 4.79e-04 | 2761.7ms/step | 8.7 samples/s
405
+ Step 2870/20005 | Loss: 3.0452 | LR: 4.79e-04 | 2763.8ms/step | 8.7 samples/s
406
+ Step 2880/20005 | Loss: 2.8760 | LR: 4.78e-04 | 2767.9ms/step | 8.7 samples/s
407
+ Step 2890/20005 | Loss: 2.8423 | LR: 4.78e-04 | 2762.7ms/step | 8.7 samples/s
408
+ Step 2900/20005 | Loss: 2.9118 | LR: 4.78e-04 | 2759.0ms/step | 8.7 samples/s
409
+ Step 2910/20005 | Loss: 2.8610 | LR: 4.78e-04 | 2761.0ms/step | 8.7 samples/s
410
+ Step 2920/20005 | Loss: 2.8671 | LR: 4.78e-04 | 2762.2ms/step | 8.7 samples/s
411
+ Step 2930/20005 | Loss: 2.9199 | LR: 4.78e-04 | 2766.6ms/step | 8.7 samples/s
412
+ Step 2940/20005 | Loss: 2.9219 | LR: 4.77e-04 | 2762.7ms/step | 8.7 samples/s
413
+ Step 2950/20005 | Loss: 2.8329 | LR: 4.77e-04 | 2765.1ms/step | 8.7 samples/s
414
+ Step 2960/20005 | Loss: 2.9131 | LR: 4.77e-04 | 2767.9ms/step | 8.7 samples/s
415
+ Step 2970/20005 | Loss: 2.9111 | LR: 4.77e-04 | 2763.7ms/step | 8.7 samples/s
416
+ Step 2980/20005 | Loss: 2.8993 | LR: 4.77e-04 | 2767.1ms/step | 8.7 samples/s
417
+ Step 2990/20005 | Loss: 2.8830 | LR: 4.77e-04 | 2766.4ms/step | 8.7 samples/s
418
+ Step 3000/20005 | Loss: 2.8453 | LR: 4.76e-04 | 2760.2ms/step | 8.7 samples/s
419
+ Validation at step 3000: Loss = 2.9063, Perplexity = 18.29
420
+ Step 3010/20005 | Loss: 2.8743 | LR: 4.76e-04 | 21966.8ms/step | 1.1 samples/s
421
+ Step 3020/20005 | Loss: 2.8530 | LR: 4.76e-04 | 2765.9ms/step | 8.7 samples/s
422
+ Step 3030/20005 | Loss: 2.8624 | LR: 4.76e-04 | 2760.9ms/step | 8.7 samples/s
423
+ Step 3040/20005 | Loss: 2.9200 | LR: 4.76e-04 | 2766.2ms/step | 8.7 samples/s
424
+ Step 3050/20005 | Loss: 2.9876 | LR: 4.76e-04 | 2766.9ms/step | 8.7 samples/s
425
+ Step 3060/20005 | Loss: 2.7992 | LR: 4.75e-04 | 2762.8ms/step | 8.7 samples/s
426
+ Step 3070/20005 | Loss: 2.9328 | LR: 4.75e-04 | 2762.1ms/step | 8.7 samples/s
427
+ Step 3080/20005 | Loss: 2.8540 | LR: 4.75e-04 | 2769.2ms/step | 8.7 samples/s
428
+ Step 3090/20005 | Loss: 2.9479 | LR: 4.75e-04 | 2767.4ms/step | 8.7 samples/s
429
+ Step 3100/20005 | Loss: 2.9021 | LR: 4.75e-04 | 2762.0ms/step | 8.7 samples/s
430
+ Step 3110/20005 | Loss: 2.8555 | LR: 4.75e-04 | 2761.7ms/step | 8.7 samples/s
431
+ Step 3120/20005 | Loss: 2.8876 | LR: 4.74e-04 | 2762.2ms/step | 8.7 samples/s
432
+ Step 3130/20005 | Loss: 2.8961 | LR: 4.74e-04 | 2764.5ms/step | 8.7 samples/s
433
+ Step 3140/20005 | Loss: 2.9067 | LR: 4.74e-04 | 2763.4ms/step | 8.7 samples/s
434
+ Step 3150/20005 | Loss: 2.8788 | LR: 4.74e-04 | 2764.3ms/step | 8.7 samples/s
435
+ Step 3160/20005 | Loss: 2.9573 | LR: 4.74e-04 | 2762.8ms/step | 8.7 samples/s
436
+ Step 3170/20005 | Loss: 2.8525 | LR: 4.74e-04 | 2762.0ms/step | 8.7 samples/s
437
+ Step 3180/20005 | Loss: 2.7888 | LR: 4.73e-04 | 2761.9ms/step | 8.7 samples/s
438
+ Step 3190/20005 | Loss: 2.8620 | LR: 4.73e-04 | 2766.2ms/step | 8.7 samples/s
439
+ Step 3200/20005 | Loss: 2.9266 | LR: 4.73e-04 | 2769.9ms/step | 8.7 samples/s
440
+ Step 3210/20005 | Loss: 2.7545 | LR: 4.73e-04 | 2762.7ms/step | 8.7 samples/s
441
+ Step 3220/20005 | Loss: 2.8694 | LR: 4.73e-04 | 2765.4ms/step | 8.7 samples/s
442
+ Step 3230/20005 | Loss: 2.8794 | LR: 4.72e-04 | 2764.2ms/step | 8.7 samples/s
443
+ Step 3240/20005 | Loss: 2.8260 | LR: 4.72e-04 | 2768.3ms/step | 8.7 samples/s
444
+ Step 3250/20005 | Loss: 2.8028 | LR: 4.72e-04 | 2761.4ms/step | 8.7 samples/s
445
+ Validation at step 3250: Loss = 2.8664, Perplexity = 17.57
446
+ Step 3260/20005 | Loss: 2.8071 | LR: 4.72e-04 | 21963.0ms/step | 1.1 samples/s
447
+ Step 3270/20005 | Loss: 2.8391 | LR: 4.72e-04 | 2764.4ms/step | 8.7 samples/s
448
+ Step 3280/20005 | Loss: 2.7943 | LR: 4.72e-04 | 2763.9ms/step | 8.7 samples/s
449
+ Step 3290/20005 | Loss: 2.8577 | LR: 4.71e-04 | 2763.8ms/step | 8.7 samples/s
450
+ Step 3300/20005 | Loss: 2.8545 | LR: 4.71e-04 | 2764.3ms/step | 8.7 samples/s
451
+ Step 3310/20005 | Loss: 2.8542 | LR: 4.71e-04 | 2764.5ms/step | 8.7 samples/s
452
+ Step 3320/20005 | Loss: 2.8469 | LR: 4.71e-04 | 2762.4ms/step | 8.7 samples/s
453
+ Step 3330/20005 | Loss: 2.8972 | LR: 4.71e-04 | 2763.8ms/step | 8.7 samples/s
454
+ Step 3340/20005 | Loss: 2.8781 | LR: 4.70e-04 | 2761.6ms/step | 8.7 samples/s
455
+ Step 3350/20005 | Loss: 2.8152 | LR: 4.70e-04 | 2762.0ms/step | 8.7 samples/s
456
+ Step 3360/20005 | Loss: 2.8059 | LR: 4.70e-04 | 2760.8ms/step | 8.7 samples/s
457
+ Step 3370/20005 | Loss: 2.8505 | LR: 4.70e-04 | 2761.3ms/step | 8.7 samples/s
458
+ Step 3380/20005 | Loss: 2.8135 | LR: 4.70e-04 | 2762.0ms/step | 8.7 samples/s
459
+ Step 3390/20005 | Loss: 2.8540 | LR: 4.69e-04 | 2760.3ms/step | 8.7 samples/s
460
+ Step 3400/20005 | Loss: 2.7695 | LR: 4.69e-04 | 2763.5ms/step | 8.7 samples/s
461
+ Step 3410/20005 | Loss: 2.7939 | LR: 4.69e-04 | 2866.2ms/step | 8.4 samples/s
462
+ Step 3420/20005 | Loss: 2.8421 | LR: 4.69e-04 | 2757.8ms/step | 8.7 samples/s
463
+ Step 3430/20005 | Loss: 2.8984 | LR: 4.69e-04 | 2760.3ms/step | 8.7 samples/s
464
+ Step 3440/20005 | Loss: 2.8325 | LR: 4.69e-04 | 2759.6ms/step | 8.7 samples/s
465
+ Step 3450/20005 | Loss: 2.8575 | LR: 4.68e-04 | 2761.9ms/step | 8.7 samples/s
466
+ Step 3460/20005 | Loss: 2.8511 | LR: 4.68e-04 | 2761.0ms/step | 8.7 samples/s
467
+ Step 3470/20005 | Loss: 2.8791 | LR: 4.68e-04 | 2767.3ms/step | 8.7 samples/s
468
+ Step 3480/20005 | Loss: 2.8410 | LR: 4.68e-04 | 2765.0ms/step | 8.7 samples/s
469
+ Step 3490/20005 | Loss: 2.8381 | LR: 4.68e-04 | 2764.3ms/step | 8.7 samples/s
470
+ Step 3500/20005 | Loss: 2.7285 | LR: 4.67e-04 | 2767.9ms/step | 8.7 samples/s
471
+ Validation at step 3500: Loss = 2.8363, Perplexity = 17.05
472
+ Step 3510/20005 | Loss: 2.8776 | LR: 4.67e-04 | 22009.6ms/step | 1.1 samples/s
473
+ Step 3520/20005 | Loss: 2.8103 | LR: 4.67e-04 | 2771.0ms/step | 8.7 samples/s
474
+ Step 3530/20005 | Loss: 2.7635 | LR: 4.67e-04 | 2767.7ms/step | 8.7 samples/s
475
+ Step 3540/20005 | Loss: 2.8527 | LR: 4.67e-04 | 2771.5ms/step | 8.7 samples/s
476
+ Step 3550/20005 | Loss: 2.8427 | LR: 4.66e-04 | 2769.3ms/step | 8.7 samples/s
477
+ Step 3560/20005 | Loss: 2.7690 | LR: 4.66e-04 | 2765.6ms/step | 8.7 samples/s
478
+ Step 3570/20005 | Loss: 2.7835 | LR: 4.66e-04 | 2764.7ms/step | 8.7 samples/s
479
+ Step 3580/20005 | Loss: 2.7442 | LR: 4.66e-04 | 2764.3ms/step | 8.7 samples/s
480
+ Step 3590/20005 | Loss: 2.8302 | LR: 4.66e-04 | 2765.2ms/step | 8.7 samples/s
481
+ Step 3600/20005 | Loss: 2.8532 | LR: 4.65e-04 | 2770.6ms/step | 8.7 samples/s
482
+ Step 3610/20005 | Loss: 2.8796 | LR: 4.65e-04 | 2770.1ms/step | 8.7 samples/s
483
+ Step 3620/20005 | Loss: 2.8090 | LR: 4.65e-04 | 2764.6ms/step | 8.7 samples/s
484
+ Step 3630/20005 | Loss: 2.9118 | LR: 4.65e-04 | 2771.4ms/step | 8.7 samples/s
485
+ Step 3640/20005 | Loss: 2.8181 | LR: 4.65e-04 | 2768.8ms/step | 8.7 samples/s
486
+ Step 3650/20005 | Loss: 2.8757 | LR: 4.64e-04 | 2770.3ms/step | 8.7 samples/s
487
+ Step 3660/20005 | Loss: 2.8329 | LR: 4.64e-04 | 2765.5ms/step | 8.7 samples/s
488
+ Step 3670/20005 | Loss: 2.7951 | LR: 4.64e-04 | 2765.2ms/step | 8.7 samples/s
489
+ Step 3680/20005 | Loss: 2.7720 | LR: 4.64e-04 | 2764.7ms/step | 8.7 samples/s
490
+ Step 3690/20005 | Loss: 2.7908 | LR: 4.64e-04 | 2764.4ms/step | 8.7 samples/s
491
+ Step 3700/20005 | Loss: 2.8953 | LR: 4.63e-04 | 2765.3ms/step | 8.7 samples/s
492
+ Step 3710/20005 | Loss: 2.8195 | LR: 4.63e-04 | 2766.3ms/step | 8.7 samples/s
493
+ Step 3720/20005 | Loss: 2.7612 | LR: 4.63e-04 | 2767.9ms/step | 8.7 samples/s
494
+ Step 3730/20005 | Loss: 2.7585 | LR: 4.63e-04 | 2769.5ms/step | 8.7 samples/s
495
+ Step 3740/20005 | Loss: 2.8642 | LR: 4.62e-04 | 2768.7ms/step | 8.7 samples/s
496
+ Step 3750/20005 | Loss: 2.7607 | LR: 4.62e-04 | 2768.1ms/step | 8.7 samples/s
497
+ Validation at step 3750: Loss = 2.8104, Perplexity = 16.62
498
+ Step 3760/20005 | Loss: 2.8152 | LR: 4.62e-04 | 21971.3ms/step | 1.1 samples/s
499
+ Step 3770/20005 | Loss: 2.8430 | LR: 4.62e-04 | 2764.8ms/step | 8.7 samples/s
500
+ Step 3780/20005 | Loss: 2.8141 | LR: 4.62e-04 | 2765.9ms/step | 8.7 samples/s
501
+ Step 3790/20005 | Loss: 2.9068 | LR: 4.61e-04 | 2765.4ms/step | 8.7 samples/s
502
+ Step 3800/20005 | Loss: 2.7418 | LR: 4.61e-04 | 2769.0ms/step | 8.7 samples/s
503
+ Step 3810/20005 | Loss: 2.8539 | LR: 4.61e-04 | 2765.4ms/step | 8.7 samples/s
504
+ Step 3820/20005 | Loss: 2.7435 | LR: 4.61e-04 | 2763.7ms/step | 8.7 samples/s
505
+ Step 3830/20005 | Loss: 2.7539 | LR: 4.61e-04 | 2766.5ms/step | 8.7 samples/s
506
+ Step 3840/20005 | Loss: 2.8248 | LR: 4.60e-04 | 2762.3ms/step | 8.7 samples/s
507
+ Step 3850/20005 | Loss: 2.8429 | LR: 4.60e-04 | 2764.1ms/step | 8.7 samples/s
508
+ Step 3860/20005 | Loss: 2.7903 | LR: 4.60e-04 | 2764.8ms/step | 8.7 samples/s
509
+ Step 3870/20005 | Loss: 2.7481 | LR: 4.60e-04 | 2762.3ms/step | 8.7 samples/s
510
+ Step 3880/20005 | Loss: 2.8574 | LR: 4.59e-04 | 2765.5ms/step | 8.7 samples/s
511
+ Step 3890/20005 | Loss: 2.7220 | LR: 4.59e-04 | 2769.2ms/step | 8.7 samples/s
512
+ Step 3900/20005 | Loss: 2.7894 | LR: 4.59e-04 | 2767.6ms/step | 8.7 samples/s
513
+ Step 3910/20005 | Loss: 2.6568 | LR: 4.59e-04 | 2767.5ms/step | 8.7 samples/s
514
+ Step 3920/20005 | Loss: 2.7097 | LR: 4.59e-04 | 2767.3ms/step | 8.7 samples/s
515
+ Step 3930/20005 | Loss: 2.7778 | LR: 4.58e-04 | 2766.1ms/step | 8.7 samples/s
516
+ Step 3940/20005 | Loss: 2.7871 | LR: 4.58e-04 | 2765.6ms/step | 8.7 samples/s
517
+ Step 3950/20005 | Loss: 2.8005 | LR: 4.58e-04 | 2766.1ms/step | 8.7 samples/s
518
+ Step 3960/20005 | Loss: 2.8264 | LR: 4.58e-04 | 2764.2ms/step | 8.7 samples/s
519
+ Step 3970/20005 | Loss: 2.6854 | LR: 4.58e-04 | 2766.4ms/step | 8.7 samples/s
520
+ Step 3980/20005 | Loss: 2.6511 | LR: 4.57e-04 | 2767.6ms/step | 8.7 samples/s
521
+ Step 3990/20005 | Loss: 2.7663 | LR: 4.57e-04 | 2764.3ms/step | 8.7 samples/s
522
+ Step 4000/20005 | Loss: 2.7743 | LR: 4.57e-04 | 2773.7ms/step | 8.7 samples/s
523
+ Validation at step 4000: Loss = 2.7805, Perplexity = 16.13
524
+ Saved checkpoint to checkpoints/run_20260506_193527/checkpoint-4000
525
+ Step 4010/20005 | Loss: 2.8014 | LR: 4.57e-04 | 22585.5ms/step | 1.1 samples/s
526
+ Step 4020/20005 | Loss: 2.7197 | LR: 4.56e-04 | 2769.9ms/step | 8.7 samples/s
527
+ Step 4030/20005 | Loss: 2.7636 | LR: 4.56e-04 | 2767.6ms/step | 8.7 samples/s
528
+ Step 4040/20005 | Loss: 2.8259 | LR: 4.56e-04 | 2762.7ms/step | 8.7 samples/s
529
+ Step 4050/20005 | Loss: 2.8130 | LR: 4.56e-04 | 2766.1ms/step | 8.7 samples/s
530
+ Step 4060/20005 | Loss: 2.8101 | LR: 4.55e-04 | 2763.8ms/step | 8.7 samples/s
531
+ Step 4070/20005 | Loss: 2.7867 | LR: 4.55e-04 | 2765.1ms/step | 8.7 samples/s
532
+ Step 4080/20005 | Loss: 2.8095 | LR: 4.55e-04 | 2765.3ms/step | 8.7 samples/s
533
+ Step 4090/20005 | Loss: 2.7426 | LR: 4.55e-04 | 2763.3ms/step | 8.7 samples/s
534
+ Step 4100/20005 | Loss: 2.7343 | LR: 4.55e-04 | 2764.5ms/step | 8.7 samples/s
535
+ Step 4110/20005 | Loss: 2.7135 | LR: 4.54e-04 | 2770.4ms/step | 8.7 samples/s
536
+ Step 4120/20005 | Loss: 2.7327 | LR: 4.54e-04 | 2772.2ms/step | 8.7 samples/s
537
+ Step 4130/20005 | Loss: 2.7300 | LR: 4.54e-04 | 2768.6ms/step | 8.7 samples/s
538
+ Step 4140/20005 | Loss: 2.8484 | LR: 4.54e-04 | 2769.0ms/step | 8.7 samples/s
539
+ Step 4150/20005 | Loss: 2.7756 | LR: 4.53e-04 | 2765.6ms/step | 8.7 samples/s
540
+ Step 4160/20005 | Loss: 2.7160 | LR: 4.53e-04 | 2768.6ms/step | 8.7 samples/s
541
+ Step 4170/20005 | Loss: 2.7787 | LR: 4.53e-04 | 2767.3ms/step | 8.7 samples/s
542
+ Step 4180/20005 | Loss: 2.6801 | LR: 4.53e-04 | 2764.1ms/step | 8.7 samples/s
543
+ Step 4190/20005 | Loss: 2.8039 | LR: 4.53e-04 | 2763.9ms/step | 8.7 samples/s
544
+ Step 4200/20005 | Loss: 2.7261 | LR: 4.52e-04 | 2767.7ms/step | 8.7 samples/s
545
+ Step 4210/20005 | Loss: 2.7982 | LR: 4.52e-04 | 2775.7ms/step | 8.6 samples/s
546
+ Step 4220/20005 | Loss: 2.7836 | LR: 4.52e-04 | 2769.6ms/step | 8.7 samples/s
547
+ Step 4230/20005 | Loss: 2.6718 | LR: 4.52e-04 | 2764.7ms/step | 8.7 samples/s
548
+ Step 4240/20005 | Loss: 2.7567 | LR: 4.51e-04 | 2767.1ms/step | 8.7 samples/s
549
+ Step 4250/20005 | Loss: 2.8348 | LR: 4.51e-04 | 2767.5ms/step | 8.7 samples/s
550
+ Validation at step 4250: Loss = 2.7618, Perplexity = 15.83
551
+ Step 4260/20005 | Loss: 2.7017 | LR: 4.51e-04 | 22131.9ms/step | 1.1 samples/s
552
+ Step 4270/20005 | Loss: 2.7846 | LR: 4.51e-04 | 2765.3ms/step | 8.7 samples/s
553
+ Step 4280/20005 | Loss: 2.7096 | LR: 4.50e-04 | 2769.6ms/step | 8.7 samples/s
554
+ Step 4290/20005 | Loss: 2.7498 | LR: 4.50e-04 | 2766.4ms/step | 8.7 samples/s
555
+ Step 4300/20005 | Loss: 2.6812 | LR: 4.50e-04 | 2769.2ms/step | 8.7 samples/s
556
+ Step 4310/20005 | Loss: 2.7629 | LR: 4.50e-04 | 2769.9ms/step | 8.7 samples/s
557
+ Step 4320/20005 | Loss: 2.6984 | LR: 4.49e-04 | 2766.0ms/step | 8.7 samples/s
558
+ Step 4330/20005 | Loss: 2.7155 | LR: 4.49e-04 | 2767.9ms/step | 8.7 samples/s
559
+ Step 4340/20005 | Loss: 2.7935 | LR: 4.49e-04 | 2765.8ms/step | 8.7 samples/s
560
+ Step 4350/20005 | Loss: 2.7341 | LR: 4.49e-04 | 2765.1ms/step | 8.7 samples/s
561
+ Step 4360/20005 | Loss: 2.6406 | LR: 4.48e-04 | 2767.1ms/step | 8.7 samples/s
562
+ Step 4370/20005 | Loss: 2.7588 | LR: 4.48e-04 | 2766.0ms/step | 8.7 samples/s
563
+ Step 4380/20005 | Loss: 2.7810 | LR: 4.48e-04 | 2763.7ms/step | 8.7 samples/s
564
+ Step 4390/20005 | Loss: 2.7709 | LR: 4.48e-04 | 2763.1ms/step | 8.7 samples/s
565
+ Step 4400/20005 | Loss: 2.7984 | LR: 4.48e-04 | 2766.7ms/step | 8.7 samples/s
566
+ Step 4410/20005 | Loss: 2.7338 | LR: 4.47e-04 | 2770.7ms/step | 8.7 samples/s
567
+ Step 4420/20005 | Loss: 2.6995 | LR: 4.47e-04 | 2765.0ms/step | 8.7 samples/s
568
+ Step 4430/20005 | Loss: 2.7740 | LR: 4.47e-04 | 2763.3ms/step | 8.7 samples/s
569
+ Step 4440/20005 | Loss: 2.6960 | LR: 4.47e-04 | 2765.9ms/step | 8.7 samples/s
570
+ Step 4450/20005 | Loss: 2.7066 | LR: 4.46e-04 | 2766.0ms/step | 8.7 samples/s
571
+ Step 4460/20005 | Loss: 2.6740 | LR: 4.46e-04 | 2764.5ms/step | 8.7 samples/s
572
+ Step 4470/20005 | Loss: 2.8155 | LR: 4.46e-04 | 2764.7ms/step | 8.7 samples/s
573
+ Step 4480/20005 | Loss: 2.7172 | LR: 4.46e-04 | 2768.0ms/step | 8.7 samples/s
574
+ Step 4490/20005 | Loss: 2.8171 | LR: 4.45e-04 | 2767.4ms/step | 8.7 samples/s
575
+ Step 4500/20005 | Loss: 2.7639 | LR: 4.45e-04 | 2769.4ms/step | 8.7 samples/s
576
+ Validation at step 4500: Loss = 2.7426, Perplexity = 15.53
577
+ Step 4510/20005 | Loss: 2.7351 | LR: 4.45e-04 | 21973.1ms/step | 1.1 samples/s
578
+ Step 4520/20005 | Loss: 2.7427 | LR: 4.45e-04 | 2763.9ms/step | 8.7 samples/s
579
+ Step 4530/20005 | Loss: 2.7839 | LR: 4.44e-04 | 2765.5ms/step | 8.7 samples/s
580
+ Step 4540/20005 | Loss: 2.7950 | LR: 4.44e-04 | 2763.9ms/step | 8.7 samples/s
581
+ Step 4550/20005 | Loss: 2.7796 | LR: 4.44e-04 | 2765.0ms/step | 8.7 samples/s
582
+ Step 4560/20005 | Loss: 2.7147 | LR: 4.44e-04 | 2765.9ms/step | 8.7 samples/s
583
+ Step 4570/20005 | Loss: 2.7487 | LR: 4.43e-04 | 2764.6ms/step | 8.7 samples/s
584
+ Step 4580/20005 | Loss: 2.7514 | LR: 4.43e-04 | 2765.8ms/step | 8.7 samples/s
585
+ Step 4590/20005 | Loss: 2.7137 | LR: 4.43e-04 | 2762.6ms/step | 8.7 samples/s
586
+ Step 4600/20005 | Loss: 2.6685 | LR: 4.43e-04 | 2764.4ms/step | 8.7 samples/s
587
+ Step 4610/20005 | Loss: 2.7308 | LR: 4.42e-04 | 2768.0ms/step | 8.7 samples/s
588
+ Step 4620/20005 | Loss: 2.7484 | LR: 4.42e-04 | 2762.6ms/step | 8.7 samples/s
589
+ Step 4630/20005 | Loss: 2.6554 | LR: 4.42e-04 | 2765.9ms/step | 8.7 samples/s
590
+ Step 4640/20005 | Loss: 2.6362 | LR: 4.42e-04 | 2765.3ms/step | 8.7 samples/s
591
+ Step 4650/20005 | Loss: 2.7495 | LR: 4.41e-04 | 2768.4ms/step | 8.7 samples/s
592
+ Step 4660/20005 | Loss: 2.6244 | LR: 4.41e-04 | 2767.4ms/step | 8.7 samples/s
593
+ Step 4670/20005 | Loss: 2.7886 | LR: 4.41e-04 | 2765.3ms/step | 8.7 samples/s
594
+ Step 4680/20005 | Loss: 2.7609 | LR: 4.40e-04 | 2762.9ms/step | 8.7 samples/s
595
+ Step 4690/20005 | Loss: 2.6595 | LR: 4.40e-04 | 2765.0ms/step | 8.7 samples/s
596
+ Step 4700/20005 | Loss: 2.8031 | LR: 4.40e-04 | 2766.9ms/step | 8.7 samples/s
597
+ Step 4710/20005 | Loss: 2.7107 | LR: 4.40e-04 | 2762.6ms/step | 8.7 samples/s
598
+ Step 4720/20005 | Loss: 2.7735 | LR: 4.39e-04 | 2766.2ms/step | 8.7 samples/s
599
+ Step 4730/20005 | Loss: 2.6731 | LR: 4.39e-04 | 2764.6ms/step | 8.7 samples/s
600
+ Step 4740/20005 | Loss: 2.7261 | LR: 4.39e-04 | 2766.9ms/step | 8.7 samples/s
601
+ Step 4750/20005 | Loss: 2.6785 | LR: 4.39e-04 | 2763.3ms/step | 8.7 samples/s
602
+ Validation at step 4750: Loss = 2.7200, Perplexity = 15.18
603
+ Step 4760/20005 | Loss: 2.6945 | LR: 4.38e-04 | 21964.4ms/step | 1.1 samples/s
604
+ Step 4770/20005 | Loss: 2.7150 | LR: 4.38e-04 | 2763.8ms/step | 8.7 samples/s
605
+ Step 4780/20005 | Loss: 2.7114 | LR: 4.38e-04 | 2766.5ms/step | 8.7 samples/s
606
+ Step 4790/20005 | Loss: 2.6597 | LR: 4.38e-04 | 2764.2ms/step | 8.7 samples/s
607
+ Step 4800/20005 | Loss: 2.7140 | LR: 4.37e-04 | 2763.3ms/step | 8.7 samples/s
608
+ Step 4810/20005 | Loss: 2.7423 | LR: 4.37e-04 | 2763.7ms/step | 8.7 samples/s
609
+ Step 4820/20005 | Loss: 2.6910 | LR: 4.37e-04 | 2766.2ms/step | 8.7 samples/s
610
+ Step 4830/20005 | Loss: 2.6518 | LR: 4.37e-04 | 2767.0ms/step | 8.7 samples/s
611
+ Step 4840/20005 | Loss: 2.7702 | LR: 4.36e-04 | 2767.5ms/step | 8.7 samples/s
612
+ Step 4850/20005 | Loss: 2.8099 | LR: 4.36e-04 | 2764.8ms/step | 8.7 samples/s
613
+ Step 4860/20005 | Loss: 2.7664 | LR: 4.36e-04 | 2765.1ms/step | 8.7 samples/s
614
+ Step 4870/20005 | Loss: 2.6859 | LR: 4.36e-04 | 2762.8ms/step | 8.7 samples/s
615
+ Step 4880/20005 | Loss: 2.6813 | LR: 4.35e-04 | 2763.0ms/step | 8.7 samples/s
616
+ Step 4890/20005 | Loss: 2.6505 | LR: 4.35e-04 | 2762.0ms/step | 8.7 samples/s
617
+ Step 4900/20005 | Loss: 2.7570 | LR: 4.35e-04 | 2762.1ms/step | 8.7 samples/s
618
+ Step 4910/20005 | Loss: 2.7519 | LR: 4.34e-04 | 2766.9ms/step | 8.7 samples/s
619
+ Step 4920/20005 | Loss: 2.6848 | LR: 4.34e-04 | 2765.9ms/step | 8.7 samples/s
620
+ Step 4930/20005 | Loss: 2.6699 | LR: 4.34e-04 | 2765.6ms/step | 8.7 samples/s
621
+ Step 4940/20005 | Loss: 2.7109 | LR: 4.34e-04 | 2765.2ms/step | 8.7 samples/s
622
+ Step 4950/20005 | Loss: 2.6629 | LR: 4.33e-04 | 2762.1ms/step | 8.7 samples/s
623
+ Step 4960/20005 | Loss: 2.6472 | LR: 4.33e-04 | 2765.3ms/step | 8.7 samples/s
624
+ Step 4970/20005 | Loss: 2.6918 | LR: 4.33e-04 | 2766.4ms/step | 8.7 samples/s
625
+ Step 4980/20005 | Loss: 2.7816 | LR: 4.33e-04 | 2765.3ms/step | 8.7 samples/s
626
+ Step 4990/20005 | Loss: 2.7503 | LR: 4.32e-04 | 2764.0ms/step | 8.7 samples/s
627
+ Step 5000/20005 | Loss: 2.7702 | LR: 4.32e-04 | 2766.6ms/step | 8.7 samples/s
628
+ Validation at step 5000: Loss = 2.7018, Perplexity = 14.91
629
+ Step 5010/20005 | Loss: 2.6438 | LR: 4.32e-04 | 21969.5ms/step | 1.1 samples/s
630
+ Step 5020/20005 | Loss: 2.6652 | LR: 4.31e-04 | 2767.1ms/step | 8.7 samples/s
631
+ Step 5030/20005 | Loss: 2.6268 | LR: 4.31e-04 | 2764.6ms/step | 8.7 samples/s
632
+ Step 5040/20005 | Loss: 2.5776 | LR: 4.31e-04 | 2766.6ms/step | 8.7 samples/s
633
+ Step 5050/20005 | Loss: 2.6962 | LR: 4.31e-04 | 2765.4ms/step | 8.7 samples/s
634
+ Step 5060/20005 | Loss: 2.6247 | LR: 4.30e-04 | 2765.7ms/step | 8.7 samples/s
635
+ Step 5070/20005 | Loss: 2.6051 | LR: 4.30e-04 | 2766.1ms/step | 8.7 samples/s
636
+ Step 5080/20005 | Loss: 2.7289 | LR: 4.30e-04 | 2764.9ms/step | 8.7 samples/s
637
+ Step 5090/20005 | Loss: 2.7074 | LR: 4.30e-04 | 2762.4ms/step | 8.7 samples/s
638
+ Step 5100/20005 | Loss: 2.7233 | LR: 4.29e-04 | 2762.3ms/step | 8.7 samples/s
639
+ Step 5110/20005 | Loss: 2.7697 | LR: 4.29e-04 | 2761.1ms/step | 8.7 samples/s
640
+ Step 5120/20005 | Loss: 2.6037 | LR: 4.29e-04 | 2865.4ms/step | 8.4 samples/s
641
+ Step 5130/20005 | Loss: 2.6507 | LR: 4.28e-04 | 2762.9ms/step | 8.7 samples/s
642
+ Step 5140/20005 | Loss: 2.6740 | LR: 4.28e-04 | 2764.5ms/step | 8.7 samples/s
643
+ Step 5150/20005 | Loss: 2.6789 | LR: 4.28e-04 | 2761.5ms/step | 8.7 samples/s
644
+ Step 5160/20005 | Loss: 2.7714 | LR: 4.28e-04 | 2764.5ms/step | 8.7 samples/s
645
+ Step 5170/20005 | Loss: 2.7007 | LR: 4.27e-04 | 2764.0ms/step | 8.7 samples/s
646
+ Step 5180/20005 | Loss: 2.6848 | LR: 4.27e-04 | 2763.3ms/step | 8.7 samples/s
647
+ Step 5190/20005 | Loss: 2.7033 | LR: 4.27e-04 | 2761.1ms/step | 8.7 samples/s
648
+ Step 5200/20005 | Loss: 2.7223 | LR: 4.26e-04 | 2761.8ms/step | 8.7 samples/s
649
+ Step 5210/20005 | Loss: 2.7041 | LR: 4.26e-04 | 2763.0ms/step | 8.7 samples/s
650
+ Step 5220/20005 | Loss: 2.6058 | LR: 4.26e-04 | 2763.7ms/step | 8.7 samples/s
651
+ Step 5230/20005 | Loss: 2.6615 | LR: 4.26e-04 | 2765.9ms/step | 8.7 samples/s
652
+ Step 5240/20005 | Loss: 2.7441 | LR: 4.25e-04 | 2767.0ms/step | 8.7 samples/s
653
+ Step 5250/20005 | Loss: 2.6789 | LR: 4.25e-04 | 2763.1ms/step | 8.7 samples/s
654
+ Validation at step 5250: Loss = 2.6861, Perplexity = 14.67
655
+ Step 5260/20005 | Loss: 2.6876 | LR: 4.25e-04 | 21964.9ms/step | 1.1 samples/s
656
+ Step 5270/20005 | Loss: 2.7078 | LR: 4.24e-04 | 2765.7ms/step | 8.7 samples/s
657
+ Step 5280/20005 | Loss: 2.6526 | LR: 4.24e-04 | 2764.5ms/step | 8.7 samples/s
658
+ Step 5290/20005 | Loss: 2.6688 | LR: 4.24e-04 | 2770.2ms/step | 8.7 samples/s
659
+ Step 5300/20005 | Loss: 2.6336 | LR: 4.24e-04 | 2766.0ms/step | 8.7 samples/s
660
+ Step 5310/20005 | Loss: 2.6459 | LR: 4.23e-04 | 2766.9ms/step | 8.7 samples/s
661
+ Step 5320/20005 | Loss: 2.6522 | LR: 4.23e-04 | 2766.0ms/step | 8.7 samples/s
662
+ Step 5330/20005 | Loss: 2.6324 | LR: 4.23e-04 | 2762.2ms/step | 8.7 samples/s
663
+ Step 5340/20005 | Loss: 2.5976 | LR: 4.22e-04 | 2762.4ms/step | 8.7 samples/s
664
+ Step 5350/20005 | Loss: 2.6870 | LR: 4.22e-04 | 2761.9ms/step | 8.7 samples/s
665
+ Step 5360/20005 | Loss: 2.6704 | LR: 4.22e-04 | 2766.8ms/step | 8.7 samples/s
666
+ Step 5370/20005 | Loss: 2.6955 | LR: 4.22e-04 | 2763.4ms/step | 8.7 samples/s
667
+ Step 5380/20005 | Loss: 2.5855 | LR: 4.21e-04 | 2765.6ms/step | 8.7 samples/s
668
+ Step 5390/20005 | Loss: 2.7164 | LR: 4.21e-04 | 2766.0ms/step | 8.7 samples/s
669
+ Step 5400/20005 | Loss: 2.6712 | LR: 4.21e-04 | 2764.2ms/step | 8.7 samples/s
670
+ Step 5410/20005 | Loss: 2.6456 | LR: 4.20e-04 | 2761.9ms/step | 8.7 samples/s
671
+ Step 5420/20005 | Loss: 2.6285 | LR: 4.20e-04 | 2764.3ms/step | 8.7 samples/s
672
+ Step 5430/20005 | Loss: 2.6408 | LR: 4.20e-04 | 2766.2ms/step | 8.7 samples/s
673
+ Step 5440/20005 | Loss: 2.6559 | LR: 4.20e-04 | 2765.6ms/step | 8.7 samples/s
674
+ Step 5450/20005 | Loss: 2.6147 | LR: 4.19e-04 | 2767.3ms/step | 8.7 samples/s
675
+ Step 5460/20005 | Loss: 2.6209 | LR: 4.19e-04 | 2764.6ms/step | 8.7 samples/s
676
+ Step 5470/20005 | Loss: 2.6592 | LR: 4.19e-04 | 2764.4ms/step | 8.7 samples/s
677
+ Step 5480/20005 | Loss: 2.6033 | LR: 4.18e-04 | 2760.0ms/step | 8.7 samples/s
678
+ Step 5490/20005 | Loss: 2.6673 | LR: 4.18e-04 | 2762.9ms/step | 8.7 samples/s
679
+ Step 5500/20005 | Loss: 2.6877 | LR: 4.18e-04 | 2763.9ms/step | 8.7 samples/s
680
+ Validation at step 5500: Loss = 2.6726, Perplexity = 14.48
681
+ Step 5510/20005 | Loss: 2.6529 | LR: 4.18e-04 | 21962.5ms/step | 1.1 samples/s
682
+ Step 5520/20005 | Loss: 2.6756 | LR: 4.17e-04 | 2766.7ms/step | 8.7 samples/s
683
+ Step 5530/20005 | Loss: 2.6052 | LR: 4.17e-04 | 2763.7ms/step | 8.7 samples/s
684
+ Step 5540/20005 | Loss: 2.6064 | LR: 4.17e-04 | 2763.5ms/step | 8.7 samples/s
685
+ Step 5550/20005 | Loss: 2.6041 | LR: 4.16e-04 | 2761.9ms/step | 8.7 samples/s
686
+ Step 5560/20005 | Loss: 2.6273 | LR: 4.16e-04 | 2770.2ms/step | 8.7 samples/s
687
+ Step 5570/20005 | Loss: 2.7686 | LR: 4.16e-04 | 2763.9ms/step | 8.7 samples/s
688
+ Step 5580/20005 | Loss: 2.6896 | LR: 4.15e-04 | 2765.1ms/step | 8.7 samples/s
689
+ Step 5590/20005 | Loss: 2.6473 | LR: 4.15e-04 | 2761.7ms/step | 8.7 samples/s
690
+ Step 5600/20005 | Loss: 2.6861 | LR: 4.15e-04 | 2762.3ms/step | 8.7 samples/s
691
+ Step 5610/20005 | Loss: 2.6353 | LR: 4.15e-04 | 2763.5ms/step | 8.7 samples/s
692
+ Step 5620/20005 | Loss: 2.6585 | LR: 4.14e-04 | 2765.8ms/step | 8.7 samples/s
693
+ Step 5630/20005 | Loss: 2.6793 | LR: 4.14e-04 | 2763.9ms/step | 8.7 samples/s
694
+ Step 5640/20005 | Loss: 2.6824 | LR: 4.14e-04 | 2764.7ms/step | 8.7 samples/s
695
+ Step 5650/20005 | Loss: 2.7052 | LR: 4.13e-04 | 2765.8ms/step | 8.7 samples/s
696
+ Step 5660/20005 | Loss: 2.5707 | LR: 4.13e-04 | 2758.3ms/step | 8.7 samples/s
697
+ Step 5670/20005 | Loss: 2.6507 | LR: 4.13e-04 | 2763.3ms/step | 8.7 samples/s
698
+ Step 5680/20005 | Loss: 2.5944 | LR: 4.12e-04 | 2768.0ms/step | 8.7 samples/s
699
+ Step 5690/20005 | Loss: 2.6114 | LR: 4.12e-04 | 2763.1ms/step | 8.7 samples/s
700
+ Step 5700/20005 | Loss: 2.7112 | LR: 4.12e-04 | 2767.3ms/step | 8.7 samples/s
701
+ Step 5710/20005 | Loss: 2.6522 | LR: 4.12e-04 | 2765.4ms/step | 8.7 samples/s
702
+ Step 5720/20005 | Loss: 2.6696 | LR: 4.11e-04 | 2763.0ms/step | 8.7 samples/s
703
+ Step 5730/20005 | Loss: 2.7241 | LR: 4.11e-04 | 2767.4ms/step | 8.7 samples/s
704
+ Step 5740/20005 | Loss: 2.6338 | LR: 4.11e-04 | 2766.1ms/step | 8.7 samples/s
705
+ Step 5750/20005 | Loss: 2.6533 | LR: 4.10e-04 | 2768.5ms/step | 8.7 samples/s
706
+ Validation at step 5750: Loss = 2.6558, Perplexity = 14.24
707
+ Step 5760/20005 | Loss: 2.6337 | LR: 4.10e-04 | 21964.0ms/step | 1.1 samples/s
708
+ Step 5770/20005 | Loss: 2.6661 | LR: 4.10e-04 | 2766.3ms/step | 8.7 samples/s
709
+ Step 5780/20005 | Loss: 2.6966 | LR: 4.09e-04 | 2763.4ms/step | 8.7 samples/s
710
+ Step 5790/20005 | Loss: 2.6534 | LR: 4.09e-04 | 2762.8ms/step | 8.7 samples/s
711
+ Step 5800/20005 | Loss: 2.7228 | LR: 4.09e-04 | 2762.7ms/step | 8.7 samples/s
712
+ Step 5810/20005 | Loss: 2.7748 | LR: 4.08e-04 | 2761.7ms/step | 8.7 samples/s
713
+ Step 5820/20005 | Loss: 2.6906 | LR: 4.08e-04 | 2761.8ms/step | 8.7 samples/s
714
+ Step 5830/20005 | Loss: 2.6230 | LR: 4.08e-04 | 2762.5ms/step | 8.7 samples/s
715
+ Step 5840/20005 | Loss: 2.6334 | LR: 4.08e-04 | 2766.7ms/step | 8.7 samples/s
716
+ Step 5850/20005 | Loss: 2.7509 | LR: 4.07e-04 | 2762.4ms/step | 8.7 samples/s
717
+ Step 5860/20005 | Loss: 2.6063 | LR: 4.07e-04 | 2762.2ms/step | 8.7 samples/s
718
+ Step 5870/20005 | Loss: 2.6165 | LR: 4.07e-04 | 2767.2ms/step | 8.7 samples/s
719
+ Step 5880/20005 | Loss: 2.6796 | LR: 4.06e-04 | 2765.8ms/step | 8.7 samples/s
720
+ Step 5890/20005 | Loss: 2.6277 | LR: 4.06e-04 | 2765.2ms/step | 8.7 samples/s
721
+ Step 5900/20005 | Loss: 2.6580 | LR: 4.06e-04 | 2766.4ms/step | 8.7 samples/s
722
+ Step 5910/20005 | Loss: 2.5867 | LR: 4.05e-04 | 2766.3ms/step | 8.7 samples/s
723
+ Step 5920/20005 | Loss: 2.6735 | LR: 4.05e-04 | 2765.2ms/step | 8.7 samples/s
724
+ Step 5930/20005 | Loss: 2.6915 | LR: 4.05e-04 | 2768.4ms/step | 8.7 samples/s
725
+ Step 5940/20005 | Loss: 2.6428 | LR: 4.04e-04 | 2767.3ms/step | 8.7 samples/s
726
+ Step 5950/20005 | Loss: 2.5878 | LR: 4.04e-04 | 2767.3ms/step | 8.7 samples/s
727
+ Step 5960/20005 | Loss: 2.5788 | LR: 4.04e-04 | 2767.4ms/step | 8.7 samples/s
728
+ Step 5970/20005 | Loss: 2.6282 | LR: 4.04e-04 | 2765.6ms/step | 8.7 samples/s
729
+ Step 5980/20005 | Loss: 2.6048 | LR: 4.03e-04 | 2864.6ms/step | 8.4 samples/s
730
+ Step 5990/20005 | Loss: 2.6510 | LR: 4.03e-04 | 2763.7ms/step | 8.7 samples/s
731
+ Step 6000/20005 | Loss: 2.7637 | LR: 4.03e-04 | 2766.4ms/step | 8.7 samples/s
732
+ Validation at step 6000: Loss = 2.6425, Perplexity = 14.05
733
+ Saved checkpoint to checkpoints/run_20260506_193527/checkpoint-6000
734
+ Step 6010/20005 | Loss: 2.6562 | LR: 4.02e-04 | 22568.8ms/step | 1.1 samples/s
735
+ Step 6020/20005 | Loss: 2.6335 | LR: 4.02e-04 | 2762.0ms/step | 8.7 samples/s
736
+ Step 6030/20005 | Loss: 2.6699 | LR: 4.02e-04 | 2767.2ms/step | 8.7 samples/s
737
+ Step 6040/20005 | Loss: 2.5419 | LR: 4.01e-04 | 2764.6ms/step | 8.7 samples/s
738
+ Step 6050/20005 | Loss: 2.5430 | LR: 4.01e-04 | 2765.5ms/step | 8.7 samples/s
739
+ Step 6060/20005 | Loss: 2.6453 | LR: 4.01e-04 | 2764.6ms/step | 8.7 samples/s
740
+ Step 6070/20005 | Loss: 2.6056 | LR: 4.00e-04 | 2770.5ms/step | 8.7 samples/s
741
+ Step 6080/20005 | Loss: 2.6331 | LR: 4.00e-04 | 2765.6ms/step | 8.7 samples/s
742
+ Step 6090/20005 | Loss: 2.6308 | LR: 4.00e-04 | 2763.2ms/step | 8.7 samples/s
743
+ Step 6100/20005 | Loss: 2.6570 | LR: 3.99e-04 | 2766.4ms/step | 8.7 samples/s
744
+ Step 6110/20005 | Loss: 2.6833 | LR: 3.99e-04 | 2765.7ms/step | 8.7 samples/s
745
+ Step 6120/20005 | Loss: 2.6174 | LR: 3.99e-04 | 2765.9ms/step | 8.7 samples/s
746
+ Step 6130/20005 | Loss: 2.7034 | LR: 3.98e-04 | 2767.1ms/step | 8.7 samples/s
747
+ Step 6140/20005 | Loss: 2.6509 | LR: 3.98e-04 | 2767.5ms/step | 8.7 samples/s
748
+ Step 6150/20005 | Loss: 2.5769 | LR: 3.98e-04 | 2767.3ms/step | 8.7 samples/s
749
+ Step 6160/20005 | Loss: 2.5485 | LR: 3.97e-04 | 2767.0ms/step | 8.7 samples/s
750
+ Step 6170/20005 | Loss: 2.6329 | LR: 3.97e-04 | 2768.4ms/step | 8.7 samples/s
751
+ Step 6180/20005 | Loss: 2.5328 | LR: 3.97e-04 | 2768.9ms/step | 8.7 samples/s
752
+ Step 6190/20005 | Loss: 2.6455 | LR: 3.97e-04 | 2763.9ms/step | 8.7 samples/s
753
+ Step 6200/20005 | Loss: 2.6515 | LR: 3.96e-04 | 2766.1ms/step | 8.7 samples/s
754
+ Step 6210/20005 | Loss: 2.5982 | LR: 3.96e-04 | 2763.8ms/step | 8.7 samples/s
755
+ Step 6220/20005 | Loss: 2.6930 | LR: 3.96e-04 | 2767.7ms/step | 8.7 samples/s
756
+ Step 6230/20005 | Loss: 2.5672 | LR: 3.95e-04 | 2765.1ms/step | 8.7 samples/s
757
+ Step 6240/20005 | Loss: 2.5973 | LR: 3.95e-04 | 2763.5ms/step | 8.7 samples/s
758
+ Step 6250/20005 | Loss: 2.5916 | LR: 3.95e-04 | 2762.5ms/step | 8.7 samples/s
759
+ Validation at step 6250: Loss = 2.6309, Perplexity = 13.89
760
+ Step 6260/20005 | Loss: 2.6581 | LR: 3.94e-04 | 21968.0ms/step | 1.1 samples/s
761
+ Step 6270/20005 | Loss: 2.6055 | LR: 3.94e-04 | 2764.7ms/step | 8.7 samples/s
762
+ Step 6280/20005 | Loss: 2.6417 | LR: 3.94e-04 | 2762.6ms/step | 8.7 samples/s
763
+ Step 6290/20005 | Loss: 2.6783 | LR: 3.93e-04 | 2761.0ms/step | 8.7 samples/s
764
+ Step 6300/20005 | Loss: 2.6729 | LR: 3.93e-04 | 2763.4ms/step | 8.7 samples/s
765
+ Step 6310/20005 | Loss: 2.6320 | LR: 3.93e-04 | 2762.7ms/step | 8.7 samples/s
766
+ Step 6320/20005 | Loss: 2.5612 | LR: 3.92e-04 | 2761.3ms/step | 8.7 samples/s
767
+ Step 6330/20005 | Loss: 2.5866 | LR: 3.92e-04 | 2761.6ms/step | 8.7 samples/s
768
+ Step 6340/20005 | Loss: 2.6301 | LR: 3.92e-04 | 2764.0ms/step | 8.7 samples/s
769
+ Step 6350/20005 | Loss: 2.5445 | LR: 3.91e-04 | 2766.1ms/step | 8.7 samples/s
770
+ Step 6360/20005 | Loss: 2.5546 | LR: 3.91e-04 | 2765.4ms/step | 8.7 samples/s
771
+ Step 6370/20005 | Loss: 2.7786 | LR: 3.91e-04 | 2763.2ms/step | 8.7 samples/s
772
+ Step 6380/20005 | Loss: 2.5914 | LR: 3.90e-04 | 2769.9ms/step | 8.7 samples/s
773
+ Step 6390/20005 | Loss: 2.6088 | LR: 3.90e-04 | 2771.8ms/step | 8.7 samples/s
774
+ Step 6400/20005 | Loss: 2.6287 | LR: 3.90e-04 | 2765.3ms/step | 8.7 samples/s
775
+ Step 6410/20005 | Loss: 2.6515 | LR: 3.89e-04 | 2764.3ms/step | 8.7 samples/s
776
+ Step 6420/20005 | Loss: 2.6463 | LR: 3.89e-04 | 2763.3ms/step | 8.7 samples/s
777
+ Step 6430/20005 | Loss: 2.6461 | LR: 3.89e-04 | 2766.7ms/step | 8.7 samples/s
778
+ Step 6440/20005 | Loss: 2.5537 | LR: 3.88e-04 | 2766.3ms/step | 8.7 samples/s
779
+ Step 6450/20005 | Loss: 2.5675 | LR: 3.88e-04 | 2765.7ms/step | 8.7 samples/s
780
+ Step 6460/20005 | Loss: 2.6336 | LR: 3.88e-04 | 2768.5ms/step | 8.7 samples/s
781
+ Step 6470/20005 | Loss: 2.6412 | LR: 3.87e-04 | 2764.8ms/step | 8.7 samples/s
782
+ Step 6480/20005 | Loss: 2.6998 | LR: 3.87e-04 | 2764.9ms/step | 8.7 samples/s
783
+ Step 6490/20005 | Loss: 2.5815 | LR: 3.87e-04 | 2765.4ms/step | 8.7 samples/s
784
+ Step 6500/20005 | Loss: 2.6082 | LR: 3.86e-04 | 2770.4ms/step | 8.7 samples/s
785
+ Validation at step 6500: Loss = 2.6169, Perplexity = 13.69
786
+ Step 6510/20005 | Loss: 2.5293 | LR: 3.86e-04 | 21968.3ms/step | 1.1 samples/s
787
+ Step 6520/20005 | Loss: 2.4751 | LR: 3.86e-04 | 2767.6ms/step | 8.7 samples/s
788
+ Step 6530/20005 | Loss: 2.6060 | LR: 3.85e-04 | 2765.7ms/step | 8.7 samples/s
789
+ Step 6540/20005 | Loss: 2.6028 | LR: 3.85e-04 | 2762.1ms/step | 8.7 samples/s
790
+ Step 6550/20005 | Loss: 2.5768 | LR: 3.85e-04 | 2766.6ms/step | 8.7 samples/s
791
+ Step 6560/20005 | Loss: 2.5611 | LR: 3.84e-04 | 2765.2ms/step | 8.7 samples/s
792
+ Step 6570/20005 | Loss: 2.6162 | LR: 3.84e-04 | 2764.1ms/step | 8.7 samples/s
793
+ Step 6580/20005 | Loss: 2.5734 | LR: 3.84e-04 | 2765.1ms/step | 8.7 samples/s
794
+ Step 6590/20005 | Loss: 2.6180 | LR: 3.83e-04 | 2766.4ms/step | 8.7 samples/s
795
+ Step 6600/20005 | Loss: 2.6397 | LR: 3.83e-04 | 2766.4ms/step | 8.7 samples/s
796
+ Step 6610/20005 | Loss: 2.5881 | LR: 3.83e-04 | 2765.5ms/step | 8.7 samples/s
797
+ Step 6620/20005 | Loss: 2.5225 | LR: 3.82e-04 | 2766.5ms/step | 8.7 samples/s
798
+ Step 6630/20005 | Loss: 2.5697 | LR: 3.82e-04 | 2764.2ms/step | 8.7 samples/s
799
+ Step 6640/20005 | Loss: 2.5826 | LR: 3.82e-04 | 2768.8ms/step | 8.7 samples/s
800
+ Step 6650/20005 | Loss: 2.6739 | LR: 3.81e-04 | 2768.9ms/step | 8.7 samples/s
801
+ Step 6660/20005 | Loss: 2.5233 | LR: 3.81e-04 | 2764.2ms/step | 8.7 samples/s
802
+ Step 6670/20005 | Loss: 2.5347 | LR: 3.81e-04 | 2763.7ms/step | 8.7 samples/s
803
+ Step 6680/20005 | Loss: 2.5876 | LR: 3.80e-04 | 2765.5ms/step | 8.7 samples/s
804
+ Step 6690/20005 | Loss: 2.5315 | LR: 3.80e-04 | 2767.4ms/step | 8.7 samples/s
805
+ Step 6700/20005 | Loss: 2.5786 | LR: 3.80e-04 | 2764.3ms/step | 8.7 samples/s
806
+ Step 6710/20005 | Loss: 2.5761 | LR: 3.79e-04 | 2763.6ms/step | 8.7 samples/s
807
+ Step 6720/20005 | Loss: 2.5733 | LR: 3.79e-04 | 2766.3ms/step | 8.7 samples/s
808
+ Step 6730/20005 | Loss: 2.5776 | LR: 3.79e-04 | 2765.6ms/step | 8.7 samples/s
809
+ Step 6740/20005 | Loss: 2.6488 | LR: 3.78e-04 | 2765.5ms/step | 8.7 samples/s
810
+ Step 6750/20005 | Loss: 2.6005 | LR: 3.78e-04 | 2765.7ms/step | 8.7 samples/s
811
+ Validation at step 6750: Loss = 2.6055, Perplexity = 13.54
812
+ Step 6760/20005 | Loss: 2.6373 | LR: 3.78e-04 | 21969.5ms/step | 1.1 samples/s
813
+ Step 6770/20005 | Loss: 2.5891 | LR: 3.77e-04 | 2762.9ms/step | 8.7 samples/s
814
+ Step 6780/20005 | Loss: 2.5608 | LR: 3.77e-04 | 2762.9ms/step | 8.7 samples/s
815
+ Step 6790/20005 | Loss: 2.5636 | LR: 3.77e-04 | 2761.1ms/step | 8.7 samples/s
816
+ Step 6800/20005 | Loss: 2.5967 | LR: 3.76e-04 | 2765.8ms/step | 8.7 samples/s
817
+ Step 6810/20005 | Loss: 2.6028 | LR: 3.76e-04 | 2766.8ms/step | 8.7 samples/s
818
+ Step 6820/20005 | Loss: 2.6290 | LR: 3.76e-04 | 2764.0ms/step | 8.7 samples/s
819
+ Step 6830/20005 | Loss: 2.6022 | LR: 3.75e-04 | 2868.3ms/step | 8.4 samples/s
820
+ Step 6840/20005 | Loss: 2.6116 | LR: 3.75e-04 | 2763.2ms/step | 8.7 samples/s
821
+ Step 6850/20005 | Loss: 2.5353 | LR: 3.74e-04 | 2765.6ms/step | 8.7 samples/s
822
+ Step 6860/20005 | Loss: 2.5284 | LR: 3.74e-04 | 2762.4ms/step | 8.7 samples/s
823
+ Step 6870/20005 | Loss: 2.6421 | LR: 3.74e-04 | 2766.9ms/step | 8.7 samples/s
824
+ Step 6880/20005 | Loss: 2.5277 | LR: 3.73e-04 | 2766.0ms/step | 8.7 samples/s
825
+ Step 6890/20005 | Loss: 2.5660 | LR: 3.73e-04 | 2768.8ms/step | 8.7 samples/s
826
+ Step 6900/20005 | Loss: 2.5597 | LR: 3.73e-04 | 2767.4ms/step | 8.7 samples/s
827
+ Step 6910/20005 | Loss: 2.6820 | LR: 3.72e-04 | 2766.2ms/step | 8.7 samples/s
828
+ Step 6920/20005 | Loss: 2.6180 | LR: 3.72e-04 | 2769.3ms/step | 8.7 samples/s
829
+ Step 6930/20005 | Loss: 2.5491 | LR: 3.72e-04 | 2768.3ms/step | 8.7 samples/s
830
+ Step 6940/20005 | Loss: 2.6398 | LR: 3.71e-04 | 2766.9ms/step | 8.7 samples/s
831
+ Step 6950/20005 | Loss: 2.6003 | LR: 3.71e-04 | 2764.2ms/step | 8.7 samples/s
832
+ Step 6960/20005 | Loss: 2.5085 | LR: 3.71e-04 | 2768.1ms/step | 8.7 samples/s
833
+ Step 6970/20005 | Loss: 2.6305 | LR: 3.70e-04 | 2768.6ms/step | 8.7 samples/s
834
+ Step 6980/20005 | Loss: 2.5444 | LR: 3.70e-04 | 2768.1ms/step | 8.7 samples/s
835
+ Step 6990/20005 | Loss: 2.5377 | LR: 3.70e-04 | 2765.8ms/step | 8.7 samples/s
836
+ Step 7000/20005 | Loss: 2.6042 | LR: 3.69e-04 | 2769.1ms/step | 8.7 samples/s
837
+ Validation at step 7000: Loss = 2.5959, Perplexity = 13.41
838
+ Step 7010/20005 | Loss: 2.5834 | LR: 3.69e-04 | 21967.4ms/step | 1.1 samples/s
839
+ Step 7020/20005 | Loss: 2.5853 | LR: 3.69e-04 | 2762.8ms/step | 8.7 samples/s
840
+ Step 7030/20005 | Loss: 2.5845 | LR: 3.68e-04 | 2763.8ms/step | 8.7 samples/s
841
+ Step 7040/20005 | Loss: 2.5510 | LR: 3.68e-04 | 2764.6ms/step | 8.7 samples/s
842
+ Step 7050/20005 | Loss: 2.5967 | LR: 3.68e-04 | 2763.9ms/step | 8.7 samples/s
843
+ Step 7060/20005 | Loss: 2.4814 | LR: 3.67e-04 | 2768.3ms/step | 8.7 samples/s
844
+ Step 7070/20005 | Loss: 2.6303 | LR: 3.67e-04 | 2765.3ms/step | 8.7 samples/s
845
+ Step 7080/20005 | Loss: 2.5425 | LR: 3.66e-04 | 2765.1ms/step | 8.7 samples/s
846
+ Step 7090/20005 | Loss: 2.5633 | LR: 3.66e-04 | 2766.3ms/step | 8.7 samples/s
847
+ Step 7100/20005 | Loss: 2.5312 | LR: 3.66e-04 | 2761.8ms/step | 8.7 samples/s
848
+ Step 7110/20005 | Loss: 2.5015 | LR: 3.65e-04 | 2762.4ms/step | 8.7 samples/s
849
+ Step 7120/20005 | Loss: 2.5057 | LR: 3.65e-04 | 2767.0ms/step | 8.7 samples/s
850
+ Step 7130/20005 | Loss: 2.5481 | LR: 3.65e-04 | 2766.7ms/step | 8.7 samples/s
851
+ Step 7140/20005 | Loss: 2.5795 | LR: 3.64e-04 | 2766.2ms/step | 8.7 samples/s
852
+ Step 7150/20005 | Loss: 2.5532 | LR: 3.64e-04 | 2765.7ms/step | 8.7 samples/s
853
+ Step 7160/20005 | Loss: 2.5427 | LR: 3.64e-04 | 2765.6ms/step | 8.7 samples/s
854
+ Step 7170/20005 | Loss: 2.6414 | LR: 3.63e-04 | 2765.2ms/step | 8.7 samples/s
855
+ Step 7180/20005 | Loss: 2.5271 | LR: 3.63e-04 | 2763.0ms/step | 8.7 samples/s
856
+ Step 7190/20005 | Loss: 2.6377 | LR: 3.63e-04 | 2761.4ms/step | 8.7 samples/s
857
+ Step 7200/20005 | Loss: 2.6102 | LR: 3.62e-04 | 2763.4ms/step | 8.7 samples/s
858
+ Step 7210/20005 | Loss: 2.6331 | LR: 3.62e-04 | 2761.6ms/step | 8.7 samples/s
859
+ Step 7220/20005 | Loss: 2.6490 | LR: 3.62e-04 | 2763.8ms/step | 8.7 samples/s
860
+ Step 7230/20005 | Loss: 2.6052 | LR: 3.61e-04 | 2765.3ms/step | 8.7 samples/s
861
+ Step 7240/20005 | Loss: 2.5892 | LR: 3.61e-04 | 2767.5ms/step | 8.7 samples/s
862
+ Step 7250/20005 | Loss: 2.5862 | LR: 3.60e-04 | 2765.2ms/step | 8.7 samples/s
863
+ Validation at step 7250: Loss = 2.5840, Perplexity = 13.25
864
+ Step 7260/20005 | Loss: 2.5554 | LR: 3.60e-04 | 21964.4ms/step | 1.1 samples/s
865
+ Step 7270/20005 | Loss: 2.5625 | LR: 3.60e-04 | 2765.7ms/step | 8.7 samples/s
866
+ Step 7280/20005 | Loss: 2.5111 | LR: 3.59e-04 | 2765.7ms/step | 8.7 samples/s
867
+ Step 7290/20005 | Loss: 2.5650 | LR: 3.59e-04 | 2764.6ms/step | 8.7 samples/s
868
+ Step 7300/20005 | Loss: 2.5752 | LR: 3.59e-04 | 2762.9ms/step | 8.7 samples/s
869
+ Step 7310/20005 | Loss: 2.5519 | LR: 3.58e-04 | 2766.3ms/step | 8.7 samples/s
870
+ Step 7320/20005 | Loss: 2.6345 | LR: 3.58e-04 | 2764.0ms/step | 8.7 samples/s
871
+ Step 7330/20005 | Loss: 2.5152 | LR: 3.58e-04 | 2764.9ms/step | 8.7 samples/s
872
+ Step 7340/20005 | Loss: 2.5122 | LR: 3.57e-04 | 2762.1ms/step | 8.7 samples/s
873
+ Step 7350/20005 | Loss: 2.5692 | LR: 3.57e-04 | 2763.6ms/step | 8.7 samples/s
874
+ Step 7360/20005 | Loss: 2.4857 | LR: 3.57e-04 | 2767.0ms/step | 8.7 samples/s
875
+ Step 7370/20005 | Loss: 2.5389 | LR: 3.56e-04 | 2765.9ms/step | 8.7 samples/s
876
+ Step 7380/20005 | Loss: 2.5983 | LR: 3.56e-04 | 2763.7ms/step | 8.7 samples/s
877
+ Step 7390/20005 | Loss: 2.5393 | LR: 3.55e-04 | 2761.2ms/step | 8.7 samples/s
878
+ Step 7400/20005 | Loss: 2.5656 | LR: 3.55e-04 | 2767.8ms/step | 8.7 samples/s
879
+ Step 7410/20005 | Loss: 2.6436 | LR: 3.55e-04 | 2764.6ms/step | 8.7 samples/s
880
+ Step 7420/20005 | Loss: 2.5985 | LR: 3.54e-04 | 2768.4ms/step | 8.7 samples/s
881
+ Step 7430/20005 | Loss: 2.5825 | LR: 3.54e-04 | 2768.4ms/step | 8.7 samples/s
882
+ Step 7440/20005 | Loss: 2.5487 | LR: 3.54e-04 | 2765.2ms/step | 8.7 samples/s
883
+ Step 7450/20005 | Loss: 2.5599 | LR: 3.53e-04 | 2763.1ms/step | 8.7 samples/s
884
+ Step 7460/20005 | Loss: 2.5808 | LR: 3.53e-04 | 2763.8ms/step | 8.7 samples/s
885
+ Step 7470/20005 | Loss: 2.5745 | LR: 3.53e-04 | 2765.7ms/step | 8.7 samples/s
886
+ Step 7480/20005 | Loss: 2.5338 | LR: 3.52e-04 | 2764.8ms/step | 8.7 samples/s
887
+ Step 7490/20005 | Loss: 2.6214 | LR: 3.52e-04 | 2764.3ms/step | 8.7 samples/s
888
+ Step 7500/20005 | Loss: 2.5342 | LR: 3.51e-04 | 2763.2ms/step | 8.7 samples/s
889
+ Validation at step 7500: Loss = 2.5745, Perplexity = 13.13
890
+ Step 7510/20005 | Loss: 2.5104 | LR: 3.51e-04 | 21971.2ms/step | 1.1 samples/s
891
+ Step 7520/20005 | Loss: 2.5854 | LR: 3.51e-04 | 2765.5ms/step | 8.7 samples/s
892
+ Step 7530/20005 | Loss: 2.5306 | LR: 3.50e-04 | 2766.9ms/step | 8.7 samples/s
893
+ Step 7540/20005 | Loss: 2.4923 | LR: 3.50e-04 | 2766.6ms/step | 8.7 samples/s
894
+ Step 7550/20005 | Loss: 2.7164 | LR: 3.50e-04 | 2765.3ms/step | 8.7 samples/s
895
+ Step 7560/20005 | Loss: 2.5250 | LR: 3.49e-04 | 2763.9ms/step | 8.7 samples/s
896
+ Step 7570/20005 | Loss: 2.5113 | LR: 3.49e-04 | 2760.6ms/step | 8.7 samples/s
897
+ Step 7580/20005 | Loss: 2.5848 | LR: 3.49e-04 | 2762.0ms/step | 8.7 samples/s
898
+ Step 7590/20005 | Loss: 2.5499 | LR: 3.48e-04 | 2766.9ms/step | 8.7 samples/s
899
+ Step 7600/20005 | Loss: 2.6436 | LR: 3.48e-04 | 2765.5ms/step | 8.7 samples/s
900
+ Step 7610/20005 | Loss: 2.5293 | LR: 3.47e-04 | 2764.3ms/step | 8.7 samples/s
901
+ Step 7620/20005 | Loss: 2.5941 | LR: 3.47e-04 | 2765.4ms/step | 8.7 samples/s
902
+ Step 7630/20005 | Loss: 2.4800 | LR: 3.47e-04 | 2767.3ms/step | 8.7 samples/s
903
+ Step 7640/20005 | Loss: 2.5380 | LR: 3.46e-04 | 2764.5ms/step | 8.7 samples/s
904
+ Step 7650/20005 | Loss: 2.5384 | LR: 3.46e-04 | 2766.9ms/step | 8.7 samples/s
905
+ Step 7660/20005 | Loss: 2.5130 | LR: 3.46e-04 | 2761.8ms/step | 8.7 samples/s
906
+ Step 7670/20005 | Loss: 2.6861 | LR: 3.45e-04 | 2765.1ms/step | 8.7 samples/s
907
+ Step 7680/20005 | Loss: 2.5725 | LR: 3.45e-04 | 2765.1ms/step | 8.7 samples/s
908
+ Step 7690/20005 | Loss: 2.5032 | LR: 3.44e-04 | 2868.8ms/step | 8.4 samples/s
909
+ Step 7700/20005 | Loss: 2.5844 | LR: 3.44e-04 | 2761.6ms/step | 8.7 samples/s
910
+ Step 7710/20005 | Loss: 2.5812 | LR: 3.44e-04 | 2761.3ms/step | 8.7 samples/s
911
+ Step 7720/20005 | Loss: 2.4975 | LR: 3.43e-04 | 2761.2ms/step | 8.7 samples/s
912
+ Step 7730/20005 | Loss: 2.5790 | LR: 3.43e-04 | 2760.8ms/step | 8.7 samples/s
913
+ Step 7740/20005 | Loss: 2.6448 | LR: 3.43e-04 | 2765.0ms/step | 8.7 samples/s
914
+ Step 7750/20005 | Loss: 2.5725 | LR: 3.42e-04 | 2765.1ms/step | 8.7 samples/s
915
+ Validation at step 7750: Loss = 2.5633, Perplexity = 12.98
916
+ Step 7760/20005 | Loss: 2.5109 | LR: 3.42e-04 | 21968.2ms/step | 1.1 samples/s
917
+ Step 7770/20005 | Loss: 2.5681 | LR: 3.42e-04 | 2765.4ms/step | 8.7 samples/s
918
+ Step 7780/20005 | Loss: 2.5744 | LR: 3.41e-04 | 2764.0ms/step | 8.7 samples/s
919
+ Step 7790/20005 | Loss: 2.6003 | LR: 3.41e-04 | 2760.3ms/step | 8.7 samples/s
920
+ Step 7800/20005 | Loss: 2.5634 | LR: 3.40e-04 | 2762.6ms/step | 8.7 samples/s
921
+ Step 7810/20005 | Loss: 2.5360 | LR: 3.40e-04 | 2765.8ms/step | 8.7 samples/s
922
+ Step 7820/20005 | Loss: 2.6090 | LR: 3.40e-04 | 2765.4ms/step | 8.7 samples/s
923
+ Step 7830/20005 | Loss: 2.5129 | LR: 3.39e-04 | 2765.6ms/step | 8.7 samples/s
924
+ Step 7840/20005 | Loss: 2.4836 | LR: 3.39e-04 | 2762.1ms/step | 8.7 samples/s
925
+ Step 7850/20005 | Loss: 2.5231 | LR: 3.39e-04 | 2762.1ms/step | 8.7 samples/s
926
+ Step 7860/20005 | Loss: 2.5570 | LR: 3.38e-04 | 2762.4ms/step | 8.7 samples/s
927
+ Step 7870/20005 | Loss: 2.6068 | LR: 3.38e-04 | 2767.8ms/step | 8.7 samples/s
928
+ Step 7880/20005 | Loss: 2.5498 | LR: 3.37e-04 | 2763.0ms/step | 8.7 samples/s
929
+ Step 7890/20005 | Loss: 2.5351 | LR: 3.37e-04 | 2767.1ms/step | 8.7 samples/s
930
+ Step 7900/20005 | Loss: 2.5360 | LR: 3.37e-04 | 2765.6ms/step | 8.7 samples/s
931
+ Step 7910/20005 | Loss: 2.5372 | LR: 3.36e-04 | 2768.5ms/step | 8.7 samples/s
932
+ Step 7920/20005 | Loss: 2.5566 | LR: 3.36e-04 | 2765.1ms/step | 8.7 samples/s
933
+ Step 7930/20005 | Loss: 2.5967 | LR: 3.36e-04 | 2764.3ms/step | 8.7 samples/s
934
+ Step 7940/20005 | Loss: 2.5661 | LR: 3.35e-04 | 2764.1ms/step | 8.7 samples/s
935
+ Step 7950/20005 | Loss: 2.5648 | LR: 3.35e-04 | 2767.3ms/step | 8.7 samples/s
936
+ Step 7960/20005 | Loss: 2.5839 | LR: 3.34e-04 | 2766.2ms/step | 8.7 samples/s
937
+ Step 7970/20005 | Loss: 2.5916 | LR: 3.34e-04 | 2769.2ms/step | 8.7 samples/s
938
+ Step 7980/20005 | Loss: 2.5633 | LR: 3.34e-04 | 2765.5ms/step | 8.7 samples/s
939
+ Step 7990/20005 | Loss: 2.5521 | LR: 3.33e-04 | 2761.0ms/step | 8.7 samples/s
940
+ Step 8000/20005 | Loss: 2.5344 | LR: 3.33e-04 | 2765.2ms/step | 8.7 samples/s
941
+ Validation at step 8000: Loss = 2.5529, Perplexity = 12.84
942
+ Saved checkpoint to checkpoints/run_20260506_193527/checkpoint-8000
943
+ Step 8010/20005 | Loss: 2.5071 | LR: 3.33e-04 | 22557.6ms/step | 1.1 samples/s
944
+ Step 8020/20005 | Loss: 2.5569 | LR: 3.32e-04 | 2764.1ms/step | 8.7 samples/s
945
+ Step 8030/20005 | Loss: 2.5553 | LR: 3.32e-04 | 2764.8ms/step | 8.7 samples/s
946
+ Step 8040/20005 | Loss: 2.4974 | LR: 3.31e-04 | 2763.9ms/step | 8.7 samples/s
947
+ Step 8050/20005 | Loss: 2.5565 | LR: 3.31e-04 | 2762.6ms/step | 8.7 samples/s
948
+ Step 8060/20005 | Loss: 2.5757 | LR: 3.31e-04 | 2762.4ms/step | 8.7 samples/s
949
+ Step 8070/20005 | Loss: 2.5794 | LR: 3.30e-04 | 2761.1ms/step | 8.7 samples/s
950
+ Step 8080/20005 | Loss: 2.5224 | LR: 3.30e-04 | 2768.8ms/step | 8.7 samples/s
951
+ Step 8090/20005 | Loss: 2.4766 | LR: 3.30e-04 | 2766.2ms/step | 8.7 samples/s
952
+ Step 8100/20005 | Loss: 2.5621 | LR: 3.29e-04 | 2766.8ms/step | 8.7 samples/s
953
+ Step 8110/20005 | Loss: 2.4788 | LR: 3.29e-04 | 2766.2ms/step | 8.7 samples/s
954
+ Step 8120/20005 | Loss: 2.5851 | LR: 3.28e-04 | 2765.6ms/step | 8.7 samples/s
955
+ Step 8130/20005 | Loss: 2.5143 | LR: 3.28e-04 | 2765.7ms/step | 8.7 samples/s
956
+ Step 8140/20005 | Loss: 2.5161 | LR: 3.28e-04 | 2764.9ms/step | 8.7 samples/s
957
+ Step 8150/20005 | Loss: 2.5738 | LR: 3.27e-04 | 2765.7ms/step | 8.7 samples/s
958
+ Step 8160/20005 | Loss: 2.4888 | LR: 3.27e-04 | 2766.0ms/step | 8.7 samples/s
959
+ Step 8170/20005 | Loss: 2.4771 | LR: 3.27e-04 | 2767.4ms/step | 8.7 samples/s
960
+ Step 8180/20005 | Loss: 2.4856 | LR: 3.26e-04 | 2765.7ms/step | 8.7 samples/s
961
+ Step 8190/20005 | Loss: 2.5484 | LR: 3.26e-04 | 2766.8ms/step | 8.7 samples/s
962
+ Step 8200/20005 | Loss: 2.4483 | LR: 3.25e-04 | 2767.8ms/step | 8.7 samples/s
963
+ Step 8210/20005 | Loss: 2.5400 | LR: 3.25e-04 | 2764.7ms/step | 8.7 samples/s
964
+ Step 8220/20005 | Loss: 2.5507 | LR: 3.25e-04 | 2763.1ms/step | 8.7 samples/s
965
+ Step 8230/20005 | Loss: 2.5590 | LR: 3.24e-04 | 2761.4ms/step | 8.7 samples/s
966
+ Step 8240/20005 | Loss: 2.5536 | LR: 3.24e-04 | 2762.1ms/step | 8.7 samples/s
967
+ Step 8250/20005 | Loss: 2.3988 | LR: 3.24e-04 | 2762.3ms/step | 8.7 samples/s
968
+ Validation at step 8250: Loss = 2.5439, Perplexity = 12.73
969
+ Step 8260/20005 | Loss: 2.5339 | LR: 3.23e-04 | 21969.2ms/step | 1.1 samples/s
970
+ Step 8270/20005 | Loss: 2.4961 | LR: 3.23e-04 | 2762.4ms/step | 8.7 samples/s
971
+ Step 8280/20005 | Loss: 2.5935 | LR: 3.22e-04 | 2768.7ms/step | 8.7 samples/s
972
+ Step 8290/20005 | Loss: 2.5530 | LR: 3.22e-04 | 2762.9ms/step | 8.7 samples/s
973
+ Step 8300/20005 | Loss: 2.5359 | LR: 3.22e-04 | 2762.4ms/step | 8.7 samples/s
974
+ Step 8310/20005 | Loss: 2.5662 | LR: 3.21e-04 | 2765.2ms/step | 8.7 samples/s
975
+ Step 8320/20005 | Loss: 2.4643 | LR: 3.21e-04 | 2764.2ms/step | 8.7 samples/s
976
+ Step 8330/20005 | Loss: 2.6074 | LR: 3.20e-04 | 2763.2ms/step | 8.7 samples/s
977
+ Step 8340/20005 | Loss: 2.5487 | LR: 3.20e-04 | 2759.7ms/step | 8.7 samples/s
978
+ Step 8350/20005 | Loss: 2.5065 | LR: 3.20e-04 | 2762.6ms/step | 8.7 samples/s
979
+ Step 8360/20005 | Loss: 2.4909 | LR: 3.19e-04 | 2767.0ms/step | 8.7 samples/s
980
+ Step 8370/20005 | Loss: 2.4509 | LR: 3.19e-04 | 2766.6ms/step | 8.7 samples/s
981
+ Step 8380/20005 | Loss: 2.4697 | LR: 3.19e-04 | 2764.8ms/step | 8.7 samples/s
982
+ Step 8390/20005 | Loss: 2.5543 | LR: 3.18e-04 | 2766.0ms/step | 8.7 samples/s
983
+ Step 8400/20005 | Loss: 2.4903 | LR: 3.18e-04 | 2764.4ms/step | 8.7 samples/s
984
+ Step 8410/20005 | Loss: 2.5760 | LR: 3.17e-04 | 2762.6ms/step | 8.7 samples/s
985
+ Step 8420/20005 | Loss: 2.5203 | LR: 3.17e-04 | 2760.1ms/step | 8.7 samples/s
986
+ Step 8430/20005 | Loss: 2.5340 | LR: 3.17e-04 | 2767.4ms/step | 8.7 samples/s
987
+ Step 8440/20005 | Loss: 2.5623 | LR: 3.16e-04 | 2766.2ms/step | 8.7 samples/s
988
+ Step 8450/20005 | Loss: 2.4311 | LR: 3.16e-04 | 2766.9ms/step | 8.7 samples/s
989
+ Step 8460/20005 | Loss: 2.5816 | LR: 3.16e-04 | 2764.0ms/step | 8.7 samples/s
990
+ Step 8470/20005 | Loss: 2.5604 | LR: 3.15e-04 | 2760.9ms/step | 8.7 samples/s
991
+ Step 8480/20005 | Loss: 2.5650 | LR: 3.15e-04 | 2763.4ms/step | 8.7 samples/s
992
+ Step 8490/20005 | Loss: 2.5883 | LR: 3.14e-04 | 2761.9ms/step | 8.7 samples/s
993
+ Step 8500/20005 | Loss: 2.5375 | LR: 3.14e-04 | 2765.8ms/step | 8.7 samples/s
994
+ Validation at step 8500: Loss = 2.5358, Perplexity = 12.63
995
+ Step 8510/20005 | Loss: 2.5626 | LR: 3.14e-04 | 21966.2ms/step | 1.1 samples/s
996
+ Step 8520/20005 | Loss: 2.5167 | LR: 3.13e-04 | 2767.4ms/step | 8.7 samples/s
997
+ Step 8530/20005 | Loss: 2.5765 | LR: 3.13e-04 | 2763.4ms/step | 8.7 samples/s
998
+ Step 8540/20005 | Loss: 2.5451 | LR: 3.12e-04 | 2866.3ms/step | 8.4 samples/s
999
+ Step 8550/20005 | Loss: 2.4950 | LR: 3.12e-04 | 2763.6ms/step | 8.7 samples/s
1000
+ Step 8560/20005 | Loss: 2.5220 | LR: 3.12e-04 | 2763.5ms/step | 8.7 samples/s
1001
+ Step 8570/20005 | Loss: 2.4943 | LR: 3.11e-04 | 2763.1ms/step | 8.7 samples/s
1002
+ Step 8580/20005 | Loss: 2.4577 | LR: 3.11e-04 | 2766.7ms/step | 8.7 samples/s
1003
+ Step 8590/20005 | Loss: 2.4540 | LR: 3.11e-04 | 2764.1ms/step | 8.7 samples/s
1004
+ Step 8600/20005 | Loss: 2.5349 | LR: 3.10e-04 | 2762.6ms/step | 8.7 samples/s
1005
+ Step 8610/20005 | Loss: 2.5736 | LR: 3.10e-04 | 2763.2ms/step | 8.7 samples/s
1006
+ Step 8620/20005 | Loss: 2.5554 | LR: 3.09e-04 | 2761.8ms/step | 8.7 samples/s
1007
+ Step 8630/20005 | Loss: 2.5669 | LR: 3.09e-04 | 2761.7ms/step | 8.7 samples/s
1008
+ Step 8640/20005 | Loss: 2.5234 | LR: 3.09e-04 | 2767.4ms/step | 8.7 samples/s
1009
+ Step 8650/20005 | Loss: 2.5592 | LR: 3.08e-04 | 2762.8ms/step | 8.7 samples/s
1010
+ Step 8660/20005 | Loss: 2.5205 | LR: 3.08e-04 | 2766.8ms/step | 8.7 samples/s
1011
+ Step 8670/20005 | Loss: 2.5035 | LR: 3.07e-04 | 2766.7ms/step | 8.7 samples/s
1012
+ Step 8680/20005 | Loss: 2.6558 | LR: 3.07e-04 | 2764.9ms/step | 8.7 samples/s
1013
+ Step 8690/20005 | Loss: 2.5407 | LR: 3.07e-04 | 2762.0ms/step | 8.7 samples/s
1014
+ Step 8700/20005 | Loss: 2.5301 | LR: 3.06e-04 | 2763.5ms/step | 8.7 samples/s
1015
+ Step 8710/20005 | Loss: 2.5130 | LR: 3.06e-04 | 2766.3ms/step | 8.7 samples/s
1016
+ Step 8720/20005 | Loss: 2.5250 | LR: 3.05e-04 | 2762.0ms/step | 8.7 samples/s
1017
+ Step 8730/20005 | Loss: 2.5493 | LR: 3.05e-04 | 2761.6ms/step | 8.7 samples/s
1018
+ Step 8740/20005 | Loss: 2.5031 | LR: 3.05e-04 | 2761.1ms/step | 8.7 samples/s
1019
+ Step 8750/20005 | Loss: 2.4913 | LR: 3.04e-04 | 2768.9ms/step | 8.7 samples/s
1020
+ Validation at step 8750: Loss = 2.5291, Perplexity = 12.54
1021
+ Step 8760/20005 | Loss: 2.5922 | LR: 3.04e-04 | 21969.1ms/step | 1.1 samples/s
1022
+ Step 8770/20005 | Loss: 2.4729 | LR: 3.04e-04 | 2761.4ms/step | 8.7 samples/s
1023
+ Step 8780/20005 | Loss: 2.4610 | LR: 3.03e-04 | 2762.9ms/step | 8.7 samples/s
1024
+ Step 8790/20005 | Loss: 2.4924 | LR: 3.03e-04 | 2766.0ms/step | 8.7 samples/s
1025
+ Step 8800/20005 | Loss: 2.3887 | LR: 3.02e-04 | 2766.5ms/step | 8.7 samples/s
1026
+ Step 8810/20005 | Loss: 2.5757 | LR: 3.02e-04 | 2765.7ms/step | 8.7 samples/s
1027
+ Step 8820/20005 | Loss: 2.5087 | LR: 3.02e-04 | 2763.7ms/step | 8.7 samples/s
1028
+ Step 8830/20005 | Loss: 2.6246 | LR: 3.01e-04 | 2766.0ms/step | 8.7 samples/s
1029
+ Step 8840/20005 | Loss: 2.5038 | LR: 3.01e-04 | 2768.1ms/step | 8.7 samples/s
1030
+ Step 8850/20005 | Loss: 2.5476 | LR: 3.00e-04 | 2769.1ms/step | 8.7 samples/s
1031
+ Step 8860/20005 | Loss: 2.4972 | LR: 3.00e-04 | 2766.7ms/step | 8.7 samples/s
1032
+ Step 8870/20005 | Loss: 2.6018 | LR: 3.00e-04 | 2769.6ms/step | 8.7 samples/s
1033
+ Step 8880/20005 | Loss: 2.5409 | LR: 2.99e-04 | 2767.6ms/step | 8.7 samples/s
1034
+ Step 8890/20005 | Loss: 2.5726 | LR: 2.99e-04 | 2766.4ms/step | 8.7 samples/s
1035
+ Step 8900/20005 | Loss: 2.5135 | LR: 2.98e-04 | 2765.6ms/step | 8.7 samples/s
1036
+ Step 8910/20005 | Loss: 2.5114 | LR: 2.98e-04 | 2769.1ms/step | 8.7 samples/s
1037
+ Step 8920/20005 | Loss: 2.3795 | LR: 2.98e-04 | 2762.6ms/step | 8.7 samples/s
1038
+ Step 8930/20005 | Loss: 2.4371 | LR: 2.97e-04 | 2760.7ms/step | 8.7 samples/s
1039
+ Step 8940/20005 | Loss: 2.5544 | LR: 2.97e-04 | 2764.2ms/step | 8.7 samples/s
1040
+ Step 8950/20005 | Loss: 2.6087 | LR: 2.97e-04 | 2766.2ms/step | 8.7 samples/s
1041
+ Step 8960/20005 | Loss: 2.5412 | LR: 2.96e-04 | 2762.2ms/step | 8.7 samples/s
1042
+ Step 8970/20005 | Loss: 2.4893 | LR: 2.96e-04 | 2763.1ms/step | 8.7 samples/s
1043
+ Step 8980/20005 | Loss: 2.3852 | LR: 2.95e-04 | 2762.0ms/step | 8.7 samples/s
1044
+ Step 8990/20005 | Loss: 2.5673 | LR: 2.95e-04 | 2763.2ms/step | 8.7 samples/s
1045
+ Step 9000/20005 | Loss: 2.5075 | LR: 2.95e-04 | 2761.5ms/step | 8.7 samples/s
epfl_final/wandb/run-20260506_200450-z06f2580/files/requirements.txt ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pytz==2026.2
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.6.3
8
+ tzdata==2026.2
9
+ typing_extensions==4.15.0
10
+ triton==3.6.0
11
+ tqdm==4.67.3
12
+ torchinfo==1.8.0
13
+ sympy==1.14.0
14
+ smmap==5.0.3
15
+ six==1.17.0
16
+ shellingham==1.5.4
17
+ safetensors==0.7.0
18
+ regex==2026.4.4
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.4.1
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.2.6
38
+ networkx==3.4.2
39
+ mdurl==0.1.2
40
+ MarkupSafe==3.0.3
41
+ idna==3.13
42
+ hf-xet==1.5.0
43
+ h11==0.16.0
44
+ fsspec==2026.2.0
45
+ frozenlist==1.8.0
46
+ filelock==3.29.0
47
+ dill==0.4.1
48
+ cuda-pathfinder==1.5.4
49
+ click==8.3.3
50
+ charset-normalizer==3.4.7
51
+ certifi==2026.4.22
52
+ attrs==26.1.0
53
+ async-timeout==5.0.1
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ typing-inspection==0.4.2
58
+ sentry-sdk==2.59.0
59
+ requests==2.33.1
60
+ python-dateutil==2.9.0.post0
61
+ pydantic_core==2.46.4
62
+ nvidia-cusparse==12.6.3.3
63
+ nvidia-cufft==12.0.0.61
64
+ nvidia-cudnn-cu13==9.19.0.56
65
+ multiprocess==0.70.19
66
+ multidict==6.7.1
67
+ markdown-it-py==4.1.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ exceptiongroup==1.3.1
72
+ cuda-bindings==13.2.0
73
+ aiosignal==1.4.0
74
+ yarl==1.23.0
75
+ rich==15.0.0
76
+ pydantic==2.13.4
77
+ pandas==2.3.3
78
+ nvidia-cusolver==12.0.4.66
79
+ GitPython==3.1.50
80
+ anyio==4.13.0
81
+ wandb==0.26.1
82
+ typer==0.25.1
83
+ httpx==0.28.1
84
+ aiohttp==3.13.5
85
+ torch==2.11.0
86
+ huggingface_hub==1.14.0
87
+ tokenizers==0.22.2
88
+ datasets==4.8.5
89
+ accelerate==1.13.0
90
+ transformers==5.8.0
91
+ PyGObject==3.42.1
92
+ dbus-python==1.2.18
93
+ distro==1.7.0
94
+ pip==22.0.2
95
+ setuptools==59.6.0
96
+ ssh-import-id==5.11
97
+ wheel==0.37.1
epfl_final/wandb/run-20260506_200450-z06f2580/files/wandb-metadata.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.10.12",
4
+ "startedAt": "2026-05-06T20:04:50.668730Z",
5
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
6
+ "codePath": "SparseAtt/train.py",
7
+ "codePathLocal": "SparseAtt/train.py",
8
+ "email": "ruben.navasardyan@mail.utoronto.ca",
9
+ "root": "/mloscratch/homes/navasard/sparse",
10
+ "host": "h100-workspace-5-0-6",
11
+ "executable": "/usr/bin/python3",
12
+ "cpu_count": 96,
13
+ "cpu_count_logical": 192,
14
+ "gpu": "NVIDIA H100 80GB HBM3",
15
+ "gpu_count": 1,
16
+ "disk": {
17
+ "/": {
18
+ "total": "9374931091456",
19
+ "used": "6724987576320"
20
+ }
21
+ },
22
+ "memory": {
23
+ "total": "1623005741056"
24
+ },
25
+ "gpu_nvidia": [
26
+ {
27
+ "name": "NVIDIA H100 80GB HBM3",
28
+ "memoryTotal": "85520809984",
29
+ "cudaCores": 16896,
30
+ "architecture": "Hopper",
31
+ "uuid": "GPU-572b71c8-9d82-4b1c-e4f6-ed5616fd870a"
32
+ }
33
+ ],
34
+ "cudaVersion": "13.0",
35
+ "writerId": "s6499ir2pntjp8vbhkid6dhttd3m7lxb"
36
+ }
epfl_final/wandb/run-20260506_200450-z06f2580/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/wandb/run-20260506_200450-z06f2580/logs/debug.log ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-05-06 20:04:50,687 INFO MainThread:18682 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-05-06 20:04:50,687 INFO MainThread:18682 [wandb_setup.py:_flush():81] Configure stats pid to 18682
3
+ 2026-05-06 20:04:50,687 INFO MainThread:18682 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-05-06 20:04:50,687 INFO MainThread:18682 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260506_200450-z06f2580/logs/debug.log
5
+ 2026-05-06 20:04:50,687 INFO MainThread:18682 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260506_200450-z06f2580/logs/debug-internal.log
6
+ 2026-05-06 20:04:50,687 INFO MainThread:18682 [wandb_init.py:init():850] calling init triggers
7
+ 2026-05-06 20:04:50,687 INFO MainThread:18682 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/the_pile_train/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': './checkpoints/run_20260506_193527', 'resume_from_checkpoint': None, 'num_epochs': 1, 'max_steps': 20005, 'per_device_batch_size': 3, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 250, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 2000, 'eval_steps': 250, 'eval_batches': 200, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': None, 'wandb_tags': ['qwen3', '0.6b'], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, '_wandb': {}}
9
+ 2026-05-06 20:04:50,687 INFO MainThread:18682 [wandb_init.py:init():898] starting backend
10
+ 2026-05-06 20:04:50,903 INFO MainThread:18682 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-05-06 20:04:51,201 INFO MainThread:18682 [wandb_init.py:init():918] backend started and connected
12
+ 2026-05-06 20:04:51,202 INFO MainThread:18682 [wandb_init.py:init():988] updated telemetry
13
+ 2026-05-06 20:04:51,203 INFO MainThread:18682 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-05-06 20:04:51,603 INFO MainThread:18682 [wandb_init.py:init():1056] starting run threads in backend
15
+ 2026-05-06 20:04:51,655 INFO MainThread:18682 [wandb_run.py:_console_start():2554] atexit reg
16
+ 2026-05-06 20:04:51,655 INFO MainThread:18682 [wandb_run.py:_redirect():2403] redirect: wrap_raw
17
+ 2026-05-06 20:04:51,655 INFO MainThread:18682 [wandb_run.py:_redirect():2472] Wrapping output streams.
18
+ 2026-05-06 20:04:51,655 INFO MainThread:18682 [wandb_run.py:_redirect():2495] Redirects installed.
19
+ 2026-05-06 20:04:51,657 INFO MainThread:18682 [wandb_init.py:init():1094] run started, returning control to user process
epfl_final/wandb/run-20260506_200450-z06f2580/run-z06f2580.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:060edd881fb51ec1f3724b490610b57934d8b35c712ca2cc627287b56f2564c7
3
+ size 5111808
epfl_final/wandb/run-20260509_201725-v6634orv/files/config.yaml ADDED
@@ -0,0 +1,134 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ _wandb:
2
+ value:
3
+ cli_version: 0.26.1
4
+ e:
5
+ kpm2cvfwleyy3zpfc2cgqadojh6mzk44:
6
+ codePath: SparseAtt/train.py
7
+ codePathLocal: SparseAtt/train.py
8
+ cpu_count: 96
9
+ cpu_count_logical: 192
10
+ cudaVersion: "13.0"
11
+ disk:
12
+ /:
13
+ total: "9374931091456"
14
+ used: "6967971639296"
15
+ email: ruben.navasardyan@mail.utoronto.ca
16
+ executable: /usr/bin/python3
17
+ gpu: NVIDIA H100 80GB HBM3
18
+ gpu_count: 1
19
+ gpu_nvidia:
20
+ - architecture: Hopper
21
+ cudaCores: 16896
22
+ memoryTotal: "85520809984"
23
+ name: NVIDIA H100 80GB HBM3
24
+ uuid: GPU-8b16a485-aa97-1545-8625-887d6079c32c
25
+ host: h100-train-13-0-4
26
+ memory:
27
+ total: "1623005847552"
28
+ os: Linux-6.8.0-59-generic-x86_64-with-glibc2.35
29
+ program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
30
+ python: CPython 3.10.12
31
+ root: /mloscratch/homes/navasard/sparse
32
+ startedAt: "2026-05-09T20:17:25.091183Z"
33
+ writerId: kpm2cvfwleyy3zpfc2cgqadojh6mzk44
34
+ m: []
35
+ python_version: 3.10.12
36
+ t:
37
+ "1":
38
+ - 1
39
+ - 11
40
+ - 49
41
+ - 51
42
+ - 71
43
+ "2":
44
+ - 1
45
+ - 11
46
+ - 49
47
+ - 51
48
+ - 71
49
+ "3":
50
+ - 2
51
+ - 15
52
+ - 16
53
+ - 61
54
+ "4": 3.10.12
55
+ "5": 0.26.1
56
+ "6": 5.8.0
57
+ "12": 0.26.1
58
+ "13": linux-x86_64
59
+ attention_type:
60
+ value: dense
61
+ checkpoint_activations:
62
+ value: false
63
+ eval_batches:
64
+ value: 200
65
+ eval_steps:
66
+ value: 250
67
+ gradient_accumulation_steps:
68
+ value: 8
69
+ init_from_scratch:
70
+ value: true
71
+ learning_rate:
72
+ value: 0.0005
73
+ logging_steps:
74
+ value: 10
75
+ lr_scheduler:
76
+ value: cosine
77
+ max_grad_norm:
78
+ value: 1
79
+ max_seq_length:
80
+ value: 8192
81
+ max_steps:
82
+ value: 20005
83
+ mixed_precision:
84
+ value: bf16
85
+ model_dtype:
86
+ value: bfloat16
87
+ model_path:
88
+ value: /mloscratch/homes/navasard/qwen3_06b_base
89
+ num_epochs:
90
+ value: 1
91
+ num_workers:
92
+ value: 8
93
+ output_dir:
94
+ value: ./checkpoints/run_20260509_201713
95
+ per_device_batch_size:
96
+ value: 3
97
+ resume_from_checkpoint:
98
+ value: null
99
+ save_steps:
100
+ value: 1000
101
+ seed:
102
+ value: 42
103
+ sparse_block_size:
104
+ value: 128
105
+ sparse_layer_indices:
106
+ value: null
107
+ sparse_mask_pool_size:
108
+ value: 1000
109
+ sparse_num_random_blocks:
110
+ value: 31
111
+ sparse_val_dense_compare:
112
+ value: false
113
+ strategy:
114
+ value: ddp
115
+ train_path:
116
+ value: /mloscratch/homes/navasard/data/fineweb/tokenized
117
+ val_path:
118
+ value: null
119
+ verbose:
120
+ value: true
121
+ wandb_enabled:
122
+ value: true
123
+ wandb_name:
124
+ value: null
125
+ wandb_project:
126
+ value: stochastic_sparsity
127
+ wandb_tags:
128
+ value:
129
+ - qwen3
130
+ - 0.6b
131
+ warmup_steps:
132
+ value: 250
133
+ weight_decay:
134
+ value: 0.01
epfl_final/wandb/run-20260509_201725-v6634orv/files/output.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/wandb/run-20260509_201725-v6634orv/files/requirements.txt ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pytz==2026.2
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.7.0
8
+ tzdata==2026.2
9
+ typing_extensions==4.15.0
10
+ triton==3.6.0
11
+ tqdm==4.67.3
12
+ torchinfo==1.8.0
13
+ sympy==1.14.0
14
+ smmap==5.0.3
15
+ six==1.17.0
16
+ shellingham==1.5.4
17
+ safetensors==0.7.0
18
+ regex==2026.4.4
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.5.2
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.2.6
38
+ networkx==3.4.2
39
+ mdurl==0.1.2
40
+ MarkupSafe==3.0.3
41
+ idna==3.13
42
+ hf-xet==1.5.0
43
+ h11==0.16.0
44
+ fsspec==2026.2.0
45
+ frozenlist==1.8.0
46
+ filelock==3.29.0
47
+ dill==0.4.1
48
+ cuda-pathfinder==1.5.4
49
+ click==8.3.3
50
+ charset-normalizer==3.4.7
51
+ certifi==2026.4.22
52
+ attrs==26.1.0
53
+ async-timeout==5.0.1
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ typing-inspection==0.4.2
58
+ sentry-sdk==2.59.0
59
+ requests==2.33.1
60
+ python-dateutil==2.9.0.post0
61
+ pydantic_core==2.46.4
62
+ nvidia-cusparse==12.6.3.3
63
+ nvidia-cufft==12.0.0.61
64
+ nvidia-cudnn-cu13==9.19.0.56
65
+ multiprocess==0.70.19
66
+ multidict==6.7.1
67
+ markdown-it-py==4.2.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ exceptiongroup==1.3.1
72
+ cuda-bindings==13.2.0
73
+ aiosignal==1.4.0
74
+ yarl==1.23.0
75
+ rich==15.0.0
76
+ pydantic==2.13.4
77
+ pandas==2.3.3
78
+ nvidia-cusolver==12.0.4.66
79
+ GitPython==3.1.50
80
+ anyio==4.13.0
81
+ wandb==0.26.1
82
+ typer==0.25.1
83
+ httpx==0.28.1
84
+ aiohttp==3.13.5
85
+ torch==2.11.0
86
+ huggingface_hub==1.14.0
87
+ tokenizers==0.22.2
88
+ datasets==4.8.5
89
+ accelerate==1.13.0
90
+ transformers==5.8.0
91
+ PyGObject==3.42.1
92
+ dbus-python==1.2.18
93
+ distro==1.7.0
94
+ pip==22.0.2
95
+ setuptools==59.6.0
96
+ ssh-import-id==5.11
97
+ wheel==0.37.1
epfl_final/wandb/run-20260509_201725-v6634orv/files/wandb-metadata.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.10.12",
4
+ "startedAt": "2026-05-09T20:17:25.091183Z",
5
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
6
+ "codePath": "SparseAtt/train.py",
7
+ "codePathLocal": "SparseAtt/train.py",
8
+ "email": "ruben.navasardyan@mail.utoronto.ca",
9
+ "root": "/mloscratch/homes/navasard/sparse",
10
+ "host": "h100-train-13-0-4",
11
+ "executable": "/usr/bin/python3",
12
+ "cpu_count": 96,
13
+ "cpu_count_logical": 192,
14
+ "gpu": "NVIDIA H100 80GB HBM3",
15
+ "gpu_count": 1,
16
+ "disk": {
17
+ "/": {
18
+ "total": "9374931091456",
19
+ "used": "6967971639296"
20
+ }
21
+ },
22
+ "memory": {
23
+ "total": "1623005847552"
24
+ },
25
+ "gpu_nvidia": [
26
+ {
27
+ "name": "NVIDIA H100 80GB HBM3",
28
+ "memoryTotal": "85520809984",
29
+ "cudaCores": 16896,
30
+ "architecture": "Hopper",
31
+ "uuid": "GPU-8b16a485-aa97-1545-8625-887d6079c32c"
32
+ }
33
+ ],
34
+ "cudaVersion": "13.0",
35
+ "writerId": "kpm2cvfwleyy3zpfc2cgqadojh6mzk44"
36
+ }
epfl_final/wandb/run-20260509_201725-v6634orv/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"_timestamp":1.7785015765837305e+09,"train/learning_rate":7.903067225645266e-11,"_step":20000,"train/epoch":0.4716981132075472,"_wandb":{"runtime":143769},"_runtime":143769.347118953,"eval/perplexity":18.142080070137933,"train/loss":2.883942210674286,"speed/ms_per_step":5779.727935791016,"speed/steps_per_sec":0.1730185245930853,"eval/loss":2.898234105706215,"speed/samples_per_sec":4.152444590234047,"train/global_step":20000}
epfl_final/wandb/run-20260509_201725-v6634orv/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/wandb/run-20260509_201725-v6634orv/logs/debug.log ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-05-09 20:17:25,114 INFO MainThread:4588 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-05-09 20:17:25,114 INFO MainThread:4588 [wandb_setup.py:_flush():81] Configure stats pid to 4588
3
+ 2026-05-09 20:17:25,115 INFO MainThread:4588 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-05-09 20:17:25,116 INFO MainThread:4588 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260509_201725-v6634orv/logs/debug.log
5
+ 2026-05-09 20:17:25,116 INFO MainThread:4588 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260509_201725-v6634orv/logs/debug-internal.log
6
+ 2026-05-09 20:17:25,117 INFO MainThread:4588 [wandb_init.py:init():850] calling init triggers
7
+ 2026-05-09 20:17:25,118 INFO MainThread:4588 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': './checkpoints/run_20260509_201713', 'resume_from_checkpoint': None, 'num_epochs': 1, 'max_steps': 20005, 'per_device_batch_size': 3, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 250, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 200, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': None, 'wandb_tags': ['qwen3', '0.6b'], 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, '_wandb': {}}
9
+ 2026-05-09 20:17:25,119 INFO MainThread:4588 [wandb_init.py:init():898] starting backend
10
+ 2026-05-09 20:17:25,337 INFO MainThread:4588 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-05-09 20:17:25,701 INFO MainThread:4588 [wandb_init.py:init():918] backend started and connected
12
+ 2026-05-09 20:17:25,707 INFO MainThread:4588 [wandb_init.py:init():988] updated telemetry
13
+ 2026-05-09 20:17:25,709 INFO MainThread:4588 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-05-09 20:17:26,143 INFO MainThread:4588 [wandb_init.py:init():1056] starting run threads in backend
15
+ 2026-05-09 20:17:26,229 INFO MainThread:4588 [wandb_run.py:_console_start():2554] atexit reg
16
+ 2026-05-09 20:17:26,229 INFO MainThread:4588 [wandb_run.py:_redirect():2403] redirect: wrap_raw
17
+ 2026-05-09 20:17:26,229 INFO MainThread:4588 [wandb_run.py:_redirect():2472] Wrapping output streams.
18
+ 2026-05-09 20:17:26,236 INFO MainThread:4588 [wandb_run.py:_redirect():2495] Redirects installed.
19
+ 2026-05-09 20:17:26,257 INFO MainThread:4588 [wandb_init.py:init():1094] run started, returning control to user process
20
+ 2026-05-11 12:13:35,489 INFO MainThread:4588 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/v6634orv
21
+ 2026-05-11 12:13:35,490 INFO MainThread:4588 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
22
+ 2026-05-11 12:13:35,490 INFO MainThread:4588 [wandb_run.py:_restore():2502] restore
23
+ 2026-05-11 12:13:35,491 INFO MainThread:4588 [wandb_run.py:_restore():2508] restore done
24
+ 2026-05-11 12:13:38,690 INFO MainThread:4588 [wandb_run.py:_footer_sync_info():3889] logging synced files
epfl_final/wandb/run-20260509_201725-v6634orv/run-v6634orv.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0901bb90244aaf89d7fe3cad7f35592863b8f90552a4b76f354eb2bd99705b38
3
+ size 21184041
epfl_final/wandb/run-20260511_155411-fii64tue/files/config.yaml ADDED
@@ -0,0 +1,133 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ _wandb:
2
+ value:
3
+ cli_version: 0.26.1
4
+ e:
5
+ 51f4lp4opnmu0k00lxp85qgvwdbstbe2:
6
+ codePath: SparseAtt/train.py
7
+ codePathLocal: SparseAtt/train.py
8
+ cpu_count: 96
9
+ cpu_count_logical: 192
10
+ cudaVersion: "13.0"
11
+ disk:
12
+ /:
13
+ total: "9374931091456"
14
+ used: "7115316539392"
15
+ email: ruben.navasardyan@mail.utoronto.ca
16
+ executable: /usr/bin/python3
17
+ gpu: NVIDIA H100 80GB HBM3
18
+ gpu_count: 1
19
+ gpu_nvidia:
20
+ - architecture: Hopper
21
+ cudaCores: 16896
22
+ memoryTotal: "85520809984"
23
+ name: NVIDIA H100 80GB HBM3
24
+ uuid: GPU-b52a5d1a-9d9a-90fe-c3b3-708dcea4146f
25
+ host: h100-train-14-0-2
26
+ memory:
27
+ total: "1623005847552"
28
+ os: Linux-6.8.0-59-generic-x86_64-with-glibc2.35
29
+ program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
30
+ python: CPython 3.10.12
31
+ root: /mloscratch/homes/navasard/sparse
32
+ startedAt: "2026-05-11T15:54:11.043050Z"
33
+ writerId: 51f4lp4opnmu0k00lxp85qgvwdbstbe2
34
+ m: []
35
+ python_version: 3.10.12
36
+ t:
37
+ "1":
38
+ - 1
39
+ - 11
40
+ - 49
41
+ - 51
42
+ - 71
43
+ "2":
44
+ - 1
45
+ - 11
46
+ - 49
47
+ - 51
48
+ - 71
49
+ "3":
50
+ - 15
51
+ - 16
52
+ - 61
53
+ "4": 3.10.12
54
+ "5": 0.26.1
55
+ "6": 5.8.0
56
+ "12": 0.26.1
57
+ "13": linux-x86_64
58
+ attention_type:
59
+ value: sparse_stochastic
60
+ checkpoint_activations:
61
+ value: false
62
+ eval_batches:
63
+ value: 200
64
+ eval_steps:
65
+ value: 250
66
+ gradient_accumulation_steps:
67
+ value: 8
68
+ init_from_scratch:
69
+ value: true
70
+ learning_rate:
71
+ value: 0.0005
72
+ logging_steps:
73
+ value: 10
74
+ lr_scheduler:
75
+ value: cosine
76
+ max_grad_norm:
77
+ value: 1
78
+ max_seq_length:
79
+ value: 8192
80
+ max_steps:
81
+ value: 20005
82
+ mixed_precision:
83
+ value: bf16
84
+ model_dtype:
85
+ value: bfloat16
86
+ model_path:
87
+ value: /mloscratch/homes/navasard/qwen3_06b_base
88
+ num_epochs:
89
+ value: 1
90
+ num_workers:
91
+ value: 8
92
+ output_dir:
93
+ value: ./checkpoints/run_20260511_144655
94
+ per_device_batch_size:
95
+ value: 3
96
+ resume_from_checkpoint:
97
+ value: null
98
+ save_steps:
99
+ value: 1000
100
+ seed:
101
+ value: 42
102
+ sparse_block_size:
103
+ value: 128
104
+ sparse_layer_indices:
105
+ value: null
106
+ sparse_mask_pool_size:
107
+ value: 2000
108
+ sparse_num_random_blocks:
109
+ value: 31
110
+ sparse_val_dense_compare:
111
+ value: false
112
+ strategy:
113
+ value: ddp
114
+ train_path:
115
+ value: /mloscratch/homes/navasard/data/fineweb/tokenized
116
+ val_path:
117
+ value: null
118
+ verbose:
119
+ value: true
120
+ wandb_enabled:
121
+ value: true
122
+ wandb_name:
123
+ value: null
124
+ wandb_project:
125
+ value: stochastic_sparsity
126
+ wandb_tags:
127
+ value:
128
+ - qwen3
129
+ - 0.6b
130
+ warmup_steps:
131
+ value: 250
132
+ weight_decay:
133
+ value: 0.01
epfl_final/wandb/run-20260511_155411-fii64tue/files/output.log ADDED
@@ -0,0 +1,467 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ ================================================================================
4
+ MODEL ARCHITECTURE
5
+ ================================================================================
6
+ Model config:
7
+ vocab_size: 151936
8
+ hidden_size: 1024
9
+ num_hidden_layers: 28
10
+ num_attention_heads: 16
11
+ num_key_value_heads: 8
12
+ head_dim: 128
13
+ intermediate_size: 3072
14
+ max_position_embeddings: 32768
15
+ rope_theta: 1000000
16
+ attention_type: sparse_stochastic
17
+ sparse_block_size: 128
18
+ sparse_num_random_blocks: 31
19
+ sparse_mask_pool_size: 2000
20
+ sparse_max_seq_length: 8192
21
+
22
+ Parameter count:
23
+ Total parameters: 596,049,920
24
+ Trainable parameters: 596,049,920
25
+ Non-trainable parameters: 0
26
+ Model size (BF16): 1.19 GB
27
+
28
+ --------------------------------------------------------------------------------
29
+ LAYER-BY-LAYER SUMMARY
30
+ --------------------------------------------------------------------------------
31
+ /home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
32
+
33
+ SOLUTION: Use torch.compile(flex_attention)(...)
34
+
35
+ If you want to debug your score_mod/mask_mod, you can set:
36
+ torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
37
+
38
+ This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
39
+ _warn_once(
40
+ ================================================================================================================================================================
41
+ Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
42
+ ================================================================================================================================================================
43
+ Qwen3ForCausalLM [3, 8192] -- -- True
44
+ ├─Qwen3Model: 1-1 -- [3, 8192, 1024] -- True
45
+ │ └─Embedding: 2-1 [3, 8192] [3, 8192, 1024] 155,582,464 True
46
+ │ └─ModuleList: 2-2 -- -- -- True
47
+ │ │ └─Qwen3DecoderLayer: 3-1 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
48
+ │ │ └─Qwen3DecoderLayer: 3-2 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
49
+ │ │ └─Qwen3DecoderLayer: 3-3 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
50
+ │ │ └─Qwen3DecoderLayer: 3-4 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
51
+ │ │ └─Qwen3DecoderLayer: 3-5 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
52
+ │ │ └─Qwen3DecoderLayer: 3-6 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
53
+ │ │ └─Qwen3DecoderLayer: 3-7 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
54
+ │ │ └─Qwen3DecoderLayer: 3-8 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
55
+ │ │ └─Qwen3DecoderLayer: 3-9 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
56
+ │ │ └─Qwen3DecoderLayer: 3-10 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
57
+ │ │ └─Qwen3DecoderLayer: 3-11 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
58
+ │ │ └─Qwen3DecoderLayer: 3-12 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
59
+ │ │ └─Qwen3DecoderLayer: 3-13 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
60
+ │ │ └─Qwen3DecoderLayer: 3-14 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
61
+ │ │ └─Qwen3DecoderLayer: 3-15 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
62
+ │ │ └─Qwen3DecoderLayer: 3-16 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
63
+ │ │ └─Qwen3DecoderLayer: 3-17 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
64
+ │ │ └─Qwen3DecoderLayer: 3-18 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
65
+ │ │ └─Qwen3DecoderLayer: 3-19 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
66
+ │ │ └─Qwen3DecoderLayer: 3-20 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
67
+ │ │ └─Qwen3DecoderLayer: 3-21 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
68
+ │ │ └─Qwen3DecoderLayer: 3-22 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
69
+ │ │ └─Qwen3DecoderLayer: 3-23 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
70
+ │ │ └─Qwen3DecoderLayer: 3-24 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
71
+ │ │ └─Qwen3DecoderLayer: 3-25 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
72
+ │ │ └─Qwen3DecoderLayer: 3-26 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
73
+ │ │ └─Qwen3DecoderLayer: 3-27 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
74
+ │ │ └─Qwen3DecoderLayer: 3-28 [3, 8192, 1024] [3, 8192, 1024] 15,730,944 True
75
+ │ └─Qwen3RMSNorm: 2-3 [3, 8192, 1024] [3, 8192, 1024] 1,024 True
76
+ ├─Linear: 1-2 [3, 8192, 1024] [3, 8192, 151936] 155,582,464 True
77
+ ================================================================================================================================================================
78
+ Total params: 751,632,384
79
+ Trainable params: 751,632,384
80
+ Non-trainable params: 0
81
+ Total mult-adds (G): 2.25
82
+ ================================================================================================================================================================
83
+ Input size (MB): 0.20
84
+ Forward/backward pass size (MB): 63052.97
85
+ Params size (MB): 1503.26
86
+ Estimated Total Size (MB): 64556.43
87
+ ================================================================================================================================================================
88
+ ================================================================================
89
+
90
+ Loading dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
91
+ Using pre-tokenized dataset (no runtime tokenization)
92
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
93
+ Loaded 5088012 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1272003)
94
+ Dataset split: 1017602 train, 254401 validation
95
+ Training examples (packed, train split): 1017602
96
+ Validation examples (packed, val split): 254401
97
+ Per-device batch size: 3
98
+ World size (GPUs): 1
99
+ Gradient accumulation steps: 8
100
+ Effective batch size (seqs/step): 24
101
+ Sequence length: 8192
102
+ Tokens per optimizer step: 196,608
103
+ Micro-batches per epoch (per rank): 339200
104
+ Optimizer steps per epoch: 42400
105
+ Total optimizer steps: 20005 (capped by max_steps; 0.47 epochs)
106
+ Starting training...
107
+ Step 10/20005 | Loss: 12.0527 | LR: 2.00e-05 | 10905.8ms/step | 2.2 samples/s
108
+ Step 20/20005 | Loss: 11.1434 | LR: 4.00e-05 | 2739.2ms/step | 8.8 samples/s
109
+ Step 30/20005 | Loss: 10.4121 | LR: 6.00e-05 | 2728.8ms/step | 8.8 samples/s
110
+ Step 40/20005 | Loss: 9.8615 | LR: 8.00e-05 | 2722.0ms/step | 8.8 samples/s
111
+ Step 50/20005 | Loss: 9.0205 | LR: 1.00e-04 | 2725.9ms/step | 8.8 samples/s
112
+ Step 60/20005 | Loss: 8.2371 | LR: 1.20e-04 | 2721.6ms/step | 8.8 samples/s
113
+ Step 70/20005 | Loss: 7.6584 | LR: 1.40e-04 | 2732.9ms/step | 8.8 samples/s
114
+ Step 80/20005 | Loss: 7.4224 | LR: 1.60e-04 | 2742.7ms/step | 8.8 samples/s
115
+ Step 90/20005 | Loss: 7.2768 | LR: 1.80e-04 | 2747.3ms/step | 8.7 samples/s
116
+ Step 100/20005 | Loss: 7.0964 | LR: 2.00e-04 | 2755.3ms/step | 8.7 samples/s
117
+ Step 110/20005 | Loss: 6.9559 | LR: 2.20e-04 | 2751.5ms/step | 8.7 samples/s
118
+ Step 120/20005 | Loss: 6.7968 | LR: 2.40e-04 | 2758.9ms/step | 8.7 samples/s
119
+ Step 130/20005 | Loss: 6.6881 | LR: 2.60e-04 | 2759.2ms/step | 8.7 samples/s
120
+ Step 140/20005 | Loss: 6.5831 | LR: 2.80e-04 | 2757.6ms/step | 8.7 samples/s
121
+ Step 150/20005 | Loss: 6.4733 | LR: 3.00e-04 | 2757.4ms/step | 8.7 samples/s
122
+ Step 160/20005 | Loss: 6.3844 | LR: 3.20e-04 | 2759.5ms/step | 8.7 samples/s
123
+ Step 170/20005 | Loss: 6.3235 | LR: 3.40e-04 | 2756.4ms/step | 8.7 samples/s
124
+ Step 180/20005 | Loss: 6.2608 | LR: 3.60e-04 | 2756.9ms/step | 8.7 samples/s
125
+ Step 190/20005 | Loss: 6.1785 | LR: 3.80e-04 | 2762.1ms/step | 8.7 samples/s
126
+ Step 200/20005 | Loss: 6.1074 | LR: 4.00e-04 | 2760.0ms/step | 8.7 samples/s
127
+ Step 210/20005 | Loss: 6.0442 | LR: 4.20e-04 | 2761.0ms/step | 8.7 samples/s
128
+ Step 220/20005 | Loss: 5.9745 | LR: 4.40e-04 | 2758.4ms/step | 8.7 samples/s
129
+ Step 230/20005 | Loss: 5.9506 | LR: 4.60e-04 | 2757.6ms/step | 8.7 samples/s
130
+ Step 240/20005 | Loss: 5.9295 | LR: 4.80e-04 | 2763.3ms/step | 8.7 samples/s
131
+ Step 250/20005 | Loss: 5.8254 | LR: 5.00e-04 | 2760.8ms/step | 8.7 samples/s
132
+ Validation at step 250: Loss = 5.8073, Perplexity = 332.71
133
+ Step 260/20005 | Loss: 5.7952 | LR: 5.00e-04 | 25512.3ms/step | 0.9 samples/s
134
+ Step 270/20005 | Loss: 5.7279 | LR: 5.00e-04 | 2765.3ms/step | 8.7 samples/s
135
+ Step 280/20005 | Loss: 5.6681 | LR: 5.00e-04 | 2765.7ms/step | 8.7 samples/s
136
+ Step 290/20005 | Loss: 5.6214 | LR: 5.00e-04 | 2764.0ms/step | 8.7 samples/s
137
+ Step 300/20005 | Loss: 5.5143 | LR: 5.00e-04 | 2765.5ms/step | 8.7 samples/s
138
+ Step 310/20005 | Loss: 5.4557 | LR: 5.00e-04 | 2763.2ms/step | 8.7 samples/s
139
+ Step 320/20005 | Loss: 5.4503 | LR: 5.00e-04 | 2765.1ms/step | 8.7 samples/s
140
+ Step 330/20005 | Loss: 5.4291 | LR: 5.00e-04 | 2763.8ms/step | 8.7 samples/s
141
+ Step 340/20005 | Loss: 5.3610 | LR: 5.00e-04 | 2767.0ms/step | 8.7 samples/s
142
+ Step 350/20005 | Loss: 5.3131 | LR: 5.00e-04 | 2766.3ms/step | 8.7 samples/s
143
+ Step 360/20005 | Loss: 5.2684 | LR: 5.00e-04 | 2765.7ms/step | 8.7 samples/s
144
+ Step 370/20005 | Loss: 5.2249 | LR: 5.00e-04 | 2767.9ms/step | 8.7 samples/s
145
+ Step 380/20005 | Loss: 5.1672 | LR: 5.00e-04 | 2764.6ms/step | 8.7 samples/s
146
+ Step 390/20005 | Loss: 5.1468 | LR: 5.00e-04 | 2761.4ms/step | 8.7 samples/s
147
+ Step 400/20005 | Loss: 5.1236 | LR: 5.00e-04 | 2762.6ms/step | 8.7 samples/s
148
+ Step 410/20005 | Loss: 5.0547 | LR: 5.00e-04 | 2762.8ms/step | 8.7 samples/s
149
+ Step 420/20005 | Loss: 5.0607 | LR: 5.00e-04 | 2761.6ms/step | 8.7 samples/s
150
+ Step 430/20005 | Loss: 5.0004 | LR: 5.00e-04 | 2763.4ms/step | 8.7 samples/s
151
+ Step 440/20005 | Loss: 4.9464 | LR: 5.00e-04 | 2761.8ms/step | 8.7 samples/s
152
+ Step 450/20005 | Loss: 4.9140 | LR: 5.00e-04 | 2760.4ms/step | 8.7 samples/s
153
+ Step 460/20005 | Loss: 4.9033 | LR: 5.00e-04 | 2762.0ms/step | 8.7 samples/s
154
+ Step 470/20005 | Loss: 4.8663 | LR: 5.00e-04 | 2764.4ms/step | 8.7 samples/s
155
+ Step 480/20005 | Loss: 4.8340 | LR: 5.00e-04 | 2762.8ms/step | 8.7 samples/s
156
+ Step 490/20005 | Loss: 4.8180 | LR: 5.00e-04 | 2763.8ms/step | 8.7 samples/s
157
+ Step 500/20005 | Loss: 4.7787 | LR: 5.00e-04 | 2765.4ms/step | 8.7 samples/s
158
+ Validation at step 500: Loss = 4.7717, Perplexity = 118.12
159
+ Step 510/20005 | Loss: 4.7716 | LR: 5.00e-04 | 21970.5ms/step | 1.1 samples/s
160
+ Step 520/20005 | Loss: 4.7357 | LR: 5.00e-04 | 2764.5ms/step | 8.7 samples/s
161
+ Step 530/20005 | Loss: 4.6742 | LR: 5.00e-04 | 2764.1ms/step | 8.7 samples/s
162
+ Step 540/20005 | Loss: 4.6742 | LR: 5.00e-04 | 2762.8ms/step | 8.7 samples/s
163
+ Step 550/20005 | Loss: 4.6391 | LR: 5.00e-04 | 2760.2ms/step | 8.7 samples/s
164
+ Step 560/20005 | Loss: 4.5814 | LR: 5.00e-04 | 2763.6ms/step | 8.7 samples/s
165
+ Step 570/20005 | Loss: 4.6031 | LR: 5.00e-04 | 2763.3ms/step | 8.7 samples/s
166
+ Step 580/20005 | Loss: 4.5612 | LR: 5.00e-04 | 2760.8ms/step | 8.7 samples/s
167
+ Step 590/20005 | Loss: 4.5202 | LR: 5.00e-04 | 2761.0ms/step | 8.7 samples/s
168
+ Step 600/20005 | Loss: 4.5007 | LR: 5.00e-04 | 2761.9ms/step | 8.7 samples/s
169
+ Step 610/20005 | Loss: 4.4848 | LR: 5.00e-04 | 2769.3ms/step | 8.7 samples/s
170
+ Step 620/20005 | Loss: 4.4061 | LR: 5.00e-04 | 2769.8ms/step | 8.7 samples/s
171
+ Step 630/20005 | Loss: 4.4174 | LR: 5.00e-04 | 2764.1ms/step | 8.7 samples/s
172
+ Step 640/20005 | Loss: 4.3735 | LR: 5.00e-04 | 2765.4ms/step | 8.7 samples/s
173
+ Step 650/20005 | Loss: 4.3873 | LR: 4.99e-04 | 2763.5ms/step | 8.7 samples/s
174
+ Step 660/20005 | Loss: 4.3536 | LR: 4.99e-04 | 2763.3ms/step | 8.7 samples/s
175
+ Step 670/20005 | Loss: 4.3517 | LR: 4.99e-04 | 2761.8ms/step | 8.7 samples/s
176
+ Step 680/20005 | Loss: 4.3266 | LR: 4.99e-04 | 2761.8ms/step | 8.7 samples/s
177
+ Step 690/20005 | Loss: 4.2936 | LR: 4.99e-04 | 2764.7ms/step | 8.7 samples/s
178
+ Step 700/20005 | Loss: 4.2970 | LR: 4.99e-04 | 2759.3ms/step | 8.7 samples/s
179
+ Step 710/20005 | Loss: 4.2761 | LR: 4.99e-04 | 2762.3ms/step | 8.7 samples/s
180
+ Step 720/20005 | Loss: 4.2406 | LR: 4.99e-04 | 2765.5ms/step | 8.7 samples/s
181
+ Step 730/20005 | Loss: 4.2322 | LR: 4.99e-04 | 2763.8ms/step | 8.7 samples/s
182
+ Step 740/20005 | Loss: 4.2136 | LR: 4.99e-04 | 2768.5ms/step | 8.7 samples/s
183
+ Step 750/20005 | Loss: 4.1953 | LR: 4.99e-04 | 2759.0ms/step | 8.7 samples/s
184
+ Validation at step 750: Loss = 4.1968, Perplexity = 66.48
185
+ Step 760/20005 | Loss: 4.1865 | LR: 4.99e-04 | 21967.0ms/step | 1.1 samples/s
186
+ Step 770/20005 | Loss: 4.1984 | LR: 4.99e-04 | 2763.4ms/step | 8.7 samples/s
187
+ Step 780/20005 | Loss: 4.1796 | LR: 4.99e-04 | 2760.6ms/step | 8.7 samples/s
188
+ Step 790/20005 | Loss: 4.1476 | LR: 4.99e-04 | 2760.3ms/step | 8.7 samples/s
189
+ Step 800/20005 | Loss: 4.1562 | LR: 4.99e-04 | 2759.9ms/step | 8.7 samples/s
190
+ Step 810/20005 | Loss: 4.1258 | LR: 4.99e-04 | 2766.0ms/step | 8.7 samples/s
191
+ Step 820/20005 | Loss: 4.1101 | LR: 4.99e-04 | 2764.8ms/step | 8.7 samples/s
192
+ Step 830/20005 | Loss: 4.1161 | LR: 4.99e-04 | 2760.1ms/step | 8.7 samples/s
193
+ Step 840/20005 | Loss: 4.0501 | LR: 4.99e-04 | 2760.0ms/step | 8.7 samples/s
194
+ Step 850/20005 | Loss: 4.0656 | LR: 4.99e-04 | 2760.2ms/step | 8.7 samples/s
195
+ Step 860/20005 | Loss: 4.0494 | LR: 4.99e-04 | 2762.2ms/step | 8.7 samples/s
196
+ Step 870/20005 | Loss: 4.0592 | LR: 4.99e-04 | 2761.6ms/step | 8.7 samples/s
197
+ Step 880/20005 | Loss: 4.0562 | LR: 4.99e-04 | 2761.6ms/step | 8.7 samples/s
198
+ Step 890/20005 | Loss: 4.0351 | LR: 4.99e-04 | 2762.6ms/step | 8.7 samples/s
199
+ Step 900/20005 | Loss: 4.0388 | LR: 4.99e-04 | 2758.5ms/step | 8.7 samples/s
200
+ Step 910/20005 | Loss: 4.0095 | LR: 4.99e-04 | 2760.4ms/step | 8.7 samples/s
201
+ Step 920/20005 | Loss: 4.0152 | LR: 4.99e-04 | 2763.2ms/step | 8.7 samples/s
202
+ Step 930/20005 | Loss: 3.9833 | LR: 4.99e-04 | 2761.1ms/step | 8.7 samples/s
203
+ Step 940/20005 | Loss: 3.9842 | LR: 4.98e-04 | 2760.9ms/step | 8.7 samples/s
204
+ Step 950/20005 | Loss: 3.9791 | LR: 4.98e-04 | 2759.2ms/step | 8.7 samples/s
205
+ Step 960/20005 | Loss: 3.9441 | LR: 4.98e-04 | 2759.0ms/step | 8.7 samples/s
206
+ Step 970/20005 | Loss: 3.9736 | LR: 4.98e-04 | 2760.3ms/step | 8.7 samples/s
207
+ Step 980/20005 | Loss: 3.9647 | LR: 4.98e-04 | 2756.9ms/step | 8.7 samples/s
208
+ Step 990/20005 | Loss: 3.9493 | LR: 4.98e-04 | 2760.7ms/step | 8.7 samples/s
209
+ Step 1000/20005 | Loss: 3.9614 | LR: 4.98e-04 | 2759.9ms/step | 8.7 samples/s
210
+ Validation at step 1000: Loss = 3.9378, Perplexity = 51.30
211
+ Saved checkpoint to checkpoints/run_20260511_144655/checkpoint-1000
212
+ Step 1010/20005 | Loss: 3.9502 | LR: 4.98e-04 | 22757.1ms/step | 1.1 samples/s
213
+ Step 1020/20005 | Loss: 3.9399 | LR: 4.98e-04 | 2773.0ms/step | 8.7 samples/s
214
+ Step 1030/20005 | Loss: 3.8943 | LR: 4.98e-04 | 2768.1ms/step | 8.7 samples/s
215
+ Step 1040/20005 | Loss: 3.9048 | LR: 4.98e-04 | 2758.9ms/step | 8.7 samples/s
216
+ Step 1050/20005 | Loss: 3.8844 | LR: 4.98e-04 | 2761.5ms/step | 8.7 samples/s
217
+ Step 1060/20005 | Loss: 3.8969 | LR: 4.98e-04 | 2760.9ms/step | 8.7 samples/s
218
+ Step 1070/20005 | Loss: 3.8798 | LR: 4.98e-04 | 2761.2ms/step | 8.7 samples/s
219
+ Step 1080/20005 | Loss: 3.8848 | LR: 4.98e-04 | 2761.4ms/step | 8.7 samples/s
220
+ Step 1090/20005 | Loss: 3.8684 | LR: 4.98e-04 | 2761.7ms/step | 8.7 samples/s
221
+ Step 1100/20005 | Loss: 3.8897 | LR: 4.98e-04 | 2760.6ms/step | 8.7 samples/s
222
+ Step 1110/20005 | Loss: 3.8610 | LR: 4.98e-04 | 2764.1ms/step | 8.7 samples/s
223
+ Step 1120/20005 | Loss: 3.8162 | LR: 4.98e-04 | 2768.2ms/step | 8.7 samples/s
224
+ Step 1130/20005 | Loss: 3.8480 | LR: 4.98e-04 | 2766.9ms/step | 8.7 samples/s
225
+ Step 1140/20005 | Loss: 3.8472 | LR: 4.98e-04 | 2762.1ms/step | 8.7 samples/s
226
+ Step 1150/20005 | Loss: 3.8400 | LR: 4.97e-04 | 2761.8ms/step | 8.7 samples/s
227
+ Step 1160/20005 | Loss: 3.8394 | LR: 4.97e-04 | 2758.6ms/step | 8.7 samples/s
228
+ Step 1170/20005 | Loss: 3.8153 | LR: 4.97e-04 | 2757.9ms/step | 8.7 samples/s
229
+ Step 1180/20005 | Loss: 3.8324 | LR: 4.97e-04 | 2760.6ms/step | 8.7 samples/s
230
+ Step 1190/20005 | Loss: 3.8034 | LR: 4.97e-04 | 2759.9ms/step | 8.7 samples/s
231
+ Step 1200/20005 | Loss: 3.8027 | LR: 4.97e-04 | 2761.3ms/step | 8.7 samples/s
232
+ Step 1210/20005 | Loss: 3.8133 | LR: 4.97e-04 | 2769.0ms/step | 8.7 samples/s
233
+ Step 1220/20005 | Loss: 3.8230 | LR: 4.97e-04 | 2762.4ms/step | 8.7 samples/s
234
+ Step 1230/20005 | Loss: 3.8067 | LR: 4.97e-04 | 2761.5ms/step | 8.7 samples/s
235
+ Step 1240/20005 | Loss: 3.8100 | LR: 4.97e-04 | 2764.9ms/step | 8.7 samples/s
236
+ Step 1250/20005 | Loss: 3.7590 | LR: 4.97e-04 | 2767.2ms/step | 8.7 samples/s
237
+ Validation at step 1250: Loss = 3.7802, Perplexity = 43.82
238
+ Step 1260/20005 | Loss: 3.7793 | LR: 4.97e-04 | 21974.8ms/step | 1.1 samples/s
239
+ Step 1270/20005 | Loss: 3.7680 | LR: 4.97e-04 | 2762.1ms/step | 8.7 samples/s
240
+ Step 1280/20005 | Loss: 3.7410 | LR: 4.97e-04 | 2764.9ms/step | 8.7 samples/s
241
+ Step 1290/20005 | Loss: 3.7464 | LR: 4.97e-04 | 2763.5ms/step | 8.7 samples/s
242
+ Step 1300/20005 | Loss: 3.7590 | LR: 4.97e-04 | 2765.3ms/step | 8.7 samples/s
243
+ Step 1310/20005 | Loss: 3.7588 | LR: 4.96e-04 | 2764.4ms/step | 8.7 samples/s
244
+ Step 1320/20005 | Loss: 3.7709 | LR: 4.96e-04 | 2763.6ms/step | 8.7 samples/s
245
+ Step 1330/20005 | Loss: 3.7533 | LR: 4.96e-04 | 2761.1ms/step | 8.7 samples/s
246
+ Step 1340/20005 | Loss: 3.7543 | LR: 4.96e-04 | 2760.0ms/step | 8.7 samples/s
247
+ Step 1350/20005 | Loss: 3.7578 | LR: 4.96e-04 | 2761.4ms/step | 8.7 samples/s
248
+ Step 1360/20005 | Loss: 3.7703 | LR: 4.96e-04 | 2760.5ms/step | 8.7 samples/s
249
+ Step 1370/20005 | Loss: 3.7302 | LR: 4.96e-04 | 2759.9ms/step | 8.7 samples/s
250
+ Step 1380/20005 | Loss: 3.7250 | LR: 4.96e-04 | 2760.7ms/step | 8.7 samples/s
251
+ Step 1390/20005 | Loss: 3.7123 | LR: 4.96e-04 | 2760.1ms/step | 8.7 samples/s
252
+ Step 1400/20005 | Loss: 3.7132 | LR: 4.96e-04 | 2762.4ms/step | 8.7 samples/s
253
+ Step 1410/20005 | Loss: 3.7260 | LR: 4.96e-04 | 2762.7ms/step | 8.7 samples/s
254
+ Step 1420/20005 | Loss: 3.7089 | LR: 4.96e-04 | 2764.7ms/step | 8.7 samples/s
255
+ Step 1430/20005 | Loss: 3.7120 | LR: 4.96e-04 | 2761.9ms/step | 8.7 samples/s
256
+ Step 1440/20005 | Loss: 3.7123 | LR: 4.96e-04 | 2761.3ms/step | 8.7 samples/s
257
+ Step 1450/20005 | Loss: 3.6496 | LR: 4.95e-04 | 2760.2ms/step | 8.7 samples/s
258
+ Step 1460/20005 | Loss: 3.7129 | LR: 4.95e-04 | 2761.8ms/step | 8.7 samples/s
259
+ Step 1470/20005 | Loss: 3.6914 | LR: 4.95e-04 | 2764.0ms/step | 8.7 samples/s
260
+ Step 1480/20005 | Loss: 3.6677 | LR: 4.95e-04 | 2767.3ms/step | 8.7 samples/s
261
+ Step 1490/20005 | Loss: 3.6743 | LR: 4.95e-04 | 2769.6ms/step | 8.7 samples/s
262
+ Step 1500/20005 | Loss: 3.6833 | LR: 4.95e-04 | 2766.2ms/step | 8.7 samples/s
263
+ Validation at step 1500: Loss = 3.6760, Perplexity = 39.49
264
+ Step 1510/20005 | Loss: 3.6669 | LR: 4.95e-04 | 21970.0ms/step | 1.1 samples/s
265
+ Step 1520/20005 | Loss: 3.6419 | LR: 4.95e-04 | 2765.1ms/step | 8.7 samples/s
266
+ Step 1530/20005 | Loss: 3.6672 | LR: 4.95e-04 | 2763.5ms/step | 8.7 samples/s
267
+ Step 1540/20005 | Loss: 3.6528 | LR: 4.95e-04 | 2762.5ms/step | 8.7 samples/s
268
+ Step 1550/20005 | Loss: 3.6560 | LR: 4.95e-04 | 2761.6ms/step | 8.7 samples/s
269
+ Step 1560/20005 | Loss: 3.6824 | LR: 4.95e-04 | 2764.9ms/step | 8.7 samples/s
270
+ Step 1570/20005 | Loss: 3.6457 | LR: 4.95e-04 | 2762.7ms/step | 8.7 samples/s
271
+ Step 1580/20005 | Loss: 3.6624 | LR: 4.94e-04 | 2764.1ms/step | 8.7 samples/s
272
+ Step 1590/20005 | Loss: 3.6307 | LR: 4.94e-04 | 2759.5ms/step | 8.7 samples/s
273
+ Step 1600/20005 | Loss: 3.6590 | LR: 4.94e-04 | 2764.9ms/step | 8.7 samples/s
274
+ Step 1610/20005 | Loss: 3.6517 | LR: 4.94e-04 | 2766.3ms/step | 8.7 samples/s
275
+ Step 1620/20005 | Loss: 3.6301 | LR: 4.94e-04 | 2764.2ms/step | 8.7 samples/s
276
+ Step 1630/20005 | Loss: 3.6345 | LR: 4.94e-04 | 2761.3ms/step | 8.7 samples/s
277
+ Step 1640/20005 | Loss: 3.6269 | LR: 4.94e-04 | 2762.7ms/step | 8.7 samples/s
278
+ Step 1650/20005 | Loss: 3.6186 | LR: 4.94e-04 | 2764.5ms/step | 8.7 samples/s
279
+ Step 1660/20005 | Loss: 3.6120 | LR: 4.94e-04 | 2765.6ms/step | 8.7 samples/s
280
+ Step 1670/20005 | Loss: 3.6311 | LR: 4.94e-04 | 2761.4ms/step | 8.7 samples/s
281
+ Step 1680/20005 | Loss: 3.5989 | LR: 4.94e-04 | 2760.9ms/step | 8.7 samples/s
282
+ Step 1690/20005 | Loss: 3.6385 | LR: 4.93e-04 | 2760.3ms/step | 8.7 samples/s
283
+ Step 1700/20005 | Loss: 3.5767 | LR: 4.93e-04 | 2759.4ms/step | 8.7 samples/s
284
+ Step 1710/20005 | Loss: 3.6037 | LR: 4.93e-04 | 2762.7ms/step | 8.7 samples/s
285
+ Step 1720/20005 | Loss: 3.6079 | LR: 4.93e-04 | 2765.5ms/step | 8.7 samples/s
286
+ Step 1730/20005 | Loss: 3.5820 | LR: 4.93e-04 | 2765.1ms/step | 8.7 samples/s
287
+ Step 1740/20005 | Loss: 3.6000 | LR: 4.93e-04 | 2758.8ms/step | 8.7 samples/s
288
+ Step 1750/20005 | Loss: 3.6089 | LR: 4.93e-04 | 2759.8ms/step | 8.7 samples/s
289
+ Validation at step 1750: Loss = 3.5972, Perplexity = 36.50
290
+ Step 1760/20005 | Loss: 3.5940 | LR: 4.93e-04 | 21967.1ms/step | 1.1 samples/s
291
+ Step 1770/20005 | Loss: 3.6168 | LR: 4.93e-04 | 2760.4ms/step | 8.7 samples/s
292
+ Step 1780/20005 | Loss: 3.5906 | LR: 4.93e-04 | 2763.4ms/step | 8.7 samples/s
293
+ Step 1790/20005 | Loss: 3.5826 | LR: 4.93e-04 | 2765.2ms/step | 8.7 samples/s
294
+ Step 1800/20005 | Loss: 3.5814 | LR: 4.92e-04 | 2766.0ms/step | 8.7 samples/s
295
+ Step 1810/20005 | Loss: 3.5834 | LR: 4.92e-04 | 2760.0ms/step | 8.7 samples/s
296
+ Step 1820/20005 | Loss: 3.5776 | LR: 4.92e-04 | 2759.6ms/step | 8.7 samples/s
297
+ Step 1830/20005 | Loss: 3.6098 | LR: 4.92e-04 | 2758.7ms/step | 8.7 samples/s
298
+ Step 1840/20005 | Loss: 3.5599 | LR: 4.92e-04 | 2764.5ms/step | 8.7 samples/s
299
+ Step 1850/20005 | Loss: 3.5589 | LR: 4.92e-04 | 2760.0ms/step | 8.7 samples/s
300
+ Step 1860/20005 | Loss: 3.6060 | LR: 4.92e-04 | 2758.6ms/step | 8.7 samples/s
301
+ Step 1870/20005 | Loss: 3.5471 | LR: 4.92e-04 | 2762.5ms/step | 8.7 samples/s
302
+ Step 1880/20005 | Loss: 3.5732 | LR: 4.92e-04 | 2764.0ms/step | 8.7 samples/s
303
+ Step 1890/20005 | Loss: 3.5642 | LR: 4.92e-04 | 2762.8ms/step | 8.7 samples/s
304
+ Step 1900/20005 | Loss: 3.5551 | LR: 4.91e-04 | 2764.6ms/step | 8.7 samples/s
305
+ Step 1910/20005 | Loss: 3.5654 | LR: 4.91e-04 | 2765.4ms/step | 8.7 samples/s
306
+ Step 1920/20005 | Loss: 3.5404 | LR: 4.91e-04 | 2766.8ms/step | 8.7 samples/s
307
+ Step 1930/20005 | Loss: 3.5564 | LR: 4.91e-04 | 2759.6ms/step | 8.7 samples/s
308
+ Step 1940/20005 | Loss: 3.5298 | LR: 4.91e-04 | 2761.4ms/step | 8.7 samples/s
309
+ Step 1950/20005 | Loss: 3.5578 | LR: 4.91e-04 | 2760.4ms/step | 8.7 samples/s
310
+ Step 1960/20005 | Loss: 3.5382 | LR: 4.91e-04 | 2767.1ms/step | 8.7 samples/s
311
+ Step 1970/20005 | Loss: 3.5331 | LR: 4.91e-04 | 2762.8ms/step | 8.7 samples/s
312
+ Step 1980/20005 | Loss: 3.5729 | LR: 4.91e-04 | 2764.6ms/step | 8.7 samples/s
313
+ Step 1990/20005 | Loss: 3.5470 | LR: 4.90e-04 | 2769.8ms/step | 8.7 samples/s
314
+ Step 2000/20005 | Loss: 3.5356 | LR: 4.90e-04 | 2762.0ms/step | 8.7 samples/s
315
+ Validation at step 2000: Loss = 3.5327, Perplexity = 34.22
316
+ Saved checkpoint to checkpoints/run_20260511_144655/checkpoint-2000
317
+ Step 2010/20005 | Loss: 3.5537 | LR: 4.90e-04 | 22546.5ms/step | 1.1 samples/s
318
+ Step 2020/20005 | Loss: 3.5415 | LR: 4.90e-04 | 2900.2ms/step | 8.3 samples/s
319
+ Step 2030/20005 | Loss: 3.5328 | LR: 4.90e-04 | 2774.9ms/step | 8.6 samples/s
320
+ Step 2040/20005 | Loss: 3.5122 | LR: 4.90e-04 | 2773.5ms/step | 8.7 samples/s
321
+ Step 2050/20005 | Loss: 3.5349 | LR: 4.90e-04 | 2774.2ms/step | 8.7 samples/s
322
+ Step 2060/20005 | Loss: 3.5354 | LR: 4.90e-04 | 2774.3ms/step | 8.7 samples/s
323
+ Step 2070/20005 | Loss: 3.4937 | LR: 4.90e-04 | 2776.1ms/step | 8.6 samples/s
324
+ Step 2080/20005 | Loss: 3.5230 | LR: 4.89e-04 | 2776.0ms/step | 8.6 samples/s
325
+ Step 2090/20005 | Loss: 3.5196 | LR: 4.89e-04 | 2779.5ms/step | 8.6 samples/s
326
+ Step 2100/20005 | Loss: 3.4990 | LR: 4.89e-04 | 2782.3ms/step | 8.6 samples/s
327
+ Step 2110/20005 | Loss: 3.5098 | LR: 4.89e-04 | 2776.2ms/step | 8.6 samples/s
328
+ Step 2120/20005 | Loss: 3.5074 | LR: 4.89e-04 | 2777.2ms/step | 8.6 samples/s
329
+ Step 2130/20005 | Loss: 3.5017 | LR: 4.89e-04 | 2779.7ms/step | 8.6 samples/s
330
+ Step 2140/20005 | Loss: 3.5224 | LR: 4.89e-04 | 2777.2ms/step | 8.6 samples/s
331
+ Step 2150/20005 | Loss: 3.5273 | LR: 4.89e-04 | 2777.6ms/step | 8.6 samples/s
332
+ Step 2160/20005 | Loss: 3.4821 | LR: 4.89e-04 | 2777.0ms/step | 8.6 samples/s
333
+ Step 2170/20005 | Loss: 3.4882 | LR: 4.88e-04 | 2775.4ms/step | 8.6 samples/s
334
+ Step 2180/20005 | Loss: 3.5056 | LR: 4.88e-04 | 2776.3ms/step | 8.6 samples/s
335
+ Step 2190/20005 | Loss: 3.4907 | LR: 4.88e-04 | 2776.5ms/step | 8.6 samples/s
336
+ Step 2200/20005 | Loss: 3.4449 | LR: 4.88e-04 | 2779.5ms/step | 8.6 samples/s
337
+ Step 2210/20005 | Loss: 3.4835 | LR: 4.88e-04 | 2778.7ms/step | 8.6 samples/s
338
+ Step 2220/20005 | Loss: 3.4972 | LR: 4.88e-04 | 2779.0ms/step | 8.6 samples/s
339
+ Step 2230/20005 | Loss: 3.4878 | LR: 4.88e-04 | 2776.1ms/step | 8.6 samples/s
340
+ Step 2240/20005 | Loss: 3.4760 | LR: 4.88e-04 | 2776.2ms/step | 8.6 samples/s
341
+ Step 2250/20005 | Loss: 3.4730 | LR: 4.87e-04 | 2776.1ms/step | 8.6 samples/s
342
+ Validation at step 2250: Loss = 3.4829, Perplexity = 32.55
343
+ Step 2260/20005 | Loss: 3.5219 | LR: 4.87e-04 | 21980.2ms/step | 1.1 samples/s
344
+ Step 2270/20005 | Loss: 3.4758 | LR: 4.87e-04 | 2779.8ms/step | 8.6 samples/s
345
+ Step 2280/20005 | Loss: 3.4983 | LR: 4.87e-04 | 2779.4ms/step | 8.6 samples/s
346
+ Step 2290/20005 | Loss: 3.4257 | LR: 4.87e-04 | 2776.0ms/step | 8.6 samples/s
347
+ Step 2300/20005 | Loss: 3.4590 | LR: 4.87e-04 | 2776.7ms/step | 8.6 samples/s
348
+ Step 2310/20005 | Loss: 3.4969 | LR: 4.87e-04 | 2775.8ms/step | 8.6 samples/s
349
+ Step 2320/20005 | Loss: 3.4762 | LR: 4.87e-04 | 2778.3ms/step | 8.6 samples/s
350
+ Step 2330/20005 | Loss: 3.4350 | LR: 4.86e-04 | 2774.7ms/step | 8.6 samples/s
351
+ Step 2340/20005 | Loss: 3.4633 | LR: 4.86e-04 | 2775.2ms/step | 8.6 samples/s
352
+ Step 2350/20005 | Loss: 3.4464 | LR: 4.86e-04 | 2775.9ms/step | 8.6 samples/s
353
+ Step 2360/20005 | Loss: 3.4851 | LR: 4.86e-04 | 2774.7ms/step | 8.6 samples/s
354
+ Step 2370/20005 | Loss: 3.4674 | LR: 4.86e-04 | 2781.2ms/step | 8.6 samples/s
355
+ Step 2380/20005 | Loss: 3.4630 | LR: 4.86e-04 | 2778.2ms/step | 8.6 samples/s
356
+ Step 2390/20005 | Loss: 3.4291 | LR: 4.86e-04 | 2780.2ms/step | 8.6 samples/s
357
+ Step 2400/20005 | Loss: 3.4631 | LR: 4.86e-04 | 2776.8ms/step | 8.6 samples/s
358
+ Step 2410/20005 | Loss: 3.4514 | LR: 4.85e-04 | 2779.0ms/step | 8.6 samples/s
359
+ Step 2420/20005 | Loss: 3.4679 | LR: 4.85e-04 | 2780.1ms/step | 8.6 samples/s
360
+ Step 2430/20005 | Loss: 3.4501 | LR: 4.85e-04 | 2779.9ms/step | 8.6 samples/s
361
+ Step 2440/20005 | Loss: 3.4360 | LR: 4.85e-04 | 2777.8ms/step | 8.6 samples/s
362
+ Step 2450/20005 | Loss: 3.4388 | LR: 4.85e-04 | 2776.1ms/step | 8.6 samples/s
363
+ Step 2460/20005 | Loss: 3.4404 | LR: 4.85e-04 | 2776.2ms/step | 8.6 samples/s
364
+ Step 2470/20005 | Loss: 3.4448 | LR: 4.85e-04 | 2780.3ms/step | 8.6 samples/s
365
+ Step 2480/20005 | Loss: 3.4615 | LR: 4.84e-04 | 2780.3ms/step | 8.6 samples/s
366
+ Step 2490/20005 | Loss: 3.4554 | LR: 4.84e-04 | 2778.7ms/step | 8.6 samples/s
367
+ Step 2500/20005 | Loss: 3.4341 | LR: 4.84e-04 | 2779.7ms/step | 8.6 samples/s
368
+ Validation at step 2500: Loss = 3.4384, Perplexity = 31.14
369
+ Step 2510/20005 | Loss: 3.4515 | LR: 4.84e-04 | 21984.1ms/step | 1.1 samples/s
370
+ Step 2520/20005 | Loss: 3.4192 | LR: 4.84e-04 | 2777.8ms/step | 8.6 samples/s
371
+ Step 2530/20005 | Loss: 3.4176 | LR: 4.84e-04 | 2776.7ms/step | 8.6 samples/s
372
+ Step 2540/20005 | Loss: 3.4397 | LR: 4.84e-04 | 2774.8ms/step | 8.6 samples/s
373
+ Step 2550/20005 | Loss: 3.4319 | LR: 4.83e-04 | 2777.4ms/step | 8.6 samples/s
374
+ Step 2560/20005 | Loss: 3.4746 | LR: 4.83e-04 | 2783.9ms/step | 8.6 samples/s
375
+ Step 2570/20005 | Loss: 3.4274 | LR: 4.83e-04 | 2782.8ms/step | 8.6 samples/s
376
+ Step 2580/20005 | Loss: 3.4110 | LR: 4.83e-04 | 2779.1ms/step | 8.6 samples/s
377
+ Step 2590/20005 | Loss: 3.4279 | LR: 4.83e-04 | 2777.2ms/step | 8.6 samples/s
378
+ Step 2600/20005 | Loss: 3.4287 | LR: 4.83e-04 | 2778.1ms/step | 8.6 samples/s
379
+ Step 2610/20005 | Loss: 3.4100 | LR: 4.83e-04 | 2777.9ms/step | 8.6 samples/s
380
+ Step 2620/20005 | Loss: 3.4379 | LR: 4.82e-04 | 2779.5ms/step | 8.6 samples/s
381
+ Step 2630/20005 | Loss: 3.4227 | LR: 4.82e-04 | 2774.8ms/step | 8.6 samples/s
382
+ Step 2640/20005 | Loss: 3.4485 | LR: 4.82e-04 | 2776.1ms/step | 8.6 samples/s
383
+ Step 2650/20005 | Loss: 3.4062 | LR: 4.82e-04 | 2775.0ms/step | 8.6 samples/s
384
+ Step 2660/20005 | Loss: 3.4128 | LR: 4.82e-04 | 2776.1ms/step | 8.6 samples/s
385
+ Step 2670/20005 | Loss: 3.4134 | LR: 4.82e-04 | 2776.4ms/step | 8.6 samples/s
386
+ Step 2680/20005 | Loss: 3.3835 | LR: 4.82e-04 | 2774.4ms/step | 8.7 samples/s
387
+ Step 2690/20005 | Loss: 3.4207 | LR: 4.81e-04 | 2775.1ms/step | 8.6 samples/s
388
+ Step 2700/20005 | Loss: 3.4129 | LR: 4.81e-04 | 2778.3ms/step | 8.6 samples/s
389
+ Step 2710/20005 | Loss: 3.4065 | LR: 4.81e-04 | 2779.5ms/step | 8.6 samples/s
390
+ Step 2720/20005 | Loss: 3.4024 | LR: 4.81e-04 | 2777.1ms/step | 8.6 samples/s
391
+ Step 2730/20005 | Loss: 3.4377 | LR: 4.81e-04 | 2777.0ms/step | 8.6 samples/s
392
+ Step 2740/20005 | Loss: 3.3884 | LR: 4.81e-04 | 2776.3ms/step | 8.6 samples/s
393
+ Step 2750/20005 | Loss: 3.4484 | LR: 4.81e-04 | 2773.7ms/step | 8.7 samples/s
394
+ Validation at step 2750: Loss = 3.4030, Perplexity = 30.05
395
+ Traceback (most recent call last):
396
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1082, in <module>
397
+ main()
398
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1078, in main
399
+ train(config)
400
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 918, in train
401
+ for step, batch in enumerate(train_loader):
402
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 741, in __next__
403
+ data = self._next_data()
404
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1548, in _next_data
405
+ return self._process_data(data, worker_id)
406
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1586, in _process_data
407
+ data.reraise()
408
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_utils.py", line 785, in reraise
409
+ raise exception
410
+ RuntimeError: Caught RuntimeError in DataLoader worker process 4.
411
+ Original Traceback (most recent call last):
412
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 358, in _worker_loop
413
+ data = fetcher.fetch(index) # type: ignore[possibly-undefined]
414
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 57, in fetch
415
+ return self.collate_fn(data)
416
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py", line 401, in default_collate
417
+ return collate(batch, collate_fn_map=default_collate_fn_map)
418
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py", line 171, in collate
419
+ {
420
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py", line 172, in <dictcomp>
421
+ key: collate(
422
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py", line 155, in collate
423
+ return collate_fn_map[elem_type](batch, collate_fn_map=collate_fn_map)
424
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py", line 273, in collate_tensor_fn
425
+ storage = elem._typed_storage()._new_shared(numel, device=elem.device)
426
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/storage.py", line 1210, in _new_shared
427
+ untyped_storage = torch.UntypedStorage._new_shared(
428
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/storage.py", line 414, in _new_shared
429
+ return cls._new_using_fd_cpu(size)
430
+ RuntimeError: unable to allocate shared memory(shm) for file </torch_3831_3158233914_8271>: Success (0)
431
+
432
+ Traceback (most recent call last):
433
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1082, in <module>
434
+ main()
435
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1078, in main
436
+ train(config)
437
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 918, in train
438
+ for step, batch in enumerate(train_loader):
439
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 741, in __next__
440
+ data = self._next_data()
441
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1548, in _next_data
442
+ return self._process_data(data, worker_id)
443
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1586, in _process_data
444
+ data.reraise()
445
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_utils.py", line 785, in reraise
446
+ raise exception
447
+ RuntimeError: Caught RuntimeError in DataLoader worker process 4.
448
+ Original Traceback (most recent call last):
449
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 358, in _worker_loop
450
+ data = fetcher.fetch(index) # type: ignore[possibly-undefined]
451
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 57, in fetch
452
+ return self.collate_fn(data)
453
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py", line 401, in default_collate
454
+ return collate(batch, collate_fn_map=default_collate_fn_map)
455
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py", line 171, in collate
456
+ {
457
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py", line 172, in <dictcomp>
458
+ key: collate(
459
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py", line 155, in collate
460
+ return collate_fn_map[elem_type](batch, collate_fn_map=collate_fn_map)
461
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/utils/data/_utils/collate.py", line 273, in collate_tensor_fn
462
+ storage = elem._typed_storage()._new_shared(numel, device=elem.device)
463
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/storage.py", line 1210, in _new_shared
464
+ untyped_storage = torch.UntypedStorage._new_shared(
465
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/storage.py", line 414, in _new_shared
466
+ return cls._new_using_fd_cpu(size)
467
+ RuntimeError: unable to allocate shared memory(shm) for file </torch_3831_3158233914_8271>: Success (0)
epfl_final/wandb/run-20260511_155411-fii64tue/files/requirements.txt ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pytz==2026.2
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.7.0
8
+ tzdata==2026.2
9
+ typing_extensions==4.15.0
10
+ triton==3.6.0
11
+ tqdm==4.67.3
12
+ torchinfo==1.8.0
13
+ sympy==1.14.0
14
+ smmap==5.0.3
15
+ six==1.17.0
16
+ shellingham==1.5.4
17
+ safetensors==0.7.0
18
+ regex==2026.5.9
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.5.2
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.2.6
38
+ networkx==3.4.2
39
+ mdurl==0.1.2
40
+ MarkupSafe==3.0.3
41
+ idna==3.14
42
+ hf-xet==1.5.0
43
+ h11==0.16.0
44
+ fsspec==2026.2.0
45
+ frozenlist==1.8.0
46
+ filelock==3.29.0
47
+ dill==0.4.1
48
+ cuda-pathfinder==1.5.4
49
+ click==8.3.3
50
+ charset-normalizer==3.4.7
51
+ certifi==2026.4.22
52
+ attrs==26.1.0
53
+ async-timeout==5.0.1
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ typing-inspection==0.4.2
58
+ sentry-sdk==2.59.0
59
+ requests==2.33.1
60
+ python-dateutil==2.9.0.post0
61
+ pydantic_core==2.46.4
62
+ nvidia-cusparse==12.6.3.3
63
+ nvidia-cufft==12.0.0.61
64
+ nvidia-cudnn-cu13==9.19.0.56
65
+ multiprocess==0.70.19
66
+ multidict==6.7.1
67
+ markdown-it-py==4.2.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ exceptiongroup==1.3.1
72
+ cuda-bindings==13.2.0
73
+ aiosignal==1.4.0
74
+ yarl==1.23.0
75
+ rich==15.0.0
76
+ pydantic==2.13.4
77
+ pandas==2.3.3
78
+ nvidia-cusolver==12.0.4.66
79
+ GitPython==3.1.50
80
+ anyio==4.13.0
81
+ wandb==0.26.1
82
+ typer==0.25.1
83
+ httpx==0.28.1
84
+ aiohttp==3.13.5
85
+ torch==2.11.0
86
+ huggingface_hub==1.14.0
87
+ tokenizers==0.22.2
88
+ datasets==4.8.5
89
+ accelerate==1.13.0
90
+ transformers==5.8.0
91
+ PyGObject==3.42.1
92
+ dbus-python==1.2.18
93
+ distro==1.7.0
94
+ pip==22.0.2
95
+ setuptools==59.6.0
96
+ ssh-import-id==5.11
97
+ wheel==0.37.1
epfl_final/wandb/run-20260511_155411-fii64tue/files/wandb-metadata.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.10.12",
4
+ "startedAt": "2026-05-11T15:54:11.043050Z",
5
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
6
+ "codePath": "SparseAtt/train.py",
7
+ "codePathLocal": "SparseAtt/train.py",
8
+ "email": "ruben.navasardyan@mail.utoronto.ca",
9
+ "root": "/mloscratch/homes/navasard/sparse",
10
+ "host": "h100-train-14-0-2",
11
+ "executable": "/usr/bin/python3",
12
+ "cpu_count": 96,
13
+ "cpu_count_logical": 192,
14
+ "gpu": "NVIDIA H100 80GB HBM3",
15
+ "gpu_count": 1,
16
+ "disk": {
17
+ "/": {
18
+ "total": "9374931091456",
19
+ "used": "7115316539392"
20
+ }
21
+ },
22
+ "memory": {
23
+ "total": "1623005847552"
24
+ },
25
+ "gpu_nvidia": [
26
+ {
27
+ "name": "NVIDIA H100 80GB HBM3",
28
+ "memoryTotal": "85520809984",
29
+ "cudaCores": 16896,
30
+ "architecture": "Hopper",
31
+ "uuid": "GPU-b52a5d1a-9d9a-90fe-c3b3-708dcea4146f"
32
+ }
33
+ ],
34
+ "cudaVersion": "13.0",
35
+ "writerId": "51f4lp4opnmu0k00lxp85qgvwdbstbe2"
36
+ }
epfl_final/wandb/run-20260511_155411-fii64tue/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"speed/samples_per_sec":8.652657991278586,"speed/ms_per_step":2773.7141609191895,"speed/steps_per_sec":0.3605274163032744,"train/epoch":0.06485849056603774,"train/global_step":2750,"_timestamp":1.7785251580679338e+09,"_step":2750,"train/loss":3.4484388798475267,"train/learning_rate":0.0004805012072697279,"_runtime":10658.195056289,"eval/perplexity":30.05284934669279,"_wandb":{"runtime":10658},"eval/loss":3.4029574766755104}
epfl_final/wandb/run-20260511_155411-fii64tue/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/wandb/run-20260511_155411-fii64tue/logs/debug.log ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-05-11 15:54:11,060 INFO MainThread:3254 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-05-11 15:54:11,060 INFO MainThread:3254 [wandb_setup.py:_flush():81] Configure stats pid to 3254
3
+ 2026-05-11 15:54:11,060 INFO MainThread:3254 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-05-11 15:54:11,060 INFO MainThread:3254 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260511_155411-fii64tue/logs/debug.log
5
+ 2026-05-11 15:54:11,060 INFO MainThread:3254 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260511_155411-fii64tue/logs/debug-internal.log
6
+ 2026-05-11 15:54:11,060 INFO MainThread:3254 [wandb_init.py:init():850] calling init triggers
7
+ 2026-05-11 15:54:11,060 INFO MainThread:3254 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': './checkpoints/run_20260511_144655', 'resume_from_checkpoint': None, 'num_epochs': 1, 'max_steps': 20005, 'per_device_batch_size': 3, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 250, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 200, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': None, 'wandb_tags': ['qwen3', '0.6b'], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 2000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, '_wandb': {}}
9
+ 2026-05-11 15:54:11,060 INFO MainThread:3254 [wandb_init.py:init():898] starting backend
10
+ 2026-05-11 15:54:11,292 INFO MainThread:3254 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-05-11 15:54:11,630 INFO MainThread:3254 [wandb_init.py:init():918] backend started and connected
12
+ 2026-05-11 15:54:11,632 INFO MainThread:3254 [wandb_init.py:init():988] updated telemetry
13
+ 2026-05-11 15:54:11,633 INFO MainThread:3254 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-05-11 15:54:12,600 INFO MainThread:3254 [wandb_init.py:init():1056] starting run threads in backend
15
+ 2026-05-11 15:54:12,710 INFO MainThread:3254 [wandb_run.py:_console_start():2554] atexit reg
16
+ 2026-05-11 15:54:12,710 INFO MainThread:3254 [wandb_run.py:_redirect():2403] redirect: wrap_raw
17
+ 2026-05-11 15:54:12,710 INFO MainThread:3254 [wandb_run.py:_redirect():2472] Wrapping output streams.
18
+ 2026-05-11 15:54:12,710 INFO MainThread:3254 [wandb_run.py:_redirect():2495] Redirects installed.
19
+ 2026-05-11 15:54:12,713 INFO MainThread:3254 [wandb_init.py:init():1094] run started, returning control to user process
20
+ 2026-05-11 18:51:50,795 INFO wandb-AsyncioManager-main:3254 [service_client.py:_forward_responses():134] Reached EOF.
21
+ 2026-05-11 18:51:50,796 INFO wandb-AsyncioManager-main:3254 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.
epfl_final/wandb/run-20260511_155411-fii64tue/run-fii64tue.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a64f536a307976a9d2aa410a4bb1925dffe0034b8e780eed28b354f4d5fff22f
3
+ size 1697706
epfl_final/wandb/run-20260511_201933-jzi5hfrk/files/config.yaml ADDED
@@ -0,0 +1,134 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ _wandb:
2
+ value:
3
+ cli_version: 0.26.1
4
+ e:
5
+ manc65tdg7e8jcg1vmj5aijxqj7zabic:
6
+ codePath: SparseAtt/train.py
7
+ codePathLocal: SparseAtt/train.py
8
+ cpu_count: 96
9
+ cpu_count_logical: 192
10
+ cudaVersion: "13.0"
11
+ disk:
12
+ /:
13
+ total: "9374931091456"
14
+ used: "7179666116608"
15
+ email: ruben.navasardyan@mail.utoronto.ca
16
+ executable: /usr/bin/python3
17
+ gpu: NVIDIA H100 80GB HBM3
18
+ gpu_count: 1
19
+ gpu_nvidia:
20
+ - architecture: Hopper
21
+ cudaCores: 16896
22
+ memoryTotal: "85520809984"
23
+ name: NVIDIA H100 80GB HBM3
24
+ uuid: GPU-b52a5d1a-9d9a-90fe-c3b3-708dcea4146f
25
+ host: h100-train-14-0-2
26
+ memory:
27
+ total: "1623005847552"
28
+ os: Linux-6.8.0-59-generic-x86_64-with-glibc2.35
29
+ program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
30
+ python: CPython 3.10.12
31
+ root: /mloscratch/homes/navasard/sparse
32
+ startedAt: "2026-05-11T20:19:33.048764Z"
33
+ writerId: manc65tdg7e8jcg1vmj5aijxqj7zabic
34
+ m: []
35
+ python_version: 3.10.12
36
+ t:
37
+ "1":
38
+ - 1
39
+ - 11
40
+ - 49
41
+ - 51
42
+ - 71
43
+ "2":
44
+ - 1
45
+ - 11
46
+ - 49
47
+ - 51
48
+ - 71
49
+ "3":
50
+ - 2
51
+ - 15
52
+ - 16
53
+ - 61
54
+ "4": 3.10.12
55
+ "5": 0.26.1
56
+ "6": 5.8.0
57
+ "12": 0.26.1
58
+ "13": linux-x86_64
59
+ attention_type:
60
+ value: sparse_stochastic
61
+ checkpoint_activations:
62
+ value: false
63
+ eval_batches:
64
+ value: 200
65
+ eval_steps:
66
+ value: 250
67
+ gradient_accumulation_steps:
68
+ value: 8
69
+ init_from_scratch:
70
+ value: true
71
+ learning_rate:
72
+ value: 0.0005
73
+ logging_steps:
74
+ value: 10
75
+ lr_scheduler:
76
+ value: cosine
77
+ max_grad_norm:
78
+ value: 1
79
+ max_seq_length:
80
+ value: 8192
81
+ max_steps:
82
+ value: 20005
83
+ mixed_precision:
84
+ value: bf16
85
+ model_dtype:
86
+ value: bfloat16
87
+ model_path:
88
+ value: /mloscratch/homes/navasard/qwen3_06b_base
89
+ num_epochs:
90
+ value: 1
91
+ num_workers:
92
+ value: 8
93
+ output_dir:
94
+ value: ./checkpoints/run_20260511_191723
95
+ per_device_batch_size:
96
+ value: 3
97
+ resume_from_checkpoint:
98
+ value: null
99
+ save_steps:
100
+ value: 1000
101
+ seed:
102
+ value: 42
103
+ sparse_block_size:
104
+ value: 128
105
+ sparse_layer_indices:
106
+ value: null
107
+ sparse_mask_pool_size:
108
+ value: 2000
109
+ sparse_num_random_blocks:
110
+ value: 31
111
+ sparse_val_dense_compare:
112
+ value: false
113
+ strategy:
114
+ value: ddp
115
+ train_path:
116
+ value: /mloscratch/homes/navasard/data/fineweb/tokenized
117
+ val_path:
118
+ value: null
119
+ verbose:
120
+ value: true
121
+ wandb_enabled:
122
+ value: true
123
+ wandb_name:
124
+ value: null
125
+ wandb_project:
126
+ value: stochastic_sparsity
127
+ wandb_tags:
128
+ value:
129
+ - qwen3
130
+ - 0.6b
131
+ warmup_steps:
132
+ value: 250
133
+ weight_decay:
134
+ value: 0.01
epfl_final/wandb/run-20260511_201933-jzi5hfrk/files/output.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/wandb/run-20260511_201933-jzi5hfrk/files/requirements.txt ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pytz==2026.2
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.7.0
8
+ tzdata==2026.2
9
+ typing_extensions==4.15.0
10
+ triton==3.6.0
11
+ tqdm==4.67.3
12
+ torchinfo==1.8.0
13
+ sympy==1.14.0
14
+ smmap==5.0.3
15
+ six==1.17.0
16
+ shellingham==1.5.4
17
+ safetensors==0.7.0
18
+ regex==2026.5.9
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.5.2
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.2.6
38
+ networkx==3.4.2
39
+ mdurl==0.1.2
40
+ MarkupSafe==3.0.3
41
+ idna==3.14
42
+ hf-xet==1.5.0
43
+ h11==0.16.0
44
+ fsspec==2026.2.0
45
+ frozenlist==1.8.0
46
+ filelock==3.29.0
47
+ dill==0.4.1
48
+ cuda-pathfinder==1.5.4
49
+ click==8.3.3
50
+ charset-normalizer==3.4.7
51
+ certifi==2026.4.22
52
+ attrs==26.1.0
53
+ async-timeout==5.0.1
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ typing-inspection==0.4.2
58
+ sentry-sdk==2.59.0
59
+ requests==2.33.1
60
+ python-dateutil==2.9.0.post0
61
+ pydantic_core==2.46.4
62
+ nvidia-cusparse==12.6.3.3
63
+ nvidia-cufft==12.0.0.61
64
+ nvidia-cudnn-cu13==9.19.0.56
65
+ multiprocess==0.70.19
66
+ multidict==6.7.1
67
+ markdown-it-py==4.2.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ exceptiongroup==1.3.1
72
+ cuda-bindings==13.2.0
73
+ aiosignal==1.4.0
74
+ yarl==1.23.0
75
+ rich==15.0.0
76
+ pydantic==2.13.4
77
+ pandas==2.3.3
78
+ nvidia-cusolver==12.0.4.66
79
+ GitPython==3.1.50
80
+ anyio==4.13.0
81
+ wandb==0.26.1
82
+ typer==0.25.1
83
+ httpx==0.28.1
84
+ aiohttp==3.13.5
85
+ torch==2.11.0
86
+ huggingface_hub==1.14.0
87
+ tokenizers==0.22.2
88
+ datasets==4.8.5
89
+ accelerate==1.13.0
90
+ transformers==5.8.0
91
+ PyGObject==3.42.1
92
+ dbus-python==1.2.18
93
+ distro==1.7.0
94
+ pip==22.0.2
95
+ setuptools==59.6.0
96
+ ssh-import-id==5.11
97
+ wheel==0.37.1
epfl_final/wandb/run-20260511_201933-jzi5hfrk/files/wandb-metadata.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.10.12",
4
+ "startedAt": "2026-05-11T20:19:33.048764Z",
5
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
6
+ "codePath": "SparseAtt/train.py",
7
+ "codePathLocal": "SparseAtt/train.py",
8
+ "email": "ruben.navasardyan@mail.utoronto.ca",
9
+ "root": "/mloscratch/homes/navasard/sparse",
10
+ "host": "h100-train-14-0-2",
11
+ "executable": "/usr/bin/python3",
12
+ "cpu_count": 96,
13
+ "cpu_count_logical": 192,
14
+ "gpu": "NVIDIA H100 80GB HBM3",
15
+ "gpu_count": 1,
16
+ "disk": {
17
+ "/": {
18
+ "total": "9374931091456",
19
+ "used": "7179666116608"
20
+ }
21
+ },
22
+ "memory": {
23
+ "total": "1623005847552"
24
+ },
25
+ "gpu_nvidia": [
26
+ {
27
+ "name": "NVIDIA H100 80GB HBM3",
28
+ "memoryTotal": "85520809984",
29
+ "cudaCores": 16896,
30
+ "architecture": "Hopper",
31
+ "uuid": "GPU-b52a5d1a-9d9a-90fe-c3b3-708dcea4146f"
32
+ }
33
+ ],
34
+ "cudaVersion": "13.0",
35
+ "writerId": "manc65tdg7e8jcg1vmj5aijxqj7zabic"
36
+ }
epfl_final/wandb/run-20260511_201933-jzi5hfrk/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"_runtime":71008.183699807,"eval/loss":2.921919618397951,"speed/ms_per_step":2801.911187171936,"_timestamp":1.7786017523139584e+09,"speed/samples_per_sec":8.565581989136499,"train/loss":2.907009869813919,"_wandb":{"runtime":71008},"train/epoch":0.4716981132075472,"train/global_step":20000,"train/learning_rate":7.903067225645266e-11,"eval/perplexity":18.576913839863586,"_step":20000,"speed/steps_per_sec":0.3568992495473541}
epfl_final/wandb/run-20260511_201933-jzi5hfrk/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/wandb/run-20260511_201933-jzi5hfrk/logs/debug.log ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-05-11 20:19:33,072 INFO MainThread:122624 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-05-11 20:19:33,072 INFO MainThread:122624 [wandb_setup.py:_flush():81] Configure stats pid to 122624
3
+ 2026-05-11 20:19:33,072 INFO MainThread:122624 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-05-11 20:19:33,072 INFO MainThread:122624 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260511_201933-jzi5hfrk/logs/debug.log
5
+ 2026-05-11 20:19:33,072 INFO MainThread:122624 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260511_201933-jzi5hfrk/logs/debug-internal.log
6
+ 2026-05-11 20:19:33,072 INFO MainThread:122624 [wandb_init.py:init():850] calling init triggers
7
+ 2026-05-11 20:19:33,072 INFO MainThread:122624 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': './checkpoints/run_20260511_191723', 'resume_from_checkpoint': None, 'num_epochs': 1, 'max_steps': 20005, 'per_device_batch_size': 3, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 250, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 200, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': None, 'wandb_tags': ['qwen3', '0.6b'], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 2000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, '_wandb': {}}
9
+ 2026-05-11 20:19:33,072 INFO MainThread:122624 [wandb_init.py:init():898] starting backend
10
+ 2026-05-11 20:19:33,296 INFO MainThread:122624 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-05-11 20:19:33,600 INFO MainThread:122624 [wandb_init.py:init():918] backend started and connected
12
+ 2026-05-11 20:19:33,602 INFO MainThread:122624 [wandb_init.py:init():988] updated telemetry
13
+ 2026-05-11 20:19:33,603 INFO MainThread:122624 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-05-11 20:19:34,161 INFO MainThread:122624 [wandb_init.py:init():1056] starting run threads in backend
15
+ 2026-05-11 20:19:34,263 INFO MainThread:122624 [wandb_run.py:_console_start():2554] atexit reg
16
+ 2026-05-11 20:19:34,263 INFO MainThread:122624 [wandb_run.py:_redirect():2403] redirect: wrap_raw
17
+ 2026-05-11 20:19:34,263 INFO MainThread:122624 [wandb_run.py:_redirect():2472] Wrapping output streams.
18
+ 2026-05-11 20:19:34,263 INFO MainThread:122624 [wandb_run.py:_redirect():2495] Redirects installed.
19
+ 2026-05-11 20:19:34,265 INFO MainThread:122624 [wandb_init.py:init():1094] run started, returning control to user process
20
+ 2026-05-12 16:03:02,339 INFO MainThread:122624 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/jzi5hfrk
21
+ 2026-05-12 16:03:02,342 INFO MainThread:122624 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
22
+ 2026-05-12 16:03:02,344 INFO MainThread:122624 [wandb_run.py:_restore():2502] restore
23
+ 2026-05-12 16:03:02,344 INFO MainThread:122624 [wandb_run.py:_restore():2508] restore done
24
+ 2026-05-12 16:03:31,851 INFO MainThread:122624 [wandb_run.py:_footer_sync_info():3889] logging synced files
epfl_final/wandb/run-20260511_201933-jzi5hfrk/run-jzi5hfrk.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:53a26149f3d8111012d2c0f0610812439d2ad6b887683dbbd7d449dc8c9a2f69
3
+ size 11445166
epfl_final/wandb/run-20260511_215604-377k1je4/files/output.log ADDED
@@ -0,0 +1,147 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ ================================================================================
4
+ MODEL ARCHITECTURE
5
+ ================================================================================
6
+ Model config:
7
+ vocab_size: 151936
8
+ hidden_size: 1024
9
+ num_hidden_layers: 28
10
+ num_attention_heads: 16
11
+ num_key_value_heads: 8
12
+ head_dim: 128
13
+ intermediate_size: 3072
14
+ max_position_embeddings: 32768
15
+ rope_theta: 1000000
16
+ attention_type: sparse_stochastic
17
+ sparse_block_size: 128
18
+ sparse_num_random_blocks: 31
19
+ sparse_mask_pool_size: 10000
20
+ sparse_max_seq_length: 8192
21
+
22
+ Parameter count:
23
+ Total parameters: 596,049,920
24
+ Trainable parameters: 596,049,920
25
+ Non-trainable parameters: 0
26
+ Model size (BF16): 1.19 GB
27
+
28
+ --------------------------------------------------------------------------------
29
+ LAYER-BY-LAYER SUMMARY
30
+ --------------------------------------------------------------------------------
31
+ /home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
32
+
33
+ SOLUTION: Use torch.compile(flex_attention)(...)
34
+
35
+ If you want to debug your score_mod/mask_mod, you can set:
36
+ torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
37
+
38
+ This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
39
+ _warn_once(
40
+ Could not generate detailed summary: Failed to run torchinfo. See above stack traces for more details. Executed layers up to: [Embedding: 2, Qwen3RMSNorm: 4, Linear: 5, Linear: 5, Linear: 5, Qwen3RMSNorm: 5, Qwen3RMSNorm: 5, Qwen3RotaryEmbedding: 5]
41
+ ================================================================================
42
+
43
+ Loading dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
44
+ Using pre-tokenized dataset (no runtime tokenization)
45
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb/tokenized...
46
+ Loaded 5088012 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=1272003)
47
+ Dataset split: 1017602 train, 254401 validation
48
+ Training examples (packed, train split): 1017602
49
+ Validation examples (packed, val split): 254401
50
+ Per-device batch size: 3
51
+ World size (GPUs): 1
52
+ Gradient accumulation steps: 8
53
+ Effective batch size (seqs/step): 24
54
+ Sequence length: 8192
55
+ Tokens per optimizer step: 196,608
56
+ Micro-batches per epoch (per rank): 339200
57
+ Optimizer steps per epoch: 42400
58
+ Total optimizer steps: 20005 (capped by max_steps; 0.47 epochs)
59
+ Starting training...
60
+ Traceback (most recent call last):
61
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1088, in <module>
62
+ main()
63
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1084, in main
64
+ train(config)
65
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 933, in train
66
+ outputs = model(
67
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 472, in __call__
68
+ return super().__call__(*args, **kwargs)
69
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1779, in _wrapped_call_impl
70
+ return self._call_impl(*args, **kwargs)
71
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1790, in _call_impl
72
+ return forward_call(*args, **kwargs)
73
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1024, in compile_wrapper
74
+ return fn(*args, **kwargs)
75
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1779, in _wrapped_call_impl
76
+ return self._call_impl(*args, **kwargs)
77
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1790, in _call_impl
78
+ return forward_call(*args, **kwargs)
79
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/model.py", line 272, in forward
80
+ hidden_states, next_cache, all_hidden_states, all_attentions = self.model(
81
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1779, in _wrapped_call_impl
82
+ return self._call_impl(*args, **kwargs)
83
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1790, in _call_impl
84
+ return forward_call(*args, **kwargs)
85
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/model.py", line 140, in forward
86
+ if has_any_sparse and (attention_mask == 0).any():
87
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/model.py", line 176, in torch_dynamo_resume_in_forward_at_140
88
+ hidden_states, attn_weights, present_key_value = decoder_layer(
89
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1779, in _wrapped_call_impl
90
+ return self._call_impl(*args, **kwargs)
91
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1790, in _call_impl
92
+ return forward_call(*args, **kwargs)
93
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/model.py", line 83, in forward
94
+ hidden_states, attn_weights, present_key_value = self.self_attn(**attn_kwargs)
95
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1779, in _wrapped_call_impl
96
+ return self._call_impl(*args, **kwargs)
97
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1790, in _call_impl
98
+ return forward_call(*args, **kwargs)
99
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/sparse_attention.py", line 242, in forward
100
+ attn_output = self._flex_attention(
101
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py", line 1627, in flex_attention
102
+ raise RuntimeError(
103
+ RuntimeError: Expect q/k/v and block_mask to be on the same device but got cuda:0 and cpu.
104
+ Traceback (most recent call last):
105
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1088, in <module>
106
+ main()
107
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1084, in main
108
+ train(config)
109
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 933, in train
110
+ outputs = model(
111
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 472, in __call__
112
+ return super().__call__(*args, **kwargs)
113
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1779, in _wrapped_call_impl
114
+ return self._call_impl(*args, **kwargs)
115
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1790, in _call_impl
116
+ return forward_call(*args, **kwargs)
117
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1024, in compile_wrapper
118
+ return fn(*args, **kwargs)
119
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1779, in _wrapped_call_impl
120
+ return self._call_impl(*args, **kwargs)
121
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1790, in _call_impl
122
+ return forward_call(*args, **kwargs)
123
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/model.py", line 272, in forward
124
+ hidden_states, next_cache, all_hidden_states, all_attentions = self.model(
125
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1779, in _wrapped_call_impl
126
+ return self._call_impl(*args, **kwargs)
127
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1790, in _call_impl
128
+ return forward_call(*args, **kwargs)
129
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/model.py", line 140, in forward
130
+ if has_any_sparse and (attention_mask == 0).any():
131
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/model.py", line 176, in torch_dynamo_resume_in_forward_at_140
132
+ hidden_states, attn_weights, present_key_value = decoder_layer(
133
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1779, in _wrapped_call_impl
134
+ return self._call_impl(*args, **kwargs)
135
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1790, in _call_impl
136
+ return forward_call(*args, **kwargs)
137
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/model.py", line 83, in forward
138
+ hidden_states, attn_weights, present_key_value = self.self_attn(**attn_kwargs)
139
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1779, in _wrapped_call_impl
140
+ return self._call_impl(*args, **kwargs)
141
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1790, in _call_impl
142
+ return forward_call(*args, **kwargs)
143
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/sparse_attention.py", line 242, in forward
144
+ attn_output = self._flex_attention(
145
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py", line 1627, in flex_attention
146
+ raise RuntimeError(
147
+ RuntimeError: Expect q/k/v and block_mask to be on the same device but got cuda:0 and cpu.
epfl_final/wandb/run-20260511_215604-377k1je4/files/requirements.txt ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pytz==2026.2
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.7.0
8
+ tzdata==2026.2
9
+ typing_extensions==4.15.0
10
+ triton==3.6.0
11
+ tqdm==4.67.3
12
+ torchinfo==1.8.0
13
+ sympy==1.14.0
14
+ smmap==5.0.3
15
+ six==1.17.0
16
+ shellingham==1.5.4
17
+ safetensors==0.7.0
18
+ regex==2026.5.9
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.5.2
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.2.6
38
+ networkx==3.4.2
39
+ mdurl==0.1.2
40
+ MarkupSafe==3.0.3
41
+ idna==3.14
42
+ hf-xet==1.5.0
43
+ h11==0.16.0
44
+ fsspec==2026.2.0
45
+ frozenlist==1.8.0
46
+ filelock==3.29.0
47
+ dill==0.4.1
48
+ cuda-pathfinder==1.5.4
49
+ click==8.3.3
50
+ charset-normalizer==3.4.7
51
+ certifi==2026.4.22
52
+ attrs==26.1.0
53
+ async-timeout==5.0.1
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ typing-inspection==0.4.2
58
+ sentry-sdk==2.59.0
59
+ requests==2.34.0
60
+ python-dateutil==2.9.0.post0
61
+ pydantic_core==2.46.4
62
+ nvidia-cusparse==12.6.3.3
63
+ nvidia-cufft==12.0.0.61
64
+ nvidia-cudnn-cu13==9.19.0.56
65
+ multiprocess==0.70.19
66
+ multidict==6.7.1
67
+ markdown-it-py==4.2.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ exceptiongroup==1.3.1
72
+ cuda-bindings==13.2.0
73
+ aiosignal==1.4.0
74
+ yarl==1.23.0
75
+ rich==15.0.0
76
+ pydantic==2.13.4
77
+ pandas==2.3.3
78
+ nvidia-cusolver==12.0.4.66
79
+ GitPython==3.1.50
80
+ anyio==4.13.0
81
+ wandb==0.26.1
82
+ typer==0.25.1
83
+ httpx==0.28.1
84
+ aiohttp==3.13.5
85
+ torch==2.11.0
86
+ huggingface_hub==1.14.0
87
+ tokenizers==0.22.2
88
+ datasets==4.8.5
89
+ accelerate==1.13.0
90
+ transformers==5.8.0
91
+ PyGObject==3.42.1
92
+ dbus-python==1.2.18
93
+ distro==1.7.0
94
+ pip==22.0.2
95
+ setuptools==59.6.0
96
+ ssh-import-id==5.11
97
+ wheel==0.37.1
epfl_final/wandb/run-20260511_215604-377k1je4/files/wandb-metadata.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.10.12",
4
+ "startedAt": "2026-05-11T21:56:04.033845Z",
5
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
6
+ "codePath": "SparseAtt/train.py",
7
+ "codePathLocal": "SparseAtt/train.py",
8
+ "email": "ruben.navasardyan@mail.utoronto.ca",
9
+ "root": "/mloscratch/homes/navasard/sparse",
10
+ "host": "h100-train-13-0-5",
11
+ "executable": "/usr/bin/python3",
12
+ "cpu_count": 96,
13
+ "cpu_count_logical": 192,
14
+ "gpu": "NVIDIA H100 80GB HBM3",
15
+ "gpu_count": 1,
16
+ "disk": {
17
+ "/": {
18
+ "total": "9374931091456",
19
+ "used": "7086919069696"
20
+ }
21
+ },
22
+ "memory": {
23
+ "total": "1623005741056"
24
+ },
25
+ "gpu_nvidia": [
26
+ {
27
+ "name": "NVIDIA H100 80GB HBM3",
28
+ "memoryTotal": "85520809984",
29
+ "cudaCores": 16896,
30
+ "architecture": "Hopper",
31
+ "uuid": "GPU-5cb9f8b7-4a13-4f85-d73e-6bcbebebefb8"
32
+ }
33
+ ],
34
+ "cudaVersion": "13.0",
35
+ "writerId": "jhi8chi0zzs19xazmkzvwdnmzvij10cx"
36
+ }
epfl_final/wandb/run-20260511_215604-377k1je4/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"_wandb":{"runtime":53},"_runtime":53}
epfl_final/wandb/run-20260511_215604-377k1je4/logs/debug-internal.log ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2026-05-11T21:56:04.277523168Z","level":"INFO","msg":"wandb-core"}
2
+ {"time":"2026-05-11T21:56:04.281373639Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
3
+ {"time":"2026-05-11T21:56:04.56718457Z","level":"INFO","msg":"stream: created new stream","id":"377k1je4"}
4
+ {"time":"2026-05-11T21:56:04.567296076Z","level":"INFO","msg":"handler: started"}
5
+ {"time":"2026-05-11T21:56:04.569072349Z","level":"INFO","msg":"stream: started"}
6
+ {"time":"2026-05-11T21:56:04.569117637Z","level":"INFO","msg":"sender: started"}
7
+ {"time":"2026-05-11T21:56:04.569113741Z","level":"INFO","msg":"writer: started","stream_id":"377k1je4"}
8
+ {"time":"2026-05-11T21:56:05.067839559Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":6}
9
+ {"time":"2026-05-11T21:56:05.319212254Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
10
+ {"time":"2026-05-11T21:56:20.068463306Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":1,"console_offset":5,"console_lines":142,"uploaded_len":2}
11
+ {"time":"2026-05-11T21:56:20.391145065Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
12
+ {"time":"2026-05-11T21:56:35.067996566Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":1,"events_lines":2}
13
+ {"time":"2026-05-11T21:56:35.325165889Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
14
+ {"time":"2026-05-11T21:56:50.068214759Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":3,"events_lines":2}
15
+ {"time":"2026-05-11T21:56:50.285565181Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
16
+ {"time":"2026-05-11T21:56:59.407169468Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
17
+ {"time":"2026-05-11T21:56:59.407293192Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":5,"events_lines":1,"uploaded_len":3,"complete":true,"exit_code":1}
18
+ {"time":"2026-05-11T21:56:59.862340795Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
19
+ {"time":"2026-05-11T21:56:59.862507764Z","level":"INFO","msg":"stream: finishing up"}
20
+ {"time":"2026-05-11T21:56:59.862520874Z","level":"INFO","msg":"handler: closed"}
21
+ {"time":"2026-05-11T21:56:59.86397766Z","level":"INFO","msg":"sender: closed"}
22
+ {"time":"2026-05-11T21:56:59.863988616Z","level":"INFO","msg":"stream: all finished"}
epfl_final/wandb/run-20260511_215604-377k1je4/logs/debug.log ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-05-11 21:56:04,050 INFO MainThread:6961 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-05-11 21:56:04,050 INFO MainThread:6961 [wandb_setup.py:_flush():81] Configure stats pid to 6961
3
+ 2026-05-11 21:56:04,051 INFO MainThread:6961 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-05-11 21:56:04,051 INFO MainThread:6961 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260511_215604-377k1je4/logs/debug.log
5
+ 2026-05-11 21:56:04,052 INFO MainThread:6961 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260511_215604-377k1je4/logs/debug-internal.log
6
+ 2026-05-11 21:56:04,053 INFO MainThread:6961 [wandb_init.py:init():850] calling init triggers
7
+ 2026-05-11 21:56:04,053 INFO MainThread:6961 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': './checkpoints/run_20260511_215307', 'resume_from_checkpoint': None, 'num_epochs': 1, 'max_steps': 20005, 'per_device_batch_size': 3, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 250, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 200, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': None, 'wandb_tags': ['qwen3', '0.6b'], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache', '_wandb': {}}
9
+ 2026-05-11 21:56:04,054 INFO MainThread:6961 [wandb_init.py:init():898] starting backend
10
+ 2026-05-11 21:56:04,270 INFO MainThread:6961 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-05-11 21:56:04,569 INFO MainThread:6961 [wandb_init.py:init():918] backend started and connected
12
+ 2026-05-11 21:56:04,571 INFO MainThread:6961 [wandb_init.py:init():988] updated telemetry
13
+ 2026-05-11 21:56:04,571 INFO MainThread:6961 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-05-11 21:56:05,000 INFO MainThread:6961 [wandb_init.py:init():1056] starting run threads in backend
15
+ 2026-05-11 21:56:05,053 INFO MainThread:6961 [wandb_run.py:_console_start():2554] atexit reg
16
+ 2026-05-11 21:56:05,054 INFO MainThread:6961 [wandb_run.py:_redirect():2403] redirect: wrap_raw
17
+ 2026-05-11 21:56:05,054 INFO MainThread:6961 [wandb_run.py:_redirect():2472] Wrapping output streams.
18
+ 2026-05-11 21:56:05,054 INFO MainThread:6961 [wandb_run.py:_redirect():2495] Redirects installed.
19
+ 2026-05-11 21:56:05,056 INFO MainThread:6961 [wandb_init.py:init():1094] run started, returning control to user process
20
+ 2026-05-11 21:56:58,920 INFO wandb-AsyncioManager-main:6961 [service_client.py:_forward_responses():134] Reached EOF.
21
+ 2026-05-11 21:56:58,921 INFO wandb-AsyncioManager-main:6961 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.