AbhishekGhosh1998 commited on
Commit
5dc6293
·
verified ·
1 Parent(s): ba4bbb2

granularity results ultraep/tierb @ 2026-08-26T18:39:55Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0

Browse files
Files changed (30) hide show
  1. ultraep/tierb/tierb_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333/rep1/mem_samples.csv +232 -0
  2. ultraep/tierb/tierb_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333/rep1/run.env +61 -0
  3. ultraep/tierb/tierb_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333/rep1/train.log +0 -0
  4. ultraep/tierb/tierb_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333/rep2/mem_samples.csv +232 -0
  5. ultraep/tierb/tierb_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333/rep2/run.env +61 -0
  6. ultraep/tierb/tierb_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333/rep2/train.log +0 -0
  7. ultraep/tierb/tierb_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333/rep3/mem_samples.csv +232 -0
  8. ultraep/tierb/tierb_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333/rep3/run.env +61 -0
  9. ultraep/tierb/tierb_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333/rep3/train.log +0 -0
  10. ultraep/tierb/tierb_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375/rep1/mem_samples.csv +192 -0
  11. ultraep/tierb/tierb_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375/rep1/run.env +61 -0
  12. ultraep/tierb/tierb_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375/rep1/train.log +1194 -0
  13. ultraep/tierb/tierb_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375/rep2/mem_samples.csv +24 -0
  14. ultraep/tierb/tierb_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375/rep2/run.env +61 -0
  15. ultraep/tierb/tierb_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375/rep2/train.log +12 -0
  16. ultraep/tierb/tierb_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p083984375/rep2/mem_samples.csv +56 -0
  17. ultraep/tierb/tierb_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p083984375/rep2/train.log +50 -0
  18. ultraep/tierb/tierb_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p083984375/rep3/mem_samples.csv +232 -0
  19. ultraep/tierb/tierb_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p083984375/rep3/run.env +61 -0
  20. ultraep/tierb/tierb_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p083984375/rep3/train.log +0 -0
  21. ultraep/tierb/tierb_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125/rep1/mem_samples.csv +240 -0
  22. ultraep/tierb/tierb_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125/rep1/run.env +61 -0
  23. ultraep/tierb/tierb_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125/rep1/train.log +0 -0
  24. ultraep/tierb/tierb_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125/rep2/mem_samples.csv +232 -0
  25. ultraep/tierb/tierb_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125/rep2/run.env +61 -0
  26. ultraep/tierb/tierb_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125/rep2/train.log +0 -0
  27. ultraep/tierb/tierb_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125/rep3/mem_samples.csv +232 -0
  28. ultraep/tierb/tierb_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125/rep3/run.env +61 -0
  29. ultraep/tierb/tierb_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125/rep3/train.log +0 -0
  30. ultraep/tierb/tierb_gbs8/results.tsv +9 -0
ultraep/tierb/tierb_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333/rep1/mem_samples.csv ADDED
@@ -0,0 +1,232 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 4
43
+ 2, 4
44
+ 3, 4
45
+ 4, 4
46
+ 5, 4
47
+ 6, 4
48
+ 7, 4
49
+ 0, 821
50
+ 1, 821
51
+ 2, 821
52
+ 3, 821
53
+ 4, 821
54
+ 5, 821
55
+ 6, 821
56
+ 7, 821
57
+ 0, 1955
58
+ 1, 1955
59
+ 2, 1955
60
+ 3, 1955
61
+ 4, 1955
62
+ 5, 1955
63
+ 6, 1955
64
+ 7, 1955
65
+ 0, 2353
66
+ 1, 2373
67
+ 2, 2357
68
+ 3, 2349
69
+ 4, 2337
70
+ 5, 2359
71
+ 6, 2353
72
+ 7, 2329
73
+ 0, 25283
74
+ 1, 25349
75
+ 2, 25327
76
+ 3, 25327
77
+ 4, 25349
78
+ 5, 25371
79
+ 6, 25393
80
+ 7, 25437
81
+ 0, 28875
82
+ 1, 28993
83
+ 2, 28799
84
+ 3, 28767
85
+ 4, 28823
86
+ 5, 28787
87
+ 6, 28907
88
+ 7, 28937
89
+ 0, 39085
90
+ 1, 39167
91
+ 2, 39019
92
+ 3, 39015
93
+ 4, 38981
94
+ 5, 39009
95
+ 6, 39115
96
+ 7, 39135
97
+ 0, 39773
98
+ 1, 39857
99
+ 2, 39707
100
+ 3, 39705
101
+ 4, 39705
102
+ 5, 39703
103
+ 6, 39839
104
+ 7, 39857
105
+ 0, 39773
106
+ 1, 39857
107
+ 2, 39707
108
+ 3, 39705
109
+ 4, 39705
110
+ 5, 39703
111
+ 6, 39839
112
+ 7, 39857
113
+ 0, 39783
114
+ 1, 39867
115
+ 2, 39717
116
+ 3, 39717
117
+ 4, 39717
118
+ 5, 39713
119
+ 6, 39849
120
+ 7, 39867
121
+ 0, 39783
122
+ 1, 39867
123
+ 2, 39717
124
+ 3, 39717
125
+ 4, 39717
126
+ 5, 39713
127
+ 6, 39849
128
+ 7, 39867
129
+ 0, 109705
130
+ 1, 109749
131
+ 2, 110123
132
+ 3, 110121
133
+ 4, 110057
134
+ 5, 110065
135
+ 6, 109739
136
+ 7, 109747
137
+ 0, 110101
138
+ 1, 110187
139
+ 2, 110513
140
+ 3, 110511
141
+ 4, 110511
142
+ 5, 110509
143
+ 6, 110167
144
+ 7, 110187
145
+ 0, 110105
146
+ 1, 110191
147
+ 2, 110517
148
+ 3, 110515
149
+ 4, 110515
150
+ 5, 110513
151
+ 6, 110171
152
+ 7, 110191
153
+ 0, 112457
154
+ 1, 112501
155
+ 2, 110755
156
+ 3, 110775
157
+ 4, 110731
158
+ 5, 110741
159
+ 6, 112459
160
+ 7, 112477
161
+ 0, 113191
162
+ 1, 113277
163
+ 2, 111555
164
+ 3, 111553
165
+ 4, 111553
166
+ 5, 111551
167
+ 6, 113257
168
+ 7, 113277
169
+ 0, 113921
170
+ 1, 113969
171
+ 2, 112277
172
+ 3, 112239
173
+ 4, 112193
174
+ 5, 112225
175
+ 6, 113921
176
+ 7, 113943
177
+ 0, 124037
178
+ 1, 126189
179
+ 2, 130157
180
+ 3, 128095
181
+ 4, 134269
182
+ 5, 128093
183
+ 6, 128229
184
+ 7, 134419
185
+ 0, 142137
186
+ 1, 142231
187
+ 2, 142081
188
+ 3, 142079
189
+ 4, 142075
190
+ 5, 142077
191
+ 6, 142211
192
+ 7, 142231
193
+ 0, 142137
194
+ 1, 142231
195
+ 2, 142081
196
+ 3, 142079
197
+ 4, 142075
198
+ 5, 142077
199
+ 6, 142211
200
+ 7, 142231
201
+ 0, 142137
202
+ 1, 142231
203
+ 2, 142081
204
+ 3, 142079
205
+ 4, 142075
206
+ 5, 142077
207
+ 6, 142211
208
+ 7, 142231
209
+ 0, 138363
210
+ 1, 138521
211
+ 2, 138953
212
+ 3, 138951
213
+ 4, 138947
214
+ 5, 138687
215
+ 6, 139083
216
+ 7, 138777
217
+ 0, 136941
218
+ 1, 137099
219
+ 2, 137531
220
+ 3, 137267
221
+ 4, 137525
222
+ 5, 137265
223
+ 6, 137661
224
+ 7, 137355
225
+ 0, 1
226
+ 1, 3
227
+ 2, 4
228
+ 3, 4
229
+ 4, 4
230
+ 5, 68
231
+ 6, 137013
232
+ 7, 324
ultraep/tierb/tierb_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=ultraep
2
+ config=DSV2_LITE_L24
3
+ family=tierB
4
+ repo=/workspace/Megatron-UltraEP
5
+ branch=ultraep-bracing-regions-through-kernels
6
+ commit=03d57ef7
7
+ num_layers=24
8
+ hidden=2048
9
+ ffn_hidden=10944
10
+ moe_ffn_hidden=1408
11
+ num_experts=64
12
+ topk=6
13
+ experts_per_rank=8
14
+ expert_params=8650752
15
+ heads=16
16
+ kv_groups=0
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=2816
20
+ shared_expert_overlap=0
21
+ mla=1
22
+ dense_first_layers=1
23
+ moe_layers=23
24
+ rotary_base=10000
25
+ rotary_scaling=40
26
+ qkv_bias=0
27
+ router_balancing=seq_aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=1
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=8192
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=49152
39
+ mean_tokens_per_expert=6144
40
+ slot_policy=matched
41
+ slots_per_rank=8
42
+ capacity_mode=static
43
+ capacity=1.0755208333333333
44
+ cuda_graph=1
45
+ probe_mode=0
46
+ async_grad_reduce=0
47
+ branch_role=timing
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=768
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
ultraep/tierb/tierb_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
ultraep/tierb/tierb_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333/rep2/mem_samples.csv ADDED
@@ -0,0 +1,232 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 629
43
+ 2, 629
44
+ 3, 629
45
+ 4, 629
46
+ 5, 629
47
+ 6, 629
48
+ 7, 629
49
+ 0, 927
50
+ 1, 929
51
+ 2, 929
52
+ 3, 939
53
+ 4, 937
54
+ 5, 949
55
+ 6, 915
56
+ 7, 941
57
+ 0, 1955
58
+ 1, 1955
59
+ 2, 1955
60
+ 3, 1955
61
+ 4, 1955
62
+ 5, 1955
63
+ 6, 1955
64
+ 7, 1955
65
+ 0, 2343
66
+ 1, 2339
67
+ 2, 2337
68
+ 3, 2331
69
+ 4, 2317
70
+ 5, 2337
71
+ 6, 2333
72
+ 7, 2341
73
+ 0, 9687
74
+ 1, 9819
75
+ 2, 9687
76
+ 3, 9665
77
+ 4, 9585
78
+ 5, 9687
79
+ 6, 9687
80
+ 7, 9433
81
+ 0, 28859
82
+ 1, 28975
83
+ 2, 28819
84
+ 3, 28801
85
+ 4, 28783
86
+ 5, 28807
87
+ 6, 28945
88
+ 7, 29001
89
+ 0, 38967
90
+ 1, 39075
91
+ 2, 38919
92
+ 3, 38903
93
+ 4, 38879
94
+ 5, 38913
95
+ 6, 39029
96
+ 7, 39091
97
+ 0, 39773
98
+ 1, 39857
99
+ 2, 39707
100
+ 3, 39705
101
+ 4, 39705
102
+ 5, 39703
103
+ 6, 39839
104
+ 7, 39857
105
+ 0, 39773
106
+ 1, 39857
107
+ 2, 39707
108
+ 3, 39705
109
+ 4, 39705
110
+ 5, 39703
111
+ 6, 39839
112
+ 7, 39857
113
+ 0, 39783
114
+ 1, 39867
115
+ 2, 39717
116
+ 3, 39717
117
+ 4, 39717
118
+ 5, 39713
119
+ 6, 39849
120
+ 7, 39867
121
+ 0, 39783
122
+ 1, 39867
123
+ 2, 39717
124
+ 3, 39717
125
+ 4, 39717
126
+ 5, 39713
127
+ 6, 39849
128
+ 7, 39867
129
+ 0, 109633
130
+ 1, 109671
131
+ 2, 110011
132
+ 3, 110037
133
+ 4, 110009
134
+ 5, 110033
135
+ 6, 109693
136
+ 7, 109679
137
+ 0, 110101
138
+ 1, 110187
139
+ 2, 110513
140
+ 3, 110511
141
+ 4, 110511
142
+ 5, 110509
143
+ 6, 110167
144
+ 7, 110187
145
+ 0, 110105
146
+ 1, 110191
147
+ 2, 110517
148
+ 3, 110515
149
+ 4, 110515
150
+ 5, 110513
151
+ 6, 110171
152
+ 7, 110191
153
+ 0, 112345
154
+ 1, 112489
155
+ 2, 110743
156
+ 3, 110719
157
+ 4, 110733
158
+ 5, 110739
159
+ 6, 112441
160
+ 7, 112475
161
+ 0, 113191
162
+ 1, 113277
163
+ 2, 111555
164
+ 3, 111553
165
+ 4, 111553
166
+ 5, 111551
167
+ 6, 113257
168
+ 7, 113277
169
+ 0, 113823
170
+ 1, 113907
171
+ 2, 112159
172
+ 3, 112209
173
+ 4, 112193
174
+ 5, 112139
175
+ 6, 113879
176
+ 7, 113913
177
+ 0, 121981
178
+ 1, 120015
179
+ 2, 123973
180
+ 3, 128085
181
+ 4, 123979
182
+ 5, 130155
183
+ 6, 128221
184
+ 7, 124131
185
+ 0, 142137
186
+ 1, 142231
187
+ 2, 142081
188
+ 3, 142079
189
+ 4, 142075
190
+ 5, 142077
191
+ 6, 142211
192
+ 7, 142231
193
+ 0, 142137
194
+ 1, 142231
195
+ 2, 142081
196
+ 3, 142079
197
+ 4, 142075
198
+ 5, 142077
199
+ 6, 142211
200
+ 7, 142231
201
+ 0, 142137
202
+ 1, 142231
203
+ 2, 142081
204
+ 3, 142079
205
+ 4, 142075
206
+ 5, 142077
207
+ 6, 142211
208
+ 7, 142231
209
+ 0, 138747
210
+ 1, 139103
211
+ 2, 138627
212
+ 3, 138369
213
+ 4, 138947
214
+ 5, 138687
215
+ 6, 139083
216
+ 7, 139103
217
+ 0, 137587
218
+ 1, 137355
219
+ 2, 136885
220
+ 3, 136947
221
+ 4, 137525
222
+ 5, 137201
223
+ 6, 137015
224
+ 7, 137099
225
+ 0, 1
226
+ 1, 4
227
+ 2, 4
228
+ 3, 4
229
+ 4, 68
230
+ 5, 136495
231
+ 6, 324
232
+ 7, 4
ultraep/tierb/tierb_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333/rep2/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=ultraep
2
+ config=DSV2_LITE_L24
3
+ family=tierB
4
+ repo=/workspace/Megatron-UltraEP
5
+ branch=ultraep-bracing-regions-through-kernels
6
+ commit=03d57ef7
7
+ num_layers=24
8
+ hidden=2048
9
+ ffn_hidden=10944
10
+ moe_ffn_hidden=1408
11
+ num_experts=64
12
+ topk=6
13
+ experts_per_rank=8
14
+ expert_params=8650752
15
+ heads=16
16
+ kv_groups=0
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=2816
20
+ shared_expert_overlap=0
21
+ mla=1
22
+ dense_first_layers=1
23
+ moe_layers=23
24
+ rotary_base=10000
25
+ rotary_scaling=40
26
+ qkv_bias=0
27
+ router_balancing=seq_aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=1
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=8192
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=49152
39
+ mean_tokens_per_expert=6144
40
+ slot_policy=matched
41
+ slots_per_rank=8
42
+ capacity_mode=static
43
+ capacity=1.0755208333333333
44
+ cuda_graph=1
45
+ probe_mode=0
46
+ async_grad_reduce=0
47
+ branch_role=timing
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=768
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
ultraep/tierb/tierb_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333/rep2/train.log ADDED
The diff for this file is too large to render. See raw diff
 
ultraep/tierb/tierb_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333/rep3/mem_samples.csv ADDED
@@ -0,0 +1,232 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 4
43
+ 2, 4
44
+ 3, 629
45
+ 4, 4
46
+ 5, 4
47
+ 6, 4
48
+ 7, 4
49
+ 0, 821
50
+ 1, 821
51
+ 2, 821
52
+ 3, 821
53
+ 4, 821
54
+ 5, 821
55
+ 6, 821
56
+ 7, 821
57
+ 0, 1955
58
+ 1, 1955
59
+ 2, 1955
60
+ 3, 1955
61
+ 4, 1955
62
+ 5, 1955
63
+ 6, 1955
64
+ 7, 1955
65
+ 0, 2343
66
+ 1, 2345
67
+ 2, 2351
68
+ 3, 2347
69
+ 4, 2339
70
+ 5, 2367
71
+ 6, 2349
72
+ 7, 2347
73
+ 0, 9263
74
+ 1, 9117
75
+ 2, 9311
76
+ 3, 9077
77
+ 4, 9759
78
+ 5, 9311
79
+ 6, 9605
80
+ 7, 9371
81
+ 0, 28851
82
+ 1, 28945
83
+ 2, 28801
84
+ 3, 28795
85
+ 4, 28775
86
+ 5, 28835
87
+ 6, 28977
88
+ 7, 28907
89
+ 0, 39067
90
+ 1, 39151
91
+ 2, 39017
92
+ 3, 39015
93
+ 4, 39001
94
+ 5, 38995
95
+ 6, 39123
96
+ 7, 39167
97
+ 0, 39773
98
+ 1, 39857
99
+ 2, 39707
100
+ 3, 39705
101
+ 4, 39705
102
+ 5, 39703
103
+ 6, 39839
104
+ 7, 39857
105
+ 0, 39773
106
+ 1, 39857
107
+ 2, 39707
108
+ 3, 39705
109
+ 4, 39705
110
+ 5, 39703
111
+ 6, 39839
112
+ 7, 39857
113
+ 0, 39783
114
+ 1, 39867
115
+ 2, 39717
116
+ 3, 39717
117
+ 4, 39717
118
+ 5, 39713
119
+ 6, 39849
120
+ 7, 39867
121
+ 0, 39783
122
+ 1, 39867
123
+ 2, 39717
124
+ 3, 39717
125
+ 4, 39717
126
+ 5, 39713
127
+ 6, 39849
128
+ 7, 39867
129
+ 0, 109693
130
+ 1, 109797
131
+ 2, 110115
132
+ 3, 110121
133
+ 4, 110111
134
+ 5, 110101
135
+ 6, 109757
136
+ 7, 109787
137
+ 0, 110101
138
+ 1, 110187
139
+ 2, 110513
140
+ 3, 110511
141
+ 4, 110511
142
+ 5, 110509
143
+ 6, 110167
144
+ 7, 110187
145
+ 0, 110105
146
+ 1, 110191
147
+ 2, 110517
148
+ 3, 110515
149
+ 4, 110515
150
+ 5, 110513
151
+ 6, 110171
152
+ 7, 110191
153
+ 0, 112269
154
+ 1, 112391
155
+ 2, 110639
156
+ 3, 110677
157
+ 4, 110667
158
+ 5, 110903
159
+ 6, 112363
160
+ 7, 112481
161
+ 0, 113191
162
+ 1, 113277
163
+ 2, 111555
164
+ 3, 111553
165
+ 4, 111553
166
+ 5, 111551
167
+ 6, 113257
168
+ 7, 113277
169
+ 0, 113901
170
+ 1, 113967
171
+ 2, 112249
172
+ 3, 112257
173
+ 4, 112217
174
+ 5, 112283
175
+ 6, 113967
176
+ 7, 113919
177
+ 0, 134333
178
+ 1, 134427
179
+ 2, 134287
180
+ 3, 134271
181
+ 4, 136331
182
+ 5, 136333
183
+ 6, 134417
184
+ 7, 134425
185
+ 0, 142137
186
+ 1, 142231
187
+ 2, 142081
188
+ 3, 142079
189
+ 4, 142075
190
+ 5, 142077
191
+ 6, 142211
192
+ 7, 142231
193
+ 0, 142137
194
+ 1, 142231
195
+ 2, 142081
196
+ 3, 142079
197
+ 4, 142075
198
+ 5, 142077
199
+ 6, 142211
200
+ 7, 142231
201
+ 0, 142137
202
+ 1, 142231
203
+ 2, 142081
204
+ 3, 142079
205
+ 4, 142075
206
+ 5, 142077
207
+ 6, 142211
208
+ 7, 142231
209
+ 0, 139009
210
+ 1, 139103
211
+ 2, 138953
212
+ 3, 138689
213
+ 4, 138947
214
+ 5, 138949
215
+ 6, 139083
216
+ 7, 139103
217
+ 0, 136999
218
+ 1, 137099
219
+ 2, 137531
220
+ 3, 137267
221
+ 4, 137913
222
+ 5, 137265
223
+ 6, 137661
224
+ 7, 138069
225
+ 0, 321
226
+ 1, 4
227
+ 2, 4
228
+ 3, 4
229
+ 4, 4
230
+ 5, 4
231
+ 6, 68
232
+ 7, 137033
ultraep/tierb/tierb_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333/rep3/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=ultraep
2
+ config=DSV2_LITE_L24
3
+ family=tierB
4
+ repo=/workspace/Megatron-UltraEP
5
+ branch=ultraep-bracing-regions-through-kernels
6
+ commit=03d57ef7
7
+ num_layers=24
8
+ hidden=2048
9
+ ffn_hidden=10944
10
+ moe_ffn_hidden=1408
11
+ num_experts=64
12
+ topk=6
13
+ experts_per_rank=8
14
+ expert_params=8650752
15
+ heads=16
16
+ kv_groups=0
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=2816
20
+ shared_expert_overlap=0
21
+ mla=1
22
+ dense_first_layers=1
23
+ moe_layers=23
24
+ rotary_base=10000
25
+ rotary_scaling=40
26
+ qkv_bias=0
27
+ router_balancing=seq_aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=1
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=8192
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=49152
39
+ mean_tokens_per_expert=6144
40
+ slot_policy=matched
41
+ slots_per_rank=8
42
+ capacity_mode=static
43
+ capacity=1.0755208333333333
44
+ cuda_graph=1
45
+ probe_mode=0
46
+ async_grad_reduce=0
47
+ branch_role=timing
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=768
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
ultraep/tierb/tierb_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333/rep3/train.log ADDED
The diff for this file is too large to render. See raw diff
 
ultraep/tierb/tierb_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375/rep1/mem_samples.csv ADDED
@@ -0,0 +1,192 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 2
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 27
43
+ 2, 4
44
+ 3, 4
45
+ 4, 37
46
+ 5, 4
47
+ 6, 4
48
+ 7, 4
49
+ 0, 821
50
+ 1, 821
51
+ 2, 821
52
+ 3, 821
53
+ 4, 821
54
+ 5, 821
55
+ 6, 821
56
+ 7, 821
57
+ 0, 1803
58
+ 1, 1803
59
+ 2, 1803
60
+ 3, 1803
61
+ 4, 1803
62
+ 5, 1803
63
+ 6, 1803
64
+ 7, 1803
65
+ 0, 2403
66
+ 1, 2375
67
+ 2, 2373
68
+ 3, 2367
69
+ 4, 2347
70
+ 5, 2361
71
+ 6, 2371
72
+ 7, 2369
73
+ 0, 46703
74
+ 1, 46137
75
+ 2, 46137
76
+ 3, 46837
77
+ 4, 46837
78
+ 5, 46837
79
+ 6, 46837
80
+ 7, 46837
81
+ 0, 47125
82
+ 1, 46555
83
+ 2, 46557
84
+ 3, 47255
85
+ 4, 47257
86
+ 5, 47263
87
+ 6, 47255
88
+ 7, 47257
89
+ 0, 48413
90
+ 1, 47843
91
+ 2, 47843
92
+ 3, 48543
93
+ 4, 48543
94
+ 5, 48543
95
+ 6, 48543
96
+ 7, 48543
97
+ 0, 48659
98
+ 1, 48089
99
+ 2, 48089
100
+ 3, 48789
101
+ 4, 48789
102
+ 5, 48789
103
+ 6, 48789
104
+ 7, 48789
105
+ 0, 48659
106
+ 1, 48089
107
+ 2, 48089
108
+ 3, 48789
109
+ 4, 48789
110
+ 5, 48789
111
+ 6, 48789
112
+ 7, 48789
113
+ 0, 50727
114
+ 1, 50157
115
+ 2, 50157
116
+ 3, 50857
117
+ 4, 50857
118
+ 5, 50857
119
+ 6, 50857
120
+ 7, 50857
121
+ 0, 73613
122
+ 1, 73627
123
+ 2, 73609
124
+ 3, 73629
125
+ 4, 73635
126
+ 5, 73633
127
+ 6, 73639
128
+ 7, 73641
129
+ 0, 74163
130
+ 1, 74141
131
+ 2, 74123
132
+ 3, 74167
133
+ 4, 74167
134
+ 5, 74167
135
+ 6, 74167
136
+ 7, 74167
137
+ 0, 74163
138
+ 1, 74143
139
+ 2, 74125
140
+ 3, 74169
141
+ 4, 74169
142
+ 5, 74169
143
+ 6, 74169
144
+ 7, 74169
145
+ 0, 74505
146
+ 1, 74377
147
+ 2, 74373
148
+ 3, 74571
149
+ 4, 74407
150
+ 5, 74355
151
+ 6, 74431
152
+ 7, 74439
153
+ 0, 82097
154
+ 1, 81703
155
+ 2, 81947
156
+ 3, 82057
157
+ 4, 82035
158
+ 5, 82043
159
+ 6, 81979
160
+ 7, 82077
161
+ 0, 96727
162
+ 1, 96777
163
+ 2, 96809
164
+ 3, 96709
165
+ 4, 96709
166
+ 5, 96709
167
+ 6, 96709
168
+ 7, 96709
169
+ 0, 98599
170
+ 1, 98233
171
+ 2, 98423
172
+ 3, 98637
173
+ 4, 98637
174
+ 5, 98637
175
+ 6, 98637
176
+ 7, 98637
177
+ 0, 98599
178
+ 1, 98233
179
+ 2, 98423
180
+ 3, 98637
181
+ 4, 98637
182
+ 5, 98637
183
+ 6, 98637
184
+ 7, 98637
185
+ 0, 95137
186
+ 1, 1284
187
+ 2, 4
188
+ 3, 4
189
+ 4, 4
190
+ 5, 4
191
+ 6, 4
192
+ 7, 260
ultraep/tierb/tierb_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=ultraep
2
+ config=QWEN2_57B
3
+ family=tierB
4
+ repo=/workspace/Megatron-UltraEP
5
+ branch=ultraep-bracing-regions-through-kernels
6
+ commit=03d57ef7
7
+ num_layers=10
8
+ hidden=3584
9
+ ffn_hidden=2560
10
+ moe_ffn_hidden=2560
11
+ num_experts=64
12
+ topk=8
13
+ experts_per_rank=8
14
+ expert_params=27525120
15
+ heads=28
16
+ kv_groups=4
17
+ kv_channels=
18
+ qk_layernorm=0
19
+ shared_ffn=20480
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=10
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=1
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-2
29
+ router_pre_softmax=1
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=4096
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=32768
39
+ mean_tokens_per_expert=4096
40
+ slot_policy=matched
41
+ slots_per_rank=8
42
+ capacity_mode=static
43
+ capacity=1.08984375
44
+ cuda_graph=1
45
+ probe_mode=0
46
+ async_grad_reduce=0
47
+ branch_role=timing
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=512
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
ultraep/tierb/tierb_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375/rep1/train.log ADDED
@@ -0,0 +1,1194 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ==============================================================
2
+ QWEN2_57B / ultraep (slots/rank=8 policy=matched)
3
+ E=64 K=8 H'=2560 m=8
4
+ L=10 seq=4096 gbs=8 (1 microbatches)
5
+ cap_mode=static cap=1.08984375 cg=1 recompute=none
6
+ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierb/tierb_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375/rep1/memory_snapshots/snapshot.pickle
7
+ repo=/workspace/Megatron-UltraEP @ 03d57ef7
8
+ ==============================================================
9
+ W0826 18:38:44.640000 538192 torch/distributed/run.py:766]
10
+ W0826 18:38:44.640000 538192 torch/distributed/run.py:766] *****************************************
11
+ W0826 18:38:44.640000 538192 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
12
+ W0826 18:38:44.640000 538192 torch/distributed/run.py:766] *****************************************
13
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
14
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
15
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
16
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
17
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
18
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
19
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
20
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
21
+ WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it
22
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
23
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
24
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
25
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
26
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
27
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
28
+ using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1
29
+ WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer
30
+ Number of virtual stages per pipeline stage: None
31
+ accumulate and all-reduce gradients in fp32 for bfloat16 data type.
32
+ using torch.bfloat16 for parameters ...
33
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
34
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
35
+ ------------------------ arguments ------------------------
36
+ account_for_embedding_in_pipeline_split ......... False
37
+ account_for_loss_in_pipeline_split .............. False
38
+ accumulate_allreduce_grads_in_fp32 .............. True
39
+ adam_beta1 ...................................... 0.9
40
+ adam_beta2 ...................................... 0.999
41
+ adam_eps ........................................ 1e-08
42
+ add_bias_linear ................................. False
43
+ add_position_embedding .......................... False
44
+ add_qkv_bias .................................... True
45
+ adlr_autoresume ................................. False
46
+ adlr_autoresume_interval ........................ 1000
47
+ align_grad_reduce ............................... True
48
+ align_param_gather .............................. False
49
+ app_tag_run_name ................................ None
50
+ app_tag_run_version ............................. 0.0.0
51
+ apply_layernorm_1p .............................. False
52
+ apply_query_key_layer_scaling ................... False
53
+ apply_residual_connection_post_layernorm ........ False
54
+ apply_rope_fusion ............................... True
55
+ async_save ...................................... None
56
+ async_tensor_model_parallel_allreduce ........... True
57
+ attention_backend ............................... AttnBackend.auto
58
+ attention_dropout ............................... 0.0
59
+ attention_softmax_in_fp32 ....................... False
60
+ auto_detect_ckpt_format ......................... False
61
+ barrier_with_L1_time ............................ True
62
+ bert_binary_head ................................ True
63
+ bert_embedder_type .............................. megatron
64
+ bert_load ....................................... None
65
+ bf16 ............................................ True
66
+ bias_dropout_fusion ............................. True
67
+ bias_gelu_fusion ................................ False
68
+ bias_swiglu_fusion .............................. True
69
+ biencoder_projection_dim ........................ 0
70
+ biencoder_shared_query_context_model ............ False
71
+ block_data_path ................................. None
72
+ calc_ft_timeouts ................................ False
73
+ calculate_per_token_loss ........................ False
74
+ check_for_large_grads ........................... False
75
+ check_for_nan_in_loss_and_grad .................. False
76
+ check_for_spiky_loss ............................ False
77
+ check_weight_hash_across_dp_replicas_interval ... None
78
+ ckpt_assume_constant_structure .................. False
79
+ ckpt_convert_format ............................. None
80
+ ckpt_convert_save ............................... None
81
+ ckpt_convert_update_legacy_dist_opt_format ...... False
82
+ ckpt_format ..................................... torch_dist
83
+ ckpt_fully_parallel_load ........................ False
84
+ ckpt_fully_parallel_save ........................ True
85
+ ckpt_fully_parallel_save_deprecated ............. False
86
+ ckpt_step ....................................... None
87
+ classes_fraction ................................ 1.0
88
+ clip_grad ....................................... 1.0
89
+ clone_scatter_output_in_embedding ............... True
90
+ config_logger_dir ...............................
91
+ consumed_train_samples .......................... 0
92
+ consumed_valid_samples .......................... 0
93
+ context_parallel_size ........................... 1
94
+ cp_comm_type .................................... ['p2p']
95
+ create_attention_mask_in_dataloader ............. True
96
+ cross_entropy_fusion_impl ....................... native
97
+ cross_entropy_loss_fusion ....................... False
98
+ cuda_graph_scope ................................ full_iteration
99
+ cuda_graph_warmup_steps ......................... 3
100
+ data_args_path .................................. None
101
+ data_cache_path ................................. None
102
+ data_parallel_random_init ....................... False
103
+ data_parallel_sharding_strategy ................. no_shard
104
+ data_parallel_size .............................. 8
105
+ data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document']
106
+ data_per_class_fraction ......................... 1.0
107
+ data_sharding ................................... True
108
+ dataloader_type ................................. single
109
+ ddp_average_in_collective ....................... False
110
+ ddp_bucket_size ................................. None
111
+ ddp_num_buckets ................................. None
112
+ ddp_pad_buckets_for_high_nccl_busbw ............. False
113
+ decoder_first_pipeline_num_layers ............... None
114
+ decoder_last_pipeline_num_layers ................ None
115
+ decoder_num_layers .............................. None
116
+ decoder_seq_length .............................. None
117
+ decoupled_lr .................................... None
118
+ decoupled_min_lr ................................ None
119
+ decrease_batch_size_if_needed ................... False
120
+ defer_embedding_wgrad_compute ................... False
121
+ delay_wgrad_compute ............................. False
122
+ deprecated_use_mcore_models ..................... True
123
+ deterministic_mode .............................. False
124
+ dino_bottleneck_size ............................ 256
125
+ dino_freeze_last_layer .......................... 1
126
+ dino_head_hidden_size ........................... 2048
127
+ dino_local_crops_number ......................... 10
128
+ dino_local_img_size ............................. 96
129
+ dino_norm_last_layer ............................ False
130
+ dino_teacher_temp ............................... 0.07
131
+ dino_warmup_teacher_temp ........................ 0.04
132
+ dino_warmup_teacher_temp_epochs ................. 30
133
+ disable_bf16_reduced_precision_matmul ........... False
134
+ disable_mamba_mem_eff_path ...................... False
135
+ disable_straggler_on_startup .................... False
136
+ dist_ckpt_format_deprecated ..................... None
137
+ dist_ckpt_strictness ............................ assume_ok_unexpected
138
+ distribute_saved_activations .................... False
139
+ distributed_backend ............................. nccl
140
+ distributed_timeout_minutes ..................... 10
141
+ embedding_init_method_std ....................... None
142
+ embedding_path .................................. None
143
+ empty_unused_memory_level ....................... 0
144
+ enable_cuda_graph ............................... True
145
+ enable_experimental ............................. False
146
+ enable_ft_package ............................... False
147
+ enable_gloo_process_groups ...................... True
148
+ enable_msc ...................................... True
149
+ enable_one_logger ............................... True
150
+ encoder_num_layers .............................. 10
151
+ encoder_seq_length .............................. 4096
152
+ end_weight_decay ................................ 0.1
153
+ eod_mask_loss ................................... False
154
+ error_injection_rate ............................ 0
155
+ error_injection_type ............................ transient_error
156
+ eval_interval ................................... 1000000
157
+ eval_iters ...................................... 0
158
+ evidence_data_path .............................. None
159
+ exit_duration_in_mins ........................... None
160
+ exit_interval ................................... None
161
+ exit_on_missing_checkpoint ...................... False
162
+ exit_signal_handler ............................. False
163
+ exp_avg_dtype ................................... torch.float32
164
+ exp_avg_sq_dtype ................................ torch.float32
165
+ expert_model_parallel_size ...................... 8
166
+ expert_tensor_parallel_size ..................... 1
167
+ export_force_local_attention .................... False
168
+ export_kd_cfg ................................... None
169
+ export_kd_teacher_ckpt_format ................... None
170
+ export_kd_teacher_load .......................... None
171
+ export_kv_cache_quant ........................... False
172
+ export_legacy_megatron .......................... False
173
+ export_model_type ............................... GPTModel
174
+ export_moe_apply_probs_on_input ................. False
175
+ export_qk_l2_norm ............................... False
176
+ export_quant_cfg ................................ None
177
+ export_real_quant_cfg ........................... None
178
+ export_te_mcore_model ........................... False
179
+ external_cuda_graph ............................. False
180
+ ffn_hidden_size ................................. 2560
181
+ finetune ........................................ False
182
+ finetune_data_split ............................. train
183
+ finetune_hf_dataset ............................. None
184
+ first_last_layers_bf16 .......................... False
185
+ flash_decode .................................... False
186
+ fp16 ............................................ False
187
+ fp16_lm_cross_entropy ........................... False
188
+ fp32_residual_connection ........................ False
189
+ fp8 ............................................. e4m3
190
+ fp8_amax_compute_algo ........................... most_recent
191
+ fp8_amax_history_len ............................ 1
192
+ fp8_interval .................................... 1
193
+ fp8_margin ...................................... 0
194
+ fp8_param_gather ................................ False
195
+ fp8_recipe ...................................... mxfp8
196
+ fp8_wgrad ....................................... True
197
+ fsdp_double_buffer .............................. False
198
+ global_batch_size ............................... 8
199
+ grad_reduce_in_bf16 ............................. False
200
+ gradient_accumulation_fusion .................... True
201
+ gradient_reduce_div_fusion ...................... True
202
+ group_query_attention ........................... True
203
+ head_lr_mult .................................... 1.0
204
+ heterogeneous_layers_config_encoded_json ........ None
205
+ heterogeneous_layers_config_path ................ None
206
+ hidden_dropout .................................. 0.0
207
+ hidden_size ..................................... 3584
208
+ hierarchical_context_parallel_sizes ............. None
209
+ high_priority_stream_groups ..................... []
210
+ hybrid_attention_ratio .......................... 0.0
211
+ hybrid_mlp_ratio ................................ 0.0
212
+ hybrid_override_pattern ......................... None
213
+ hysteresis ...................................... 2
214
+ ict_head_size ................................... None
215
+ ict_load ........................................ None
216
+ img_h ........................................... 224
217
+ img_w ........................................... 224
218
+ indexer_batch_size .............................. 128
219
+ indexer_log_interval ............................ 1000
220
+ inference_batch_times_seqlen_threshold .......... -1
221
+ inference_dynamic_batching ...................... False
222
+ inference_dynamic_batching_buffer_guaranteed_fraction 0.2
223
+ inference_dynamic_batching_buffer_overflow_factor None
224
+ inference_dynamic_batching_buffer_size_gb ....... 40.0
225
+ inference_dynamic_batching_chunk_size ........... 256
226
+ inference_dynamic_batching_max_requests_override None
227
+ inference_dynamic_batching_max_tokens_override .. None
228
+ inference_dynamic_batching_num_cuda_graphs ...... 16
229
+ inference_max_batch_size ........................ 8
230
+ inference_max_seq_length ........................ 2560
231
+ inference_rng_tracker ........................... False
232
+ init_method_std ................................. 0.01
233
+ init_method_xavier_uniform ...................... False
234
+ init_model_with_meta_device ..................... False
235
+ initial_loss_scale .............................. 4294967296
236
+ inprocess_active_world_size ..................... 8
237
+ inprocess_barrier_timeout ....................... 120
238
+ inprocess_completion_timeout .................... 120
239
+ inprocess_empty_cuda_cache ...................... False
240
+ inprocess_granularity ........................... node
241
+ inprocess_hard_timeout .......................... 90
242
+ inprocess_heartbeat_interval .................... 30
243
+ inprocess_heartbeat_timeout ..................... 60
244
+ inprocess_last_call_wait ........................ 1
245
+ inprocess_max_iterations ........................ None
246
+ inprocess_monitor_process_interval .............. 1.0
247
+ inprocess_monitor_thread_interval ............... 1.0
248
+ inprocess_progress_watchdog_interval ............ 1.0
249
+ inprocess_restart ............................... False
250
+ inprocess_soft_timeout .......................... 60
251
+ inprocess_termination_grace_time ................ 1
252
+ is_hybrid_model ................................. False
253
+ iter_per_epoch .................................. 1250
254
+ iterations_to_skip .............................. []
255
+ keep_fp8_transpose_cache_when_using_custom_fsdp . False
256
+ kitchen_config_file ............................. None
257
+ kitchen_recipe_number ........................... None
258
+ kv_channels ..................................... 128
259
+ kv_lora_rank .................................... 32
260
+ lazy_mpu_init ................................... None
261
+ load ............................................ None
262
+ load_main_params_from_ckpt ...................... None
263
+ load_model_opt_format ........................... False
264
+ local_rank ...................................... 0
265
+ log_energy ...................................... False
266
+ log_interval .................................... 1
267
+ log_loss_scale_to_tensorboard ................... True
268
+ log_memory_to_tensorboard ....................... False
269
+ log_num_zeros_in_grad ........................... False
270
+ log_params_norm ................................. False
271
+ log_progress .................................... False
272
+ log_straggler ................................... False
273
+ log_throughput .................................. True
274
+ log_timers_to_tensorboard ....................... False
275
+ log_validation_ppl_to_tensorboard ............... False
276
+ log_world_size_to_tensorboard ................... False
277
+ logging_level ................................... None
278
+ loss_scale ...................................... None
279
+ loss_scale_window ............................... 1000
280
+ lr .............................................. 0.0001
281
+ lr_decay_iters .................................. 8
282
+ lr_decay_samples ................................ None
283
+ lr_decay_style .................................. cosine
284
+ lr_warmup_fraction .............................. None
285
+ lr_warmup_init .................................. 0.0
286
+ lr_warmup_iters ................................. 5
287
+ lr_warmup_samples ............................... 0
288
+ lr_wsd_decay_iters .............................. None
289
+ lr_wsd_decay_samples ............................ None
290
+ lr_wsd_decay_style .............................. exponential
291
+ main_grads_dtype ................................ torch.float32
292
+ main_params_dtype ............................... torch.float32
293
+ make_vocab_size_divisible_by .................... 128
294
+ mamba_head_dim .................................. 64
295
+ mamba_num_groups ................................ 8
296
+ mamba_num_heads ................................. None
297
+ mamba_state_dim ................................. 128
298
+ manual_gc ....................................... False
299
+ manual_gc_eval .................................. True
300
+ manual_gc_interval .............................. 0
301
+ mask_factor ..................................... 1.0
302
+ mask_prob ....................................... 0.15
303
+ mask_type ....................................... random
304
+ masked_softmax_fusion ........................... False
305
+ max_position_embeddings ......................... 32768
306
+ max_tokens_to_oom ............................... 12000
307
+ memory_snapshot_path ............................ snapshot.pickle
308
+ merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt
309
+ micro_batch_size ................................ 1
310
+ microbatch_group_size_per_vp_stage .............. None
311
+ mid_level_dataset_surplus ....................... 0.005
312
+ min_loss_scale .................................. 1.0
313
+ min_lr .......................................... 1e-05
314
+ mlp_chunks_for_prefill .......................... 1
315
+ mmap_bin_files .................................. True
316
+ mock_data ....................................... False
317
+ moe_apply_probs_on_input ........................ False
318
+ moe_aux_loss_coeff .............................. 0.01
319
+ moe_capacity_monitor ............................ True
320
+ moe_deepep_num_sms .............................. 20
321
+ moe_echo_enable_random_offloading ............... False
322
+ moe_echo_expert_dispatch_overlap ................ False
323
+ moe_echo_expert_dispatcher_type ................. hybridep
324
+ moe_echo_recompute_expert_dispatch .............. False
325
+ moe_enable_deepep ............................... False
326
+ moe_enable_echo ................................. False
327
+ moe_enable_hybridep ............................. True
328
+ moe_enable_ultraep .............................. True
329
+ moe_expert_capacity_factor ...................... None
330
+ moe_extended_tp ................................. False
331
+ moe_ffn_hidden_size ............................. 2560
332
+ moe_grouped_gemm ................................ True
333
+ moe_hybridep_num_sms ............................ 16
334
+ moe_input_jitter_eps ............................ None
335
+ moe_layer_freq .................................. 1
336
+ moe_layer_recompute ............................. False
337
+ moe_num_echo_experts ............................ None
338
+ moe_num_redundant_experts_per_rank .............. 8
339
+ moe_pad_expert_input_to_capacity ................ False
340
+ moe_per_layer_logging ........................... False
341
+ moe_permute_fusion .............................. False
342
+ moe_received_token_capacity ..................... 1.08984375
343
+ moe_router_bias_update_rate ..................... 0.001
344
+ moe_router_dtype ................................ fp32
345
+ moe_router_enable_expert_bias ................... False
346
+ moe_router_force_load_balancing ................. False
347
+ moe_router_group_topk ........................... None
348
+ moe_router_load_balancing_type .................. aux_loss
349
+ moe_router_num_groups ........................... None
350
+ moe_router_padding_for_fp8 ...................... False
351
+ moe_router_pre_softmax .......................... True
352
+ moe_router_score_function ....................... softmax
353
+ moe_router_topk ................................. 8
354
+ moe_router_topk_scaling_factor .................. None
355
+ moe_shared_expert_intermediate_size ............. 20480
356
+ moe_shared_expert_overlap ....................... False
357
+ moe_token_dispatcher_type ....................... flex
358
+ moe_token_drop_policy ........................... probs
359
+ moe_ultraep_max_microbatches .................... None
360
+ moe_ultraep_overlap_weight_sync ................. True
361
+ moe_ultraep_weight_sync_join .................... before_dispatch
362
+ moe_upcycling_granularity ....................... 1
363
+ moe_use_device_initiated_grouped_gemm ........... True
364
+ moe_use_legacy_grouped_gemm ..................... False
365
+ moe_use_upcycling ............................... False
366
+ moe_z_loss_coeff ................................ None
367
+ mrope_section ................................... None
368
+ mscale .......................................... 1.0
369
+ mscale_all_dim .................................. 0.0
370
+ mtp_loss_scaling_factor ......................... 0.1
371
+ mtp_num_layers .................................. None
372
+ multi_latent_attention .......................... False
373
+ nccl_all_reduce_for_prefill ..................... False
374
+ nccl_communicator_config_path ................... None
375
+ nccl_ub ......................................... False
376
+ no_load_optim ................................... None
377
+ no_load_rng ..................................... None
378
+ no_persist_layer_norm ........................... False
379
+ no_rope_freq .................................... None
380
+ no_save_optim ................................... None
381
+ no_save_rng ..................................... None
382
+ non_persistent_ckpt_type ........................ None
383
+ non_persistent_global_ckpt_dir .................. None
384
+ non_persistent_local_ckpt_algo .................. fully_parallel
385
+ non_persistent_local_ckpt_dir ................... None
386
+ non_persistent_save_interval .................... None
387
+ norm_epsilon .................................... 1e-06
388
+ normalization ................................... RMSNorm
389
+ num_attention_heads ............................. 28
390
+ num_channels .................................... 3
391
+ num_classes ..................................... 1000
392
+ num_dataset_builder_threads ..................... 1
393
+ num_distributed_optimizer_instances ............. 1
394
+ num_experts ..................................... 64
395
+ num_layers ...................................... 10
396
+ num_layers_at_end_in_bf16 ....................... 1
397
+ num_layers_at_start_in_bf16 ..................... 1
398
+ num_layers_per_virtual_pipeline_stage ........... None
399
+ num_query_groups ................................ 4
400
+ num_virtual_stages_per_pipeline_rank ............ None
401
+ num_workers ..................................... 2
402
+ object_storage_cache_path ....................... None
403
+ one_logger_async ................................ False
404
+ one_logger_project .............................. megatron-lm
405
+ one_logger_run_name ............................. None
406
+ onnx_safe ....................................... None
407
+ openai_gelu ..................................... False
408
+ optimizer ....................................... adam
409
+ optimizer_cpu_offload ........................... False
410
+ optimizer_offload_fraction ...................... 1.0
411
+ output_bert_embeddings .......................... False
412
+ overlap_cpu_optimizer_d2h_h2d ................... False
413
+ overlap_grad_reduce ............................. True
414
+ overlap_moe_expert_parallel_comm ................ False
415
+ overlap_p2p_comm ................................ False
416
+ overlap_p2p_comm_warmup_flush ................... False
417
+ overlap_param_gather ............................ True
418
+ overlap_param_gather_with_optimizer_step ........ False
419
+ override_opt_param_scheduler .................... False
420
+ params_dtype .................................... torch.bfloat16
421
+ patch_dim ....................................... 16
422
+ per_split_data_args_path ........................ None
423
+ perform_initialization .......................... True
424
+ pin_cpu_grads ................................... True
425
+ pin_cpu_params .................................. True
426
+ pipeline_model_parallel_comm_backend ............ None
427
+ pipeline_model_parallel_layout .................. None
428
+ pipeline_model_parallel_size .................... 1
429
+ position_embedding_type ......................... rope
430
+ pretrained_checkpoint ........................... None
431
+ profile ......................................... False
432
+ profile_ranks ................................... [0]
433
+ profile_step_end ................................ 12
434
+ profile_step_start .............................. 10
435
+ q_lora_rank ..................................... None
436
+ qk_head_dim ..................................... 128
437
+ qk_l2_norm ...................................... False
438
+ qk_layernorm .................................... False
439
+ qk_pos_emb_head_dim ............................. 64
440
+ query_in_block_prob ............................. 0.1
441
+ rampup_batch_size ............................... None
442
+ rank ............................................ 0
443
+ recompute_granularity ........................... None
444
+ recompute_method ................................ None
445
+ recompute_modules ............................... None
446
+ recompute_num_layers ............................ None
447
+ record_memory_history ........................... False
448
+ relative_attention_max_distance ................. 128
449
+ relative_attention_num_buckets .................. 32
450
+ replication ..................................... False
451
+ replication_factor .............................. 2
452
+ replication_jump ................................ None
453
+ rerun_mode ...................................... validate_results
454
+ reset_attention_mask ............................ False
455
+ reset_position_ids .............................. False
456
+ result_rejected_tracker_filename ................ None
457
+ retriever_report_topk_accuracies ................ []
458
+ retriever_score_scaling ......................... False
459
+ retriever_seq_length ............................ 256
460
+ retro_add_retriever ............................. False
461
+ retro_attention_gate ............................ 1
462
+ retro_cyclic_train_iters ........................ None
463
+ retro_encoder_attention_dropout ................. 0.1
464
+ retro_encoder_hidden_dropout .................... 0.1
465
+ retro_encoder_layers ............................ 2
466
+ retro_num_neighbors ............................. 2
467
+ retro_num_retrieved_chunks ...................... 2
468
+ retro_project_dir ............................... None
469
+ retro_verify_neighbor_count ..................... True
470
+ reuse_grad_buf_for_mxfp8_param_ag ............... False
471
+ rope_scaling_factor ............................. 8.0
472
+ rope_type ....................................... None
473
+ rotary_base ..................................... 1000000
474
+ rotary_interleaved .............................. False
475
+ rotary_percent .................................. 1.0
476
+ rotary_scaling_factor ........................... 1.0
477
+ rotary_seq_len_interpolation_factor ............. None
478
+ run_workload_inspector_server ................... False
479
+ sample_rate ..................................... 1.0
480
+ save ............................................ None
481
+ save_interval ................................... None
482
+ save_retain_interval ............................ None
483
+ scatter_gather_tensors_in_pipeline .............. True
484
+ seed ............................................ 1234
485
+ seq_length ...................................... 4096
486
+ sequence_parallel ............................... False
487
+ sft ............................................. False
488
+ sft_tokenizer_prompt_format ..................... nemotron-h-aligned
489
+ sgd_momentum .................................... 0.9
490
+ short_seq_prob .................................. 0.1
491
+ skip_train ...................................... False
492
+ skipped_train_samples ........................... 0
493
+ spec ............................................ None
494
+ split ........................................... 100,0,0
495
+ squared_relu .................................... False
496
+ start_weight_decay .............................. 0.1
497
+ straggler_ctrlr_port ............................ 65535
498
+ straggler_minmax_count .......................... 1
499
+ suggested_communication_unit_size ............... None
500
+ swiglu .......................................... True
501
+ swin_backbone_type .............................. tiny
502
+ symmetric_ar_type ............................... None
503
+ te_rng_tracker .................................. True
504
+ tensor_model_parallel_size ...................... 1
505
+ tensorboard_dir ................................. None
506
+ tensorboard_log_interval ........................ 1
507
+ tensorboard_queue_size .......................... 1000
508
+ test_data_path .................................. None
509
+ test_mode ....................................... False
510
+ tiktoken_num_special_tokens ..................... 1000
511
+ tiktoken_pattern ................................ None
512
+ tiktoken_special_tokens ......................... None
513
+ timing_log_level ................................ 0
514
+ timing_log_option ............................... minmax
515
+ titles_data_path ................................ None
516
+ tokenizer_model ................................. None
517
+ tokenizer_type .................................. GPT2BPETokenizer
518
+ torch_fsdp2_reshard_after_forward ............... True
519
+ tp_comm_bootstrap_backend ....................... nccl
520
+ tp_comm_bulk_dgrad .............................. True
521
+ tp_comm_bulk_wgrad .............................. True
522
+ tp_comm_overlap ................................. False
523
+ tp_comm_overlap_ag .............................. True
524
+ tp_comm_overlap_cfg ............................. None
525
+ tp_comm_overlap_rs .............................. True
526
+ tp_comm_overlap_rs_dgrad ........................ False
527
+ tp_comm_split_ag ................................ True
528
+ tp_comm_split_rs ................................ True
529
+ train_data_path ................................. None
530
+ train_iters ..................................... 12
531
+ train_samples ................................... None
532
+ train_sync_interval ............................. None
533
+ transformer_impl ................................ transformer_engine
534
+ transformer_pipeline_model_parallel_size ........ 1
535
+ untie_embeddings_and_output_weights ............. True
536
+ use_checkpoint_args ............................. False
537
+ use_checkpoint_opt_param_scheduler .............. False
538
+ use_cpu_initialization .......................... None
539
+ use_custom_fsdp ................................. False
540
+ use_dist_ckpt ................................... True
541
+ use_dist_ckpt_deprecated ........................ False
542
+ use_distributed_optimizer ....................... True
543
+ use_flash_attn .................................. False
544
+ use_legacy_models ............................... False
545
+ use_mp_args_from_checkpoint_args ................ False
546
+ use_one_sent_docs ............................... False
547
+ use_persistent_ckpt_worker ...................... False
548
+ use_precision_aware_optimizer ................... False
549
+ use_pytorch_profiler ............................ False
550
+ use_ring_exchange_p2p ........................... False
551
+ use_rope_scaling ................................ False
552
+ use_rotary_position_embeddings .................. False
553
+ use_sharp ....................................... False
554
+ use_tokenizer_model_from_checkpoint_args ........ True
555
+ use_torch_fsdp2 ................................. False
556
+ use_torch_optimizer_for_cpu_offload ............. False
557
+ use_tp_pp_dp_mapping ............................ False
558
+ v_head_dim ...................................... 128
559
+ valid_data_path ................................. None
560
+ variable_seq_lengths ............................ False
561
+ virtual_pipeline_model_parallel_size ............ None
562
+ vision_backbone_type ............................ vit
563
+ vision_pretraining .............................. False
564
+ vision_pretraining_type ......................... classify
565
+ vocab_extra_ids ................................. 0
566
+ vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json
567
+ vocab_size ...................................... None
568
+ wandb_exp_name ..................................
569
+ wandb_project ...................................
570
+ wandb_save_dir ..................................
571
+ weight_decay .................................... 0.1
572
+ weight_decay_incr_style ......................... constant
573
+ wgrad_deferral_limit ............................ 0
574
+ world_size ...................................... 8
575
+ yaml_cfg ........................................ None
576
+ -------------------- end of arguments ---------------------
577
+ INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1
578
+ > building GPT2BPETokenizer tokenizer ...
579
+ > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304)
580
+ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS
581
+ > initializing torch distributed ...
582
+ [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
583
+ [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
584
+ [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
585
+ [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
586
+ [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
587
+ [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
588
+ [Gloo] Rank [Gloo] Rank 7 is connected to 76 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks.
589
+ Expected number of connected peer ranks is : 7
590
+ [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
591
+ [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
592
+ [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
593
+ [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
594
+ [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
595
+ [Gloo] Rank [Gloo] Rank [Gloo] Rank 567 is connected to is connected to is connected to 777 peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77
596
+ 7
597
+
598
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
599
+ > initialized tensor model parallel with size 1
600
+ > initialized pipeline model parallel with size 1
601
+ > setting random seeds to 1234 ...
602
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
603
+ [Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00
604
+
605
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
606
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
607
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
608
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
609
+ [rank1]:[W826 18:39:02.758192730 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
610
+ [rank2]:[W826 18:39:02.758218559 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
611
+ [rank4]:[W826 18:39:02.758224235 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
612
+ [rank7]:[W826 18:39:02.758264494 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
613
+ [rank3]:[W826 18:39:02.758263509 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
614
+ [rank5]:[W826 18:39:02.758284080 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
615
+ [rank6]:[W826 18:39:02.758367472 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
616
+ > compiling dataset index builder ...
617
+ make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets'
618
+ make: Nothing to be done for 'default'.
619
+ make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets'
620
+ >>> done with dataset index builder. Compilation time: 0.036 seconds
621
+ WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations.
622
+ > compiling and loading fused kernels ...
623
+ [rank0]:[W826 18:39:02.928009101 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
624
+ >>> done with compiling and loading fused kernels. Compilation time: 3.911 seconds
625
+ time to initialize megatron (seconds): 7.130
626
+ [after megatron is initialized] datetime: 2026-08-26 18:39:08
627
+ building GPT model ...
628
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
629
+ warnings.warn(
630
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
631
+ warnings.warn(
632
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
633
+ warnings.warn(
634
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
635
+ warnings.warn(
636
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
637
+ warnings.warn(
638
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
639
+ warnings.warn(
640
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
641
+ warnings.warn(
642
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
643
+ warnings.warn(
644
+ [INFO] Use MNNVL fabric: False
645
+ [WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8.
646
+ NVSHMEM v3.7.2
647
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
648
+
649
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
650
+
651
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
652
+
653
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
654
+
655
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
656
+
657
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
658
+
659
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
660
+
661
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
662
+
663
+ UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive).
664
+ > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 7262624256
665
+ INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False)
666
+ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 22
667
+ Params for bucket 1 (180289536 elements, 180289536 padded size):
668
+ module.output_layer.weight
669
+ Params for bucket 2 (73403904 elements, 73403904 padded size):
670
+ module.decoder.final_layernorm.weight
671
+ module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight
672
+ Params for bucket 3 (146800640 elements, 146800640 padded size):
673
+ module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight
674
+ Params for bucket 4 (103001600 elements, 103001600 padded size):
675
+ module.decoder.layers.9.mlp.router.weight
676
+ module.decoder.layers.9.self_attention.linear_qkv.weight
677
+ module.decoder.layers.9.self_attention.linear_proj.weight
678
+ module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight
679
+ module.decoder.layers.9.pre_mlp_layernorm.weight
680
+ module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight
681
+ module.decoder.layers.9.self_attention.linear_qkv.bias
682
+ Params for bucket 5 (146800640 elements, 146800640 padded size):
683
+ module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight
684
+ Params for bucket 6 (103001600 elements, 103001600 padded size):
685
+ module.decoder.layers.8.mlp.router.weight
686
+ module.decoder.layers.8.self_attention.linear_qkv.weight
687
+ module.decoder.layers.8.self_attention.linear_proj.weight
688
+ module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight
689
+ module.decoder.layers.8.pre_mlp_layernorm.weight
690
+ module.decoder.layers.8.self_attention.linear_qkv.bias
691
+ module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight
692
+ Params for bucket 7 (146800640 elements, 146800640 padded size):
693
+ module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight
694
+ Params for bucket 8 (103001600 elements, 103001600 padded size):
695
+ module.decoder.layers.7.mlp.router.weight
696
+ module.decoder.layers.7.self_attention.linear_qkv.weight
697
+ module.decoder.layers.7.self_attention.linear_proj.weight
698
+ module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight
699
+ module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight
700
+ module.decoder.layers.7.pre_mlp_layernorm.weight
701
+ module.decoder.layers.7.self_attention.linear_qkv.bias
702
+ Params for bucket 9 (146800640 elements, 146800640 padded size):
703
+ module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight
704
+ Params for bucket 10 (103001600 elements, 103001600 padded size):
705
+ module.decoder.layers.6.mlp.router.weight
706
+ module.decoder.layers.6.self_attention.linear_qkv.weight
707
+ module.decoder.layers.6.self_attention.linear_proj.weight
708
+ module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight
709
+ module.decoder.layers.6.self_attention.linear_qkv.bias
710
+ module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight
711
+ module.decoder.layers.6.pre_mlp_layernorm.weight
712
+ Params for bucket 11 (146800640 elements, 146800640 padded size):
713
+ module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight
714
+ Params for bucket 12 (103001600 elements, 103001600 padded size):
715
+ module.decoder.layers.5.mlp.router.weight
716
+ module.decoder.layers.5.self_attention.linear_qkv.weight
717
+ module.decoder.layers.5.self_attention.linear_proj.weight
718
+ module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight
719
+ module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight
720
+ module.decoder.layers.5.pre_mlp_layernorm.weight
721
+ module.decoder.layers.5.self_attention.linear_qkv.bias
722
+ Params for bucket 13 (146800640 elements, 146800640 padded size):
723
+ module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight
724
+ Params for bucket 14 (103001600 elements, 103001600 padded size):
725
+ module.decoder.layers.4.mlp.router.weight
726
+ module.decoder.layers.4.self_attention.linear_qkv.weight
727
+ module.decoder.layers.4.self_attention.linear_proj.weight
728
+ module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight
729
+ module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight
730
+ module.decoder.layers.4.self_attention.linear_qkv.bias
731
+ module.decoder.layers.4.pre_mlp_layernorm.weight
732
+ Params for bucket 15 (146800640 elements, 146800640 padded size):
733
+ module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight
734
+ Params for bucket 16 (103001600 elements, 103001600 padded size):
735
+ module.decoder.layers.3.mlp.router.weight
736
+ module.decoder.layers.3.self_attention.linear_qkv.weight
737
+ module.decoder.layers.3.self_attention.linear_proj.weight
738
+ module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight
739
+ module.decoder.layers.3.self_attention.linear_qkv.bias
740
+ module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight
741
+ module.decoder.layers.3.pre_mlp_layernorm.weight
742
+ Params for bucket 17 (146800640 elements, 146800640 padded size):
743
+ module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight
744
+ Params for bucket 18 (103001600 elements, 103001600 padded size):
745
+ module.decoder.layers.2.mlp.router.weight
746
+ module.decoder.layers.2.self_attention.linear_qkv.weight
747
+ module.decoder.layers.2.self_attention.linear_proj.weight
748
+ module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight
749
+ module.decoder.layers.2.pre_mlp_layernorm.weight
750
+ module.decoder.layers.2.self_attention.linear_qkv.bias
751
+ module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight
752
+ Params for bucket 19 (146800640 elements, 146800640 padded size):
753
+ module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight
754
+ Params for bucket 20 (103001600 elements, 103001600 padded size):
755
+ module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight
756
+ module.decoder.layers.1.self_attention.linear_qkv.bias
757
+ module.decoder.layers.1.pre_mlp_layernorm.weight
758
+ module.decoder.layers.1.mlp.router.weight
759
+ module.decoder.layers.1.self_attention.linear_qkv.weight
760
+ module.decoder.layers.1.self_attention.linear_proj.weight
761
+ module.decoder.layers.0.mlp.shared_experts.linear_fc2.weight
762
+ Params for bucket 21 (146800640 elements, 146800640 padded size):
763
+ module.decoder.layers.0.mlp.shared_experts.linear_fc1.weight
764
+ Params for bucket 22 (209890816 elements, 209890816 padded size):
765
+ module.decoder.layers.0.self_attention.linear_qkv.bias
766
+ module.decoder.layers.0.mlp.router.weight
767
+ module.decoder.layers.0.pre_mlp_layernorm.weight
768
+ module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight
769
+ module.decoder.layers.0.self_attention.linear_qkv.weight
770
+ module.decoder.layers.0.self_attention.linear_proj.weight
771
+ module.embedding.word_embeddings.weight
772
+ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 44
773
+ Params for bucket 1 (45875200 elements, 45875200 padded size):
774
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight7
775
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight5
776
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight3
777
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight4
778
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight6
779
+ Params for bucket 2 (45875200 elements, 45875200 padded size):
780
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight2
781
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight7
782
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight1
783
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight0
784
+ Params for bucket 3 (55050240 elements, 55050240 padded size):
785
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight6
786
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight4
787
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight5
788
+ Params for bucket 4 (55050240 elements, 55050240 padded size):
789
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight3
790
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight1
791
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight2
792
+ Params for bucket 5 (45875200 elements, 45875200 padded size):
793
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight0
794
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight6
795
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight7
796
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight5
797
+ Params for bucket 6 (45875200 elements, 45875200 padded size):
798
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight3
799
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight1
800
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight4
801
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight0
802
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight2
803
+ Params for bucket 7 (55050240 elements, 55050240 padded size):
804
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight7
805
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight6
806
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight5
807
+ Params for bucket 8 (55050240 elements, 55050240 padded size):
808
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight4
809
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight3
810
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight2
811
+ Params for bucket 9 (45875200 elements, 45875200 padded size):
812
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight1
813
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight0
814
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight7
815
+ Params for bucket 10 (45875200 elements, 45875200 padded size):
816
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight5
817
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight3
818
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight6
819
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight2
820
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight4
821
+ Params for bucket 11 (55050240 elements, 55050240 padded size):
822
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight1
823
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight6
824
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight0
825
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight7
826
+ Params for bucket 12 (55050240 elements, 55050240 padded size):
827
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight5
828
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight4
829
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight3
830
+ Params for bucket 13 (55050240 elements, 55050240 padded size):
831
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight2
832
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight1
833
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight0
834
+ Params for bucket 14 (45875200 elements, 45875200 padded size):
835
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight7
836
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight3
837
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight5
838
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight6
839
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight4
840
+ Params for bucket 15 (45875200 elements, 45875200 padded size):
841
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight2
842
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight0
843
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight7
844
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight1
845
+ Params for bucket 16 (55050240 elements, 55050240 padded size):
846
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight6
847
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight4
848
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight5
849
+ Params for bucket 17 (55050240 elements, 55050240 padded size):
850
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight3
851
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight2
852
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight1
853
+ Params for bucket 18 (45875200 elements, 45875200 padded size):
854
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight0
855
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight5
856
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight6
857
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight7
858
+ Params for bucket 19 (45875200 elements, 45875200 padded size):
859
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight1
860
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight3
861
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight4
862
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight2
863
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight0
864
+ Params for bucket 20 (55050240 elements, 55050240 padded size):
865
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight7
866
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight5
867
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight6
868
+ Params for bucket 21 (55050240 elements, 55050240 padded size):
869
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight4
870
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight2
871
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight3
872
+ Params for bucket 22 (45875200 elements, 45875200 padded size):
873
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight1
874
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight7
875
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight0
876
+ Params for bucket 23 (45875200 elements, 45875200 padded size):
877
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight3
878
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight5
879
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight6
880
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight4
881
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight2
882
+ Params for bucket 24 (55050240 elements, 55050240 padded size):
883
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight1
884
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight0
885
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight6
886
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight7
887
+ Params for bucket 25 (55050240 elements, 55050240 padded size):
888
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight5
889
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight4
890
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight3
891
+ Params for bucket 26 (55050240 elements, 55050240 padded size):
892
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight2
893
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight0
894
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight1
895
+ Params for bucket 27 (45875200 elements, 45875200 padded size):
896
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight6
897
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight4
898
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight7
899
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight3
900
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight5
901
+ Params for bucket 28 (45875200 elements, 45875200 padded size):
902
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight2
903
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight7
904
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight0
905
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight1
906
+ Params for bucket 29 (55050240 elements, 55050240 padded size):
907
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight6
908
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight4
909
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight5
910
+ Params for bucket 30 (55050240 elements, 55050240 padded size):
911
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight3
912
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight1
913
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight2
914
+ Params for bucket 31 (45875200 elements, 45875200 padded size):
915
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight0
916
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight6
917
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight7
918
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight5
919
+ Params for bucket 32 (45875200 elements, 45875200 padded size):
920
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight3
921
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight1
922
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight4
923
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight0
924
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight2
925
+ Params for bucket 33 (55050240 elements, 55050240 padded size):
926
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight7
927
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight6
928
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight5
929
+ Params for bucket 34 (55050240 elements, 55050240 padded size):
930
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight4
931
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight2
932
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight3
933
+ Params for bucket 35 (45875200 elements, 45875200 padded size):
934
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight1
935
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight7
936
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight0
937
+ Params for bucket 36 (45875200 elements, 45875200 padded size):
938
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight6
939
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight2
940
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight4
941
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight3
942
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight5
943
+ Params for bucket 37 (55050240 elements, 55050240 padded size):
944
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight1
945
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight6
946
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight0
947
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight7
948
+ Params for bucket 38 (55050240 elements, 55050240 padded size):
949
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight5
950
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight3
951
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight4
952
+ Params for bucket 39 (55050240 elements, 55050240 padded size):
953
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight2
954
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight0
955
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight1
956
+ Params for bucket 40 (45875200 elements, 45875200 padded size):
957
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight7
958
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight3
959
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight5
960
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight4
961
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight6
962
+ Params for bucket 41 (45875200 elements, 45875200 padded size):
963
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight2
964
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight1
965
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight0
966
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight7
967
+ Params for bucket 42 (55050240 elements, 55050240 padded size):
968
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight6
969
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight5
970
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight4
971
+ Params for bucket 43 (55050240 elements, 55050240 padded size):
972
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight3
973
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight2
974
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight1
975
+ Params for bucket 44 (18350080 elements, 18350080 padded size):
976
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight0
977
+ INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=<megatron.core.timers.Timers object at 0x71b8d532ce00>, config_logger_dir='')
978
+ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine
979
+ [after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 18:39:12
980
+ > building train, validation, and test datasets ...
981
+ > datasets target sizes (minimum size):
982
+ train: 96
983
+ validation: 0
984
+ test: 0
985
+ INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None]
986
+ > building train, validation, and test datasets for GPT ...
987
+ INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=<megatron.training.tokenizer.tokenizer._GPT2BPETokenizer object at 0x71b8d582b5c0>, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None)
988
+ INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx
989
+ INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths
990
+ INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers
991
+ INFO:megatron.core.datasets.indexed_dataset: Extract the document indices
992
+ INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029
993
+ INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350
994
+ INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices
995
+ INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy
996
+ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy
997
+ INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy
998
+ INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073
999
+ > finished creating GPT datasets ...
1000
+ [after dataloaders are built] datetime: 2026-08-26 18:39:12
1001
+ done with setup ...
1002
+ (min, max) time across ranks (ms):
1003
+ model-and-optimizer-setup ......................: (3467.51, 3475.61)
1004
+ train/valid/test-data-iterators-setup ..........: (71.25, 91.10)
1005
+ training ...
1006
+ Overwriting rerun_state_machine.current_iteration from -1 to 0...
1007
+ [before the start of training step] datetime: 2026-08-26 18:39:12
1008
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1009
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1010
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1011
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1012
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1013
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1014
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1015
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1016
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1017
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1018
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1019
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1020
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1021
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1022
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1023
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1024
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1025
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1026
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1027
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1028
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1029
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1030
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1031
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1032
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1033
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1034
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1035
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1036
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1037
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1038
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1039
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1040
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1041
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1042
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1043
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1044
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1045
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1046
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1047
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1048
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1049
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1050
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1051
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1052
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1053
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1054
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1055
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1056
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1057
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1058
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1059
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1060
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1061
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1062
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1063
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1064
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1065
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1066
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1067
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1068
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1069
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1070
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1071
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1072
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1073
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1074
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1075
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1076
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1077
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1078
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1079
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1080
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1081
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1082
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1083
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1084
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1085
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1086
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1087
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1088
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1089
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1090
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1091
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1092
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1093
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1094
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1095
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1096
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1097
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1098
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1099
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1100
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1101
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1102
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1103
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1104
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1105
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1106
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1107
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1108
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1109
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1110
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1111
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1112
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1113
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1114
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1115
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1116
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1117
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1118
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1119
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1120
+ [2026-08-26 18:39:37] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 25184.3 | throughput per GPU (TFLOP/s/GPU): 4.9 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.097922E+01 | loss scale: 1.0 | grad norm: 29.878 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 |
1121
+ Number of parameters in transformer block in billions: 20.11
1122
+ Number of parameters in embedding layers in billions: 0.36
1123
+ Total number of parameters in billions: 20.47
1124
+ Number of parameters in most loaded shard in billions: 20.4724
1125
+ Theoretical memory footprints: weight and optimizer=146430.14 MB
1126
+ [Rank 0] (after 1 iterations) memory (MB) | allocated: 64278.42138671875 | max allocated: 65496.45947265625 | reserved: 73306.0 | max reserved: 73306.0
1127
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1128
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1129
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1130
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1131
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1132
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1133
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1134
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1135
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1136
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1137
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1138
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1139
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1140
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1141
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1142
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1143
+ [2026-08-26 18:39:37] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 329.5 | throughput per GPU (TFLOP/s/GPU): 374.8 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.097874E+01 | loss scale: 1.0 | grad norm: 30.618 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33792 | min safe capacity: 1.0312 |
1144
+ [2026-08-26 18:39:38] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 215.4 | throughput per GPU (TFLOP/s/GPU): 573.3 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.539364E+00 | loss scale: 1.0 | grad norm: 32.415 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33792 | min safe capacity: 1.0312 |
1145
+ Capture CUDA graph for training!!!
1146
+ CUDA graph capture done!!!
1147
+ [2026-08-26 18:39:39] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 1731.4 | throughput per GPU (TFLOP/s/GPU): 71.3 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.109718E+01 | loss scale: 1.0 | grad norm: 107.119 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33920 | min safe capacity: 1.0352 |
1148
+ [2026-08-26 18:39:40] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 198.9 | throughput per GPU (TFLOP/s/GPU): 620.7 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.000232E+01 | loss scale: 1.0 | grad norm: 26.383 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 |
1149
+ [2026-08-26 18:39:40] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 192.7 | throughput per GPU (TFLOP/s/GPU): 640.8 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.049848E+01 | loss scale: 1.0 | grad norm: 8.728 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 |
1150
+ [2026-08-26 18:39:40] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 184.4 | throughput per GPU (TFLOP/s/GPU): 669.7 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.443075E+00 | loss scale: 1.0 | grad norm: 5.409 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 |
1151
+ [2026-08-26 18:39:40] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 190.9 | throughput per GPU (TFLOP/s/GPU): 646.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.965197E+00 | loss scale: 1.0 | grad norm: 4.552 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 |
1152
+ [2026-08-26 18:39:40] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 193.1 | throughput per GPU (TFLOP/s/GPU): 639.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.749504E+00 | loss scale: 1.0 | grad norm: 4.299 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 |
1153
+ [2026-08-26 18:39:41] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 197.5 | throughput per GPU (TFLOP/s/GPU): 625.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.689064E+00 | loss scale: 1.0 | grad norm: 4.427 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 |
1154
+ [2026-08-26 18:39:41] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 190.9 | throughput per GPU (TFLOP/s/GPU): 646.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.628358E+00 | loss scale: 1.0 | grad norm: 4.348 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 |
1155
+ [2026-08-26 18:39:41] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 199.0 | throughput per GPU (TFLOP/s/GPU): 620.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.552548E+00 | loss scale: 1.0 | grad norm: 4.285 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 |
1156
+ [after training is done] datetime: 2026-08-26 18:39:41
1157
+ [rank0]:[W826 18:39:42.511122045 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1158
+ W0826 18:39:43.309000 538192 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers
1159
+ W0826 18:39:43.310000 538192 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 538278 closing signal SIGTERM
1160
+ W0826 18:39:43.310000 538192 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 538279 closing signal SIGTERM
1161
+ W0826 18:39:43.316000 538192 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 538280 closing signal SIGTERM
1162
+ W0826 18:39:43.318000 538192 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 538281 closing signal SIGTERM
1163
+ W0826 18:39:43.319000 538192 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 538282 closing signal SIGTERM
1164
+ W0826 18:39:43.323000 538192 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 538283 closing signal SIGTERM
1165
+ W0826 18:39:43.323000 538192 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 538284 closing signal SIGTERM
1166
+ W0826 18:39:43.327000 538192 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 538285 closing signal SIGTERM
1167
+ Traceback (most recent call last):
1168
+ File "/usr/local/bin/torchrun", line 8, in <module>
1169
+ sys.exit(main())
1170
+ ^^^^^^
1171
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
1172
+ return f(*args, **kwargs)
1173
+ ^^^^^^^^^^^^^^^^^^
1174
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main
1175
+ run(args)
1176
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run
1177
+ elastic_launch(
1178
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__
1179
+ return launch_agent(self._config, self._entrypoint, list(args))
1180
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1181
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent
1182
+ result = agent.run()
1183
+ ^^^^^^^^^^^
1184
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
1185
+ result = f(*args, **kwargs)
1186
+ ^^^^^^^^^^^^^^^^^^
1187
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run
1188
+ result = self._invoke_run(role)
1189
+ ^^^^^^^^^^^^^^^^^^^^^^
1190
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run
1191
+ time.sleep(monitor_interval)
1192
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler
1193
+ raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
1194
+ torch.distributed.elastic.multiprocessing.api.SignalException: Process 538192 got signal: 15
ultraep/tierb/tierb_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375/rep2/mem_samples.csv ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
ultraep/tierb/tierb_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375/rep2/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=ultraep
2
+ config=QWEN2_57B
3
+ family=tierB
4
+ repo=/workspace/Megatron-UltraEP
5
+ branch=ultraep-bracing-regions-through-kernels
6
+ commit=03d57ef7
7
+ num_layers=10
8
+ hidden=3584
9
+ ffn_hidden=2560
10
+ moe_ffn_hidden=2560
11
+ num_experts=64
12
+ topk=8
13
+ experts_per_rank=8
14
+ expert_params=27525120
15
+ heads=28
16
+ kv_groups=4
17
+ kv_channels=
18
+ qk_layernorm=0
19
+ shared_ffn=20480
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=10
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=1
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-2
29
+ router_pre_softmax=1
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=4096
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=32768
39
+ mean_tokens_per_expert=4096
40
+ slot_policy=matched
41
+ slots_per_rank=8
42
+ capacity_mode=static
43
+ capacity=1.08984375
44
+ cuda_graph=1
45
+ probe_mode=0
46
+ async_grad_reduce=0
47
+ branch_role=timing
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=512
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
ultraep/tierb/tierb_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375/rep2/train.log ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ==============================================================
2
+ QWEN2_57B / ultraep (slots/rank=8 policy=matched)
3
+ E=64 K=8 H'=2560 m=8
4
+ L=10 seq=4096 gbs=8 (1 microbatches)
5
+ cap_mode=static cap=1.08984375 cg=1 recompute=none
6
+ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierb/tierb_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375/rep2/memory_snapshots/snapshot.pickle
7
+ repo=/workspace/Megatron-UltraEP @ 03d57ef7
8
+ ==============================================================
9
+ W0826 18:39:49.837000 541113 torch/distributed/run.py:766]
10
+ W0826 18:39:49.837000 541113 torch/distributed/run.py:766] *****************************************
11
+ W0826 18:39:49.837000 541113 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
12
+ W0826 18:39:49.837000 541113 torch/distributed/run.py:766] *****************************************
ultraep/tierb/tierb_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p083984375/rep2/mem_samples.csv CHANGED
@@ -174,3 +174,59 @@
174
  5, 121231
175
  6, 121231
176
  7, 121231
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
174
  5, 121231
175
  6, 121231
176
  7, 121231
177
+ 0, 93203
178
+ 1, 93173
179
+ 2, 94093
180
+ 3, 94115
181
+ 4, 94319
182
+ 5, 106529
183
+ 6, 96785
184
+ 7, 96701
185
+ 0, 121957
186
+ 1, 121963
187
+ 2, 121963
188
+ 3, 121963
189
+ 4, 122229
190
+ 5, 121963
191
+ 6, 121963
192
+ 7, 121963
193
+ 0, 121957
194
+ 1, 121963
195
+ 2, 121963
196
+ 3, 121963
197
+ 4, 122229
198
+ 5, 121963
199
+ 6, 121963
200
+ 7, 121963
201
+ 0, 116655
202
+ 1, 116341
203
+ 2, 116405
204
+ 3, 116987
205
+ 4, 116865
206
+ 5, 116725
207
+ 6, 116987
208
+ 7, 116987
209
+ 0, 115233
210
+ 1, 114919
211
+ 2, 114983
212
+ 3, 115565
213
+ 4, 115243
214
+ 5, 114599
215
+ 6, 114983
216
+ 7, 115565
217
+ 0, 4
218
+ 1, 4
219
+ 2, 4
220
+ 3, 113885
221
+ 4, 4
222
+ 5, 4
223
+ 6, 4
224
+ 7, 113885
225
+ 0, 0
226
+ 1, 0
227
+ 2, 0
228
+ 3, 0
229
+ 4, 0
230
+ 5, 0
231
+ 6, 0
232
+ 7, 0
ultraep/tierb/tierb_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p083984375/rep2/train.log CHANGED
@@ -1396,3 +1396,53 @@ Theoretical memory footprints: weight and optimizer=216428.96 MB
1396
  [2026-08-26 18:29:45] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 855.4 | throughput per GPU (TFLOP/s/GPU): 211.4 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117607E+01 | loss scale: 1.0 | grad norm: 44.342 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67712 | min safe capacity: 1.0332 |
1397
  [2026-08-26 18:29:45] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 309.0 | throughput per GPU (TFLOP/s/GPU): 585.2 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.686025E+00 | loss scale: 1.0 | grad norm: 62.884 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67968 | min safe capacity: 1.0371 |
1398
  Capture CUDA graph for training!!!
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1396
  [2026-08-26 18:29:45] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 855.4 | throughput per GPU (TFLOP/s/GPU): 211.4 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117607E+01 | loss scale: 1.0 | grad norm: 44.342 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67712 | min safe capacity: 1.0332 |
1397
  [2026-08-26 18:29:45] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 309.0 | throughput per GPU (TFLOP/s/GPU): 585.2 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.686025E+00 | loss scale: 1.0 | grad norm: 62.884 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67968 | min safe capacity: 1.0371 |
1398
  Capture CUDA graph for training!!!
1399
+ CUDA graph capture done!!!
1400
+ [2026-08-26 18:29:48] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 2799.6 | throughput per GPU (TFLOP/s/GPU): 64.6 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338096E+01 | loss scale: 1.0 | grad norm: 20.302 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67968 | min safe capacity: 1.0371 |
1401
+ [2026-08-26 18:29:48] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 299.9 | throughput per GPU (TFLOP/s/GPU): 603.0 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.049768E+01 | loss scale: 1.0 | grad norm: 95.165 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67968 | min safe capacity: 1.0371 |
1402
+ [2026-08-26 18:29:49] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 299.0 | throughput per GPU (TFLOP/s/GPU): 604.9 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.044046E+01 | loss scale: 1.0 | grad norm: 51.769 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67968 | min safe capacity: 1.0371 |
1403
+ [2026-08-26 18:29:49] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 300.2 | throughput per GPU (TFLOP/s/GPU): 602.4 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.027005E+01 | loss scale: 1.0 | grad norm: 6.321 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67968 | min safe capacity: 1.0371 |
1404
+ [2026-08-26 18:29:49] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 297.8 | throughput per GPU (TFLOP/s/GPU): 607.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.808748E+00 | loss scale: 1.0 | grad norm: 5.544 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67968 | min safe capacity: 1.0371 |
1405
+ [2026-08-26 18:29:50] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 296.9 | throughput per GPU (TFLOP/s/GPU): 609.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.545315E+00 | loss scale: 1.0 | grad norm: 5.293 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67968 | min safe capacity: 1.0371 |
1406
+ [2026-08-26 18:29:50] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 298.0 | throughput per GPU (TFLOP/s/GPU): 606.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.446304E+00 | loss scale: 1.0 | grad norm: 5.022 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67968 | min safe capacity: 1.0371 |
1407
+ [2026-08-26 18:29:50] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 298.5 | throughput per GPU (TFLOP/s/GPU): 605.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.324305E+00 | loss scale: 1.0 | grad norm: 4.703 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67968 | min safe capacity: 1.0371 |
1408
+ [2026-08-26 18:29:51] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 296.2 | throughput per GPU (TFLOP/s/GPU): 610.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.210243E+00 | loss scale: 1.0 | grad norm: 4.466 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67968 | min safe capacity: 1.0371 |
1409
+ [after training is done] datetime: 2026-08-26 18:29:51
1410
+ [rank0]:[W826 18:29:52.290646458 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1411
+ [rank6]:[W826 18:29:53.486765266 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1412
+ [rank4]:[W826 18:29:53.488339453 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1413
+ [rank2]:[W826 18:29:53.489308748 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1414
+ [rank3]:[W826 18:29:53.490253156 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1415
+ [rank5]:[W826 18:29:53.020287247 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1416
+ [rank1]:[W826 18:29:54.066830618 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1417
+ [rank7]:[W826 18:29:54.353518346 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1418
+ W0826 18:29:59.556000 513115 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers
1419
+ W0826 18:29:59.557000 513115 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 513207 closing signal SIGTERM
1420
+ W0826 18:29:59.557000 513115 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 513211 closing signal SIGTERM
1421
+ Traceback (most recent call last):
1422
+ File "/usr/local/bin/torchrun", line 8, in <module>
1423
+ sys.exit(main())
1424
+ ^^^^^^
1425
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
1426
+ return f(*args, **kwargs)
1427
+ ^^^^^^^^^^^^^^^^^^
1428
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main
1429
+ run(args)
1430
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run
1431
+ elastic_launch(
1432
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__
1433
+ return launch_agent(self._config, self._entrypoint, list(args))
1434
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1435
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent
1436
+ result = agent.run()
1437
+ ^^^^^^^^^^^
1438
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
1439
+ result = f(*args, **kwargs)
1440
+ ^^^^^^^^^^^^^^^^^^
1441
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run
1442
+ result = self._invoke_run(role)
1443
+ ^^^^^^^^^^^^^^^^^^^^^^
1444
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run
1445
+ time.sleep(monitor_interval)
1446
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler
1447
+ raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
1448
+ torch.distributed.elastic.multiprocessing.api.SignalException: Process 513115 got signal: 15
ultraep/tierb/tierb_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p083984375/rep3/mem_samples.csv ADDED
@@ -0,0 +1,232 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 2
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 821
42
+ 1, 819
43
+ 2, 821
44
+ 3, 821
45
+ 4, 821
46
+ 5, 821
47
+ 6, 821
48
+ 7, 821
49
+ 0, 1537
50
+ 1, 1537
51
+ 2, 1537
52
+ 3, 1537
53
+ 4, 1537
54
+ 5, 1537
55
+ 6, 1537
56
+ 7, 1537
57
+ 0, 2551
58
+ 1, 2551
59
+ 2, 2551
60
+ 3, 2551
61
+ 4, 2551
62
+ 5, 2551
63
+ 6, 2551
64
+ 7, 2551
65
+ 0, 2959
66
+ 1, 2959
67
+ 2, 2959
68
+ 3, 2959
69
+ 4, 2959
70
+ 5, 2959
71
+ 6, 2959
72
+ 7, 2959
73
+ 0, 51537
74
+ 1, 51537
75
+ 2, 51537
76
+ 3, 51537
77
+ 4, 51249
78
+ 5, 51537
79
+ 6, 51537
80
+ 7, 51537
81
+ 0, 53019
82
+ 1, 53017
83
+ 2, 53017
84
+ 3, 53017
85
+ 4, 52729
86
+ 5, 53017
87
+ 6, 53017
88
+ 7, 53017
89
+ 0, 54651
90
+ 1, 54651
91
+ 2, 54651
92
+ 3, 54651
93
+ 4, 54363
94
+ 5, 54651
95
+ 6, 54651
96
+ 7, 54651
97
+ 0, 55207
98
+ 1, 55207
99
+ 2, 55207
100
+ 3, 55207
101
+ 4, 54919
102
+ 5, 55207
103
+ 6, 55207
104
+ 7, 55207
105
+ 0, 55207
106
+ 1, 55207
107
+ 2, 55207
108
+ 3, 55207
109
+ 4, 54919
110
+ 5, 55207
111
+ 6, 55207
112
+ 7, 55207
113
+ 0, 56875
114
+ 1, 56875
115
+ 2, 56875
116
+ 3, 56875
117
+ 4, 56587
118
+ 5, 56875
119
+ 6, 56875
120
+ 7, 56875
121
+ 0, 88753
122
+ 1, 88809
123
+ 2, 88809
124
+ 3, 88809
125
+ 4, 88793
126
+ 5, 88809
127
+ 6, 88809
128
+ 7, 88809
129
+ 0, 89465
130
+ 1, 89521
131
+ 2, 89521
132
+ 3, 89521
133
+ 4, 89505
134
+ 5, 89521
135
+ 6, 89521
136
+ 7, 89521
137
+ 0, 89465
138
+ 1, 89521
139
+ 2, 89521
140
+ 3, 89521
141
+ 4, 89505
142
+ 5, 89521
143
+ 6, 89521
144
+ 7, 89521
145
+ 0, 89465
146
+ 1, 89521
147
+ 2, 89521
148
+ 3, 89521
149
+ 4, 89505
150
+ 5, 89521
151
+ 6, 89521
152
+ 7, 89521
153
+ 0, 89771
154
+ 1, 89921
155
+ 2, 89861
156
+ 3, 89883
157
+ 4, 89877
158
+ 5, 89815
159
+ 6, 89873
160
+ 7, 89859
161
+ 0, 94923
162
+ 1, 94971
163
+ 2, 95021
164
+ 3, 94997
165
+ 4, 95137
166
+ 5, 94989
167
+ 6, 94969
168
+ 7, 95005
169
+ 0, 121033
170
+ 1, 111455
171
+ 2, 109251
172
+ 3, 111165
173
+ 4, 107317
174
+ 5, 111767
175
+ 6, 108277
176
+ 7, 121231
177
+ 0, 88409
178
+ 1, 93185
179
+ 2, 91351
180
+ 3, 90235
181
+ 4, 94155
182
+ 5, 90137
183
+ 6, 93185
184
+ 7, 90371
185
+ 0, 121957
186
+ 1, 121963
187
+ 2, 121963
188
+ 3, 121963
189
+ 4, 122229
190
+ 5, 121963
191
+ 6, 121963
192
+ 7, 121963
193
+ 0, 121957
194
+ 1, 121963
195
+ 2, 121963
196
+ 3, 121963
197
+ 4, 122229
198
+ 5, 121963
199
+ 6, 121963
200
+ 7, 121963
201
+ 0, 116719
202
+ 1, 116987
203
+ 2, 116987
204
+ 3, 116661
205
+ 4, 116671
206
+ 5, 116987
207
+ 6, 116987
208
+ 7, 116987
209
+ 0, 114651
210
+ 1, 114983
211
+ 2, 115565
212
+ 3, 115303
213
+ 4, 115831
214
+ 5, 115303
215
+ 6, 115565
216
+ 7, 115437
217
+ 0, 4
218
+ 1, 113885
219
+ 2, 113885
220
+ 3, 4
221
+ 4, 114151
222
+ 5, 113883
223
+ 6, 114273
224
+ 7, 113883
225
+ 0, 1
226
+ 1, 1
227
+ 2, 1
228
+ 3, 1
229
+ 4, 1
230
+ 5, 3
231
+ 6, 114185
232
+ 7, 4
ultraep/tierb/tierb_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p083984375/rep3/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=ultraep
2
+ config=QWEN3_235B
3
+ family=tierB
4
+ repo=/workspace/Megatron-UltraEP
5
+ branch=ultraep-bracing-regions-through-kernels
6
+ commit=03d57ef7
7
+ num_layers=12
8
+ hidden=4096
9
+ ffn_hidden=12288
10
+ moe_ffn_hidden=1536
11
+ num_experts=128
12
+ topk=8
13
+ experts_per_rank=16
14
+ expert_params=18874368
15
+ heads=64
16
+ kv_groups=4
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=0
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=12
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=0
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=0
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=8192
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=65536
39
+ mean_tokens_per_expert=4096
40
+ slot_policy=matched
41
+ slots_per_rank=16
42
+ capacity_mode=static
43
+ capacity=1.083984375
44
+ cuda_graph=1
45
+ probe_mode=0
46
+ async_grad_reduce=0
47
+ branch_role=timing
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=512
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
ultraep/tierb/tierb_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p083984375/rep3/train.log ADDED
The diff for this file is too large to render. See raw diff
 
ultraep/tierb/tierb_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125/rep1/mem_samples.csv ADDED
@@ -0,0 +1,240 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 1
41
+ 0, 4
42
+ 1, 4
43
+ 2, 4
44
+ 3, 4
45
+ 4, 4
46
+ 5, 4
47
+ 6, 4
48
+ 7, 4
49
+ 0, 27
50
+ 1, 633
51
+ 2, 633
52
+ 3, 633
53
+ 4, 633
54
+ 5, 633
55
+ 6, 633
56
+ 7, 725
57
+ 0, 905
58
+ 1, 917
59
+ 2, 931
60
+ 3, 937
61
+ 4, 913
62
+ 5, 921
63
+ 6, 913
64
+ 7, 917
65
+ 0, 1831
66
+ 1, 1831
67
+ 2, 1831
68
+ 3, 1831
69
+ 4, 1831
70
+ 5, 1831
71
+ 6, 1831
72
+ 7, 1831
73
+ 0, 2147
74
+ 1, 2159
75
+ 2, 2137
76
+ 3, 2151
77
+ 4, 2133
78
+ 5, 2131
79
+ 6, 2129
80
+ 7, 2129
81
+ 0, 11671
82
+ 1, 11671
83
+ 2, 11611
84
+ 3, 11811
85
+ 4, 11811
86
+ 5, 11811
87
+ 6, 12011
88
+ 7, 11611
89
+ 0, 46961
90
+ 1, 46953
91
+ 2, 47007
92
+ 3, 46939
93
+ 4, 47003
94
+ 5, 47011
95
+ 6, 46973
96
+ 7, 46991
97
+ 0, 48353
98
+ 1, 48395
99
+ 2, 48345
100
+ 3, 48357
101
+ 4, 48381
102
+ 5, 48395
103
+ 6, 48405
104
+ 7, 48375
105
+ 0, 48647
106
+ 1, 48649
107
+ 2, 48659
108
+ 3, 48621
109
+ 4, 48645
110
+ 5, 48659
111
+ 6, 48649
112
+ 7, 48659
113
+ 0, 48791
114
+ 1, 48793
115
+ 2, 48803
116
+ 3, 48765
117
+ 4, 48789
118
+ 5, 48803
119
+ 6, 48793
120
+ 7, 48803
121
+ 0, 49447
122
+ 1, 49447
123
+ 2, 49451
124
+ 3, 49421
125
+ 4, 49419
126
+ 5, 49451
127
+ 6, 49405
128
+ 7, 49451
129
+ 0, 56991
130
+ 1, 57093
131
+ 2, 57049
132
+ 3, 57129
133
+ 4, 57045
134
+ 5, 57133
135
+ 6, 57051
136
+ 7, 57133
137
+ 0, 85219
138
+ 1, 85241
139
+ 2, 85217
140
+ 3, 85233
141
+ 4, 85233
142
+ 5, 85217
143
+ 6, 85217
144
+ 7, 85217
145
+ 0, 85607
146
+ 1, 85629
147
+ 2, 85605
148
+ 3, 85621
149
+ 4, 85621
150
+ 5, 85605
151
+ 6, 85605
152
+ 7, 85605
153
+ 0, 85607
154
+ 1, 85629
155
+ 2, 85605
156
+ 3, 85621
157
+ 4, 85621
158
+ 5, 85605
159
+ 6, 85605
160
+ 7, 85605
161
+ 0, 85845
162
+ 1, 85849
163
+ 2, 85821
164
+ 3, 85933
165
+ 4, 85895
166
+ 5, 85881
167
+ 6, 85865
168
+ 7, 85881
169
+ 0, 86657
170
+ 1, 86677
171
+ 2, 86655
172
+ 3, 86671
173
+ 4, 86669
174
+ 5, 86655
175
+ 6, 86655
176
+ 7, 86655
177
+ 0, 87423
178
+ 1, 87371
179
+ 2, 87409
180
+ 3, 87395
181
+ 4, 87407
182
+ 5, 87373
183
+ 6, 87357
184
+ 7, 87343
185
+ 0, 116721
186
+ 1, 116759
187
+ 2, 116699
188
+ 3, 116727
189
+ 4, 116771
190
+ 5, 116699
191
+ 6, 116743
192
+ 7, 116699
193
+ 0, 116981
194
+ 1, 116881
195
+ 2, 117931
196
+ 3, 117141
197
+ 4, 118163
198
+ 5, 120025
199
+ 6, 117225
200
+ 7, 120025
201
+ 0, 120103
202
+ 1, 120129
203
+ 2, 120099
204
+ 3, 119829
205
+ 4, 120121
206
+ 5, 120099
207
+ 6, 120101
208
+ 7, 120099
209
+ 0, 120103
210
+ 1, 120129
211
+ 2, 120099
212
+ 3, 119829
213
+ 4, 120121
214
+ 5, 120099
215
+ 6, 120101
216
+ 7, 120099
217
+ 0, 117099
218
+ 1, 117125
219
+ 2, 117095
220
+ 3, 116825
221
+ 4, 117117
222
+ 5, 117095
223
+ 6, 117097
224
+ 7, 117095
225
+ 0, 772
226
+ 1, 115703
227
+ 2, 772
228
+ 3, 115083
229
+ 4, 115695
230
+ 5, 116061
231
+ 6, 115675
232
+ 7, 115673
233
+ 0, 641
234
+ 1, 1
235
+ 2, 1
236
+ 3, 1
237
+ 4, 1
238
+ 5, 4
239
+ 6, 132
240
+ 7, 115623
ultraep/tierb/tierb_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=ultraep
2
+ config=QWEN3_30B
3
+ family=tierB
4
+ repo=/workspace/Megatron-UltraEP
5
+ branch=ultraep-bracing-regions-through-kernels
6
+ commit=03d57ef7
7
+ num_layers=48
8
+ hidden=2048
9
+ ffn_hidden=6144
10
+ moe_ffn_hidden=768
11
+ num_experts=128
12
+ topk=8
13
+ experts_per_rank=16
14
+ expert_params=4718592
15
+ heads=32
16
+ kv_groups=4
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=0
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=48
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=0
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=0
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=4096
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=32768
39
+ mean_tokens_per_expert=2048
40
+ slot_policy=matched
41
+ slots_per_rank=16
42
+ capacity_mode=static
43
+ capacity=1.11328125
44
+ cuda_graph=1
45
+ probe_mode=0
46
+ async_grad_reduce=0
47
+ branch_role=timing
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=256
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
ultraep/tierb/tierb_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
ultraep/tierb/tierb_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125/rep2/mem_samples.csv ADDED
@@ -0,0 +1,232 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 4
43
+ 2, 4
44
+ 3, 4
45
+ 4, 35
46
+ 5, 4
47
+ 6, 4
48
+ 7, 4
49
+ 0, 821
50
+ 1, 821
51
+ 2, 821
52
+ 3, 821
53
+ 4, 821
54
+ 5, 821
55
+ 6, 821
56
+ 7, 821
57
+ 0, 1831
58
+ 1, 1831
59
+ 2, 1831
60
+ 3, 1831
61
+ 4, 1831
62
+ 5, 1831
63
+ 6, 1831
64
+ 7, 1831
65
+ 0, 2171
66
+ 1, 2195
67
+ 2, 2191
68
+ 3, 2183
69
+ 4, 2183
70
+ 5, 2205
71
+ 6, 2195
72
+ 7, 2201
73
+ 0, 10591
74
+ 1, 10631
75
+ 2, 10431
76
+ 3, 10451
77
+ 4, 10571
78
+ 5, 10631
79
+ 6, 10251
80
+ 7, 10631
81
+ 0, 47051
82
+ 1, 47059
83
+ 2, 47015
84
+ 3, 47027
85
+ 4, 47003
86
+ 5, 47077
87
+ 6, 47039
88
+ 7, 47055
89
+ 0, 48477
90
+ 1, 48479
91
+ 2, 48489
92
+ 3, 48451
93
+ 4, 48475
94
+ 5, 48489
95
+ 6, 48479
96
+ 7, 48489
97
+ 0, 48647
98
+ 1, 48649
99
+ 2, 48659
100
+ 3, 48621
101
+ 4, 48645
102
+ 5, 48659
103
+ 6, 48649
104
+ 7, 48659
105
+ 0, 48791
106
+ 1, 48793
107
+ 2, 48803
108
+ 3, 48765
109
+ 4, 48789
110
+ 5, 48803
111
+ 6, 48793
112
+ 7, 48803
113
+ 0, 49447
114
+ 1, 49447
115
+ 2, 49451
116
+ 3, 49421
117
+ 4, 49419
118
+ 5, 49451
119
+ 6, 49405
120
+ 7, 49451
121
+ 0, 59355
122
+ 1, 59503
123
+ 2, 59501
124
+ 3, 59389
125
+ 4, 59387
126
+ 5, 59559
127
+ 6, 59539
128
+ 7, 59529
129
+ 0, 85147
130
+ 1, 85155
131
+ 2, 85155
132
+ 3, 85175
133
+ 4, 85145
134
+ 5, 85125
135
+ 6, 85149
136
+ 7, 85123
137
+ 0, 85607
138
+ 1, 85629
139
+ 2, 85605
140
+ 3, 85621
141
+ 4, 85621
142
+ 5, 85605
143
+ 6, 85605
144
+ 7, 85605
145
+ 0, 85607
146
+ 1, 85629
147
+ 2, 85605
148
+ 3, 85621
149
+ 4, 85621
150
+ 5, 85605
151
+ 6, 85605
152
+ 7, 85605
153
+ 0, 85869
154
+ 1, 85925
155
+ 2, 85843
156
+ 3, 85871
157
+ 4, 85901
158
+ 5, 85861
159
+ 6, 85871
160
+ 7, 85867
161
+ 0, 86919
162
+ 1, 86911
163
+ 2, 86927
164
+ 3, 86911
165
+ 4, 86865
166
+ 5, 86861
167
+ 6, 86871
168
+ 7, 86871
169
+ 0, 107195
170
+ 1, 107519
171
+ 2, 107885
172
+ 3, 107893
173
+ 4, 106799
174
+ 5, 107437
175
+ 6, 107361
176
+ 7, 107721
177
+ 0, 106561
178
+ 1, 106635
179
+ 2, 106819
180
+ 3, 106783
181
+ 4, 105027
182
+ 5, 105727
183
+ 6, 102315
184
+ 7, 105615
185
+ 0, 120103
186
+ 1, 120129
187
+ 2, 120099
188
+ 3, 119829
189
+ 4, 120121
190
+ 5, 120099
191
+ 6, 120101
192
+ 7, 120099
193
+ 0, 120103
194
+ 1, 120129
195
+ 2, 120099
196
+ 3, 119829
197
+ 4, 120121
198
+ 5, 120099
199
+ 6, 120101
200
+ 7, 120099
201
+ 0, 120103
202
+ 1, 120129
203
+ 2, 120099
204
+ 3, 119829
205
+ 4, 120121
206
+ 5, 120099
207
+ 6, 120101
208
+ 7, 120099
209
+ 0, 117099
210
+ 1, 116799
211
+ 2, 116423
212
+ 3, 116179
213
+ 4, 116535
214
+ 5, 117095
215
+ 6, 116835
216
+ 7, 117095
217
+ 0, 115239
218
+ 1, 114481
219
+ 2, 114445
220
+ 3, 114757
221
+ 4, 115003
222
+ 5, 115135
223
+ 6, 115093
224
+ 7, 115305
225
+ 0, 1
226
+ 1, 4
227
+ 2, 4
228
+ 3, 4
229
+ 4, 68
230
+ 5, 115027
231
+ 6, 324
232
+ 7, 4
ultraep/tierb/tierb_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125/rep2/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=ultraep
2
+ config=QWEN3_30B
3
+ family=tierB
4
+ repo=/workspace/Megatron-UltraEP
5
+ branch=ultraep-bracing-regions-through-kernels
6
+ commit=03d57ef7
7
+ num_layers=48
8
+ hidden=2048
9
+ ffn_hidden=6144
10
+ moe_ffn_hidden=768
11
+ num_experts=128
12
+ topk=8
13
+ experts_per_rank=16
14
+ expert_params=4718592
15
+ heads=32
16
+ kv_groups=4
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=0
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=48
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=0
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=0
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=4096
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=32768
39
+ mean_tokens_per_expert=2048
40
+ slot_policy=matched
41
+ slots_per_rank=16
42
+ capacity_mode=static
43
+ capacity=1.11328125
44
+ cuda_graph=1
45
+ probe_mode=0
46
+ async_grad_reduce=0
47
+ branch_role=timing
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=256
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
ultraep/tierb/tierb_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125/rep2/train.log ADDED
The diff for this file is too large to render. See raw diff
 
ultraep/tierb/tierb_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125/rep3/mem_samples.csv ADDED
@@ -0,0 +1,232 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 1
40
+ 7, 1
41
+ 0, 4
42
+ 1, 4
43
+ 2, 4
44
+ 3, 4
45
+ 4, 4
46
+ 5, 4
47
+ 6, 4
48
+ 7, 4
49
+ 0, 821
50
+ 1, 821
51
+ 2, 821
52
+ 3, 821
53
+ 4, 821
54
+ 5, 821
55
+ 6, 821
56
+ 7, 821
57
+ 0, 1831
58
+ 1, 1831
59
+ 2, 1831
60
+ 3, 1831
61
+ 4, 1831
62
+ 5, 1831
63
+ 6, 1831
64
+ 7, 1831
65
+ 0, 2151
66
+ 1, 2161
67
+ 2, 2163
68
+ 3, 2155
69
+ 4, 2153
70
+ 5, 2145
71
+ 6, 2157
72
+ 7, 2141
73
+ 0, 9851
74
+ 1, 10251
75
+ 2, 10051
76
+ 3, 10051
77
+ 4, 10291
78
+ 5, 10131
79
+ 6, 9851
80
+ 7, 10251
81
+ 0, 46999
82
+ 1, 47003
83
+ 2, 47021
84
+ 3, 46939
85
+ 4, 47019
86
+ 5, 46983
87
+ 6, 47007
88
+ 7, 46989
89
+ 0, 48123
90
+ 1, 48145
91
+ 2, 48205
92
+ 3, 48147
93
+ 4, 48211
94
+ 5, 48175
95
+ 6, 48225
96
+ 7, 48205
97
+ 0, 48647
98
+ 1, 48649
99
+ 2, 48659
100
+ 3, 48621
101
+ 4, 48645
102
+ 5, 48659
103
+ 6, 48649
104
+ 7, 48659
105
+ 0, 48791
106
+ 1, 48793
107
+ 2, 48803
108
+ 3, 48765
109
+ 4, 48789
110
+ 5, 48803
111
+ 6, 48793
112
+ 7, 48803
113
+ 0, 49447
114
+ 1, 49447
115
+ 2, 49451
116
+ 3, 49421
117
+ 4, 49419
118
+ 5, 49451
119
+ 6, 49405
120
+ 7, 49451
121
+ 0, 56771
122
+ 1, 56791
123
+ 2, 56769
124
+ 3, 56785
125
+ 4, 56781
126
+ 5, 56767
127
+ 6, 56769
128
+ 7, 56767
129
+ 0, 85175
130
+ 1, 85227
131
+ 2, 85177
132
+ 3, 85201
133
+ 4, 85215
134
+ 5, 85163
135
+ 6, 85171
136
+ 7, 85167
137
+ 0, 85607
138
+ 1, 85629
139
+ 2, 85605
140
+ 3, 85621
141
+ 4, 85621
142
+ 5, 85605
143
+ 6, 85605
144
+ 7, 85605
145
+ 0, 85607
146
+ 1, 85629
147
+ 2, 85605
148
+ 3, 85621
149
+ 4, 85621
150
+ 5, 85605
151
+ 6, 85605
152
+ 7, 85605
153
+ 0, 85907
154
+ 1, 85897
155
+ 2, 85913
156
+ 3, 85923
157
+ 4, 85933
158
+ 5, 85845
159
+ 6, 85929
160
+ 7, 85863
161
+ 0, 86817
162
+ 1, 86879
163
+ 2, 86857
164
+ 3, 86843
165
+ 4, 86847
166
+ 5, 86807
167
+ 6, 86855
168
+ 7, 86871
169
+ 0, 108415
170
+ 1, 108783
171
+ 2, 108069
172
+ 3, 108077
173
+ 4, 108447
174
+ 5, 109085
175
+ 6, 108093
176
+ 7, 108577
177
+ 0, 105469
178
+ 1, 105543
179
+ 2, 105615
180
+ 3, 104923
181
+ 4, 105683
182
+ 5, 95563
183
+ 6, 104547
184
+ 7, 103735
185
+ 0, 120103
186
+ 1, 120129
187
+ 2, 120099
188
+ 3, 119829
189
+ 4, 120121
190
+ 5, 120099
191
+ 6, 120101
192
+ 7, 120099
193
+ 0, 120103
194
+ 1, 120129
195
+ 2, 120099
196
+ 3, 119829
197
+ 4, 120121
198
+ 5, 120099
199
+ 6, 120101
200
+ 7, 120099
201
+ 0, 120103
202
+ 1, 120129
203
+ 2, 120099
204
+ 3, 119829
205
+ 4, 120121
206
+ 5, 120099
207
+ 6, 120101
208
+ 7, 120099
209
+ 0, 116837
210
+ 1, 117125
211
+ 2, 117095
212
+ 3, 116825
213
+ 4, 117117
214
+ 5, 117095
215
+ 6, 117097
216
+ 7, 117095
217
+ 0, 115613
218
+ 1, 115377
219
+ 2, 115091
220
+ 3, 115403
221
+ 4, 115433
222
+ 5, 115673
223
+ 6, 115413
224
+ 7, 115353
225
+ 0, 641
226
+ 1, 4
227
+ 2, 4
228
+ 3, 4
229
+ 4, 4
230
+ 5, 4
231
+ 6, 132
232
+ 7, 115289
ultraep/tierb/tierb_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125/rep3/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=ultraep
2
+ config=QWEN3_30B
3
+ family=tierB
4
+ repo=/workspace/Megatron-UltraEP
5
+ branch=ultraep-bracing-regions-through-kernels
6
+ commit=03d57ef7
7
+ num_layers=48
8
+ hidden=2048
9
+ ffn_hidden=6144
10
+ moe_ffn_hidden=768
11
+ num_experts=128
12
+ topk=8
13
+ experts_per_rank=16
14
+ expert_params=4718592
15
+ heads=32
16
+ kv_groups=4
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=0
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=48
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=0
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=0
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=4096
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=32768
39
+ mean_tokens_per_expert=2048
40
+ slot_policy=matched
41
+ slots_per_rank=16
42
+ capacity_mode=static
43
+ capacity=1.11328125
44
+ cuda_graph=1
45
+ probe_mode=0
46
+ async_grad_reduce=0
47
+ branch_role=timing
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=256
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
ultraep/tierb/tierb_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125/rep3/train.log ADDED
The diff for this file is too large to render. See raw diff
 
ultraep/tierb/tierb_gbs8/results.tsv CHANGED
@@ -1,2 +1,11 @@
1
  config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
2
  QWEN3_235B ultraep matched 16 8192 8 1.083984375 none 1 OK 297.73 295.80 122229 /workspace/tierb_study_20260826_144644/ultraep/tierb/tierb_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p083984375/rep1
 
 
 
 
 
 
 
 
 
 
1
  config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
2
  QWEN3_235B ultraep matched 16 8192 8 1.083984375 none 1 OK 297.73 295.80 122229 /workspace/tierb_study_20260826_144644/ultraep/tierb/tierb_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p083984375/rep1
3
+ QWEN3_235B ultraep matched 16 8192 8 1.083984375 none 2 OK 297.91 295.80 122229 /workspace/tierb_study_20260826_144644/ultraep/tierb/tierb_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p083984375/rep2
4
+ QWEN3_235B ultraep matched 16 8192 8 1.083984375 none 3 OK 297.96 295.70 122229 /workspace/tierb_study_20260826_144644/ultraep/tierb/tierb_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p083984375/rep3
5
+ QWEN3_30B ultraep matched 16 4096 8 1.11328125 none 1 OK 352.90 351.40 120129 /workspace/tierb_study_20260826_144644/ultraep/tierb/tierb_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125/rep1
6
+ QWEN3_30B ultraep matched 16 4096 8 1.11328125 none 2 OK 353.63 351.40 120129 /workspace/tierb_study_20260826_144644/ultraep/tierb/tierb_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125/rep2
7
+ QWEN3_30B ultraep matched 16 4096 8 1.11328125 none 3 OK 354.55 351.40 120129 /workspace/tierb_study_20260826_144644/ultraep/tierb/tierb_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125/rep3
8
+ DSV2_LITE_L24 ultraep matched 8 8192 8 1.0755208333333333 none 1 OK 579.87 578.60 142231 /workspace/tierb_study_20260826_144644/ultraep/tierb/tierb_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333/rep1
9
+ DSV2_LITE_L24 ultraep matched 8 8192 8 1.0755208333333333 none 2 OK 580.05 578.50 142231 /workspace/tierb_study_20260826_144644/ultraep/tierb/tierb_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333/rep2
10
+ DSV2_LITE_L24 ultraep matched 8 8192 8 1.0755208333333333 none 3 OK 580.05 578.50 142231 /workspace/tierb_study_20260826_144644/ultraep/tierb/tierb_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333/rep3
11
+ QWEN2_57B ultraep matched 8 4096 8 1.08984375 none 1 OK 192.64 184.40 98637 /workspace/tierb_study_20260826_144644/ultraep/tierb/tierb_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375/rep1