AbhishekGhosh1998 commited on
Commit
914146a
·
verified ·
1 Parent(s): 1bb110e

granularity results tierb_study_20260826_144644 @ 2026-08-26T16:16:44Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0

Browse files
Files changed (31) hide show
  1. tierb_study_20260826_144644/STATUS.txt +2 -5
  2. tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv +256 -0
  3. tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/run.env +61 -0
  4. tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log +0 -0
  5. tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep2/mem_samples.csv +256 -0
  6. tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep2/run.env +61 -0
  7. tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep2/train.log +0 -0
  8. tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep3/mem_samples.csv +96 -0
  9. tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep3/run.env +61 -0
  10. tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep3/train.log +0 -0
  11. tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep2/mem_samples.csv +176 -0
  12. tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep2/train.log +150 -0
  13. tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep3/mem_samples.csv +248 -0
  14. tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep3/run.env +61 -0
  15. tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep3/train.log +0 -0
  16. tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/mem_samples.csv +256 -0
  17. tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/run.env +61 -0
  18. tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/train.log +0 -0
  19. tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep2/mem_samples.csv +264 -0
  20. tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep2/run.env +61 -0
  21. tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep2/train.log +0 -0
  22. tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep3/mem_samples.csv +256 -0
  23. tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep3/run.env +61 -0
  24. tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep3/train.log +0 -0
  25. tierb_study_20260826_144644/echo/tierb/tierb_gbs8/results.tsv +7 -0
  26. tierb_study_20260826_144644/echo/tierbprobe/provenance.txt +1 -62
  27. tierb_study_20260826_144644/hf_watcher.log +5 -0
  28. tierb_study_20260826_144644/hf_watcher.pid +1 -1
  29. tierb_study_20260826_144644/study.log +33 -0
  30. tierb_study_20260826_144644/tierb_timing_capacities.env.tmp.168529 +2 -0
  31. tierb_study_20260826_144644/tierb_timing_capacities.tsv.tmp.168529 +2 -0
tierb_study_20260826_144644/STATUS.txt CHANGED
@@ -1,12 +1,9 @@
1
  study_root: /workspace/tierb_study_20260826_144644
2
  gbs: 8
3
- updated: 2026-08-26T16:00:42+00:00
4
- current: echo__tierb__gbs8
5
 
6
  completed steps:
7
- echo__tierbprobe__gbs8
8
- moonep__tierbprobe__gbs8
9
- ultraep__tierbprobe__gbs8
10
 
11
  failed steps:
12
  FAIL ultraep tierbprobe 8 2026-08-26T14:57:57+00:00 2026-08-26T15:05:21+00:00 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe
 
1
  study_root: /workspace/tierb_study_20260826_144644
2
  gbs: 8
3
+ updated: 2026-08-26T16:16:43+00:00
4
+ current: echo__tierbprobe__gbs8
5
 
6
  completed steps:
 
 
 
7
 
8
  failed steps:
9
  FAIL ultraep tierbprobe 8 2026-08-26T14:57:57+00:00 2026-08-26T15:05:21+00:00 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe
tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv ADDED
@@ -0,0 +1,256 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 4
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 2
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 4
43
+ 2, 4
44
+ 3, 4
45
+ 4, 4
46
+ 5, 4
47
+ 6, 4
48
+ 7, 4
49
+ 0, 821
50
+ 1, 821
51
+ 2, 821
52
+ 3, 821
53
+ 4, 821
54
+ 5, 821
55
+ 6, 821
56
+ 7, 821
57
+ 0, 1955
58
+ 1, 1955
59
+ 2, 1955
60
+ 3, 1955
61
+ 4, 1955
62
+ 5, 1955
63
+ 6, 1955
64
+ 7, 1955
65
+ 0, 5833
66
+ 1, 5857
67
+ 2, 6015
68
+ 3, 6167
69
+ 4, 5869
70
+ 5, 6015
71
+ 6, 6015
72
+ 7, 5993
73
+ 0, 26681
74
+ 1, 26761
75
+ 2, 26635
76
+ 3, 26625
77
+ 4, 26611
78
+ 5, 26639
79
+ 6, 26777
80
+ 7, 26767
81
+ 0, 36379
82
+ 1, 36457
83
+ 2, 36323
84
+ 3, 36323
85
+ 4, 36301
86
+ 5, 36317
87
+ 6, 36455
88
+ 7, 36477
89
+ 0, 37785
90
+ 1, 37869
91
+ 2, 37719
92
+ 3, 37717
93
+ 4, 37717
94
+ 5, 37715
95
+ 6, 37851
96
+ 7, 37869
97
+ 0, 37873
98
+ 1, 37957
99
+ 2, 37807
100
+ 3, 37805
101
+ 4, 37805
102
+ 5, 37803
103
+ 6, 37939
104
+ 7, 37957
105
+ 0, 37937
106
+ 1, 38037
107
+ 2, 37871
108
+ 3, 37883
109
+ 4, 37877
110
+ 5, 37889
111
+ 6, 38017
112
+ 7, 38051
113
+ 0, 38821
114
+ 1, 38905
115
+ 2, 38755
116
+ 3, 38755
117
+ 4, 38755
118
+ 5, 38751
119
+ 6, 38887
120
+ 7, 38905
121
+ 0, 38821
122
+ 1, 38905
123
+ 2, 38755
124
+ 3, 38755
125
+ 4, 38755
126
+ 5, 38751
127
+ 6, 38887
128
+ 7, 38905
129
+ 0, 38827
130
+ 1, 38911
131
+ 2, 38761
132
+ 3, 38761
133
+ 4, 38761
134
+ 5, 38757
135
+ 6, 38893
136
+ 7, 38911
137
+ 0, 38829
138
+ 1, 38915
139
+ 2, 38763
140
+ 3, 38763
141
+ 4, 38763
142
+ 5, 38761
143
+ 6, 38895
144
+ 7, 38915
145
+ 0, 121367
146
+ 1, 121723
147
+ 2, 121507
148
+ 3, 121477
149
+ 4, 121505
150
+ 5, 121611
151
+ 6, 121651
152
+ 7, 121683
153
+ 0, 121857
154
+ 1, 122153
155
+ 2, 121983
156
+ 3, 121983
157
+ 4, 121983
158
+ 5, 122063
159
+ 6, 122133
160
+ 7, 122153
161
+ 0, 121857
162
+ 1, 122153
163
+ 2, 121983
164
+ 3, 121983
165
+ 4, 121983
166
+ 5, 122063
167
+ 6, 122133
168
+ 7, 122153
169
+ 0, 121857
170
+ 1, 122153
171
+ 2, 121983
172
+ 3, 121983
173
+ 4, 121983
174
+ 5, 122063
175
+ 6, 122133
176
+ 7, 122153
177
+ 0, 121857
178
+ 1, 122153
179
+ 2, 121983
180
+ 3, 121983
181
+ 4, 121983
182
+ 5, 122063
183
+ 6, 122133
184
+ 7, 122153
185
+ 0, 122541
186
+ 1, 122837
187
+ 2, 122667
188
+ 3, 124717
189
+ 4, 122667
190
+ 5, 122747
191
+ 6, 122817
192
+ 7, 122837
193
+ 0, 123719
194
+ 1, 126063
195
+ 2, 125893
196
+ 3, 125893
197
+ 4, 125893
198
+ 5, 125973
199
+ 6, 126043
200
+ 7, 126063
201
+ 0, 124369
202
+ 1, 126725
203
+ 2, 126521
204
+ 3, 126487
205
+ 4, 126511
206
+ 5, 126599
207
+ 6, 126665
208
+ 7, 126673
209
+ 0, 97651
210
+ 1, 99299
211
+ 2, 99913
212
+ 3, 99737
213
+ 4, 104029
214
+ 5, 99213
215
+ 6, 99287
216
+ 7, 99309
217
+ 0, 168117
218
+ 1, 169669
219
+ 2, 170291
220
+ 3, 170291
221
+ 4, 170291
222
+ 5, 169581
223
+ 6, 169657
224
+ 7, 169669
225
+ 0, 168117
226
+ 1, 169669
227
+ 2, 170291
228
+ 3, 170291
229
+ 4, 170291
230
+ 5, 169581
231
+ 6, 169657
232
+ 7, 169669
233
+ 0, 168117
234
+ 1, 169669
235
+ 2, 170291
236
+ 3, 170291
237
+ 4, 170291
238
+ 5, 169581
239
+ 6, 169657
240
+ 7, 169669
241
+ 0, 2058
242
+ 1, 1034
243
+ 2, 169343
244
+ 3, 2314
245
+ 4, 169343
246
+ 5, 168633
247
+ 6, 168709
248
+ 7, 168721
249
+ 0, 0
250
+ 1, 0
251
+ 2, 0
252
+ 3, 0
253
+ 4, 0
254
+ 5, 0
255
+ 6, 0
256
+ 7, 0
tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=echo
2
+ config=DSV2_LITE_L24
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=bracing-regions-through-kernels
6
+ commit=9a236793
7
+ num_layers=24
8
+ hidden=2048
9
+ ffn_hidden=10944
10
+ moe_ffn_hidden=1408
11
+ num_experts=64
12
+ topk=6
13
+ experts_per_rank=8
14
+ expert_params=8650752
15
+ heads=16
16
+ kv_groups=0
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=2816
20
+ shared_expert_overlap=0
21
+ mla=1
22
+ dense_first_layers=1
23
+ moe_layers=23
24
+ rotary_base=10000
25
+ rotary_scaling=40
26
+ qkv_bias=0
27
+ router_balancing=seq_aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=1
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=8192
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=49152
39
+ mean_tokens_per_expert=6144
40
+ slot_policy=matched
41
+ slots_per_rank=8
42
+ capacity_mode=static
43
+ capacity=2.00
44
+ cuda_graph=1
45
+ probe_mode=0
46
+ async_grad_reduce=0
47
+ branch_role=timing
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep2/mem_samples.csv ADDED
@@ -0,0 +1,256 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 4
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 629
43
+ 2, 629
44
+ 3, 629
45
+ 4, 629
46
+ 5, 629
47
+ 6, 629
48
+ 7, 629
49
+ 0, 823
50
+ 1, 829
51
+ 2, 827
52
+ 3, 829
53
+ 4, 833
54
+ 5, 829
55
+ 6, 833
56
+ 7, 837
57
+ 0, 1955
58
+ 1, 1955
59
+ 2, 1955
60
+ 3, 1955
61
+ 4, 1955
62
+ 5, 1955
63
+ 6, 1955
64
+ 7, 1955
65
+ 0, 26289
66
+ 1, 26377
67
+ 2, 26223
68
+ 3, 26223
69
+ 4, 26223
70
+ 5, 26223
71
+ 6, 26355
72
+ 7, 26377
73
+ 0, 36251
74
+ 1, 36335
75
+ 2, 36185
76
+ 3, 36183
77
+ 4, 36183
78
+ 5, 36181
79
+ 6, 36317
80
+ 7, 36335
81
+ 0, 36901
82
+ 1, 36985
83
+ 2, 36835
84
+ 3, 36833
85
+ 4, 36833
86
+ 5, 36831
87
+ 6, 36967
88
+ 7, 36985
89
+ 0, 37785
90
+ 1, 37869
91
+ 2, 37719
92
+ 3, 37717
93
+ 4, 37717
94
+ 5, 37715
95
+ 6, 37851
96
+ 7, 37869
97
+ 0, 37873
98
+ 1, 37957
99
+ 2, 37807
100
+ 3, 37805
101
+ 4, 37805
102
+ 5, 37803
103
+ 6, 37939
104
+ 7, 37957
105
+ 0, 38051
106
+ 1, 38133
107
+ 2, 37993
108
+ 3, 37997
109
+ 4, 37995
110
+ 5, 38007
111
+ 6, 38161
112
+ 7, 38143
113
+ 0, 38821
114
+ 1, 38905
115
+ 2, 38755
116
+ 3, 38755
117
+ 4, 38755
118
+ 5, 38751
119
+ 6, 38887
120
+ 7, 38905
121
+ 0, 38821
122
+ 1, 38905
123
+ 2, 38755
124
+ 3, 38755
125
+ 4, 38755
126
+ 5, 38751
127
+ 6, 38887
128
+ 7, 38905
129
+ 0, 38829
130
+ 1, 38915
131
+ 2, 38763
132
+ 3, 38763
133
+ 4, 38763
134
+ 5, 38761
135
+ 6, 38895
136
+ 7, 38915
137
+ 0, 38829
138
+ 1, 38915
139
+ 2, 38763
140
+ 3, 38763
141
+ 4, 38763
142
+ 5, 38761
143
+ 6, 38895
144
+ 7, 38915
145
+ 0, 121325
146
+ 1, 121647
147
+ 2, 121475
148
+ 3, 121461
149
+ 4, 121473
150
+ 5, 121573
151
+ 6, 121625
152
+ 7, 121621
153
+ 0, 121857
154
+ 1, 122153
155
+ 2, 121983
156
+ 3, 121983
157
+ 4, 121983
158
+ 5, 122063
159
+ 6, 122133
160
+ 7, 122153
161
+ 0, 121857
162
+ 1, 122153
163
+ 2, 121983
164
+ 3, 121983
165
+ 4, 121983
166
+ 5, 122063
167
+ 6, 122133
168
+ 7, 122153
169
+ 0, 121857
170
+ 1, 122153
171
+ 2, 121983
172
+ 3, 121983
173
+ 4, 121983
174
+ 5, 122063
175
+ 6, 122133
176
+ 7, 122153
177
+ 0, 121857
178
+ 1, 122153
179
+ 2, 121983
180
+ 3, 121983
181
+ 4, 121983
182
+ 5, 122063
183
+ 6, 122133
184
+ 7, 122153
185
+ 0, 122541
186
+ 1, 122837
187
+ 2, 122667
188
+ 3, 122667
189
+ 4, 122667
190
+ 5, 122747
191
+ 6, 122817
192
+ 7, 122837
193
+ 0, 123719
194
+ 1, 126063
195
+ 2, 125893
196
+ 3, 125893
197
+ 4, 125893
198
+ 5, 125973
199
+ 6, 126043
200
+ 7, 126063
201
+ 0, 124107
202
+ 1, 126451
203
+ 2, 126281
204
+ 3, 126281
205
+ 4, 126281
206
+ 5, 126361
207
+ 6, 126431
208
+ 7, 126451
209
+ 0, 139009
210
+ 1, 140705
211
+ 2, 140545
212
+ 3, 140541
213
+ 4, 140541
214
+ 5, 138567
215
+ 6, 140757
216
+ 7, 140705
217
+ 0, 168117
218
+ 1, 169669
219
+ 2, 170291
220
+ 3, 170291
221
+ 4, 170291
222
+ 5, 169581
223
+ 6, 169657
224
+ 7, 169669
225
+ 0, 168117
226
+ 1, 169669
227
+ 2, 170291
228
+ 3, 170291
229
+ 4, 170291
230
+ 5, 169581
231
+ 6, 169657
232
+ 7, 169669
233
+ 0, 168117
234
+ 1, 169669
235
+ 2, 170291
236
+ 3, 170291
237
+ 4, 170291
238
+ 5, 169581
239
+ 6, 169657
240
+ 7, 169669
241
+ 0, 168117
242
+ 1, 169669
243
+ 2, 170291
244
+ 3, 170291
245
+ 4, 170291
246
+ 5, 169581
247
+ 6, 169657
248
+ 7, 169669
249
+ 0, 778
250
+ 1, 778
251
+ 2, 778
252
+ 3, 778
253
+ 4, 778
254
+ 5, 1034
255
+ 6, 168709
256
+ 7, 2058
tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep2/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=echo
2
+ config=DSV2_LITE_L24
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=bracing-regions-through-kernels
6
+ commit=9a236793
7
+ num_layers=24
8
+ hidden=2048
9
+ ffn_hidden=10944
10
+ moe_ffn_hidden=1408
11
+ num_experts=64
12
+ topk=6
13
+ experts_per_rank=8
14
+ expert_params=8650752
15
+ heads=16
16
+ kv_groups=0
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=2816
20
+ shared_expert_overlap=0
21
+ mla=1
22
+ dense_first_layers=1
23
+ moe_layers=23
24
+ rotary_base=10000
25
+ rotary_scaling=40
26
+ qkv_bias=0
27
+ router_balancing=seq_aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=1
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=8192
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=49152
39
+ mean_tokens_per_expert=6144
40
+ slot_policy=matched
41
+ slots_per_rank=8
42
+ capacity_mode=static
43
+ capacity=2.00
44
+ cuda_graph=1
45
+ probe_mode=0
46
+ async_grad_reduce=0
47
+ branch_role=timing
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep2/train.log ADDED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep3/mem_samples.csv ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 2
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 4
43
+ 2, 4
44
+ 3, 4
45
+ 4, 27
46
+ 5, 4
47
+ 6, 4
48
+ 7, 4
49
+ 0, 821
50
+ 1, 821
51
+ 2, 821
52
+ 3, 821
53
+ 4, 821
54
+ 5, 821
55
+ 6, 821
56
+ 7, 821
57
+ 0, 1955
58
+ 1, 1955
59
+ 2, 1955
60
+ 3, 1955
61
+ 4, 1955
62
+ 5, 1955
63
+ 6, 1955
64
+ 7, 1955
65
+ 0, 26289
66
+ 1, 26377
67
+ 2, 26223
68
+ 3, 26223
69
+ 4, 26223
70
+ 5, 26223
71
+ 6, 26355
72
+ 7, 26377
73
+ 0, 32145
74
+ 1, 32231
75
+ 2, 32081
76
+ 3, 32079
77
+ 4, 32079
78
+ 5, 32077
79
+ 6, 32211
80
+ 7, 32231
81
+ 0, 36901
82
+ 1, 36985
83
+ 2, 36835
84
+ 3, 36833
85
+ 4, 36833
86
+ 5, 36831
87
+ 6, 36967
88
+ 7, 36985
89
+ 0, 37785
90
+ 1, 37869
91
+ 2, 37719
92
+ 3, 37717
93
+ 4, 37717
94
+ 5, 37715
95
+ 6, 37851
96
+ 7, 37869
tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep3/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=echo
2
+ config=DSV2_LITE_L24
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=bracing-regions-through-kernels
6
+ commit=9a236793
7
+ num_layers=24
8
+ hidden=2048
9
+ ffn_hidden=10944
10
+ moe_ffn_hidden=1408
11
+ num_experts=64
12
+ topk=6
13
+ experts_per_rank=8
14
+ expert_params=8650752
15
+ heads=16
16
+ kv_groups=0
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=2816
20
+ shared_expert_overlap=0
21
+ mla=1
22
+ dense_first_layers=1
23
+ moe_layers=23
24
+ rotary_base=10000
25
+ rotary_scaling=40
26
+ qkv_bias=0
27
+ router_balancing=seq_aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=1
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=8192
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=49152
39
+ mean_tokens_per_expert=6144
40
+ slot_policy=matched
41
+ slots_per_rank=8
42
+ capacity_mode=static
43
+ capacity=2.00
44
+ cuda_graph=1
45
+ probe_mode=0
46
+ async_grad_reduce=0
47
+ branch_role=timing
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep3/train.log ADDED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep2/mem_samples.csv CHANGED
@@ -78,3 +78,179 @@
78
  5, 48127
79
  6, 48143
80
  7, 48171
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
78
  5, 48127
79
  6, 48143
80
  7, 48171
81
+ 0, 49929
82
+ 1, 49929
83
+ 2, 49929
84
+ 3, 49929
85
+ 4, 49641
86
+ 5, 49929
87
+ 6, 49929
88
+ 7, 49929
89
+ 0, 53699
90
+ 1, 53699
91
+ 2, 53699
92
+ 3, 53699
93
+ 4, 53411
94
+ 5, 53699
95
+ 6, 53699
96
+ 7, 53699
97
+ 0, 54107
98
+ 1, 54107
99
+ 2, 54107
100
+ 3, 54107
101
+ 4, 53819
102
+ 5, 54107
103
+ 6, 54107
104
+ 7, 54107
105
+ 0, 54385
106
+ 1, 54401
107
+ 2, 54393
108
+ 3, 54409
109
+ 4, 54139
110
+ 5, 54421
111
+ 6, 54413
112
+ 7, 54391
113
+ 0, 55545
114
+ 1, 55545
115
+ 2, 55545
116
+ 3, 55545
117
+ 4, 55257
118
+ 5, 55545
119
+ 6, 55545
120
+ 7, 55545
121
+ 0, 56569
122
+ 1, 56569
123
+ 2, 56569
124
+ 3, 56569
125
+ 4, 56281
126
+ 5, 56569
127
+ 6, 56569
128
+ 7, 56569
129
+ 0, 59801
130
+ 1, 59801
131
+ 2, 59801
132
+ 3, 59801
133
+ 4, 59513
134
+ 5, 59801
135
+ 6, 59801
136
+ 7, 59801
137
+ 0, 59801
138
+ 1, 59801
139
+ 2, 59801
140
+ 3, 59801
141
+ 4, 59513
142
+ 5, 59801
143
+ 6, 59801
144
+ 7, 59801
145
+ 0, 105819
146
+ 1, 105839
147
+ 2, 105881
148
+ 3, 105851
149
+ 4, 105831
150
+ 5, 105837
151
+ 6, 105869
152
+ 7, 105839
153
+ 0, 106307
154
+ 1, 106347
155
+ 2, 106347
156
+ 3, 106347
157
+ 4, 106327
158
+ 5, 106347
159
+ 6, 106347
160
+ 7, 106347
161
+ 0, 106307
162
+ 1, 106347
163
+ 2, 106347
164
+ 3, 106347
165
+ 4, 106327
166
+ 5, 106347
167
+ 6, 106347
168
+ 7, 106347
169
+ 0, 106307
170
+ 1, 106347
171
+ 2, 106347
172
+ 3, 106347
173
+ 4, 106327
174
+ 5, 106347
175
+ 6, 106347
176
+ 7, 106347
177
+ 0, 106307
178
+ 1, 106347
179
+ 2, 106347
180
+ 3, 106347
181
+ 4, 106327
182
+ 5, 106347
183
+ 6, 106347
184
+ 7, 106347
185
+ 0, 110745
186
+ 1, 110785
187
+ 2, 110785
188
+ 3, 110785
189
+ 4, 110765
190
+ 5, 110785
191
+ 6, 110401
192
+ 7, 110785
193
+ 0, 111393
194
+ 1, 111433
195
+ 2, 111433
196
+ 3, 111433
197
+ 4, 111413
198
+ 5, 111433
199
+ 6, 111049
200
+ 7, 111433
201
+ 0, 112097
202
+ 1, 112121
203
+ 2, 112107
204
+ 3, 112155
205
+ 4, 112119
206
+ 5, 112117
207
+ 6, 111771
208
+ 7, 112135
209
+ 0, 130679
210
+ 1, 130847
211
+ 2, 120531
212
+ 3, 131887
213
+ 4, 130567
214
+ 5, 128851
215
+ 6, 129427
216
+ 7, 112545
217
+ 0, 155643
218
+ 1, 155809
219
+ 2, 155809
220
+ 3, 155425
221
+ 4, 155807
222
+ 5, 155809
223
+ 6, 155809
224
+ 7, 155809
225
+ 0, 155643
226
+ 1, 155809
227
+ 2, 155809
228
+ 3, 155425
229
+ 4, 155807
230
+ 5, 155809
231
+ 6, 155809
232
+ 7, 155809
233
+ 0, 154397
234
+ 1, 154563
235
+ 2, 154563
236
+ 3, 154179
237
+ 4, 154561
238
+ 5, 154563
239
+ 6, 154563
240
+ 7, 154563
241
+ 0, 3363
242
+ 1, 3366
243
+ 2, 3366
244
+ 3, 3366
245
+ 4, 3366
246
+ 5, 3622
247
+ 6, 154561
248
+ 7, 4646
249
+ 0, 0
250
+ 1, 0
251
+ 2, 0
252
+ 3, 0
253
+ 4, 0
254
+ 5, 0
255
+ 6, 0
256
+ 7, 0
tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep2/train.log CHANGED
@@ -1232,3 +1232,153 @@ done with setup ...
1232
  training ...
1233
  Overwriting rerun_state_machine.current_iteration from -1 to 0...
1234
  [before the start of training step] datetime: 2026-08-26 16:02:44
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1232
  training ...
1233
  Overwriting rerun_state_machine.current_iteration from -1 to 0...
1234
  [before the start of training step] datetime: 2026-08-26 16:02:44
1235
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1236
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1237
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1238
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1239
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1240
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1241
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1242
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1243
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1244
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1245
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1246
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1247
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1248
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1249
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1250
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1251
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1252
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1253
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1254
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1255
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1256
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1257
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1258
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1259
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1260
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1261
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1262
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1263
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1264
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1265
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1266
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1267
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1268
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1269
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1270
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1271
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1272
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1273
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1274
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1275
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1276
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1277
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1278
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1279
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1280
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1281
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1282
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1283
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1284
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1285
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1286
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1287
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1288
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1289
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1290
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1291
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1292
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1293
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1294
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1295
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1296
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1297
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1298
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1299
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1300
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1301
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1302
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1303
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1304
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1305
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1306
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1307
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1308
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1309
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1310
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1311
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1312
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1313
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1314
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1315
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1316
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1317
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1318
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1319
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1320
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1321
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1322
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1323
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1324
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1325
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1326
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1327
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1328
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1329
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1330
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1331
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1332
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1333
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1334
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1335
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1336
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1337
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1338
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1339
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1340
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1341
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1342
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1343
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1344
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1345
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1346
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1347
+ [2026-08-26 16:03:22] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 38132.5 | throughput per GPU (TFLOP/s/GPU): 4.7 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.115992E+01 | loss scale: 1.0 | grad norm: 44.779 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 29.85
1348
+
1349
+ Number of parameters in embedding layers in billions: 0.41
1350
+ Total number of parameters in billions: 30.26
1351
+ Number of parameters in most loaded shard in billions: 30.2590
1352
+ Theoretical memory footprints: weight and optimizer=216428.96 MB
1353
+ [Rank 0] (after 1 iterations) memory (MB) | allocated: 81341.77783203125 | max allocated: 96217.232421875 | reserved: 102124.0 | max reserved: 102124.0
1354
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1355
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1356
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1357
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1358
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1359
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1360
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1361
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1362
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1363
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1364
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1365
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1366
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1367
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1368
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1369
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1370
+ [2026-08-26 16:03:23] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 588.1 | throughput per GPU (TFLOP/s/GPU): 307.5 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117604E+01 | loss scale: 1.0 | grad norm: 44.306 | number of skipped iterations: 0 | number of nan iterations: 0 |
1371
+ [2026-08-26 16:03:23] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 644.4 | throughput per GPU (TFLOP/s/GPU): 280.6 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.694104E+00 | loss scale: 1.0 | grad norm: 66.384 | number of skipped iterations: 0 | number of nan iterations: 0 |
1372
+ Capture CUDA graph for training!!!
1373
+ CUDA graph capture done!!!
1374
+ [2026-08-26 16:03:26] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 2761.9 | throughput per GPU (TFLOP/s/GPU): 65.5 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338530E+01 | loss scale: 1.0 | grad norm: 19.947 | number of skipped iterations: 0 | number of nan iterations: 0 |
1375
+ [2026-08-26 16:03:26] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 324.9 | throughput per GPU (TFLOP/s/GPU): 556.6 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.011544E+01 | loss scale: 1.0 | grad norm: 84.156 | number of skipped iterations: 0 | number of nan iterations: 0 |
1376
+ [2026-08-26 16:03:27] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 323.1 | throughput per GPU (TFLOP/s/GPU): 559.7 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.339844E+01 | loss scale: 1.0 | grad norm: 121.227 | number of skipped iterations: 0 | number of nan iterations: 0 |
1377
+ [2026-08-26 16:03:27] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 329.3 | throughput per GPU (TFLOP/s/GPU): 549.2 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.031030E+01 | loss scale: 1.0 | grad norm: 6.141 | number of skipped iterations: 0 | number of nan iterations: 0 |
1378
+ [2026-08-26 16:03:27] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 323.8 | throughput per GPU (TFLOP/s/GPU): 558.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.867838E+00 | loss scale: 1.0 | grad norm: 5.538 | number of skipped iterations: 0 | number of nan iterations: 0 |
1379
+ [2026-08-26 16:03:28] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 323.1 | throughput per GPU (TFLOP/s/GPU): 559.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.620957E+00 | loss scale: 1.0 | grad norm: 5.434 | number of skipped iterations: 0 | number of nan iterations: 0 |
1380
+ [2026-08-26 16:03:28] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 327.5 | throughput per GPU (TFLOP/s/GPU): 552.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.527014E+00 | loss scale: 1.0 | grad norm: 5.273 | number of skipped iterations: 0 | number of nan iterations: 0 |
1381
+ [2026-08-26 16:03:28] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 322.6 | throughput per GPU (TFLOP/s/GPU): 560.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.403372E+00 | loss scale: 1.0 | grad norm: 4.972 | number of skipped iterations: 0 | number of nan iterations: 0 |
1382
+ [2026-08-26 16:03:29] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 329.4 | throughput per GPU (TFLOP/s/GPU): 549.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.283264E+00 | loss scale: 1.0 | grad norm: 4.664 | number of skipped iterations: 0 | number of nan iterations: 0 |
1383
+ [after training is done] datetime: 2026-08-26 16:03:29
1384
+ [rank0]:[W826 16:03:30.426110892 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep3/mem_samples.csv ADDED
@@ -0,0 +1,248 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 4
43
+ 2, 4
44
+ 3, 4
45
+ 4, 4
46
+ 5, 4
47
+ 6, 4
48
+ 7, 4
49
+ 0, 821
50
+ 1, 821
51
+ 2, 821
52
+ 3, 821
53
+ 4, 821
54
+ 5, 821
55
+ 6, 821
56
+ 7, 821
57
+ 0, 1975
58
+ 1, 1975
59
+ 2, 1975
60
+ 3, 1975
61
+ 4, 1975
62
+ 5, 1975
63
+ 6, 1975
64
+ 7, 1975
65
+ 0, 47799
66
+ 1, 47799
67
+ 2, 47799
68
+ 3, 47799
69
+ 4, 47511
70
+ 5, 47799
71
+ 6, 47799
72
+ 7, 47799
73
+ 0, 49047
74
+ 1, 49057
75
+ 2, 49049
76
+ 3, 49047
77
+ 4, 48769
78
+ 5, 49053
79
+ 6, 49047
80
+ 7, 49057
81
+ 0, 49929
82
+ 1, 49929
83
+ 2, 49929
84
+ 3, 49929
85
+ 4, 49641
86
+ 5, 49929
87
+ 6, 49929
88
+ 7, 49929
89
+ 0, 53699
90
+ 1, 53699
91
+ 2, 53699
92
+ 3, 53699
93
+ 4, 53411
94
+ 5, 53699
95
+ 6, 53699
96
+ 7, 53699
97
+ 0, 54107
98
+ 1, 54107
99
+ 2, 54107
100
+ 3, 54107
101
+ 4, 53819
102
+ 5, 54107
103
+ 6, 54107
104
+ 7, 54107
105
+ 0, 54433
106
+ 1, 54419
107
+ 2, 54425
108
+ 3, 54437
109
+ 4, 54117
110
+ 5, 54429
111
+ 6, 54425
112
+ 7, 54411
113
+ 0, 55545
114
+ 1, 55545
115
+ 2, 55545
116
+ 3, 55545
117
+ 4, 55257
118
+ 5, 55545
119
+ 6, 55545
120
+ 7, 55545
121
+ 0, 56569
122
+ 1, 56569
123
+ 2, 56569
124
+ 3, 56569
125
+ 4, 56281
126
+ 5, 56569
127
+ 6, 56569
128
+ 7, 56569
129
+ 0, 59801
130
+ 1, 59801
131
+ 2, 59801
132
+ 3, 59801
133
+ 4, 59513
134
+ 5, 59801
135
+ 6, 59801
136
+ 7, 59801
137
+ 0, 59801
138
+ 1, 59801
139
+ 2, 59801
140
+ 3, 59801
141
+ 4, 59513
142
+ 5, 59801
143
+ 6, 59801
144
+ 7, 59801
145
+ 0, 105771
146
+ 1, 105791
147
+ 2, 105783
148
+ 3, 105773
149
+ 4, 105757
150
+ 5, 105781
151
+ 6, 105775
152
+ 7, 105785
153
+ 0, 106307
154
+ 1, 106347
155
+ 2, 106347
156
+ 3, 106347
157
+ 4, 106327
158
+ 5, 106347
159
+ 6, 106347
160
+ 7, 106347
161
+ 0, 106307
162
+ 1, 106347
163
+ 2, 106347
164
+ 3, 106347
165
+ 4, 106327
166
+ 5, 106347
167
+ 6, 106347
168
+ 7, 106347
169
+ 0, 106307
170
+ 1, 106347
171
+ 2, 106347
172
+ 3, 106347
173
+ 4, 106327
174
+ 5, 106347
175
+ 6, 106347
176
+ 7, 106347
177
+ 0, 106307
178
+ 1, 106347
179
+ 2, 106347
180
+ 3, 106347
181
+ 4, 106327
182
+ 5, 106347
183
+ 6, 106347
184
+ 7, 106347
185
+ 0, 110843
186
+ 1, 110887
187
+ 2, 110893
188
+ 3, 110931
189
+ 4, 110877
190
+ 5, 110919
191
+ 6, 110897
192
+ 7, 110885
193
+ 0, 111783
194
+ 1, 111823
195
+ 2, 111823
196
+ 3, 111823
197
+ 4, 111803
198
+ 5, 111823
199
+ 6, 111823
200
+ 7, 111823
201
+ 0, 142195
202
+ 1, 142307
203
+ 2, 142307
204
+ 3, 142307
205
+ 4, 142335
206
+ 5, 142307
207
+ 6, 142307
208
+ 7, 142307
209
+ 0, 143677
210
+ 1, 141263
211
+ 2, 155809
212
+ 3, 143799
213
+ 4, 142513
214
+ 5, 143917
215
+ 6, 144203
216
+ 7, 144043
217
+ 0, 155259
218
+ 1, 155809
219
+ 2, 155809
220
+ 3, 155809
221
+ 4, 155807
222
+ 5, 155809
223
+ 6, 155809
224
+ 7, 155809
225
+ 0, 155259
226
+ 1, 155809
227
+ 2, 155809
228
+ 3, 155809
229
+ 4, 155807
230
+ 5, 155809
231
+ 6, 155809
232
+ 7, 155809
233
+ 0, 3366
234
+ 1, 3366
235
+ 2, 3366
236
+ 3, 3366
237
+ 4, 3622
238
+ 5, 154561
239
+ 6, 4646
240
+ 7, 3366
241
+ 0, 0
242
+ 1, 0
243
+ 2, 0
244
+ 3, 0
245
+ 4, 0
246
+ 5, 0
247
+ 6, 0
248
+ 7, 0
tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep3/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=echo
2
+ config=QWEN3_235B
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=bracing-regions-through-kernels
6
+ commit=9a236793
7
+ num_layers=12
8
+ hidden=4096
9
+ ffn_hidden=12288
10
+ moe_ffn_hidden=1536
11
+ num_experts=128
12
+ topk=8
13
+ experts_per_rank=16
14
+ expert_params=18874368
15
+ heads=64
16
+ kv_groups=4
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=0
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=12
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=0
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=0
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=8192
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=65536
39
+ mean_tokens_per_expert=4096
40
+ slot_policy=matched
41
+ slots_per_rank=16
42
+ capacity_mode=static
43
+ capacity=2.00
44
+ cuda_graph=1
45
+ probe_mode=0
46
+ async_grad_reduce=0
47
+ branch_role=timing
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep3/train.log ADDED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/mem_samples.csv ADDED
@@ -0,0 +1,256 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 4
43
+ 2, 4
44
+ 3, 4
45
+ 4, 4
46
+ 5, 4
47
+ 6, 4
48
+ 7, 4
49
+ 0, 821
50
+ 1, 821
51
+ 2, 821
52
+ 3, 821
53
+ 4, 821
54
+ 5, 821
55
+ 6, 821
56
+ 7, 821
57
+ 0, 1831
58
+ 1, 1831
59
+ 2, 1831
60
+ 3, 1831
61
+ 4, 1831
62
+ 5, 1831
63
+ 6, 1831
64
+ 7, 1831
65
+ 0, 7637
66
+ 1, 7657
67
+ 2, 7357
68
+ 3, 7557
69
+ 4, 7697
70
+ 5, 7757
71
+ 6, 7757
72
+ 7, 7757
73
+ 0, 44781
74
+ 1, 44843
75
+ 2, 44869
76
+ 3, 44803
77
+ 4, 44823
78
+ 5, 44845
79
+ 6, 44827
80
+ 7, 44827
81
+ 0, 45509
82
+ 1, 45511
83
+ 2, 45521
84
+ 3, 45483
85
+ 4, 45507
86
+ 5, 45521
87
+ 6, 45511
88
+ 7, 45521
89
+ 0, 46919
90
+ 1, 46921
91
+ 2, 46931
92
+ 3, 46893
93
+ 4, 46917
94
+ 5, 46931
95
+ 6, 46921
96
+ 7, 46931
97
+ 0, 47015
98
+ 1, 47017
99
+ 2, 47027
100
+ 3, 46989
101
+ 4, 47013
102
+ 5, 47027
103
+ 6, 47017
104
+ 7, 47027
105
+ 0, 47073
106
+ 1, 47079
107
+ 2, 47093
108
+ 3, 47061
109
+ 4, 47083
110
+ 5, 47101
111
+ 6, 47085
112
+ 7, 47099
113
+ 0, 47879
114
+ 1, 47881
115
+ 2, 47891
116
+ 3, 47853
117
+ 4, 47877
118
+ 5, 47891
119
+ 6, 47881
120
+ 7, 47891
121
+ 0, 48135
122
+ 1, 48137
123
+ 2, 48147
124
+ 3, 48109
125
+ 4, 48133
126
+ 5, 48147
127
+ 6, 48137
128
+ 7, 48147
129
+ 0, 48779
130
+ 1, 48801
131
+ 2, 48795
132
+ 3, 48773
133
+ 4, 48777
134
+ 5, 48795
135
+ 6, 48765
136
+ 7, 48795
137
+ 0, 49133
138
+ 1, 49153
139
+ 2, 49147
140
+ 3, 49127
141
+ 4, 49129
142
+ 5, 49147
143
+ 6, 49119
144
+ 7, 49147
145
+ 0, 100837
146
+ 1, 100859
147
+ 2, 100843
148
+ 3, 100859
149
+ 4, 100839
150
+ 5, 100845
151
+ 6, 100847
152
+ 7, 100835
153
+ 0, 101347
154
+ 1, 101367
155
+ 2, 101353
156
+ 3, 101361
157
+ 4, 101363
158
+ 5, 101353
159
+ 6, 101353
160
+ 7, 101353
161
+ 0, 101347
162
+ 1, 101367
163
+ 2, 101353
164
+ 3, 101361
165
+ 4, 101363
166
+ 5, 101353
167
+ 6, 101353
168
+ 7, 101353
169
+ 0, 101347
170
+ 1, 101367
171
+ 2, 101353
172
+ 3, 101361
173
+ 4, 101363
174
+ 5, 101353
175
+ 6, 101353
176
+ 7, 101353
177
+ 0, 101347
178
+ 1, 101367
179
+ 2, 101353
180
+ 3, 101361
181
+ 4, 101363
182
+ 5, 101353
183
+ 6, 101353
184
+ 7, 101353
185
+ 0, 102061
186
+ 1, 102849
187
+ 2, 102067
188
+ 3, 102075
189
+ 4, 102077
190
+ 5, 102259
191
+ 6, 102259
192
+ 7, 102259
193
+ 0, 102997
194
+ 1, 103497
195
+ 2, 102715
196
+ 3, 102915
197
+ 4, 102917
198
+ 5, 103099
199
+ 6, 102907
200
+ 7, 103003
201
+ 0, 103687
202
+ 1, 104149
203
+ 2, 103367
204
+ 3, 103593
205
+ 4, 103597
206
+ 5, 103737
207
+ 6, 103587
208
+ 7, 103659
209
+ 0, 133045
210
+ 1, 133533
211
+ 2, 132963
212
+ 3, 133047
213
+ 4, 132993
214
+ 5, 133251
215
+ 6, 132943
216
+ 7, 133155
217
+ 0, 140013
218
+ 1, 140011
219
+ 2, 139051
220
+ 3, 140219
221
+ 4, 140203
222
+ 5, 139617
223
+ 6, 147795
224
+ 7, 140337
225
+ 0, 151641
226
+ 1, 151639
227
+ 2, 151641
228
+ 3, 151651
229
+ 4, 151635
230
+ 5, 151641
231
+ 6, 151643
232
+ 7, 151641
233
+ 0, 151641
234
+ 1, 151639
235
+ 2, 151641
236
+ 3, 151651
237
+ 4, 151635
238
+ 5, 151641
239
+ 6, 151643
240
+ 7, 151641
241
+ 0, 150825
242
+ 1, 150823
243
+ 2, 150825
244
+ 3, 150835
245
+ 4, 150819
246
+ 5, 150825
247
+ 6, 150827
248
+ 7, 150825
249
+ 0, 2186
250
+ 1, 906
251
+ 2, 906
252
+ 3, 906
253
+ 4, 906
254
+ 5, 906
255
+ 6, 1162
256
+ 7, 150825
tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=echo
2
+ config=QWEN3_30B
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=bracing-regions-through-kernels
6
+ commit=9a236793
7
+ num_layers=48
8
+ hidden=2048
9
+ ffn_hidden=6144
10
+ moe_ffn_hidden=768
11
+ num_experts=128
12
+ topk=8
13
+ experts_per_rank=16
14
+ expert_params=4718592
15
+ heads=32
16
+ kv_groups=4
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=0
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=48
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=0
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=0
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=4096
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=32768
39
+ mean_tokens_per_expert=2048
40
+ slot_policy=matched
41
+ slots_per_rank=16
42
+ capacity_mode=static
43
+ capacity=2.00
44
+ cuda_graph=1
45
+ probe_mode=0
46
+ async_grad_reduce=0
47
+ branch_role=timing
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep2/mem_samples.csv ADDED
@@ -0,0 +1,264 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 4
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 629
43
+ 2, 629
44
+ 3, 629
45
+ 4, 629
46
+ 5, 629
47
+ 6, 629
48
+ 7, 629
49
+ 0, 907
50
+ 1, 879
51
+ 2, 875
52
+ 3, 865
53
+ 4, 871
54
+ 5, 879
55
+ 6, 909
56
+ 7, 873
57
+ 0, 1831
58
+ 1, 1831
59
+ 2, 1831
60
+ 3, 1831
61
+ 4, 1831
62
+ 5, 1831
63
+ 6, 1831
64
+ 7, 1831
65
+ 0, 39169
66
+ 1, 38995
67
+ 2, 39197
68
+ 3, 38995
69
+ 4, 38995
70
+ 5, 39197
71
+ 6, 38995
72
+ 7, 39197
73
+ 0, 44991
74
+ 1, 44993
75
+ 2, 45003
76
+ 3, 44965
77
+ 4, 44989
78
+ 5, 45003
79
+ 6, 44993
80
+ 7, 45003
81
+ 0, 45897
82
+ 1, 45899
83
+ 2, 45909
84
+ 3, 45871
85
+ 4, 45895
86
+ 5, 45909
87
+ 6, 45899
88
+ 7, 45909
89
+ 0, 46919
90
+ 1, 46921
91
+ 2, 46931
92
+ 3, 46893
93
+ 4, 46917
94
+ 5, 46931
95
+ 6, 46921
96
+ 7, 46931
97
+ 0, 47015
98
+ 1, 47017
99
+ 2, 47027
100
+ 3, 46989
101
+ 4, 47013
102
+ 5, 47027
103
+ 6, 47017
104
+ 7, 47027
105
+ 0, 47221
106
+ 1, 47247
107
+ 2, 47233
108
+ 3, 47199
109
+ 4, 47221
110
+ 5, 47229
111
+ 6, 47201
112
+ 7, 47219
113
+ 0, 47879
114
+ 1, 47881
115
+ 2, 47891
116
+ 3, 47853
117
+ 4, 47877
118
+ 5, 47891
119
+ 6, 47881
120
+ 7, 47891
121
+ 0, 48135
122
+ 1, 48137
123
+ 2, 48147
124
+ 3, 48109
125
+ 4, 48133
126
+ 5, 48147
127
+ 6, 48137
128
+ 7, 48147
129
+ 0, 49133
130
+ 1, 49153
131
+ 2, 49147
132
+ 3, 49127
133
+ 4, 49129
134
+ 5, 49147
135
+ 6, 49119
136
+ 7, 49147
137
+ 0, 49133
138
+ 1, 49153
139
+ 2, 49147
140
+ 3, 49127
141
+ 4, 49129
142
+ 5, 49147
143
+ 6, 49119
144
+ 7, 49147
145
+ 0, 100817
146
+ 1, 100865
147
+ 2, 100819
148
+ 3, 100825
149
+ 4, 100823
150
+ 5, 100819
151
+ 6, 100833
152
+ 7, 100831
153
+ 0, 101347
154
+ 1, 101367
155
+ 2, 101353
156
+ 3, 101361
157
+ 4, 101363
158
+ 5, 101353
159
+ 6, 101353
160
+ 7, 101353
161
+ 0, 101347
162
+ 1, 101367
163
+ 2, 101353
164
+ 3, 101361
165
+ 4, 101363
166
+ 5, 101353
167
+ 6, 101353
168
+ 7, 101353
169
+ 0, 101347
170
+ 1, 101367
171
+ 2, 101353
172
+ 3, 101361
173
+ 4, 101363
174
+ 5, 101353
175
+ 6, 101353
176
+ 7, 101353
177
+ 0, 101347
178
+ 1, 101367
179
+ 2, 101353
180
+ 3, 101361
181
+ 4, 101363
182
+ 5, 101353
183
+ 6, 101353
184
+ 7, 101353
185
+ 0, 102253
186
+ 1, 102081
187
+ 2, 102067
188
+ 3, 102075
189
+ 4, 102077
190
+ 5, 102067
191
+ 6, 102067
192
+ 7, 102067
193
+ 0, 103477
194
+ 1, 103209
195
+ 2, 102715
196
+ 3, 103299
197
+ 4, 103301
198
+ 5, 103291
199
+ 6, 102907
200
+ 7, 103291
201
+ 0, 104149
202
+ 1, 103919
203
+ 2, 103385
204
+ 3, 103969
205
+ 4, 103969
206
+ 5, 103961
207
+ 6, 103597
208
+ 7, 103951
209
+ 0, 133525
210
+ 1, 133533
211
+ 2, 132771
212
+ 3, 133527
213
+ 4, 133569
214
+ 5, 133539
215
+ 6, 133519
216
+ 7, 133539
217
+ 0, 139125
218
+ 1, 143003
219
+ 2, 132365
220
+ 3, 134511
221
+ 4, 132503
222
+ 5, 133587
223
+ 6, 137973
224
+ 7, 134455
225
+ 0, 151737
226
+ 1, 151735
227
+ 2, 151641
228
+ 3, 151747
229
+ 4, 151731
230
+ 5, 151737
231
+ 6, 151643
232
+ 7, 151641
233
+ 0, 151737
234
+ 1, 151735
235
+ 2, 151641
236
+ 3, 151747
237
+ 4, 151731
238
+ 5, 151737
239
+ 6, 151643
240
+ 7, 151641
241
+ 0, 150921
242
+ 1, 150919
243
+ 2, 150825
244
+ 3, 150931
245
+ 4, 150915
246
+ 5, 150921
247
+ 6, 150827
248
+ 7, 150825
249
+ 0, 906
250
+ 1, 906
251
+ 2, 906
252
+ 3, 906
253
+ 4, 1162
254
+ 5, 150921
255
+ 6, 2186
256
+ 7, 906
257
+ 0, 0
258
+ 1, 0
259
+ 2, 0
260
+ 3, 0
261
+ 4, 0
262
+ 5, 0
263
+ 6, 0
264
+ 7, 0
tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep2/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=echo
2
+ config=QWEN3_30B
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=bracing-regions-through-kernels
6
+ commit=9a236793
7
+ num_layers=48
8
+ hidden=2048
9
+ ffn_hidden=6144
10
+ moe_ffn_hidden=768
11
+ num_experts=128
12
+ topk=8
13
+ experts_per_rank=16
14
+ expert_params=4718592
15
+ heads=32
16
+ kv_groups=4
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=0
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=48
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=0
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=0
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=4096
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=32768
39
+ mean_tokens_per_expert=2048
40
+ slot_policy=matched
41
+ slots_per_rank=16
42
+ capacity_mode=static
43
+ capacity=2.00
44
+ cuda_graph=1
45
+ probe_mode=0
46
+ async_grad_reduce=0
47
+ branch_role=timing
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep2/train.log ADDED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep3/mem_samples.csv ADDED
@@ -0,0 +1,256 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 2
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 4
43
+ 2, 4
44
+ 3, 4
45
+ 4, 4
46
+ 5, 629
47
+ 6, 4
48
+ 7, 4
49
+ 0, 821
50
+ 1, 821
51
+ 2, 821
52
+ 3, 821
53
+ 4, 821
54
+ 5, 821
55
+ 6, 821
56
+ 7, 821
57
+ 0, 1831
58
+ 1, 1831
59
+ 2, 1831
60
+ 3, 1831
61
+ 4, 1831
62
+ 5, 1831
63
+ 6, 1831
64
+ 7, 1831
65
+ 0, 38993
66
+ 1, 38993
67
+ 2, 38993
68
+ 3, 38993
69
+ 4, 38993
70
+ 5, 38993
71
+ 6, 38993
72
+ 7, 38993
73
+ 0, 44939
74
+ 1, 44919
75
+ 2, 44951
76
+ 3, 44929
77
+ 4, 44937
78
+ 5, 44947
79
+ 6, 44957
80
+ 7, 44967
81
+ 0, 45453
82
+ 1, 45479
83
+ 2, 45477
84
+ 3, 45435
85
+ 4, 45429
86
+ 5, 45455
87
+ 6, 45439
88
+ 7, 45445
89
+ 0, 46919
90
+ 1, 46921
91
+ 2, 46931
92
+ 3, 46893
93
+ 4, 46917
94
+ 5, 46931
95
+ 6, 46921
96
+ 7, 46931
97
+ 0, 47015
98
+ 1, 47017
99
+ 2, 47027
100
+ 3, 46989
101
+ 4, 47013
102
+ 5, 47027
103
+ 6, 47017
104
+ 7, 47027
105
+ 0, 47213
106
+ 1, 47215
107
+ 2, 47241
108
+ 3, 47197
109
+ 4, 47209
110
+ 5, 47221
111
+ 6, 47213
112
+ 7, 47237
113
+ 0, 47879
114
+ 1, 47881
115
+ 2, 47891
116
+ 3, 47853
117
+ 4, 47877
118
+ 5, 47891
119
+ 6, 47881
120
+ 7, 47891
121
+ 0, 48135
122
+ 1, 48137
123
+ 2, 48147
124
+ 3, 48109
125
+ 4, 48133
126
+ 5, 48147
127
+ 6, 48137
128
+ 7, 48147
129
+ 0, 49133
130
+ 1, 49153
131
+ 2, 49147
132
+ 3, 49127
133
+ 4, 49129
134
+ 5, 49147
135
+ 6, 49119
136
+ 7, 49147
137
+ 0, 49133
138
+ 1, 49153
139
+ 2, 49147
140
+ 3, 49127
141
+ 4, 49129
142
+ 5, 49147
143
+ 6, 49119
144
+ 7, 49147
145
+ 0, 100859
146
+ 1, 100887
147
+ 2, 100871
148
+ 3, 100851
149
+ 4, 100845
150
+ 5, 100851
151
+ 6, 100907
152
+ 7, 100865
153
+ 0, 101347
154
+ 1, 101367
155
+ 2, 101353
156
+ 3, 101361
157
+ 4, 101363
158
+ 5, 101353
159
+ 6, 101353
160
+ 7, 101353
161
+ 0, 101347
162
+ 1, 101367
163
+ 2, 101353
164
+ 3, 101361
165
+ 4, 101363
166
+ 5, 101353
167
+ 6, 101353
168
+ 7, 101353
169
+ 0, 101347
170
+ 1, 101367
171
+ 2, 101353
172
+ 3, 101361
173
+ 4, 101363
174
+ 5, 101353
175
+ 6, 101353
176
+ 7, 101353
177
+ 0, 101347
178
+ 1, 101367
179
+ 2, 101353
180
+ 3, 101361
181
+ 4, 101363
182
+ 5, 101353
183
+ 6, 101353
184
+ 7, 101353
185
+ 0, 102061
186
+ 1, 102273
187
+ 2, 102067
188
+ 3, 102363
189
+ 4, 102269
190
+ 5, 102835
191
+ 6, 102259
192
+ 7, 102355
193
+ 0, 102709
194
+ 1, 102921
195
+ 2, 102715
196
+ 3, 103011
197
+ 4, 102917
198
+ 5, 103483
199
+ 6, 102907
200
+ 7, 103003
201
+ 0, 103369
202
+ 1, 103575
203
+ 2, 103343
204
+ 3, 103669
205
+ 4, 103559
206
+ 5, 104131
207
+ 6, 103557
208
+ 7, 103643
209
+ 0, 133045
210
+ 1, 133053
211
+ 2, 133251
212
+ 3, 133527
213
+ 4, 133377
214
+ 5, 133539
215
+ 6, 132943
216
+ 7, 133059
217
+ 0, 141287
218
+ 1, 137011
219
+ 2, 139149
220
+ 3, 142357
221
+ 4, 139145
222
+ 5, 140113
223
+ 6, 147219
224
+ 7, 146353
225
+ 0, 151641
226
+ 1, 151639
227
+ 2, 151641
228
+ 3, 151747
229
+ 4, 151635
230
+ 5, 151641
231
+ 6, 151643
232
+ 7, 151641
233
+ 0, 151641
234
+ 1, 151639
235
+ 2, 151641
236
+ 3, 151747
237
+ 4, 151635
238
+ 5, 151641
239
+ 6, 151643
240
+ 7, 151641
241
+ 0, 150825
242
+ 1, 150823
243
+ 2, 150825
244
+ 3, 150931
245
+ 4, 150819
246
+ 5, 150825
247
+ 6, 150827
248
+ 7, 150825
249
+ 0, 905
250
+ 1, 1162
251
+ 2, 150825
252
+ 3, 2186
253
+ 4, 906
254
+ 5, 906
255
+ 6, 906
256
+ 7, 906
tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep3/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=echo
2
+ config=QWEN3_30B
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=bracing-regions-through-kernels
6
+ commit=9a236793
7
+ num_layers=48
8
+ hidden=2048
9
+ ffn_hidden=6144
10
+ moe_ffn_hidden=768
11
+ num_experts=128
12
+ topk=8
13
+ experts_per_rank=16
14
+ expert_params=4718592
15
+ heads=32
16
+ kv_groups=4
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=0
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=48
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=0
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=0
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=4096
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=32768
39
+ mean_tokens_per_expert=2048
40
+ slot_policy=matched
41
+ slots_per_rank=16
42
+ capacity_mode=static
43
+ capacity=2.00
44
+ cuda_graph=1
45
+ probe_mode=0
46
+ async_grad_reduce=0
47
+ branch_role=timing
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep3/train.log ADDED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/echo/tierb/tierb_gbs8/results.tsv CHANGED
@@ -1,2 +1,9 @@
1
  config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
2
  QWEN3_235B echo matched 16 8192 8 2.00 none 1 OK 323.26 322.10 155809 /workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1
 
 
 
 
 
 
 
 
1
  config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
2
  QWEN3_235B echo matched 16 8192 8 2.00 none 1 OK 323.26 322.10 155809 /workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1
3
+ QWEN3_235B echo matched 16 8192 8 2.00 none 2 OK 324.40 322.10 155809 /workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep2
4
+ QWEN3_235B echo matched 16 8192 8 2.00 none 3 OK 324.13 322.10 155809 /workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep3
5
+ QWEN3_30B echo matched 16 4096 8 2.00 none 1 OK 399.01 396.50 151651 /workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1
6
+ QWEN3_30B echo matched 16 4096 8 2.00 none 2 OK 399.94 396.50 151747 /workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep2
7
+ QWEN3_30B echo matched 16 4096 8 2.00 none 3 OK 400.01 396.50 151747 /workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep3
8
+ DSV2_LITE_L24 echo matched 8 8192 8 2.00 none 1 OK 600.57 598.10 170291 /workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1
9
+ DSV2_LITE_L24 echo matched 8 8192 8 2.00 none 2 OK 600.63 598.10 170291 /workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep2
tierb_study_20260826_144644/echo/tierbprobe/provenance.txt CHANGED
@@ -1,7 +1,7 @@
1
  ==============================================================
2
  PROVENANCE: echo__tierbprobe__gbs8
3
  ==============================================================
4
- captured_at : 2026-08-26T15:32:33+00:00
5
  hostname : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0
6
  kernel : 6.8.0-1043-nvidia
7
  cwd : /workspace/blackwell_run_scripts
@@ -27,64 +27,3 @@ command : run_ladder.sh --phase tierbprobe --gbs 8 --out /workspace/tierb_
27
  7, 120 MHz, 1965 MHz, 1000.00 W, 28
28
 
29
  NVLink topology:
30
- GPU0 GPU1 GPU2 GPU3 GPU4 GPU5 GPU6 GPU7 NIC0 NIC1 NIC2 NIC3 NIC4 NIC5 NIC6 NIC7 NIC8 CPU Affinity NUMA Affinity GPU NUMA ID
31
- GPU0 X NV18 NV18 NV18 NV18 NV18 NV18 NV18 PIX NODE SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A
32
- GPU1 NV18 X NV18 NV18 NV18 NV18 NV18 NV18 NODE PIX SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A
33
- GPU2 NV18 NV18 X NV18 NV18 NV18 NV18 NV18 SYS SYS PIX NODE SYS SYS SYS SYS SYS 28-55 1 N/A
34
- GPU3 NV18 NV18 NV18 X NV18 NV18 NV18 NV18 SYS SYS NODE PIX SYS SYS SYS SYS SYS 28-55 1 N/A
35
- GPU4 NV18 NV18 NV18 NV18 X NV18 NV18 NV18 SYS SYS SYS SYS PIX NODE SYS SYS SYS 56-83 2 N/A
36
- GPU5 NV18 NV18 NV18 NV18 NV18 X NV18 NV18 SYS SYS SYS SYS NODE PIX SYS SYS SYS 56-83 2 N/A
37
- GPU6 NV18 NV18 NV18 NV18 NV18 NV18 X NV18 SYS SYS SYS SYS SYS SYS PIX NODE SYS 84-111 3 N/A
38
- GPU7 NV18 NV18 NV18 NV18 NV18 NV18 NV18 X SYS SYS SYS SYS SYS SYS NODE PIX SYS 84-111 3 N/A
39
- NIC0 PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS SYS NODE
40
- NIC1 NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS SYS NODE
41
- NIC2 SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS
42
- NIC3 SYS SYS NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS
43
- NIC4 SYS SYS SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS
44
- unavailable
45
-
46
- ---- toolchain -----------------------------------------------
47
- nvcc : release 12.9, V12.9.86
48
- gcc : gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0
49
- python : 3.12.3
50
- torch : 2.8.0a0+5228986c39.nv25.06 cuda=12.9 nccl=2.27.3
51
- torch arch : ['sm_75', 'sm_80', 'sm_86', 'sm_90', 'sm_100', 'sm_120', 'compute_120']
52
- transformer_engine: 2.10.0.dev0+10f6128d [/usr/local/lib/python3.12/dist-packages/transformer_engine/__init__.py]
53
- deep_ep : 1.2.1+34152ae [/workspace/DeepEP/deep_ep/__init__.py]
54
- ultra_ep : 1.0.0 [/workspace/UltraEP/ultra_ep/__init__.py]
55
- moonep : 0.0.1 [/workspace/MoonEP/moonep/__init__.py]
56
- megatron : 0.14.0rc5 [None]
57
- flash_attn : 2.7.4.post1 [/usr/local/lib/python3.12/dist-packages/flash_attn/__init__.py]
58
- numpy : 1.26.4 [/usr/local/lib/python3.12/dist-packages/numpy/__init__.py]
59
- huggingface_hub: 1.28.0 [/usr/local/lib/python3.12/dist-packages/huggingface_hub/__init__.py]
60
-
61
- ---- repositories --------------------------------------------
62
- Megatron-MoE-Echo token-drop-analysis b099fd8494b0
63
- Megatron-UltraEP ultraep-token-drop-analysis eaf829e7ad21
64
- UltraEP main 94cab099b44f
65
- DeepEP HEAD 34152ae28f80 (+3 untracked: build output)
66
- MoonEP megatron-integration c27d35da15f2
67
- blackwell_run_scripts granularity-echo d11391fc9103
68
-
69
- ---- environment ---------------------------------------------
70
- CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0
71
- CUDA_DRIVER_VERSION=575.57.08
72
- CUDA_HOME=/usr/local/cuda
73
- CUDA_MODULE_LOADING=LAZY
74
- CUDA_VERSION=12.9.1.010
75
- HF_REMOTE_ROOT=tierb_study_20260826_144644
76
- NCCL_VERSION=2.27.3
77
- NVSHMEM_VERSION=3.2.5
78
- PYTORCH_BUILD_NUMBER=0
79
- PYTORCH_BUILD_VERSION=2.8.0a0+5228986
80
- PYTORCH_HOME=/opt/pytorch/pytorch
81
- PYTORCH_VERSION=2.8.0a0+5228986
82
- TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1
83
- TORCH_CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0+PTX
84
- TORCH_NCCL_USE_COMM_NONBLOCKING=0
85
-
86
- ---- shared library resolution -------------------------------
87
- libnvshmem_host.so: not in ldconfig
88
- libnccl.so: /lib/x86_64-linux-gnu/libnccl.so.2
89
- lrwxrwxrwx 1 root root 20 Aug 26 14:37 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so -> libnvshmem_host.so.3
90
- -rw-r--r-- 1 root root 179045744 Aug 26 14:09 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so.3
 
1
  ==============================================================
2
  PROVENANCE: echo__tierbprobe__gbs8
3
  ==============================================================
4
+ captured_at : 2026-08-26T16:16:43+00:00
5
  hostname : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0
6
  kernel : 6.8.0-1043-nvidia
7
  cwd : /workspace/blackwell_run_scripts
 
27
  7, 120 MHz, 1965 MHz, 1000.00 W, 28
28
 
29
  NVLink topology:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
tierb_study_20260826_144644/hf_watcher.log CHANGED
@@ -26,3 +26,8 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
26
  uploaded: /workspace/tierb_study_20260826_144644 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tierb_study_20260826_144644
27
  https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
28
  [2026-08-26T16:02:44+00:00] backup pass
 
 
 
 
 
 
26
  uploaded: /workspace/tierb_study_20260826_144644 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tierb_study_20260826_144644
27
  https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
28
  [2026-08-26T16:02:44+00:00] backup pass
29
+ uploaded: /workspace/tierb_study_20260826_144644 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tierb_study_20260826_144644
30
+ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
31
+ Already logged in to Hugging Face as: AbhishekGhosh1998
32
+ Watching /workspace/tierb_study_20260826_144644 -> HF every 600s (kill to stop)
33
+ [2026-08-26T16:16:44+00:00] backup pass
tierb_study_20260826_144644/hf_watcher.pid CHANGED
@@ -1 +1 @@
1
- 99772
 
1
+ 203270
tierb_study_20260826_144644/study.log CHANGED
@@ -853,3 +853,36 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
853
  [16:00:58] CELL QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00 (slots=16)
854
  [guard] iter 8/12 (70s)
855
  [guard] iter 12/12 (80s)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
853
  [16:00:58] CELL QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00 (slots=16)
854
  [guard] iter 8/12 (70s)
855
  [guard] iter 12/12 (80s)
856
+ [guard] iter 9/12 (70s)
857
+ [guard] iter 12/12 (80s)
858
+ [guard] iter 10/12 (70s)
859
+ [guard] iter 12/12 (80s)
860
+ --------------------------------------------------------------------------
861
+ [16:04:58] CELL QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00 (slots=16)
862
+ [guard] iter 3/12 (70s)
863
+ [guard] iter 12/12 (80s)
864
+ [guard] tearing down process group (outcome=DONE)
865
+ [guard] iter 3/12 (70s)
866
+ [guard] iter 12/12 (80s)
867
+ [guard] tearing down process group (outcome=DONE)
868
+ [guard] iter 3/12 (70s)
869
+ [guard] iter 12/12 (80s)
870
+ [guard] tearing down process group (outcome=DONE)
871
+ --------------------------------------------------------------------------
872
+ [16:09:04] CELL DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00 (slots=8)
873
+ [guard] iter 7/12 (70s)
874
+ [guard] iter 12/12 (80s)
875
+ [guard] iter 7/12 (70s)
876
+ [guard] iter 12/12 (80s)
877
+ study_root: /workspace/tierb_study_20260826_144644
878
+ gbs: 8
879
+ stages: 7,4,8,9
880
+ nsys: 1
881
+ hf_repo: MoE-CUDA-Graph-Analysis/tierb-b200-gbs8
882
+ hf_remote_root: tierb_study_20260826_144644
883
+ started: 2026-08-26T16:16:43+00:00
884
+ driver_repo: /workspace/blackwell_run_scripts @ b99bce1
885
+
886
+ === STAGE 7 -- Tier B probe and zero-drop capacity selection
887
+
888
+ === STEP echo__tierbprobe__gbs8
tierb_study_20260826_144644/tierb_timing_capacities.env.tmp.168529 CHANGED
@@ -0,0 +1,2 @@
 
 
 
1
+ TIERB_TIMING_CAP_QWEN3_235B=2.00
2
+ TIERB_TIMING_CAP_QWEN3_30B=2.00
tierb_study_20260826_144644/tierb_timing_capacities.tsv.tmp.168529 CHANGED
@@ -1 +1,3 @@
1
  model common_timing_capacity echo_probe_required_capacity echo_full_cg_status echo_run_dir
 
 
 
1
  model common_timing_capacity echo_probe_required_capacity echo_full_cg_status echo_run_dir
2
+ QWEN3_235B 2.00 1.0 OK /workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep3
3
+ QWEN3_30B 2.00 1.0 OK /workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep3