AbhishekGhosh1998 commited on
Commit
8db9c63
·
verified ·
1 Parent(s): 84c07c5

granularity results tierb-fixed-moonep-full @ 2026-08-29T23:43:52Z from sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0

Browse files
Files changed (22) hide show
  1. tierb-fixed-moonep-full/hf_watcher.log +1 -0
  2. tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/mem_samples.csv +264 -0
  3. tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/run.env +73 -0
  4. tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/train.log +0 -0
  5. tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1/mem_samples.csv +216 -0
  6. tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1/run.env +73 -0
  7. tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1/train.log +0 -0
  8. tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/rep1/mem_samples.csv +216 -0
  9. tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/rep1/run.env +73 -0
  10. tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/rep1/train.log +0 -0
  11. tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/mem_samples.csv +96 -0
  12. tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/run.env +73 -0
  13. tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/train.log +752 -0
  14. tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/mem_samples.csv +216 -0
  15. tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/train.log +0 -0
  16. tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/mem_samples.csv +264 -0
  17. tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/run.env +73 -0
  18. tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/train.log +0 -0
  19. tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv +5 -0
  20. tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv +2 -0
  21. tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/results.tsv +5 -0
  22. tierb-fixed-moonep-full/study.log +24 -0
tierb-fixed-moonep-full/hf_watcher.log CHANGED
@@ -3,3 +3,4 @@ Watching /workspace/tierb-fixed-moonep-full -> HF every 600s (kill to stop)
3
  [2026-08-29T23:33:50+00:00] backup pass
4
  uploaded: /workspace/tierb-fixed-moonep-full -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tierb-fixed-moonep-full
5
  https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
 
 
3
  [2026-08-29T23:33:50+00:00] backup pass
4
  uploaded: /workspace/tierb-fixed-moonep-full -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tierb-fixed-moonep-full
5
  https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
6
+ [2026-08-29T23:43:52+00:00] backup pass
tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/mem_samples.csv ADDED
@@ -0,0 +1,264 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 4
10
+ 1, 4
11
+ 2, 1
12
+ 3, 1
13
+ 4, 1
14
+ 5, 1
15
+ 6, 1
16
+ 7, 1
17
+ 0, 4
18
+ 1, 4
19
+ 2, 4
20
+ 3, 4
21
+ 4, 4
22
+ 5, 4
23
+ 6, 4
24
+ 7, 1
25
+ 0, 1
26
+ 1, 1
27
+ 2, 1
28
+ 3, 1
29
+ 4, 1
30
+ 5, 4
31
+ 6, 4
32
+ 7, 4
33
+ 0, 0
34
+ 1, 0
35
+ 2, 0
36
+ 3, 0
37
+ 4, 0
38
+ 5, 0
39
+ 6, 0
40
+ 7, 0
41
+ 0, 1
42
+ 1, 1
43
+ 2, 1
44
+ 3, 1
45
+ 4, 1
46
+ 5, 1
47
+ 6, 1
48
+ 7, 1
49
+ 0, 4
50
+ 1, 4
51
+ 2, 4
52
+ 3, 4
53
+ 4, 1
54
+ 5, 1
55
+ 6, 1
56
+ 7, 1
57
+ 0, 4
58
+ 1, 4
59
+ 2, 4
60
+ 3, 4
61
+ 4, 4
62
+ 5, 4
63
+ 6, 4
64
+ 7, 4
65
+ 0, 4
66
+ 1, 4
67
+ 2, 4
68
+ 3, 4
69
+ 4, 4
70
+ 5, 4
71
+ 6, 4
72
+ 7, 4
73
+ 0, 783
74
+ 1, 725
75
+ 2, 727
76
+ 3, 743
77
+ 4, 725
78
+ 5, 725
79
+ 6, 725
80
+ 7, 725
81
+ 0, 1537
82
+ 1, 1537
83
+ 2, 1537
84
+ 3, 1537
85
+ 4, 1537
86
+ 5, 1537
87
+ 6, 1537
88
+ 7, 1537
89
+ 0, 1955
90
+ 1, 1955
91
+ 2, 1955
92
+ 3, 1955
93
+ 4, 1955
94
+ 5, 1955
95
+ 6, 1955
96
+ 7, 1955
97
+ 0, 5835
98
+ 1, 5823
99
+ 2, 5847
100
+ 3, 5823
101
+ 4, 5847
102
+ 5, 6371
103
+ 6, 6157
104
+ 7, 6133
105
+ 0, 26773
106
+ 1, 26773
107
+ 2, 26773
108
+ 3, 26773
109
+ 4, 26773
110
+ 5, 26773
111
+ 6, 26773
112
+ 7, 26773
113
+ 0, 30417
114
+ 1, 30547
115
+ 2, 30375
116
+ 3, 30411
117
+ 4, 30385
118
+ 5, 30397
119
+ 6, 30509
120
+ 7, 30555
121
+ 0, 40611
122
+ 1, 40689
123
+ 2, 40543
124
+ 3, 40531
125
+ 4, 40527
126
+ 5, 40537
127
+ 6, 40685
128
+ 7, 40681
129
+ 0, 41131
130
+ 1, 41215
131
+ 2, 41065
132
+ 3, 41065
133
+ 4, 41065
134
+ 5, 41061
135
+ 6, 41197
136
+ 7, 41215
137
+ 0, 41135
138
+ 1, 41219
139
+ 2, 41069
140
+ 3, 41069
141
+ 4, 41069
142
+ 5, 41065
143
+ 6, 41201
144
+ 7, 41219
145
+ 0, 45269
146
+ 1, 45355
147
+ 2, 45205
148
+ 3, 45203
149
+ 4, 45203
150
+ 5, 45201
151
+ 6, 45335
152
+ 7, 45355
153
+ 0, 108983
154
+ 1, 109077
155
+ 2, 108899
156
+ 3, 108903
157
+ 4, 108917
158
+ 5, 108909
159
+ 6, 109067
160
+ 7, 109085
161
+ 0, 109391
162
+ 1, 109477
163
+ 2, 109325
164
+ 3, 109325
165
+ 4, 109325
166
+ 5, 109323
167
+ 6, 109457
168
+ 7, 109475
169
+ 0, 109391
170
+ 1, 109477
171
+ 2, 109325
172
+ 3, 109325
173
+ 4, 109325
174
+ 5, 109323
175
+ 6, 109457
176
+ 7, 109477
177
+ 0, 109781
178
+ 1, 109867
179
+ 2, 109715
180
+ 3, 109715
181
+ 4, 109715
182
+ 5, 109713
183
+ 6, 109847
184
+ 7, 109867
185
+ 0, 109905
186
+ 1, 109991
187
+ 2, 109857
188
+ 3, 109845
189
+ 4, 109859
190
+ 5, 109865
191
+ 6, 109987
192
+ 7, 110007
193
+ 0, 111071
194
+ 1, 111111
195
+ 2, 110953
196
+ 3, 110975
197
+ 4, 110979
198
+ 5, 110973
199
+ 6, 111125
200
+ 7, 111153
201
+ 0, 125077
202
+ 1, 125157
203
+ 2, 125013
204
+ 3, 125007
205
+ 4, 125009
206
+ 5, 125003
207
+ 6, 125143
208
+ 7, 125157
209
+ 0, 125077
210
+ 1, 125157
211
+ 2, 125013
212
+ 3, 125007
213
+ 4, 125009
214
+ 5, 125003
215
+ 6, 125143
216
+ 7, 125157
217
+ 0, 125077
218
+ 1, 125157
219
+ 2, 125013
220
+ 3, 125007
221
+ 4, 125009
222
+ 5, 125003
223
+ 6, 125143
224
+ 7, 125157
225
+ 0, 125077
226
+ 1, 125157
227
+ 2, 125013
228
+ 3, 125007
229
+ 4, 125009
230
+ 5, 125003
231
+ 6, 125143
232
+ 7, 125157
233
+ 0, 123195
234
+ 1, 123663
235
+ 2, 123319
236
+ 3, 123513
237
+ 4, 123515
238
+ 5, 123509
239
+ 6, 123525
240
+ 7, 123663
241
+ 0, 121637
242
+ 1, 122629
243
+ 2, 121835
244
+ 3, 122479
245
+ 4, 121831
246
+ 5, 122087
247
+ 6, 121965
248
+ 7, 122241
249
+ 0, 4
250
+ 1, 120553
251
+ 2, 4
252
+ 3, 120403
253
+ 4, 4
254
+ 5, 120399
255
+ 6, 111043
256
+ 7, 120553
257
+ 0, 1
258
+ 1, 1
259
+ 2, 1
260
+ 3, 1
261
+ 4, 1
262
+ 5, 1
263
+ 6, 1
264
+ 7, 120550
tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/run.env ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=moonep
2
+ config=DSV2_LITE_L24
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=moonep_backend_async_grad_reduce_token_routing_analysis
6
+ commit=22fd6865
7
+ moonep_source_dir=/workspace/MoonEP
8
+ moonep_source_branch=megatron-ddp-symmetric-params
9
+ moonep_source_commit=d8bc666
10
+ num_layers=24
11
+ hidden=2048
12
+ ffn_hidden=10944
13
+ moe_ffn_hidden=1408
14
+ num_experts=64
15
+ topk=6
16
+ experts_per_rank=8
17
+ expert_params=8650752
18
+ heads=16
19
+ kv_groups=0
20
+ kv_channels=128
21
+ qk_layernorm=1
22
+ shared_ffn=2816
23
+ shared_expert_overlap=0
24
+ moe_token_dispatcher=flex
25
+ mla=1
26
+ dense_first_layers=1
27
+ moe_layers=23
28
+ rotary_base=10000
29
+ rotary_scaling=40
30
+ qkv_bias=0
31
+ router_balancing=seq_aux_loss
32
+ aux_loss_coeff=1e-3
33
+ router_pre_softmax=1
34
+ router_fusion=0
35
+ router_dtype=fp32
36
+ device_initiated_grouped_gemm=1
37
+ norm_epsilon=1e-6
38
+ seq_length=8192
39
+ micro_batch_size=1
40
+ global_batch_size=8
41
+ num_microbatches=1
42
+ assignments_per_rank=49152
43
+ mean_tokens_per_expert=6144
44
+ slot_policy=matched
45
+ slots_per_rank=8
46
+ capacity_mode=none
47
+ capacity=none
48
+ cuda_graph=0
49
+ probe_mode=1
50
+ async_grad_reduce=1
51
+ branch_role=probe
52
+ memory_dump_mode=none
53
+ memory_snapshot_path=
54
+ cuda_graph_warmup_steps=3
55
+ recompute=none
56
+ force_balance=0
57
+ train_iters=12
58
+ ep_size=8
59
+ echo_dispatch_overlap=1
60
+ echo_dispatch_recompute=0
61
+ ultraep_quota_min_tokens=
62
+ ultraep_weight_sync_join=
63
+ profile_enable=0
64
+ kernel_trace=0
65
+ fp8_mode=mxfp8
66
+ fp8_recipe=mxfp8
67
+ training_seed=1234
68
+ learning_rate=1e-4
69
+ min_learning_rate=1.0e-5
70
+ lr_warmup_iters=5
71
+ state_dump_enable=0
72
+ state_dump_dir=
73
+ moonep_assert_no_pending_reduce=1
tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1/mem_samples.csv ADDED
@@ -0,0 +1,216 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 4
10
+ 1, 4
11
+ 2, 1
12
+ 3, 1
13
+ 4, 1
14
+ 5, 1
15
+ 6, 1
16
+ 7, 1
17
+ 0, 4
18
+ 1, 4
19
+ 2, 4
20
+ 3, 4
21
+ 4, 4
22
+ 5, 4
23
+ 6, 4
24
+ 7, 1
25
+ 0, 1
26
+ 1, 1
27
+ 2, 1
28
+ 3, 1
29
+ 4, 4
30
+ 5, 4
31
+ 6, 4
32
+ 7, 4
33
+ 0, 0
34
+ 1, 0
35
+ 2, 0
36
+ 3, 0
37
+ 4, 0
38
+ 5, 0
39
+ 6, 0
40
+ 7, 0
41
+ 0, 1
42
+ 1, 1
43
+ 2, 1
44
+ 3, 1
45
+ 4, 1
46
+ 5, 1
47
+ 6, 1
48
+ 7, 1
49
+ 0, 4
50
+ 1, 4
51
+ 2, 4
52
+ 3, 4
53
+ 4, 1
54
+ 5, 1
55
+ 6, 1
56
+ 7, 1
57
+ 0, 4
58
+ 1, 4
59
+ 2, 4
60
+ 3, 4
61
+ 4, 4
62
+ 5, 4
63
+ 6, 4
64
+ 7, 4
65
+ 0, 4
66
+ 1, 4
67
+ 2, 4
68
+ 3, 4
69
+ 4, 4
70
+ 5, 4
71
+ 6, 4
72
+ 7, 629
73
+ 0, 821
74
+ 1, 821
75
+ 2, 821
76
+ 3, 821
77
+ 4, 821
78
+ 5, 821
79
+ 6, 821
80
+ 7, 821
81
+ 0, 1955
82
+ 1, 1955
83
+ 2, 1955
84
+ 3, 1955
85
+ 4, 1955
86
+ 5, 1955
87
+ 6, 1955
88
+ 7, 1955
89
+ 0, 20231
90
+ 1, 17899
91
+ 2, 22695
92
+ 3, 18031
93
+ 4, 17767
94
+ 5, 18031
95
+ 6, 17767
96
+ 7, 17987
97
+ 0, 27085
98
+ 1, 27085
99
+ 2, 27085
100
+ 3, 27085
101
+ 4, 27085
102
+ 5, 27085
103
+ 6, 27085
104
+ 7, 27085
105
+ 0, 30589
106
+ 1, 30655
107
+ 2, 30547
108
+ 3, 30585
109
+ 4, 30535
110
+ 5, 30539
111
+ 6, 30667
112
+ 7, 30683
113
+ 0, 40621
114
+ 1, 40695
115
+ 2, 40581
116
+ 3, 40565
117
+ 4, 40587
118
+ 5, 40563
119
+ 6, 40689
120
+ 7, 40757
121
+ 0, 41131
122
+ 1, 41215
123
+ 2, 41065
124
+ 3, 41065
125
+ 4, 41065
126
+ 5, 41061
127
+ 6, 41197
128
+ 7, 41215
129
+ 0, 41135
130
+ 1, 41221
131
+ 2, 41069
132
+ 3, 41069
133
+ 4, 41069
134
+ 5, 41067
135
+ 6, 41201
136
+ 7, 41221
137
+ 0, 108921
138
+ 1, 109077
139
+ 2, 108897
140
+ 3, 108871
141
+ 4, 108923
142
+ 5, 108921
143
+ 6, 109057
144
+ 7, 109085
145
+ 0, 109391
146
+ 1, 109477
147
+ 2, 109325
148
+ 3, 109325
149
+ 4, 109325
150
+ 5, 109323
151
+ 6, 109457
152
+ 7, 109477
153
+ 0, 109659
154
+ 1, 109793
155
+ 2, 109647
156
+ 3, 109643
157
+ 4, 109693
158
+ 5, 109669
159
+ 6, 109819
160
+ 7, 109807
161
+ 0, 110407
162
+ 1, 110495
163
+ 2, 110327
164
+ 3, 110353
165
+ 4, 110347
166
+ 5, 110343
167
+ 6, 110473
168
+ 7, 110491
169
+ 0, 111027
170
+ 1, 111139
171
+ 2, 110989
172
+ 3, 110983
173
+ 4, 111023
174
+ 5, 111037
175
+ 6, 111133
176
+ 7, 111157
177
+ 0, 125553
178
+ 1, 125633
179
+ 2, 125013
180
+ 3, 125007
181
+ 4, 125009
182
+ 5, 125003
183
+ 6, 125143
184
+ 7, 125633
185
+ 0, 125553
186
+ 1, 125633
187
+ 2, 125013
188
+ 3, 125007
189
+ 4, 125009
190
+ 5, 125003
191
+ 6, 125143
192
+ 7, 125633
193
+ 0, 125553
194
+ 1, 125633
195
+ 2, 125013
196
+ 3, 125007
197
+ 4, 125009
198
+ 5, 125003
199
+ 6, 125143
200
+ 7, 125633
201
+ 0, 125553
202
+ 1, 125633
203
+ 2, 125013
204
+ 3, 125007
205
+ 4, 125009
206
+ 5, 125003
207
+ 6, 125143
208
+ 7, 125633
209
+ 0, 9575
210
+ 1, 9578
211
+ 2, 9578
212
+ 3, 9898
213
+ 4, 125007
214
+ 5, 11178
215
+ 6, 9578
216
+ 7, 9578
tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1/run.env ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=moonep
2
+ config=DSV2_LITE_L24
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=moonep_backend_async_grad_reduce_token_routing_analysis
6
+ commit=22fd6865
7
+ moonep_source_dir=/workspace/MoonEP
8
+ moonep_source_branch=megatron-ddp-symmetric-params
9
+ moonep_source_commit=d8bc666
10
+ num_layers=24
11
+ hidden=2048
12
+ ffn_hidden=10944
13
+ moe_ffn_hidden=1408
14
+ num_experts=64
15
+ topk=6
16
+ experts_per_rank=8
17
+ expert_params=8650752
18
+ heads=16
19
+ kv_groups=0
20
+ kv_channels=128
21
+ qk_layernorm=1
22
+ shared_ffn=2816
23
+ shared_expert_overlap=0
24
+ moe_token_dispatcher=flex
25
+ mla=1
26
+ dense_first_layers=1
27
+ moe_layers=23
28
+ rotary_base=10000
29
+ rotary_scaling=40
30
+ qkv_bias=0
31
+ router_balancing=seq_aux_loss
32
+ aux_loss_coeff=1e-3
33
+ router_pre_softmax=1
34
+ router_fusion=0
35
+ router_dtype=fp32
36
+ device_initiated_grouped_gemm=1
37
+ norm_epsilon=1e-6
38
+ seq_length=8192
39
+ micro_batch_size=1
40
+ global_batch_size=8
41
+ num_microbatches=1
42
+ assignments_per_rank=49152
43
+ mean_tokens_per_expert=6144
44
+ slot_policy=matched
45
+ slots_per_rank=8
46
+ capacity_mode=none
47
+ capacity=none
48
+ cuda_graph=0
49
+ probe_mode=1
50
+ async_grad_reduce=1
51
+ branch_role=probe
52
+ memory_dump_mode=none
53
+ memory_snapshot_path=
54
+ cuda_graph_warmup_steps=3
55
+ recompute=none
56
+ force_balance=1
57
+ train_iters=12
58
+ ep_size=8
59
+ echo_dispatch_overlap=1
60
+ echo_dispatch_recompute=0
61
+ ultraep_quota_min_tokens=
62
+ ultraep_weight_sync_join=
63
+ profile_enable=0
64
+ kernel_trace=0
65
+ fp8_mode=mxfp8
66
+ fp8_recipe=mxfp8
67
+ training_seed=1234
68
+ learning_rate=1e-4
69
+ min_learning_rate=1.0e-5
70
+ lr_warmup_iters=5
71
+ state_dump_enable=0
72
+ state_dump_dir=
73
+ moonep_assert_no_pending_reduce=1
tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/rep1/mem_samples.csv ADDED
@@ -0,0 +1,216 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 4
10
+ 1, 4
11
+ 2, 1
12
+ 3, 1
13
+ 4, 1
14
+ 5, 1
15
+ 6, 1
16
+ 7, 1
17
+ 0, 4
18
+ 1, 4
19
+ 2, 4
20
+ 3, 4
21
+ 4, 4
22
+ 5, 4
23
+ 6, 4
24
+ 7, 1
25
+ 0, 1
26
+ 1, 1
27
+ 2, 1
28
+ 3, 3
29
+ 4, 4
30
+ 5, 4
31
+ 6, 4
32
+ 7, 4
33
+ 0, 0
34
+ 1, 0
35
+ 2, 0
36
+ 3, 0
37
+ 4, 0
38
+ 5, 0
39
+ 6, 0
40
+ 7, 0
41
+ 0, 1
42
+ 1, 1
43
+ 2, 1
44
+ 3, 1
45
+ 4, 1
46
+ 5, 1
47
+ 6, 1
48
+ 7, 1
49
+ 0, 4
50
+ 1, 4
51
+ 2, 1
52
+ 3, 1
53
+ 4, 1
54
+ 5, 1
55
+ 6, 1
56
+ 7, 1
57
+ 0, 4
58
+ 1, 4
59
+ 2, 4
60
+ 3, 4
61
+ 4, 1
62
+ 5, 1
63
+ 6, 1
64
+ 7, 1
65
+ 0, 4
66
+ 1, 4
67
+ 2, 4
68
+ 3, 4
69
+ 4, 4
70
+ 5, 4
71
+ 6, 4
72
+ 7, 4
73
+ 0, 819
74
+ 1, 807
75
+ 2, 801
76
+ 3, 817
77
+ 4, 797
78
+ 5, 819
79
+ 6, 811
80
+ 7, 797
81
+ 0, 1079
82
+ 1, 1079
83
+ 2, 1079
84
+ 3, 1079
85
+ 4, 1079
86
+ 5, 1079
87
+ 6, 1079
88
+ 7, 1079
89
+ 0, 1803
90
+ 1, 1803
91
+ 2, 1803
92
+ 3, 1803
93
+ 4, 1803
94
+ 5, 1803
95
+ 6, 1803
96
+ 7, 1803
97
+ 0, 45191
98
+ 1, 45185
99
+ 2, 45197
100
+ 3, 45191
101
+ 4, 45205
102
+ 5, 45195
103
+ 6, 45191
104
+ 7, 45179
105
+ 0, 49415
106
+ 1, 48859
107
+ 2, 48843
108
+ 3, 49553
109
+ 4, 49593
110
+ 5, 49607
111
+ 6, 49593
112
+ 7, 49571
113
+ 0, 49987
114
+ 1, 49445
115
+ 2, 49463
116
+ 3, 50145
117
+ 4, 50171
118
+ 5, 50169
119
+ 6, 50143
120
+ 7, 50153
121
+ 0, 50687
122
+ 1, 50117
123
+ 2, 50119
124
+ 3, 50819
125
+ 4, 50817
126
+ 5, 50817
127
+ 6, 50817
128
+ 7, 50817
129
+ 0, 51387
130
+ 1, 50817
131
+ 2, 50817
132
+ 3, 51517
133
+ 4, 51517
134
+ 5, 51517
135
+ 6, 51517
136
+ 7, 51517
137
+ 0, 72055
138
+ 1, 72089
139
+ 2, 72083
140
+ 3, 72035
141
+ 4, 72061
142
+ 5, 72079
143
+ 6, 72067
144
+ 7, 72083
145
+ 0, 72595
146
+ 1, 72619
147
+ 2, 72617
148
+ 3, 72581
149
+ 4, 72581
150
+ 5, 72581
151
+ 6, 72581
152
+ 7, 72581
153
+ 0, 72893
154
+ 1, 72923
155
+ 2, 72901
156
+ 3, 72913
157
+ 4, 72911
158
+ 5, 72885
159
+ 6, 72905
160
+ 7, 72949
161
+ 0, 82427
162
+ 1, 82351
163
+ 2, 82373
164
+ 3, 82309
165
+ 4, 82345
166
+ 5, 82331
167
+ 6, 82363
168
+ 7, 82349
169
+ 0, 95855
170
+ 1, 95789
171
+ 2, 95823
172
+ 3, 95787
173
+ 4, 95787
174
+ 5, 95787
175
+ 6, 95787
176
+ 7, 95787
177
+ 0, 95855
178
+ 1, 95789
179
+ 2, 95823
180
+ 3, 95787
181
+ 4, 95787
182
+ 5, 95787
183
+ 6, 95787
184
+ 7, 95787
185
+ 0, 95529
186
+ 1, 95143
187
+ 2, 95177
188
+ 3, 95205
189
+ 4, 95787
190
+ 5, 95461
191
+ 6, 95205
192
+ 7, 95787
193
+ 0, 93609
194
+ 1, 93223
195
+ 2, 93645
196
+ 3, 93609
197
+ 4, 93673
198
+ 5, 93867
199
+ 6, 93605
200
+ 7, 93867
201
+ 0, 87021
202
+ 1, 78619
203
+ 2, 92223
204
+ 3, 91925
205
+ 4, 92187
206
+ 5, 91861
207
+ 6, 91541
208
+ 7, 91541
209
+ 0, 4
210
+ 1, 4
211
+ 2, 77327
212
+ 3, 4
213
+ 4, 4
214
+ 5, 4
215
+ 6, 4
216
+ 7, 4
tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/rep1/run.env ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=moonep
2
+ config=QWEN2_57B
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=moonep_backend_async_grad_reduce_token_routing_analysis
6
+ commit=22fd6865
7
+ moonep_source_dir=/workspace/MoonEP
8
+ moonep_source_branch=megatron-ddp-symmetric-params
9
+ moonep_source_commit=d8bc666
10
+ num_layers=10
11
+ hidden=3584
12
+ ffn_hidden=2560
13
+ moe_ffn_hidden=2560
14
+ num_experts=64
15
+ topk=8
16
+ experts_per_rank=8
17
+ expert_params=27525120
18
+ heads=28
19
+ kv_groups=4
20
+ kv_channels=
21
+ qk_layernorm=0
22
+ shared_ffn=20480
23
+ shared_expert_overlap=0
24
+ moe_token_dispatcher=flex
25
+ mla=0
26
+ dense_first_layers=0
27
+ moe_layers=10
28
+ rotary_base=1000000
29
+ rotary_scaling=
30
+ qkv_bias=1
31
+ router_balancing=aux_loss
32
+ aux_loss_coeff=1e-2
33
+ router_pre_softmax=1
34
+ router_fusion=0
35
+ router_dtype=fp32
36
+ device_initiated_grouped_gemm=1
37
+ norm_epsilon=1e-6
38
+ seq_length=4096
39
+ micro_batch_size=1
40
+ global_batch_size=8
41
+ num_microbatches=1
42
+ assignments_per_rank=32768
43
+ mean_tokens_per_expert=4096
44
+ slot_policy=matched
45
+ slots_per_rank=8
46
+ capacity_mode=none
47
+ capacity=none
48
+ cuda_graph=0
49
+ probe_mode=1
50
+ async_grad_reduce=1
51
+ branch_role=probe
52
+ memory_dump_mode=none
53
+ memory_snapshot_path=
54
+ cuda_graph_warmup_steps=3
55
+ recompute=none
56
+ force_balance=0
57
+ train_iters=12
58
+ ep_size=8
59
+ echo_dispatch_overlap=1
60
+ echo_dispatch_recompute=0
61
+ ultraep_quota_min_tokens=
62
+ ultraep_weight_sync_join=
63
+ profile_enable=0
64
+ kernel_trace=0
65
+ fp8_mode=mxfp8
66
+ fp8_recipe=mxfp8
67
+ training_seed=1234
68
+ learning_rate=1e-4
69
+ min_learning_rate=1.0e-5
70
+ lr_warmup_iters=5
71
+ state_dump_enable=0
72
+ state_dump_dir=
73
+ moonep_assert_no_pending_reduce=1
tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/mem_samples.csv ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 4
10
+ 1, 4
11
+ 2, 1
12
+ 3, 1
13
+ 4, 1
14
+ 5, 1
15
+ 6, 1
16
+ 7, 1
17
+ 0, 4
18
+ 1, 4
19
+ 2, 4
20
+ 3, 4
21
+ 4, 4
22
+ 5, 4
23
+ 6, 4
24
+ 7, 1
25
+ 0, 1
26
+ 1, 1
27
+ 2, 1
28
+ 3, 1
29
+ 4, 4
30
+ 5, 4
31
+ 6, 4
32
+ 7, 4
33
+ 0, 0
34
+ 1, 0
35
+ 2, 0
36
+ 3, 0
37
+ 4, 0
38
+ 5, 0
39
+ 6, 0
40
+ 7, 0
41
+ 0, 1
42
+ 1, 1
43
+ 2, 1
44
+ 3, 1
45
+ 4, 1
46
+ 5, 1
47
+ 6, 1
48
+ 7, 1
49
+ 0, 4
50
+ 1, 4
51
+ 2, 4
52
+ 3, 1
53
+ 4, 1
54
+ 5, 1
55
+ 6, 1
56
+ 7, 1
57
+ 0, 4
58
+ 1, 4
59
+ 2, 4
60
+ 3, 4
61
+ 4, 4
62
+ 5, 4
63
+ 6, 1
64
+ 7, 1
65
+ 0, 4
66
+ 1, 4
67
+ 2, 4
68
+ 3, 4
69
+ 4, 4
70
+ 5, 4
71
+ 6, 4
72
+ 7, 4
73
+ 0, 821
74
+ 1, 819
75
+ 2, 819
76
+ 3, 819
77
+ 4, 819
78
+ 5, 821
79
+ 6, 819
80
+ 7, 819
81
+ 0, 1537
82
+ 1, 1537
83
+ 2, 1537
84
+ 3, 1537
85
+ 4, 1537
86
+ 5, 1537
87
+ 6, 1537
88
+ 7, 1537
89
+ 0, 1919
90
+ 1, 1919
91
+ 2, 1919
92
+ 3, 1919
93
+ 4, 1919
94
+ 5, 1975
95
+ 6, 1975
96
+ 7, 1919
tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/run.env ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=moonep
2
+ config=QWEN2_57B
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=moonep_backend_async_grad_reduce_token_routing_analysis
6
+ commit=22fd6865
7
+ moonep_source_dir=/workspace/MoonEP
8
+ moonep_source_branch=megatron-ddp-symmetric-params
9
+ moonep_source_commit=d8bc666
10
+ num_layers=10
11
+ hidden=3584
12
+ ffn_hidden=2560
13
+ moe_ffn_hidden=2560
14
+ num_experts=64
15
+ topk=8
16
+ experts_per_rank=8
17
+ expert_params=27525120
18
+ heads=28
19
+ kv_groups=4
20
+ kv_channels=
21
+ qk_layernorm=0
22
+ shared_ffn=20480
23
+ shared_expert_overlap=0
24
+ moe_token_dispatcher=flex
25
+ mla=0
26
+ dense_first_layers=0
27
+ moe_layers=10
28
+ rotary_base=1000000
29
+ rotary_scaling=
30
+ qkv_bias=1
31
+ router_balancing=aux_loss
32
+ aux_loss_coeff=1e-2
33
+ router_pre_softmax=1
34
+ router_fusion=0
35
+ router_dtype=fp32
36
+ device_initiated_grouped_gemm=1
37
+ norm_epsilon=1e-6
38
+ seq_length=4096
39
+ micro_batch_size=1
40
+ global_batch_size=8
41
+ num_microbatches=1
42
+ assignments_per_rank=32768
43
+ mean_tokens_per_expert=4096
44
+ slot_policy=matched
45
+ slots_per_rank=8
46
+ capacity_mode=none
47
+ capacity=none
48
+ cuda_graph=0
49
+ probe_mode=1
50
+ async_grad_reduce=1
51
+ branch_role=probe
52
+ memory_dump_mode=none
53
+ memory_snapshot_path=
54
+ cuda_graph_warmup_steps=3
55
+ recompute=none
56
+ force_balance=1
57
+ train_iters=12
58
+ ep_size=8
59
+ echo_dispatch_overlap=1
60
+ echo_dispatch_recompute=0
61
+ ultraep_quota_min_tokens=
62
+ ultraep_weight_sync_join=
63
+ profile_enable=0
64
+ kernel_trace=0
65
+ fp8_mode=mxfp8
66
+ fp8_recipe=mxfp8
67
+ training_seed=1234
68
+ learning_rate=1e-4
69
+ min_learning_rate=1.0e-5
70
+ lr_warmup_iters=5
71
+ state_dump_enable=0
72
+ state_dump_dir=
73
+ moonep_assert_no_pending_reduce=1
tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/train.log ADDED
@@ -0,0 +1,752 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [23:43:17] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe)
2
+ ==============================================================
3
+ QWEN2_57B / moonep (slots/rank=8 policy=matched)
4
+ E=64 K=8 H'=2560 m=8
5
+ L=10 seq=4096 gbs=8 (1 microbatches)
6
+ cap_mode=none cap=none cg=0 recompute=none
7
+ memory_dump=none snapshots=/workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/memory_snapshots/snapshot.pickle
8
+ repo=/workspace/Megatron-MoE-Echo @ 22fd6865
9
+ ==============================================================
10
+ W0829 23:43:27.686000 234329 torch/distributed/run.py:766]
11
+ W0829 23:43:27.686000 234329 torch/distributed/run.py:766] *****************************************
12
+ W0829 23:43:27.686000 234329 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
13
+ W0829 23:43:27.686000 234329 torch/distributed/run.py:766] *****************************************
14
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
15
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
16
+ WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it
17
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
18
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
19
+ using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1
20
+ WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer
21
+ Number of virtual stages per pipeline stage: None
22
+ accumulate and all-reduce gradients in fp32 for bfloat16 data type.
23
+ using torch.bfloat16 for parameters ...
24
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
25
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
26
+ ------------------------ arguments ------------------------
27
+ account_for_embedding_in_pipeline_split ......... False
28
+ account_for_loss_in_pipeline_split .............. False
29
+ accumulate_allreduce_grads_in_fp32 .............. True
30
+ adam_beta1 ...................................... 0.9
31
+ adam_beta2 ...................................... 0.999
32
+ adam_eps ........................................ 1e-08
33
+ add_bias_linear ................................. False
34
+ add_position_embedding .......................... False
35
+ add_qkv_bias .................................... True
36
+ adlr_autoresume ................................. False
37
+ adlr_autoresume_interval ........................ 1000
38
+ align_grad_reduce ............................... True
39
+ align_param_gather .............................. False
40
+ app_tag_run_name ................................ None
41
+ app_tag_run_version ............................. 0.0.0
42
+ apply_layernorm_1p .............................. False
43
+ apply_query_key_layer_scaling ................... False
44
+ apply_residual_connection_post_layernorm ........ False
45
+ apply_rope_fusion ............................... True
46
+ async_save ...................................... None
47
+ async_tensor_model_parallel_allreduce ........... True
48
+ attention_backend ............................... AttnBackend.auto
49
+ attention_dropout ............................... 0.0
50
+ attention_softmax_in_fp32 ....................... False
51
+ auto_detect_ckpt_format ......................... False
52
+ barrier_with_L1_time ............................ True
53
+ bert_binary_head ................................ True
54
+ bert_embedder_type .............................. megatron
55
+ bert_load ....................................... None
56
+ bf16 ............................................ True
57
+ bias_dropout_fusion ............................. True
58
+ bias_gelu_fusion ................................ False
59
+ bias_swiglu_fusion .............................. True
60
+ biencoder_projection_dim ........................ 0
61
+ biencoder_shared_query_context_model ............ False
62
+ block_data_path ................................. None
63
+ calc_ft_timeouts ................................ False
64
+ calculate_per_token_loss ........................ False
65
+ check_for_large_grads ........................... False
66
+ check_for_nan_in_loss_and_grad .................. True
67
+ check_for_spiky_loss ............................ False
68
+ check_weight_hash_across_dp_replicas_interval ... None
69
+ ckpt_assume_constant_structure .................. False
70
+ ckpt_convert_format ............................. None
71
+ ckpt_convert_save ............................... None
72
+ ckpt_convert_update_legacy_dist_opt_format ...... False
73
+ ckpt_format ..................................... torch_dist
74
+ ckpt_fully_parallel_load ........................ False
75
+ ckpt_fully_parallel_save ........................ True
76
+ ckpt_fully_parallel_save_deprecated ............. False
77
+ ckpt_step ....................................... None
78
+ classes_fraction ................................ 1.0
79
+ clip_grad ....................................... 1.0
80
+ clone_scatter_output_in_embedding ............... True
81
+ config_logger_dir ...............................
82
+ consumed_train_samples .......................... 0
83
+ consumed_valid_samples .......................... 0
84
+ context_parallel_size ........................... 1
85
+ cp_comm_type .................................... ['p2p']
86
+ create_attention_mask_in_dataloader ............. True
87
+ cross_entropy_fusion_impl ....................... native
88
+ cross_entropy_loss_fusion ....................... False
89
+ cuda_graph_scope ................................ full
90
+ cuda_graph_warmup_steps ......................... 3
91
+ data_args_path .................................. None
92
+ data_cache_path ................................. None
93
+ data_parallel_random_init ....................... False
94
+ data_parallel_sharding_strategy ................. no_shard
95
+ data_parallel_size .............................. 8
96
+ data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document']
97
+ data_per_class_fraction ......................... 1.0
98
+ data_sharding ................................... True
99
+ dataloader_type ................................. single
100
+ ddp_average_in_collective ....................... False
101
+ ddp_bucket_size ................................. None
102
+ ddp_num_buckets ................................. None
103
+ ddp_pad_buckets_for_high_nccl_busbw ............. False
104
+ decoder_first_pipeline_num_layers ............... None
105
+ decoder_last_pipeline_num_layers ................ None
106
+ decoder_num_layers .............................. None
107
+ decoder_seq_length .............................. None
108
+ decoupled_lr .................................... None
109
+ decoupled_min_lr ................................ None
110
+ decrease_batch_size_if_needed ................... False
111
+ defer_embedding_wgrad_compute ................... False
112
+ delay_wgrad_compute ............................. False
113
+ deprecated_use_mcore_models ..................... True
114
+ deterministic_mode .............................. False
115
+ dino_bottleneck_size ............................ 256
116
+ dino_freeze_last_layer .......................... 1
117
+ dino_head_hidden_size ........................... 2048
118
+ dino_local_crops_number ......................... 10
119
+ dino_local_img_size ............................. 96
120
+ dino_norm_last_layer ............................ False
121
+ dino_teacher_temp ............................... 0.07
122
+ dino_warmup_teacher_temp ........................ 0.04
123
+ dino_warmup_teacher_temp_epochs ................. 30
124
+ disable_bf16_reduced_precision_matmul ........... False
125
+ disable_mamba_mem_eff_path ...................... False
126
+ disable_straggler_on_startup .................... False
127
+ dist_ckpt_format_deprecated ..................... None
128
+ dist_ckpt_strictness ............................ assume_ok_unexpected
129
+ distribute_saved_activations .................... False
130
+ distributed_backend ............................. nccl
131
+ distributed_timeout_minutes ..................... 10
132
+ embedding_init_method_std ....................... None
133
+ embedding_path .................................. None
134
+ empty_unused_memory_level ....................... 0
135
+ enable_cuda_graph ............................... False
136
+ enable_experimental ............................. False
137
+ enable_ft_package ............................... False
138
+ enable_gloo_process_groups ...................... True
139
+ enable_msc ...................................... True
140
+ enable_one_logger ............................... True
141
+ encoder_num_layers .............................. 10
142
+ encoder_seq_length .............................. 4096
143
+ end_weight_decay ................................ 0.1
144
+ eod_mask_loss ................................... False
145
+ error_injection_rate ............................ 0
146
+ error_injection_type ............................ transient_error
147
+ eval_interval ................................... 1000000
148
+ eval_iters ...................................... 0
149
+ evidence_data_path .............................. None
150
+ exit_duration_in_mins ........................... None
151
+ exit_interval ................................... None
152
+ exit_on_missing_checkpoint ...................... False
153
+ exit_signal_handler ............................. False
154
+ exp_avg_dtype ................................... torch.float32
155
+ exp_avg_sq_dtype ................................ torch.float32
156
+ expert_model_parallel_size ...................... 8
157
+ expert_tensor_parallel_size ..................... 1
158
+ export_force_local_attention .................... False
159
+ export_kd_cfg ................................... None
160
+ export_kd_teacher_ckpt_format ................... None
161
+ export_kd_teacher_load .......................... None
162
+ export_kv_cache_quant ........................... False
163
+ export_legacy_megatron .......................... False
164
+ export_model_type ............................... GPTModel
165
+ export_moe_apply_probs_on_input ................. False
166
+ export_qk_l2_norm ............................... False
167
+ export_quant_cfg ................................ None
168
+ export_real_quant_cfg ........................... None
169
+ export_te_mcore_model ........................... False
170
+ external_cuda_graph ............................. False
171
+ ffn_hidden_size ................................. 2560
172
+ finetune ........................................ False
173
+ finetune_data_split ............................. train
174
+ finetune_hf_dataset ............................. None
175
+ first_last_layers_bf16 .......................... False
176
+ flash_decode .................................... False
177
+ fp16 ............................................ False
178
+ fp16_lm_cross_entropy ........................... False
179
+ fp32_residual_connection ........................ False
180
+ fp8 ............................................. e4m3
181
+ fp8_amax_compute_algo ........................... most_recent
182
+ fp8_amax_history_len ............................ 1
183
+ fp8_interval .................................... 1
184
+ fp8_margin ...................................... 0
185
+ fp8_param_gather ................................ False
186
+ fp8_recipe ...................................... mxfp8
187
+ fp8_wgrad ....................................... True
188
+ fsdp_double_buffer .............................. False
189
+ global_batch_size ............................... 8
190
+ grad_reduce_in_bf16 ............................. False
191
+ gradient_accumulation_fusion .................... True
192
+ gradient_reduce_div_fusion ...................... True
193
+ group_query_attention ........................... True
194
+ head_lr_mult .................................... 1.0
195
+ heterogeneous_layers_config_encoded_json ........ None
196
+ heterogeneous_layers_config_path ................ None
197
+ hidden_dropout .................................. 0.0
198
+ hidden_size ..................................... 3584
199
+ hierarchical_context_parallel_sizes ............. None
200
+ high_priority_stream_groups ..................... []
201
+ hybrid_attention_ratio .......................... 0.0
202
+ hybrid_mlp_ratio ................................ 0.0
203
+ hybrid_override_pattern ......................... None
204
+ hysteresis ...................................... 2
205
+ ict_head_size ................................... None
206
+ ict_load ........................................ None
207
+ img_h ........................................... 224
208
+ img_w ........................................... 224
209
+ indexer_batch_size .............................. 128
210
+ indexer_log_interval ............................ 1000
211
+ inference_batch_times_seqlen_threshold .......... -1
212
+ inference_dynamic_batching ...................... False
213
+ inference_dynamic_batching_buffer_guaranteed_fraction 0.2
214
+ inference_dynamic_batching_buffer_overflow_factor None
215
+ inference_dynamic_batching_buffer_size_gb ....... 40.0
216
+ inference_dynamic_batching_chunk_size ........... 256
217
+ inference_dynamic_batching_max_requests_override None
218
+ inference_dynamic_batching_max_tokens_override .. None
219
+ inference_dynamic_batching_num_cuda_graphs ...... 16
220
+ inference_max_batch_size ........................ 8
221
+ inference_max_seq_length ........................ 2560
222
+ inference_rng_tracker ........................... False
223
+ init_method_std ................................. 0.01
224
+ init_method_xavier_uniform ...................... False
225
+ init_model_with_meta_device ..................... False
226
+ initial_loss_scale .............................. 4294967296
227
+ inprocess_active_world_size ..................... 8
228
+ inprocess_barrier_timeout ....................... 120
229
+ inprocess_completion_timeout .................... 120
230
+ inprocess_empty_cuda_cache ...................... False
231
+ inprocess_granularity ........................... node
232
+ inprocess_hard_timeout .......................... 90
233
+ inprocess_heartbeat_interval .................... 30
234
+ inprocess_heartbeat_timeout ..................... 60
235
+ inprocess_last_call_wait ........................ 1
236
+ inprocess_max_iterations ........................ None
237
+ inprocess_monitor_process_interval .............. 1.0
238
+ inprocess_monitor_thread_interval ............... 1.0
239
+ inprocess_progress_watchdog_interval ............ 1.0
240
+ inprocess_restart ............................... False
241
+ inprocess_soft_timeout .......................... 60
242
+ inprocess_termination_grace_time ................ 1
243
+ is_hybrid_model ................................. False
244
+ iter_per_epoch .................................. 1250
245
+ iterations_to_skip .............................. []
246
+ keep_fp8_transpose_cache_when_using_custom_fsdp . False
247
+ kitchen_config_file ............................. None
248
+ kitchen_recipe_number ........................... None
249
+ kv_channels ..................................... 128
250
+ kv_lora_rank .................................... 32
251
+ lazy_mpu_init ................................... None
252
+ load ............................................ None
253
+ load_main_params_from_ckpt ...................... None
254
+ load_model_opt_format ........................... False
255
+ local_rank ...................................... 0
256
+ log_energy ...................................... False
257
+ log_interval .................................... 1
258
+ log_loss_scale_to_tensorboard ................... True
259
+ log_memory_to_tensorboard ....................... False
260
+ log_num_zeros_in_grad ........................... False
261
+ log_params_norm ................................. False
262
+ log_progress .................................... False
263
+ log_straggler ................................... False
264
+ log_throughput .................................. True
265
+ log_timers_to_tensorboard ....................... False
266
+ log_validation_ppl_to_tensorboard ............... False
267
+ log_world_size_to_tensorboard ................... False
268
+ logging_level ................................... None
269
+ loss_scale ...................................... None
270
+ loss_scale_window ............................... 1000
271
+ lr .............................................. 0.0001
272
+ lr_decay_iters .................................. 8
273
+ lr_decay_samples ................................ None
274
+ lr_decay_style .................................. cosine
275
+ lr_warmup_fraction .............................. None
276
+ lr_warmup_init .................................. 0.0
277
+ lr_warmup_iters ................................. 5
278
+ lr_warmup_samples ............................... 0
279
+ lr_wsd_decay_iters .............................. None
280
+ lr_wsd_decay_samples ............................ None
281
+ lr_wsd_decay_style .............................. exponential
282
+ main_grads_dtype ................................ torch.float32
283
+ main_params_dtype ............................... torch.float32
284
+ make_vocab_size_divisible_by .................... 128
285
+ mamba_head_dim .................................. 64
286
+ mamba_num_groups ................................ 8
287
+ mamba_num_heads ................................. None
288
+ mamba_state_dim ................................. 128
289
+ manual_gc ....................................... False
290
+ manual_gc_eval .................................. True
291
+ manual_gc_interval .............................. 0
292
+ mask_factor ..................................... 1.0
293
+ mask_prob ....................................... 0.15
294
+ mask_type ....................................... random
295
+ masked_softmax_fusion ........................... False
296
+ max_position_embeddings ......................... 32768
297
+ max_tokens_to_oom ............................... 12000
298
+ memory_snapshot_path ............................ snapshot.pickle
299
+ merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt
300
+ micro_batch_size ................................ 1
301
+ microbatch_group_size_per_vp_stage .............. None
302
+ mid_level_dataset_surplus ....................... 0.005
303
+ min_loss_scale .................................. 1.0
304
+ min_lr .......................................... 1e-05
305
+ mlp_chunks_for_prefill .......................... 1
306
+ mmap_bin_files .................................. True
307
+ mock_data ....................................... False
308
+ moe_apply_probs_on_input ........................ False
309
+ moe_aux_loss_coeff .............................. 0.01
310
+ moe_deepep_num_sms .............................. 20
311
+ moe_echo_enable_random_offloading ............... False
312
+ moe_echo_expert_dispatch_overlap ................ False
313
+ moe_echo_expert_dispatcher_type ................. hybridep
314
+ moe_echo_recompute_expert_dispatch .............. False
315
+ moe_enable_deepep ............................... False
316
+ moe_enable_echo ................................. False
317
+ moe_enable_hybridep ............................. False
318
+ moe_enable_moonep ............................... True
319
+ moe_expert_capacity_factor ...................... None
320
+ moe_extended_tp ................................. False
321
+ moe_ffn_hidden_size ............................. 2560
322
+ moe_grouped_gemm ................................ True
323
+ moe_hybridep_num_sms ............................ 16
324
+ moe_input_jitter_eps ............................ None
325
+ moe_layer_freq .................................. 1
326
+ moe_layer_recompute ............................. False
327
+ moe_moonep_async_grad_reduce .................... True
328
+ moe_moonep_num_sms .............................. None
329
+ moe_moonep_traffic_logging ...................... True
330
+ moe_moonep_traffic_logging_interval ............. 1
331
+ moe_moonep_traffic_warmup_iters ................. 2
332
+ moe_num_echo_experts ............................ None
333
+ moe_pad_expert_input_to_capacity ................ False
334
+ moe_per_layer_logging ........................... False
335
+ moe_permute_fusion .............................. False
336
+ moe_received_token_capacity ..................... None
337
+ moe_router_bias_update_rate ..................... 0.001
338
+ moe_router_dtype ................................ fp32
339
+ moe_router_enable_expert_bias ................... False
340
+ moe_router_force_load_balancing ................. True
341
+ moe_router_group_topk ........................... None
342
+ moe_router_load_balancing_type .................. aux_loss
343
+ moe_router_num_groups ........................... None
344
+ moe_router_padding_for_fp8 ...................... False
345
+ moe_router_pre_softmax .......................... True
346
+ moe_router_score_function ....................... softmax
347
+ moe_router_topk ................................. 8
348
+ moe_router_topk_scaling_factor .................. None
349
+ moe_shared_expert_intermediate_size ............. 20480
350
+ moe_shared_expert_overlap ....................... False
351
+ moe_token_dispatcher_type ....................... flex
352
+ moe_token_drop_policy ........................... probs
353
+ moe_upcycling_granularity ....................... 1
354
+ moe_use_device_initiated_grouped_gemm ........... True
355
+ moe_use_legacy_grouped_gemm ..................... False
356
+ moe_use_upcycling ............................... False
357
+ moe_z_loss_coeff ................................ None
358
+ mrope_section ................................... None
359
+ mscale .......................................... 1.0
360
+ mscale_all_dim .................................. 0.0
361
+ mtp_loss_scaling_factor ......................... 0.1
362
+ mtp_num_layers .................................. None
363
+ multi_latent_attention .......................... False
364
+ nccl_all_reduce_for_prefill ..................... False
365
+ nccl_communicator_config_path ................... None
366
+ nccl_ub ......................................... False
367
+ no_load_optim ................................... None
368
+ no_load_rng ..................................... None
369
+ no_persist_layer_norm ........................... False
370
+ no_rope_freq .................................... None
371
+ no_save_optim ................................... None
372
+ no_save_rng ..................................... None
373
+ non_persistent_ckpt_type ........................ None
374
+ non_persistent_global_ckpt_dir .................. None
375
+ non_persistent_local_ckpt_algo .................. fully_parallel
376
+ non_persistent_local_ckpt_dir ................... None
377
+ non_persistent_save_interval .................... None
378
+ norm_epsilon .................................... 1e-06
379
+ normalization ................................... RMSNorm
380
+ num_attention_heads ............................. 28
381
+ num_channels .................................... 3
382
+ num_classes ..................................... 1000
383
+ num_dataset_builder_threads ..................... 1
384
+ num_distributed_optimizer_instances ............. 1
385
+ num_experts ..................................... 64
386
+ num_layers ...................................... 10
387
+ num_layers_at_end_in_bf16 ....................... 1
388
+ num_layers_at_start_in_bf16 ..................... 1
389
+ num_layers_per_virtual_pipeline_stage ........... None
390
+ num_query_groups ................................ 4
391
+ num_virtual_stages_per_pipeline_rank ............ None
392
+ num_workers ..................................... 2
393
+ object_storage_cache_path ....................... None
394
+ one_logger_async ................................ False
395
+ one_logger_project .............................. megatron-lm
396
+ one_logger_run_name ............................. None
397
+ onnx_safe ....................................... None
398
+ openai_gelu ..................................... False
399
+ optimizer ....................................... adam
400
+ optimizer_cpu_offload ........................... False
401
+ optimizer_offload_fraction ...................... 1.0
402
+ output_bert_embeddings .......................... False
403
+ overlap_cpu_optimizer_d2h_h2d ................... False
404
+ overlap_grad_reduce ............................. True
405
+ overlap_moe_expert_parallel_comm ................ False
406
+ overlap_p2p_comm ................................ False
407
+ overlap_p2p_comm_warmup_flush ................... False
408
+ overlap_param_gather ............................ True
409
+ overlap_param_gather_with_optimizer_step ........ False
410
+ override_opt_param_scheduler .................... False
411
+ params_dtype .................................... torch.bfloat16
412
+ patch_dim ....................................... 16
413
+ per_split_data_args_path ........................ None
414
+ perform_initialization .......................... True
415
+ pin_cpu_grads ................................... True
416
+ pin_cpu_params .................................. True
417
+ pipeline_model_parallel_comm_backend ............ None
418
+ pipeline_model_parallel_layout .................. None
419
+ pipeline_model_parallel_size .................... 1
420
+ position_embedding_type ......................... rope
421
+ pretrained_checkpoint ........................... None
422
+ profile ......................................... False
423
+ profile_ranks ................................... [0]
424
+ profile_step_end ................................ 12
425
+ profile_step_start .............................. 10
426
+ q_lora_rank ..................................... None
427
+ qk_head_dim ..................................... 128
428
+ qk_l2_norm ...................................... False
429
+ qk_layernorm .................................... False
430
+ qk_pos_emb_head_dim ............................. 64
431
+ query_in_block_prob ............................. 0.1
432
+ rampup_batch_size ............................... None
433
+ rank ............................................ 0
434
+ recompute_granularity ........................... None
435
+ recompute_method ................................ None
436
+ recompute_modules ............................... None
437
+ recompute_num_layers ............................ None
438
+ record_memory_history ........................... False
439
+ relative_attention_max_distance ................. 128
440
+ relative_attention_num_buckets .................. 32
441
+ replication ..................................... False
442
+ replication_factor .............................. 2
443
+ replication_jump ................................ None
444
+ rerun_mode ...................................... validate_results
445
+ reset_attention_mask ............................ False
446
+ reset_position_ids .............................. False
447
+ result_rejected_tracker_filename ................ None
448
+ retriever_report_topk_accuracies ................ []
449
+ retriever_score_scaling ......................... False
450
+ retriever_seq_length ............................ 256
451
+ retro_add_retriever ............................. False
452
+ retro_attention_gate ............................ 1
453
+ retro_cyclic_train_iters ........................ None
454
+ retro_encoder_attention_dropout ................. 0.1
455
+ retro_encoder_hidden_dropout .................... 0.1
456
+ retro_encoder_layers ............................ 2
457
+ retro_num_neighbors ............................. 2
458
+ retro_num_retrieved_chunks ...................... 2
459
+ retro_project_dir ............................... None
460
+ retro_verify_neighbor_count ..................... True
461
+ reuse_grad_buf_for_mxfp8_param_ag ............... False
462
+ rope_scaling_factor ............................. 8.0
463
+ rope_type ....................................... None
464
+ rotary_base ..................................... 1000000
465
+ rotary_interleaved .............................. False
466
+ rotary_percent .................................. 1.0
467
+ rotary_scaling_factor ........................... 1.0
468
+ rotary_seq_len_interpolation_factor ............. None
469
+ run_workload_inspector_server ................... False
470
+ sample_rate ..................................... 1.0
471
+ save ............................................ None
472
+ save_interval ................................... None
473
+ save_retain_interval ............................ None
474
+ scatter_gather_tensors_in_pipeline .............. True
475
+ seed ............................................ 1234
476
+ seq_length ...................................... 4096
477
+ sequence_parallel ............................... False
478
+ sft ............................................. False
479
+ sft_tokenizer_prompt_format ..................... nemotron-h-aligned
480
+ sgd_momentum .................................... 0.9
481
+ short_seq_prob .................................. 0.1
482
+ skip_train ...................................... False
483
+ skipped_train_samples ........................... 0
484
+ spec ............................................ None
485
+ split ........................................... 100,0,0
486
+ squared_relu .................................... False
487
+ start_weight_decay .............................. 0.1
488
+ straggler_ctrlr_port ............................ 65535
489
+ straggler_minmax_count .......................... 1
490
+ suggested_communication_unit_size ............... None
491
+ swiglu .......................................... True
492
+ swin_backbone_type .............................. tiny
493
+ symmetric_ar_type ............................... None
494
+ te_rng_tracker .................................. False
495
+ tensor_model_parallel_size ...................... 1
496
+ tensorboard_dir ................................. None
497
+ tensorboard_log_interval ........................ 1
498
+ tensorboard_queue_size .......................... 1000
499
+ test_data_path .................................. None
500
+ test_mode ....................................... False
501
+ tiktoken_num_special_tokens ..................... 1000
502
+ tiktoken_pattern ................................ None
503
+ tiktoken_special_tokens ......................... None
504
+ timing_log_level ................................ 0
505
+ timing_log_option ............................... minmax
506
+ titles_data_path ................................ None
507
+ tokenizer_model ................................. None
508
+ tokenizer_type .................................. GPT2BPETokenizer
509
+ torch_fsdp2_reshard_after_forward ............... True
510
+ tp_comm_bootstrap_backend ....................... nccl
511
+ tp_comm_bulk_dgrad .............................. True
512
+ tp_comm_bulk_wgrad .............................. True
513
+ tp_comm_overlap ................................. False
514
+ tp_comm_overlap_ag .............................. True
515
+ tp_comm_overlap_cfg ............................. None
516
+ tp_comm_overlap_rs .............................. True
517
+ tp_comm_overlap_rs_dgrad ........................ False
518
+ tp_comm_split_ag ................................ True
519
+ tp_comm_split_rs ................................ True
520
+ train_data_path ................................. None
521
+ train_iters ..................................... 12
522
+ train_samples ................................... None
523
+ train_sync_interval ............................. None
524
+ transformer_impl ................................ transformer_engine
525
+ transformer_pipeline_model_parallel_size ........ 1
526
+ untie_embeddings_and_output_weights ............. True
527
+ use_checkpoint_args ............................. False
528
+ use_checkpoint_opt_param_scheduler .............. False
529
+ use_cpu_initialization .......................... None
530
+ use_custom_fsdp ................................. False
531
+ use_dist_ckpt ................................... True
532
+ use_dist_ckpt_deprecated ........................ False
533
+ use_distributed_optimizer ....................... True
534
+ use_flash_attn .................................. False
535
+ use_legacy_models ............................... False
536
+ use_mp_args_from_checkpoint_args ................ False
537
+ use_one_sent_docs ............................... False
538
+ use_persistent_ckpt_worker ...................... False
539
+ use_precision_aware_optimizer ................... False
540
+ use_pytorch_profiler ............................ False
541
+ use_ring_exchange_p2p ........................... False
542
+ use_rope_scaling ................................ False
543
+ use_rotary_position_embeddings .................. False
544
+ use_sharp ....................................... False
545
+ use_tokenizer_model_from_checkpoint_args ........ True
546
+ use_torch_fsdp2 ................................. False
547
+ use_torch_optimizer_for_cpu_offload ............. False
548
+ use_tp_pp_dp_mapping ............................ False
549
+ v_head_dim ...................................... 128
550
+ valid_data_path ................................. None
551
+ variable_seq_lengths ............................ False
552
+ virtual_pipeline_model_parallel_size ............ None
553
+ vision_backbone_type ............................ vit
554
+ vision_pretraining .............................. False
555
+ vision_pretraining_type ......................... classify
556
+ vocab_extra_ids ................................. 0
557
+ vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json
558
+ vocab_size ...................................... None
559
+ wandb_exp_name ..................................
560
+ wandb_project ...................................
561
+ wandb_save_dir ..................................
562
+ weight_decay .................................... 0.1
563
+ weight_decay_incr_style ......................... constant
564
+ wgrad_deferral_limit ............................ 0
565
+ world_size ...................................... 8
566
+ yaml_cfg ........................................ None
567
+ -------------------- end of arguments ---------------------
568
+ INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1
569
+ > building GPT2BPETokenizer tokenizer ...
570
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
571
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
572
+ > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304)
573
+ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS
574
+ > initializing torch distributed ...
575
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
576
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
577
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
578
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
579
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
580
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
581
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
582
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
583
+ [Gloo] Rank [Gloo] Rank 01 is connected to is connected to 7[Gloo] Rank 7 peer ranks. peer ranks. Expected number of connected peer ranks is : 2Expected number of connected peer ranks is : 7 is connected to 77
584
+
585
+ peer ranks. Expected number of connected peer ranks is : 7
586
+ [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
587
+ [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
588
+ [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
589
+ [Gloo] Rank [Gloo] Rank 7 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is :
590
+ 7
591
+ [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 01324 is connected to is connected to is connected to is connected to is connected to 77777 peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77777
592
+
593
+
594
+
595
+
596
+ [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
597
+ [Gloo] Rank 7 is connected to [Gloo] Rank 76 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks.
598
+ Expected number of connected peer ranks is : 7
599
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
600
+ > initialized tensor model parallel with size 1
601
+ > initialized pipeline model parallel with size 1
602
+ > setting random seeds to 1234 ...
603
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
604
+ [Gloo] Rank 0[Gloo] Rank is connected to 00 is connected to peer ranks. [Gloo] Rank 0Expected number of connected peer ranks is : peer ranks. 00Expected number of connected peer ranks is : is connected to
605
+ 00
606
+ peer ranks. Expected number of connected peer ranks is : 0
607
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
608
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
609
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
610
+ [rank6]:[W829 23:43:46.690436463 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
611
+ [rank4]:[W829 23:43:46.690442828 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
612
+ [rank7]:[W829 23:43:46.690509297 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
613
+ [rank1]:[W829 23:43:46.690565089 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
614
+ [rank3]:[W829 23:43:46.690608406 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
615
+ [rank2]:[W829 23:43:46.690613276 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
616
+ [rank5]:[W829 23:43:46.692216962 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
617
+ > compiling dataset index builder ...
618
+ make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets'
619
+ make: Nothing to be done for 'default'.
620
+ make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets'
621
+ >>> done with dataset index builder. Compilation time: 0.035 seconds
622
+ WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations.
623
+ > compiling and loading fused kernels ...
624
+ [rank0]:[W829 23:43:46.847415590 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
625
+ >>> done with compiling and loading fused kernels. Compilation time: 3.763 seconds
626
+ time to initialize megatron (seconds): 7.415
627
+ [after megatron is initialized] datetime: 2026-08-29 23:43:52
628
+ building GPT model ...
629
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
630
+ warnings.warn(
631
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
632
+ warnings.warn(
633
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
634
+ warnings.warn(
635
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
636
+ warnings.warn(
637
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
638
+ warnings.warn(
639
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
640
+ warnings.warn(
641
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
642
+ warnings.warn(
643
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
644
+ warnings.warn(
645
+ > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 5060614656
646
+ INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False)
647
+ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 22
648
+ Params for bucket 1 (180289536 elements, 180289536 padded size):
649
+ module.output_layer.weight
650
+ Params for bucket 2 (73403904 elements, 73403904 padded size):
651
+ module.decoder.final_layernorm.weight
652
+ module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight
653
+ Params for bucket 3 (146800640 elements, 146800640 padded size):
654
+ module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight
655
+ Params for bucket 4 (103001600 elements, 103001600 padded size):
656
+ module.decoder.layers.9.mlp.router.weight
657
+ module.decoder.layers.9.pre_mlp_layernorm.weight
658
+ module.decoder.layers.9.self_attention.linear_qkv.bias
659
+ module.decoder.layers.9.self_attention.linear_proj.weight
660
+ module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight
661
+ module.decoder.layers.9.self_attention.linear_qkv.weight
662
+ module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight
663
+ Params for bucket 5 (146800640 elements, 146800640 padded size):
664
+ module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight
665
+ Params for bucket 6 (103001600 elements, 103001600 padded size):
666
+ module.decoder.layers.8.mlp.router.weight
667
+ module.decoder.layers.8.pre_mlp_layernorm.weight
668
+ module.decoder.layers.8.self_attention.linear_qkv.bias
669
+ module.decoder.layers.8.self_attention.linear_proj.weight
670
+ module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight
671
+ module.decoder.layers.8.self_attention.linear_qkv.weight
672
+ module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight
673
+ Params for bucket 7 (146800640 elements, 146800640 padded size):
674
+ module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight
675
+ Params for bucket 8 (103001600 elements, 103001600 padded size):
676
+ module.decoder.layers.7.mlp.router.weight
677
+ module.decoder.layers.7.self_attention.linear_qkv.bias
678
+ module.decoder.layers.7.pre_mlp_layernorm.weight
679
+ module.decoder.layers.7.self_attention.linear_proj.weight
680
+ module.decoder.layers.7.self_attention.linear_qkv.weight
681
+ module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight
682
+ module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight
683
+ Params for bucket 9 (146800640 elements, 146800640 padded size):
684
+ module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight
685
+ Params for bucket 10 (103001600 elements, 103001600 padded size):
686
+ module.decoder.layers.6.self_attention.linear_qkv.weight
687
+ module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight
688
+ module.decoder.layers.6.mlp.router.weight
689
+ module.decoder.layers.6.self_attention.linear_qkv.bias
690
+ module.decoder.layers.6.pre_mlp_layernorm.weight
691
+ module.decoder.layers.6.self_attention.linear_proj.weight
692
+ module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight
693
+ Params for bucket 11 (146800640 elements, 146800640 padded size):
694
+ module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight
695
+ Params for bucket 12 (103001600 elements, 103001600 padded size):
696
+ module.decoder.layers.5.mlp.router.weight
697
+ module.decoder.layers.5.pre_mlp_layernorm.weight
698
+ module.decoder.layers.5.self_attention.linear_qkv.bias
699
+ module.decoder.layers.5.self_attention.linear_proj.weight
700
+ module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight
701
+ module.decoder.layers.5.self_attention.linear_qkv.weight
702
+ module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight
703
+ Params for bucket 13 (146800640 elements, 146800640 padded size):
704
+ module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight
705
+ Params for bucket 14 (103001600 elements, 103001600 padded size):
706
+ module.decoder.layers.4.mlp.router.weight
707
+ module.decoder.layers.4.self_attention.linear_qkv.bias
708
+ module.decoder.layers.4.pre_mlp_layernorm.weight
709
+ module.decoder.layers.4.self_attention.linear_proj.weight
710
+ module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight
711
+ module.decoder.layers.4.self_attention.linear_qkv.weight
712
+ module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight
713
+ Params for bucket 15 (146800640 elements, 146800640 padded size):
714
+ module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight
715
+ Params for bucket 16 (103001600 elements, 103001600 padded size):
716
+ module.decoder.layers.3.mlp.router.weight
717
+ module.decoder.layers.3.self_attention.linear_qkv.bias
718
+ module.decoder.layers.3.pre_mlp_layernorm.weight
719
+ module.decoder.layers.3.self_attention.linear_proj.weight
720
+ module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight
721
+ module.decoder.layers.3.self_attention.linear_qkv.weight
722
+ module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight
723
+ Params for bucket 17 (146800640 elements, 146800640 padded size):
724
+ module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight
725
+ Params for bucket 18 (103001600 elements, 103001600 padded size):
726
+ module.decoder.layers.2.self_attention.linear_qkv.weight
727
+ module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight
728
+ module.decoder.layers.2.mlp.router.weight
729
+ module.decoder.layers.2.self_attention.linear_qkv.bias
730
+ module.decoder.layers.2.pre_mlp_layernorm.weight
731
+ module.decoder.layers.2.self_attention.linear_proj.weight
732
+ module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight
733
+ Params for bucket 19 (146800640 elements, 146800640 padded size):
734
+ module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight
735
+ Params for bucket 20 (103001600 elements, 103001600 padded size):
736
+ module.decoder.layers.1.mlp.router.weight
737
+ module.decoder.layers.1.pre_mlp_layernorm.weight
738
+ module.decoder.layers.1.self_attention.linear_qkv.bias
739
+ module.decoder.layers.1.self_attention.linear_proj.weight
740
+ module.decoder.layers.0.mlp.shared_experts.linear_fc2.weight
741
+ module.decoder.layers.1.self_attention.linear_qkv.weight
742
+ module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight
743
+ Params for bucket 21 (146800640 elements, 146800640 padded size):
744
+ module.decoder.layers.0.mlp.shared_experts.linear_fc1.weight
745
+ Params for bucket 22 (209890816 elements, 209890816 padded size):
746
+ module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight
747
+ module.decoder.layers.0.self_attention.linear_qkv.bias
748
+ module.decoder.layers.0.mlp.router.weight
749
+ module.decoder.layers.0.self_attention.linear_qkv.weight
750
+ module.decoder.layers.0.pre_mlp_layernorm.weight
751
+ module.decoder.layers.0.self_attention.linear_proj.weight
752
+ module.embedding.word_embeddings.weight
tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/mem_samples.csv CHANGED
@@ -46,3 +46,219 @@
46
  5, 1
47
  6, 1
48
  7, 1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
46
  5, 1
47
  6, 1
48
  7, 1
49
+ 0, 4
50
+ 1, 4
51
+ 2, 4
52
+ 3, 4
53
+ 4, 1
54
+ 5, 1
55
+ 6, 1
56
+ 7, 1
57
+ 0, 4
58
+ 1, 4
59
+ 2, 4
60
+ 3, 4
61
+ 4, 4
62
+ 5, 4
63
+ 6, 4
64
+ 7, 4
65
+ 0, 4
66
+ 1, 45
67
+ 2, 35
68
+ 3, 629
69
+ 4, 629
70
+ 5, 35
71
+ 6, 629
72
+ 7, 629
73
+ 0, 939
74
+ 1, 949
75
+ 2, 941
76
+ 3, 949
77
+ 4, 977
78
+ 5, 951
79
+ 6, 959
80
+ 7, 959
81
+ 0, 1831
82
+ 1, 1831
83
+ 2, 1831
84
+ 3, 1831
85
+ 4, 1831
86
+ 5, 1831
87
+ 6, 1831
88
+ 7, 1831
89
+ 0, 2135
90
+ 1, 2135
91
+ 2, 2135
92
+ 3, 1783
93
+ 4, 1783
94
+ 5, 1783
95
+ 6, 1783
96
+ 7, 1783
97
+ 0, 46689
98
+ 1, 46703
99
+ 2, 46687
100
+ 3, 46675
101
+ 4, 46679
102
+ 5, 46715
103
+ 6, 46693
104
+ 7, 46709
105
+ 0, 47547
106
+ 1, 47549
107
+ 2, 47549
108
+ 3, 47549
109
+ 4, 47549
110
+ 5, 47549
111
+ 6, 47549
112
+ 7, 47549
113
+ 0, 48021
114
+ 1, 47845
115
+ 2, 48027
116
+ 3, 47871
117
+ 4, 47929
118
+ 5, 48093
119
+ 6, 47905
120
+ 7, 48083
121
+ 0, 49085
122
+ 1, 48741
123
+ 2, 49113
124
+ 3, 48743
125
+ 4, 48879
126
+ 5, 49113
127
+ 6, 48741
128
+ 7, 49115
129
+ 0, 49231
130
+ 1, 48889
131
+ 2, 49259
132
+ 3, 48889
133
+ 4, 49025
134
+ 5, 49259
135
+ 6, 48887
136
+ 7, 49261
137
+ 0, 49233
138
+ 1, 48889
139
+ 2, 49261
140
+ 3, 48889
141
+ 4, 49025
142
+ 5, 49261
143
+ 6, 48889
144
+ 7, 49261
145
+ 0, 59143
146
+ 1, 59125
147
+ 2, 59135
148
+ 3, 59147
149
+ 4, 59145
150
+ 5, 59135
151
+ 6, 59147
152
+ 7, 59135
153
+ 0, 83553
154
+ 1, 83539
155
+ 2, 83571
156
+ 3, 83557
157
+ 4, 83563
158
+ 5, 83577
159
+ 6, 83563
160
+ 7, 83577
161
+ 0, 83949
162
+ 1, 83933
163
+ 2, 83967
164
+ 3, 83953
165
+ 4, 83953
166
+ 5, 83967
167
+ 6, 83953
168
+ 7, 83967
169
+ 0, 84199
170
+ 1, 84175
171
+ 2, 84183
172
+ 3, 84225
173
+ 4, 84205
174
+ 5, 84319
175
+ 6, 84241
176
+ 7, 84253
177
+ 0, 84999
178
+ 1, 84983
179
+ 2, 85017
180
+ 3, 85003
181
+ 4, 85003
182
+ 5, 85017
183
+ 6, 85003
184
+ 7, 85017
185
+ 0, 95481
186
+ 1, 95423
187
+ 2, 95525
188
+ 3, 95335
189
+ 4, 95437
190
+ 5, 95557
191
+ 6, 95407
192
+ 7, 95533
193
+ 0, 116419
194
+ 1, 116321
195
+ 2, 116403
196
+ 3, 116243
197
+ 4, 116329
198
+ 5, 116403
199
+ 6, 116291
200
+ 7, 116403
201
+ 0, 116419
202
+ 1, 116321
203
+ 2, 116403
204
+ 3, 116243
205
+ 4, 116329
206
+ 5, 116403
207
+ 6, 116291
208
+ 7, 116403
209
+ 0, 116419
210
+ 1, 116321
211
+ 2, 116403
212
+ 3, 116243
213
+ 4, 116329
214
+ 5, 116403
215
+ 6, 116291
216
+ 7, 116403
217
+ 0, 116419
218
+ 1, 116321
219
+ 2, 116403
220
+ 3, 116243
221
+ 4, 116329
222
+ 5, 116403
223
+ 6, 116291
224
+ 7, 116403
225
+ 0, 116419
226
+ 1, 116321
227
+ 2, 116403
228
+ 3, 115981
229
+ 4, 116009
230
+ 5, 116403
231
+ 6, 116291
232
+ 7, 116403
233
+ 0, 114981
234
+ 1, 114883
235
+ 2, 114965
236
+ 3, 114543
237
+ 4, 114891
238
+ 5, 114965
239
+ 6, 114853
240
+ 7, 114965
241
+ 0, 113297
242
+ 1, 113849
243
+ 2, 113931
244
+ 3, 112795
245
+ 4, 112887
246
+ 5, 113223
247
+ 6, 113819
248
+ 7, 113543
249
+ 0, 111701
250
+ 1, 111747
251
+ 2, 112027
252
+ 3, 64
253
+ 4, 4
254
+ 5, 110581
255
+ 6, 111991
256
+ 7, 112137
257
+ 0, 1
258
+ 1, 13
259
+ 2, 112024
260
+ 3, 61
261
+ 4, 1
262
+ 5, 1
263
+ 6, 1
264
+ 7, 4
tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/train.log CHANGED
The diff for this file is too large to render. See raw diff
 
tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/mem_samples.csv ADDED
@@ -0,0 +1,264 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 4
10
+ 1, 4
11
+ 2, 1
12
+ 3, 1
13
+ 4, 1
14
+ 5, 1
15
+ 6, 1
16
+ 7, 1
17
+ 0, 4
18
+ 1, 4
19
+ 2, 4
20
+ 3, 4
21
+ 4, 4
22
+ 5, 4
23
+ 6, 4
24
+ 7, 1
25
+ 0, 1
26
+ 1, 1
27
+ 2, 1
28
+ 3, 1
29
+ 4, 1
30
+ 5, 3
31
+ 6, 4
32
+ 7, 4
33
+ 0, 0
34
+ 1, 0
35
+ 2, 0
36
+ 3, 0
37
+ 4, 0
38
+ 5, 0
39
+ 6, 0
40
+ 7, 0
41
+ 0, 1
42
+ 1, 1
43
+ 2, 1
44
+ 3, 1
45
+ 4, 1
46
+ 5, 1
47
+ 6, 1
48
+ 7, 1
49
+ 0, 4
50
+ 1, 4
51
+ 2, 4
52
+ 3, 2
53
+ 4, 1
54
+ 5, 1
55
+ 6, 1
56
+ 7, 1
57
+ 0, 4
58
+ 1, 4
59
+ 2, 4
60
+ 3, 4
61
+ 4, 4
62
+ 5, 4
63
+ 6, 4
64
+ 7, 4
65
+ 0, 4
66
+ 1, 629
67
+ 2, 27
68
+ 3, 4
69
+ 4, 629
70
+ 5, 629
71
+ 6, 629
72
+ 7, 629
73
+ 0, 921
74
+ 1, 925
75
+ 2, 903
76
+ 3, 923
77
+ 4, 915
78
+ 5, 919
79
+ 6, 919
80
+ 7, 913
81
+ 0, 1831
82
+ 1, 1831
83
+ 2, 1831
84
+ 3, 1831
85
+ 4, 1831
86
+ 5, 1831
87
+ 6, 1831
88
+ 7, 1831
89
+ 0, 15509
90
+ 1, 15509
91
+ 2, 15509
92
+ 3, 15765
93
+ 4, 15509
94
+ 5, 15369
95
+ 6, 15665
96
+ 7, 16101
97
+ 0, 47207
98
+ 1, 47207
99
+ 2, 47207
100
+ 3, 47207
101
+ 4, 47207
102
+ 5, 47207
103
+ 6, 47207
104
+ 7, 47207
105
+ 0, 47549
106
+ 1, 47551
107
+ 2, 47551
108
+ 3, 47551
109
+ 4, 47551
110
+ 5, 47551
111
+ 6, 47551
112
+ 7, 47551
113
+ 0, 48115
114
+ 1, 47941
115
+ 2, 48143
116
+ 3, 47941
117
+ 4, 47941
118
+ 5, 48143
119
+ 6, 47941
120
+ 7, 48143
121
+ 0, 49011
122
+ 1, 48689
123
+ 2, 49061
124
+ 3, 48691
125
+ 4, 48803
126
+ 5, 49039
127
+ 6, 48685
128
+ 7, 49065
129
+ 0, 49231
130
+ 1, 48889
131
+ 2, 49259
132
+ 3, 48887
133
+ 4, 49023
134
+ 5, 49259
135
+ 6, 48887
136
+ 7, 49259
137
+ 0, 49233
138
+ 1, 48889
139
+ 2, 49261
140
+ 3, 48889
141
+ 4, 49025
142
+ 5, 49261
143
+ 6, 48889
144
+ 7, 49261
145
+ 0, 59143
146
+ 1, 59129
147
+ 2, 59143
148
+ 3, 59147
149
+ 4, 59149
150
+ 5, 59143
151
+ 6, 59131
152
+ 7, 59143
153
+ 0, 83561
154
+ 1, 83497
155
+ 2, 83513
156
+ 3, 83535
157
+ 4, 83539
158
+ 5, 83541
159
+ 6, 83513
160
+ 7, 83517
161
+ 0, 83953
162
+ 1, 83933
163
+ 2, 83951
164
+ 3, 83957
165
+ 4, 83957
166
+ 5, 83951
167
+ 6, 83937
168
+ 7, 83951
169
+ 0, 84343
170
+ 1, 84323
171
+ 2, 84341
172
+ 3, 84347
173
+ 4, 84347
174
+ 5, 84341
175
+ 6, 84327
176
+ 7, 84341
177
+ 0, 87015
178
+ 1, 86951
179
+ 2, 87141
180
+ 3, 87255
181
+ 4, 86939
182
+ 5, 87081
183
+ 6, 87079
184
+ 7, 87021
185
+ 0, 95373
186
+ 1, 95267
187
+ 2, 95385
188
+ 3, 95135
189
+ 4, 95275
190
+ 5, 95395
191
+ 6, 95223
192
+ 7, 95415
193
+ 0, 116515
194
+ 1, 116417
195
+ 2, 116511
196
+ 3, 116283
197
+ 4, 116405
198
+ 5, 116511
199
+ 6, 116359
200
+ 7, 116511
201
+ 0, 116515
202
+ 1, 116417
203
+ 2, 116511
204
+ 3, 116283
205
+ 4, 116405
206
+ 5, 116511
207
+ 6, 116359
208
+ 7, 116511
209
+ 0, 116515
210
+ 1, 116417
211
+ 2, 116511
212
+ 3, 116283
213
+ 4, 116405
214
+ 5, 116511
215
+ 6, 116359
216
+ 7, 116511
217
+ 0, 116515
218
+ 1, 116417
219
+ 2, 116511
220
+ 3, 116283
221
+ 4, 116405
222
+ 5, 116511
223
+ 6, 116359
224
+ 7, 116511
225
+ 0, 116515
226
+ 1, 116417
227
+ 2, 116447
228
+ 3, 116021
229
+ 4, 116405
230
+ 5, 116511
231
+ 6, 116359
232
+ 7, 116511
233
+ 0, 115077
234
+ 1, 114979
235
+ 2, 115073
236
+ 3, 114583
237
+ 4, 114967
238
+ 5, 115073
239
+ 6, 114921
240
+ 7, 115073
241
+ 0, 114043
242
+ 1, 113945
243
+ 2, 113651
244
+ 3, 113161
245
+ 4, 113545
246
+ 5, 114039
247
+ 6, 113499
248
+ 7, 113389
249
+ 0, 109335
250
+ 1, 111833
251
+ 2, 108659
252
+ 3, 4
253
+ 4, 108505
254
+ 5, 111927
255
+ 6, 90649
256
+ 7, 4
257
+ 0, 1
258
+ 1, 111828
259
+ 2, 1
260
+ 3, 1
261
+ 4, 4
262
+ 5, 4
263
+ 6, 4
264
+ 7, 4
tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/run.env ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=moonep
2
+ config=QWEN3_30B
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=moonep_backend_async_grad_reduce_token_routing_analysis
6
+ commit=22fd6865
7
+ moonep_source_dir=/workspace/MoonEP
8
+ moonep_source_branch=megatron-ddp-symmetric-params
9
+ moonep_source_commit=d8bc666
10
+ num_layers=48
11
+ hidden=2048
12
+ ffn_hidden=6144
13
+ moe_ffn_hidden=768
14
+ num_experts=128
15
+ topk=8
16
+ experts_per_rank=16
17
+ expert_params=4718592
18
+ heads=32
19
+ kv_groups=4
20
+ kv_channels=128
21
+ qk_layernorm=1
22
+ shared_ffn=0
23
+ shared_expert_overlap=0
24
+ moe_token_dispatcher=flex
25
+ mla=0
26
+ dense_first_layers=0
27
+ moe_layers=48
28
+ rotary_base=1000000
29
+ rotary_scaling=
30
+ qkv_bias=0
31
+ router_balancing=aux_loss
32
+ aux_loss_coeff=1e-3
33
+ router_pre_softmax=0
34
+ router_fusion=0
35
+ router_dtype=fp32
36
+ device_initiated_grouped_gemm=1
37
+ norm_epsilon=1e-6
38
+ seq_length=4096
39
+ micro_batch_size=1
40
+ global_batch_size=8
41
+ num_microbatches=1
42
+ assignments_per_rank=32768
43
+ mean_tokens_per_expert=2048
44
+ slot_policy=matched
45
+ slots_per_rank=16
46
+ capacity_mode=none
47
+ capacity=none
48
+ cuda_graph=0
49
+ probe_mode=1
50
+ async_grad_reduce=1
51
+ branch_role=probe
52
+ memory_dump_mode=none
53
+ memory_snapshot_path=
54
+ cuda_graph_warmup_steps=3
55
+ recompute=none
56
+ force_balance=1
57
+ train_iters=12
58
+ ep_size=8
59
+ echo_dispatch_overlap=1
60
+ echo_dispatch_recompute=0
61
+ ultraep_quota_min_tokens=
62
+ ultraep_weight_sync_join=
63
+ profile_enable=0
64
+ kernel_trace=0
65
+ fp8_mode=mxfp8
66
+ fp8_recipe=mxfp8
67
+ training_seed=1234
68
+ learning_rate=1e-4
69
+ min_learning_rate=1.0e-5
70
+ lr_warmup_iters=5
71
+ state_dump_enable=0
72
+ state_dump_dir=
73
+ moonep_assert_no_pending_reduce=1
tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv CHANGED
@@ -1,3 +1,8 @@
1
  model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir
2
  QWEN3_235B moonep natural none OK 120129 1 0 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1
3
  QWEN3_235B moonep forced none OK 120081 1 0 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1
 
 
 
 
 
 
1
  model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir
2
  QWEN3_235B moonep natural none OK 120129 1 0 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1
3
  QWEN3_235B moonep forced none OK 120081 1 0 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1
4
+ QWEN3_30B moonep natural none OK 116419 1 0 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1
5
+ QWEN3_30B moonep forced none OK 116515 1 0 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1
6
+ DSV2_LITE_L24 moonep natural none OK 125157 1 0 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1
7
+ DSV2_LITE_L24 moonep forced none OK 125633 1 0 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1
8
+ QWEN2_57B moonep natural none OK 95855 1 0 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/rep1
tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv CHANGED
@@ -1,2 +1,4 @@
1
  model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir
2
  QWEN3_235B moonep SELECTED none 0 120129 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1
 
 
 
1
  model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir
2
  QWEN3_235B moonep SELECTED none 0 120129 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1
3
+ QWEN3_30B moonep SELECTED none 0 116419 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1
4
+ DSV2_LITE_L24 moonep SELECTED none 0 125157 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1
tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/results.tsv CHANGED
@@ -1,3 +1,8 @@
1
  config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
2
  QWEN3_235B moonep matched 16 8192 8 none none 1 OK 370.33 368.00 120129 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1
3
  QWEN3_235B moonep matched 16 8192 8 none none+forcebal 1 OK 399.19 354.00 120081 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1
 
 
 
 
 
 
1
  config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
2
  QWEN3_235B moonep matched 16 8192 8 none none 1 OK 370.33 368.00 120129 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1
3
  QWEN3_235B moonep matched 16 8192 8 none none+forcebal 1 OK 399.19 354.00 120081 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1
4
+ QWEN3_30B moonep matched 16 4096 8 none none 1 OK 829.20 747.60 116419 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1
5
+ QWEN3_30B moonep matched 16 4096 8 none none+forcebal 1 OK 819.26 774.40 116515 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1
6
+ DSV2_LITE_L24 moonep matched 8 8192 8 none none 1 OK 673.29 669.50 125157 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1
7
+ DSV2_LITE_L24 moonep matched 8 8192 8 none none+forcebal 1 OK 674.59 672.10 125633 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1
8
+ QWEN2_57B moonep matched 8 4096 8 none none 1 OK 238.47 236.30 95855 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/rep1
tierb-fixed-moonep-full/study.log CHANGED
@@ -26,3 +26,27 @@ driver_repo: /workspace/blackwell_run_scripts @ 5054c89
26
  [guard] tearing down process group (outcome=DONE)
27
  --------------------------------------------------------------------------
28
  [23:36:36] CELL QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async (slots=16)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
26
  [guard] tearing down process group (outcome=DONE)
27
  --------------------------------------------------------------------------
28
  [23:36:36] CELL QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async (slots=16)
29
+ [guard] iter 8/12 (70s)
30
+ [guard] iter 12/12 (80s)
31
+ [guard] tearing down process group (outcome=DONE)
32
+ --------------------------------------------------------------------------
33
+ [23:38:01] CELL QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__forcebal__async (slots=16)
34
+ [guard] iter 9/12 (70s)
35
+ [guard] iter 12/12 (80s)
36
+ [guard] tearing down process group (outcome=DONE)
37
+ --------------------------------------------------------------------------
38
+ [23:39:25] CELL DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async (slots=8)
39
+ [guard] iter 7/12 (70s)
40
+ [guard] iter 12/12 (80s)
41
+ [guard] tearing down process group (outcome=DONE)
42
+ --------------------------------------------------------------------------
43
+ [23:40:49] CELL DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal__async (slots=8)
44
+ [guard] iter 12/12 (70s)
45
+ [guard] tearing down process group (outcome=DONE)
46
+ --------------------------------------------------------------------------
47
+ [23:42:04] CELL QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async (slots=8)
48
+ [guard] iter 6/12 (60s)
49
+ [guard] iter 12/12 (70s)
50
+ [guard] tearing down process group (outcome=DONE)
51
+ --------------------------------------------------------------------------
52
+ [23:43:17] CELL QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal__async (slots=8)