AbhishekGhosh1998 commited on
Commit
2a772b7
·
verified ·
1 Parent(s): 24020af

granularity results tierb_study_20260826_144644/moonep/tierbprobe @ 2026-08-26T15:12:53Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0

Browse files
Files changed (20) hide show
  1. tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/mem_samples.csv +224 -0
  2. tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/run.env +61 -0
  3. tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/train.log +0 -0
  4. tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/mem_samples.csv +224 -0
  5. tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/run.env +61 -0
  6. tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/train.log +0 -0
  7. tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv +192 -0
  8. tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/run.env +61 -0
  9. tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log +0 -0
  10. tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/mem_samples.csv +192 -0
  11. tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/run.env +61 -0
  12. tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/train.log +0 -0
  13. tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv +224 -0
  14. tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/run.env +61 -0
  15. tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log +0 -0
  16. tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv +5 -0
  17. tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv +3 -0
  18. tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.json +454 -0
  19. tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv +5 -0
  20. tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/results.tsv +5 -0
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/mem_samples.csv ADDED
@@ -0,0 +1,224 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 629
43
+ 2, 629
44
+ 3, 629
45
+ 4, 629
46
+ 5, 629
47
+ 6, 629
48
+ 7, 629
49
+ 0, 891
50
+ 1, 901
51
+ 2, 887
52
+ 3, 889
53
+ 4, 899
54
+ 5, 885
55
+ 6, 897
56
+ 7, 901
57
+ 0, 1955
58
+ 1, 1955
59
+ 2, 1955
60
+ 3, 1955
61
+ 4, 1955
62
+ 5, 1955
63
+ 6, 1955
64
+ 7, 1955
65
+ 0, 3925
66
+ 1, 3937
67
+ 2, 3903
68
+ 3, 3925
69
+ 4, 3941
70
+ 5, 3909
71
+ 6, 3921
72
+ 7, 3895
73
+ 0, 22631
74
+ 1, 22631
75
+ 2, 22631
76
+ 3, 22631
77
+ 4, 22631
78
+ 5, 22631
79
+ 6, 22631
80
+ 7, 22631
81
+ 0, 35805
82
+ 1, 35959
83
+ 2, 35811
84
+ 3, 35851
85
+ 4, 35751
86
+ 5, 35885
87
+ 6, 35829
88
+ 7, 35893
89
+ 0, 45895
90
+ 1, 45969
91
+ 2, 45845
92
+ 3, 45907
93
+ 4, 45879
94
+ 5, 45919
95
+ 6, 45893
96
+ 7, 45985
97
+ 0, 46289
98
+ 1, 46409
99
+ 2, 46283
100
+ 3, 46343
101
+ 4, 46295
102
+ 5, 46339
103
+ 6, 46331
104
+ 7, 46409
105
+ 0, 46291
106
+ 1, 46411
107
+ 2, 46285
108
+ 3, 46345
109
+ 4, 46297
110
+ 5, 46341
111
+ 6, 46333
112
+ 7, 46411
113
+ 0, 69799
114
+ 1, 69857
115
+ 2, 69793
116
+ 3, 69789
117
+ 4, 69869
118
+ 5, 69851
119
+ 6, 69905
120
+ 7, 71969
121
+ 0, 138955
122
+ 1, 138949
123
+ 2, 138923
124
+ 3, 138927
125
+ 4, 138937
126
+ 5, 138947
127
+ 6, 139737
128
+ 7, 138937
129
+ 0, 139647
130
+ 1, 139651
131
+ 2, 139669
132
+ 3, 139659
133
+ 4, 139641
134
+ 5, 139623
135
+ 6, 140457
136
+ 7, 139645
137
+ 0, 139801
138
+ 1, 139793
139
+ 2, 139795
140
+ 3, 139791
141
+ 4, 139807
142
+ 5, 139787
143
+ 6, 140565
144
+ 7, 139793
145
+ 0, 142039
146
+ 1, 142059
147
+ 2, 142053
148
+ 3, 142049
149
+ 4, 142031
150
+ 5, 142003
151
+ 6, 142783
152
+ 7, 142047
153
+ 0, 143185
154
+ 1, 143209
155
+ 2, 143177
156
+ 3, 143173
157
+ 4, 143199
158
+ 5, 143113
159
+ 6, 143935
160
+ 7, 143135
161
+ 0, 157149
162
+ 1, 157133
163
+ 2, 157143
164
+ 3, 157135
165
+ 4, 157151
166
+ 5, 157127
167
+ 6, 156339
168
+ 7, 157133
169
+ 0, 157149
170
+ 1, 157133
171
+ 2, 157143
172
+ 3, 157135
173
+ 4, 157151
174
+ 5, 157127
175
+ 6, 156339
176
+ 7, 157133
177
+ 0, 157149
178
+ 1, 157133
179
+ 2, 157143
180
+ 3, 157135
181
+ 4, 157151
182
+ 5, 157127
183
+ 6, 156339
184
+ 7, 157133
185
+ 0, 157149
186
+ 1, 157133
187
+ 2, 157143
188
+ 3, 157135
189
+ 4, 157151
190
+ 5, 157127
191
+ 6, 156339
192
+ 7, 157133
193
+ 0, 157149
194
+ 1, 157133
195
+ 2, 157143
196
+ 3, 157135
197
+ 4, 157151
198
+ 5, 157127
199
+ 6, 156339
200
+ 7, 157133
201
+ 0, 154621
202
+ 1, 155057
203
+ 2, 155261
204
+ 3, 155315
205
+ 4, 154623
206
+ 5, 154839
207
+ 6, 154845
208
+ 7, 155313
209
+ 0, 152791
210
+ 1, 154217
211
+ 2, 153639
212
+ 3, 153573
213
+ 4, 152875
214
+ 5, 152983
215
+ 6, 152777
216
+ 7, 153459
217
+ 0, 9111
218
+ 1, 9112
219
+ 2, 9112
220
+ 3, 9112
221
+ 4, 9112
222
+ 5, 9112
223
+ 6, 152103
224
+ 7, 9112
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=moonep
2
+ config=DSV2_LITE_L24
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=moonep_backend_token_routing_analysis
6
+ commit=35ae6e7f
7
+ num_layers=24
8
+ hidden=2048
9
+ ffn_hidden=10944
10
+ moe_ffn_hidden=1408
11
+ num_experts=64
12
+ topk=6
13
+ experts_per_rank=8
14
+ expert_params=8650752
15
+ heads=16
16
+ kv_groups=0
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=2816
20
+ shared_expert_overlap=0
21
+ mla=1
22
+ dense_first_layers=1
23
+ moe_layers=23
24
+ rotary_base=10000
25
+ rotary_scaling=40
26
+ qkv_bias=0
27
+ router_balancing=seq_aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=1
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=8192
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=49152
39
+ mean_tokens_per_expert=6144
40
+ slot_policy=matched
41
+ slots_per_rank=8
42
+ capacity_mode=none
43
+ capacity=none
44
+ cuda_graph=0
45
+ probe_mode=1
46
+ async_grad_reduce=0
47
+ branch_role=probe
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/mem_samples.csv ADDED
@@ -0,0 +1,224 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 4
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 4
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 629
42
+ 1, 725
43
+ 2, 725
44
+ 3, 725
45
+ 4, 725
46
+ 5, 725
47
+ 6, 725
48
+ 7, 725
49
+ 0, 969
50
+ 1, 983
51
+ 2, 975
52
+ 3, 997
53
+ 4, 967
54
+ 5, 1007
55
+ 6, 975
56
+ 7, 963
57
+ 0, 1955
58
+ 1, 1955
59
+ 2, 1955
60
+ 3, 1955
61
+ 4, 1955
62
+ 5, 1955
63
+ 6, 1955
64
+ 7, 1955
65
+ 0, 3875
66
+ 1, 3881
67
+ 2, 3897
68
+ 3, 3855
69
+ 4, 3869
70
+ 5, 3873
71
+ 6, 3895
72
+ 7, 3883
73
+ 0, 31783
74
+ 1, 31827
75
+ 2, 31849
76
+ 3, 31805
77
+ 4, 31849
78
+ 5, 31871
79
+ 6, 31871
80
+ 7, 31893
81
+ 0, 35867
82
+ 1, 35991
83
+ 2, 35861
84
+ 3, 35921
85
+ 4, 35873
86
+ 5, 35921
87
+ 6, 35909
88
+ 7, 35991
89
+ 0, 45843
90
+ 1, 46017
91
+ 2, 45833
92
+ 3, 45925
93
+ 4, 45857
94
+ 5, 45869
95
+ 6, 45869
96
+ 7, 45963
97
+ 0, 46289
98
+ 1, 46409
99
+ 2, 46283
100
+ 3, 46343
101
+ 4, 46295
102
+ 5, 46339
103
+ 6, 46331
104
+ 7, 46409
105
+ 0, 46293
106
+ 1, 46411
107
+ 2, 46287
108
+ 3, 46347
109
+ 4, 46299
110
+ 5, 46345
111
+ 6, 46333
112
+ 7, 46415
113
+ 0, 111863
114
+ 1, 111853
115
+ 2, 111859
116
+ 3, 111859
117
+ 4, 111869
118
+ 5, 111853
119
+ 6, 111841
120
+ 7, 111857
121
+ 0, 139413
122
+ 1, 139405
123
+ 2, 139407
124
+ 3, 139401
125
+ 4, 139417
126
+ 5, 139399
127
+ 6, 140177
128
+ 7, 139405
129
+ 0, 139679
130
+ 1, 139691
131
+ 2, 139657
132
+ 3, 139655
133
+ 4, 139695
134
+ 5, 139655
135
+ 6, 140417
136
+ 7, 139705
137
+ 0, 139801
138
+ 1, 139793
139
+ 2, 139795
140
+ 3, 139791
141
+ 4, 139807
142
+ 5, 139787
143
+ 6, 140565
144
+ 7, 139793
145
+ 0, 142111
146
+ 1, 142103
147
+ 2, 142105
148
+ 3, 142101
149
+ 4, 142117
150
+ 5, 142097
151
+ 6, 142875
152
+ 7, 142103
153
+ 0, 143115
154
+ 1, 143077
155
+ 2, 143095
156
+ 3, 143075
157
+ 4, 143125
158
+ 5, 143105
159
+ 6, 143863
160
+ 7, 143101
161
+ 0, 157149
162
+ 1, 157133
163
+ 2, 157143
164
+ 3, 157135
165
+ 4, 157151
166
+ 5, 157127
167
+ 6, 156339
168
+ 7, 157133
169
+ 0, 157149
170
+ 1, 157133
171
+ 2, 157143
172
+ 3, 157135
173
+ 4, 157151
174
+ 5, 157127
175
+ 6, 156339
176
+ 7, 157133
177
+ 0, 157149
178
+ 1, 157133
179
+ 2, 157143
180
+ 3, 157135
181
+ 4, 157151
182
+ 5, 157127
183
+ 6, 156339
184
+ 7, 157133
185
+ 0, 157149
186
+ 1, 157133
187
+ 2, 157143
188
+ 3, 157135
189
+ 4, 157151
190
+ 5, 157127
191
+ 6, 156339
192
+ 7, 157133
193
+ 0, 156503
194
+ 1, 156551
195
+ 2, 157143
196
+ 3, 156809
197
+ 4, 156505
198
+ 5, 156545
199
+ 6, 156339
200
+ 7, 156807
201
+ 0, 154621
202
+ 1, 154669
203
+ 2, 155261
204
+ 3, 154539
205
+ 4, 154623
206
+ 5, 155051
207
+ 6, 154457
208
+ 7, 154925
209
+ 0, 152157
210
+ 1, 152925
211
+ 2, 152935
212
+ 3, 152143
213
+ 4, 152649
214
+ 5, 153155
215
+ 6, 152321
216
+ 7, 152925
217
+ 0, 1
218
+ 1, 4
219
+ 2, 4
220
+ 3, 4
221
+ 4, 4
222
+ 5, 4
223
+ 6, 142237
224
+ 7, 4
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=moonep
2
+ config=DSV2_LITE_L24
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=moonep_backend_token_routing_analysis
6
+ commit=35ae6e7f
7
+ num_layers=24
8
+ hidden=2048
9
+ ffn_hidden=10944
10
+ moe_ffn_hidden=1408
11
+ num_experts=64
12
+ topk=6
13
+ experts_per_rank=8
14
+ expert_params=8650752
15
+ heads=16
16
+ kv_groups=0
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=2816
20
+ shared_expert_overlap=0
21
+ mla=1
22
+ dense_first_layers=1
23
+ moe_layers=23
24
+ rotary_base=10000
25
+ rotary_scaling=40
26
+ qkv_bias=0
27
+ router_balancing=seq_aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=1
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=8192
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=49152
39
+ mean_tokens_per_expert=6144
40
+ slot_policy=matched
41
+ slots_per_rank=8
42
+ capacity_mode=none
43
+ capacity=none
44
+ cuda_graph=0
45
+ probe_mode=1
46
+ async_grad_reduce=0
47
+ branch_role=probe
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=1
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv ADDED
@@ -0,0 +1,192 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 629
43
+ 2, 629
44
+ 3, 629
45
+ 4, 629
46
+ 5, 629
47
+ 6, 629
48
+ 7, 629
49
+ 0, 975
50
+ 1, 971
51
+ 2, 981
52
+ 3, 987
53
+ 4, 973
54
+ 5, 965
55
+ 6, 957
56
+ 7, 953
57
+ 0, 1803
58
+ 1, 1803
59
+ 2, 1803
60
+ 3, 1803
61
+ 4, 1803
62
+ 5, 1803
63
+ 6, 1803
64
+ 7, 1803
65
+ 0, 6689
66
+ 1, 6699
67
+ 2, 6695
68
+ 3, 6701
69
+ 4, 6683
70
+ 5, 6687
71
+ 6, 6693
72
+ 7, 6695
73
+ 0, 58449
74
+ 1, 57903
75
+ 2, 57859
76
+ 3, 58599
77
+ 4, 58603
78
+ 5, 58577
79
+ 6, 58575
80
+ 7, 58567
81
+ 0, 59447
82
+ 1, 58893
83
+ 2, 58873
84
+ 3, 59577
85
+ 4, 59577
86
+ 5, 59575
87
+ 6, 59581
88
+ 7, 59587
89
+ 0, 59719
90
+ 1, 59149
91
+ 2, 59149
92
+ 3, 59849
93
+ 4, 59849
94
+ 5, 59849
95
+ 6, 59849
96
+ 7, 59849
97
+ 0, 60419
98
+ 1, 59851
99
+ 2, 59851
100
+ 3, 60551
101
+ 4, 60551
102
+ 5, 60551
103
+ 6, 60551
104
+ 7, 60551
105
+ 0, 67691
106
+ 1, 67695
107
+ 2, 68433
108
+ 3, 69157
109
+ 4, 68407
110
+ 5, 76049
111
+ 6, 75091
112
+ 7, 69115
113
+ 0, 112523
114
+ 1, 113183
115
+ 2, 114755
116
+ 3, 113143
117
+ 4, 113013
118
+ 5, 117111
119
+ 6, 117111
120
+ 7, 113573
121
+ 0, 118643
122
+ 1, 118685
123
+ 2, 118655
124
+ 3, 118669
125
+ 4, 118675
126
+ 5, 118659
127
+ 6, 118667
128
+ 7, 118707
129
+ 0, 118941
130
+ 1, 118987
131
+ 2, 118967
132
+ 3, 119001
133
+ 4, 119001
134
+ 5, 119001
135
+ 6, 119001
136
+ 7, 119001
137
+ 0, 119103
138
+ 1, 119113
139
+ 2, 119139
140
+ 3, 119151
141
+ 4, 119127
142
+ 5, 119135
143
+ 6, 119151
144
+ 7, 119129
145
+ 0, 128615
146
+ 1, 128353
147
+ 2, 128475
148
+ 3, 128499
149
+ 4, 128445
150
+ 5, 128485
151
+ 6, 128443
152
+ 7, 128461
153
+ 0, 141323
154
+ 1, 141553
155
+ 2, 141527
156
+ 3, 141353
157
+ 4, 141353
158
+ 5, 141353
159
+ 6, 141353
160
+ 7, 141353
161
+ 0, 141323
162
+ 1, 141553
163
+ 2, 141527
164
+ 3, 141353
165
+ 4, 141353
166
+ 5, 141353
167
+ 6, 141353
168
+ 7, 141353
169
+ 0, 140677
170
+ 1, 140581
171
+ 2, 140299
172
+ 3, 140707
173
+ 4, 140707
174
+ 5, 140707
175
+ 6, 140707
176
+ 7, 140707
177
+ 0, 139463
178
+ 1, 139437
179
+ 2, 139239
180
+ 3, 139477
181
+ 4, 139819
182
+ 5, 139557
183
+ 6, 139819
184
+ 7, 139431
185
+ 0, 13561
186
+ 1, 12604
187
+ 2, 12604
188
+ 3, 12604
189
+ 4, 12604
190
+ 5, 12604
191
+ 6, 12796
192
+ 7, 139431
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=moonep
2
+ config=QWEN2_57B
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=moonep_backend_token_routing_analysis
6
+ commit=35ae6e7f
7
+ num_layers=10
8
+ hidden=3584
9
+ ffn_hidden=2560
10
+ moe_ffn_hidden=2560
11
+ num_experts=64
12
+ topk=8
13
+ experts_per_rank=8
14
+ expert_params=27525120
15
+ heads=28
16
+ kv_groups=4
17
+ kv_channels=
18
+ qk_layernorm=0
19
+ shared_ffn=20480
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=10
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=1
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-2
29
+ router_pre_softmax=1
30
+ router_fusion=0
31
+ router_dtype=
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=4096
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=32768
39
+ mean_tokens_per_expert=4096
40
+ slot_policy=matched
41
+ slots_per_rank=8
42
+ capacity_mode=none
43
+ capacity=none
44
+ cuda_graph=0
45
+ probe_mode=1
46
+ async_grad_reduce=0
47
+ branch_role=probe
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/mem_samples.csv ADDED
@@ -0,0 +1,192 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 725
42
+ 1, 725
43
+ 2, 725
44
+ 3, 725
45
+ 4, 725
46
+ 5, 725
47
+ 6, 725
48
+ 7, 725
49
+ 0, 953
50
+ 1, 945
51
+ 2, 959
52
+ 3, 943
53
+ 4, 953
54
+ 5, 955
55
+ 6, 923
56
+ 7, 945
57
+ 0, 1803
58
+ 1, 1803
59
+ 2, 1803
60
+ 3, 1803
61
+ 4, 1803
62
+ 5, 1803
63
+ 6, 1803
64
+ 7, 1803
65
+ 0, 6793
66
+ 1, 6799
67
+ 2, 6801
68
+ 3, 6819
69
+ 4, 6785
70
+ 5, 6795
71
+ 6, 6803
72
+ 7, 6801
73
+ 0, 58463
74
+ 1, 57881
75
+ 2, 57909
76
+ 3, 58565
77
+ 4, 58555
78
+ 5, 58609
79
+ 6, 58607
80
+ 7, 58573
81
+ 0, 59479
82
+ 1, 58911
83
+ 2, 58911
84
+ 3, 59611
85
+ 4, 59611
86
+ 5, 59611
87
+ 6, 59611
88
+ 7, 59611
89
+ 0, 59719
90
+ 1, 59151
91
+ 2, 59151
92
+ 3, 59851
93
+ 4, 59851
94
+ 5, 59851
95
+ 6, 59851
96
+ 7, 59851
97
+ 0, 60419
98
+ 1, 59851
99
+ 2, 59851
100
+ 3, 60551
101
+ 4, 60551
102
+ 5, 60551
103
+ 6, 60551
104
+ 7, 60551
105
+ 0, 86691
106
+ 1, 86427
107
+ 2, 86883
108
+ 3, 85141
109
+ 4, 86917
110
+ 5, 86117
111
+ 6, 86711
112
+ 7, 89399
113
+ 0, 117915
114
+ 1, 117957
115
+ 2, 117943
116
+ 3, 117989
117
+ 4, 117977
118
+ 5, 117985
119
+ 6, 117987
120
+ 7, 117993
121
+ 0, 118939
122
+ 1, 118983
123
+ 2, 118967
124
+ 3, 119001
125
+ 4, 119001
126
+ 5, 119001
127
+ 6, 119001
128
+ 7, 119001
129
+ 0, 118939
130
+ 1, 118983
131
+ 2, 118967
132
+ 3, 119001
133
+ 4, 119001
134
+ 5, 119001
135
+ 6, 119001
136
+ 7, 119001
137
+ 0, 119537
138
+ 1, 119599
139
+ 2, 119569
140
+ 3, 119615
141
+ 4, 119607
142
+ 5, 119617
143
+ 6, 119609
144
+ 7, 119607
145
+ 0, 128707
146
+ 1, 128371
147
+ 2, 128547
148
+ 3, 128579
149
+ 4, 128525
150
+ 5, 128523
151
+ 6, 128521
152
+ 7, 128469
153
+ 0, 141321
154
+ 1, 141551
155
+ 2, 141525
156
+ 3, 141351
157
+ 4, 141351
158
+ 5, 141351
159
+ 6, 141351
160
+ 7, 141351
161
+ 0, 141321
162
+ 1, 141551
163
+ 2, 141525
164
+ 3, 141351
165
+ 4, 141351
166
+ 5, 141351
167
+ 6, 141351
168
+ 7, 141351
169
+ 0, 140675
170
+ 1, 140905
171
+ 2, 140879
172
+ 3, 140705
173
+ 4, 140705
174
+ 5, 140705
175
+ 6, 140705
176
+ 7, 140705
177
+ 0, 139401
178
+ 1, 139305
179
+ 2, 138959
180
+ 3, 138785
181
+ 4, 138785
182
+ 5, 138785
183
+ 6, 138785
184
+ 7, 138849
185
+ 0, 12601
186
+ 1, 12604
187
+ 2, 12732
188
+ 3, 138783
189
+ 4, 13244
190
+ 5, 12604
191
+ 6, 12604
192
+ 7, 12604
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=moonep
2
+ config=QWEN2_57B
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=moonep_backend_token_routing_analysis
6
+ commit=35ae6e7f
7
+ num_layers=10
8
+ hidden=3584
9
+ ffn_hidden=2560
10
+ moe_ffn_hidden=2560
11
+ num_experts=64
12
+ topk=8
13
+ experts_per_rank=8
14
+ expert_params=27525120
15
+ heads=28
16
+ kv_groups=4
17
+ kv_channels=
18
+ qk_layernorm=0
19
+ shared_ffn=20480
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=10
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=1
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-2
29
+ router_pre_softmax=1
30
+ router_fusion=0
31
+ router_dtype=
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=4096
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=32768
39
+ mean_tokens_per_expert=4096
40
+ slot_policy=matched
41
+ slots_per_rank=8
42
+ capacity_mode=none
43
+ capacity=none
44
+ cuda_graph=0
45
+ probe_mode=1
46
+ async_grad_reduce=0
47
+ branch_role=probe
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=1
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv ADDED
@@ -0,0 +1,224 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 4
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 4
43
+ 2, 4
44
+ 3, 4
45
+ 4, 4
46
+ 5, 4
47
+ 6, 4
48
+ 7, 4
49
+ 0, 821
50
+ 1, 821
51
+ 2, 821
52
+ 3, 821
53
+ 4, 821
54
+ 5, 821
55
+ 6, 821
56
+ 7, 821
57
+ 0, 1831
58
+ 1, 1831
59
+ 2, 1831
60
+ 3, 1831
61
+ 4, 1831
62
+ 5, 1831
63
+ 6, 1831
64
+ 7, 1831
65
+ 0, 3735
66
+ 1, 3745
67
+ 2, 3723
68
+ 3, 3727
69
+ 4, 3737
70
+ 5, 3743
71
+ 6, 3733
72
+ 7, 3725
73
+ 0, 15371
74
+ 1, 15373
75
+ 2, 15373
76
+ 3, 15373
77
+ 4, 15373
78
+ 5, 15373
79
+ 6, 15373
80
+ 7, 15373
81
+ 0, 55341
82
+ 1, 55343
83
+ 2, 55343
84
+ 3, 55343
85
+ 4, 55343
86
+ 5, 55343
87
+ 6, 55343
88
+ 7, 55343
89
+ 0, 67357
90
+ 1, 67409
91
+ 2, 67367
92
+ 3, 67315
93
+ 4, 67353
94
+ 5, 67413
95
+ 6, 67383
96
+ 7, 67383
97
+ 0, 68527
98
+ 1, 68565
99
+ 2, 68543
100
+ 3, 68493
101
+ 4, 68529
102
+ 5, 68543
103
+ 6, 68565
104
+ 7, 68543
105
+ 0, 68673
106
+ 1, 68711
107
+ 2, 68689
108
+ 3, 68639
109
+ 4, 68675
110
+ 5, 68689
111
+ 6, 68711
112
+ 7, 68689
113
+ 0, 68675
114
+ 1, 68713
115
+ 2, 68691
116
+ 3, 68641
117
+ 4, 68677
118
+ 5, 68691
119
+ 6, 68713
120
+ 7, 68691
121
+ 0, 71871
122
+ 1, 71625
123
+ 2, 71653
124
+ 3, 71883
125
+ 4, 71605
126
+ 5, 72187
127
+ 6, 71755
128
+ 7, 71857
129
+ 0, 117205
130
+ 1, 114877
131
+ 2, 116895
132
+ 3, 117519
133
+ 4, 114367
134
+ 5, 117101
135
+ 6, 116825
136
+ 7, 116873
137
+ 0, 152069
138
+ 1, 149561
139
+ 2, 150911
140
+ 3, 152077
141
+ 4, 148449
142
+ 5, 150911
143
+ 6, 150223
144
+ 7, 150475
145
+ 0, 156087
146
+ 1, 156099
147
+ 2, 156085
148
+ 3, 156093
149
+ 4, 156089
150
+ 5, 156085
151
+ 6, 156101
152
+ 7, 156085
153
+ 0, 156777
154
+ 1, 156735
155
+ 2, 156791
156
+ 3, 156803
157
+ 4, 156741
158
+ 5, 156731
159
+ 6, 156711
160
+ 7, 156763
161
+ 0, 156877
162
+ 1, 156889
163
+ 2, 156875
164
+ 3, 156883
165
+ 4, 156879
166
+ 5, 156875
167
+ 6, 156891
168
+ 7, 156875
169
+ 0, 157525
170
+ 1, 157537
171
+ 2, 157523
172
+ 3, 157531
173
+ 4, 157527
174
+ 5, 157523
175
+ 6, 157539
176
+ 7, 157523
177
+ 0, 167421
178
+ 1, 167335
179
+ 2, 167449
180
+ 3, 167299
181
+ 4, 167619
182
+ 5, 167435
183
+ 6, 167361
184
+ 7, 167425
185
+ 0, 182487
186
+ 1, 182571
187
+ 2, 182505
188
+ 3, 182525
189
+ 4, 182511
190
+ 5, 182505
191
+ 6, 182625
192
+ 7, 182505
193
+ 0, 182487
194
+ 1, 182571
195
+ 2, 182505
196
+ 3, 182525
197
+ 4, 182511
198
+ 5, 182505
199
+ 6, 182625
200
+ 7, 182505
201
+ 0, 182487
202
+ 1, 182571
203
+ 2, 182505
204
+ 3, 182525
205
+ 4, 182511
206
+ 5, 182505
207
+ 6, 182625
208
+ 7, 182505
209
+ 0, 182487
210
+ 1, 182571
211
+ 2, 182505
212
+ 3, 182525
213
+ 4, 182511
214
+ 5, 182505
215
+ 6, 182625
216
+ 7, 182505
217
+ 0, 182487
218
+ 1, 182571
219
+ 2, 182505
220
+ 3, 182525
221
+ 4, 182511
222
+ 5, 182505
223
+ 6, 182625
224
+ 7, 182505
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=moonep
2
+ config=QWEN3_30B
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=moonep_backend_token_routing_analysis
6
+ commit=35ae6e7f
7
+ num_layers=48
8
+ hidden=2048
9
+ ffn_hidden=6144
10
+ moe_ffn_hidden=768
11
+ num_experts=128
12
+ topk=8
13
+ experts_per_rank=16
14
+ expert_params=4718592
15
+ heads=32
16
+ kv_groups=4
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=0
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=48
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=0
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=0
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=4096
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=32768
39
+ mean_tokens_per_expert=2048
40
+ slot_policy=matched
41
+ slots_per_rank=16
42
+ capacity_mode=none
43
+ capacity=none
44
+ cuda_graph=0
45
+ probe_mode=1
46
+ async_grad_reduce=0
47
+ branch_role=probe
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv CHANGED
@@ -1,2 +1,7 @@
1
  model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir
2
  QWEN3_235B moonep natural none OOM 182587 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1
 
 
 
 
 
 
1
  model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir
2
  QWEN3_235B moonep natural none OOM 182587 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1
3
+ QWEN3_30B moonep natural none OOM 182625 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1
4
+ DSV2_LITE_L24 moonep natural none OK 157151 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1
5
+ DSV2_LITE_L24 moonep forced none OK 157151 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1
6
+ QWEN2_57B moonep natural none OK 141553 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1
7
+ QWEN2_57B moonep forced none OK 141551 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv CHANGED
@@ -1,2 +1,5 @@
1
  model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir
2
  QWEN3_235B moonep ERROR_OOM 182587 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1
 
 
 
 
1
  model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir
2
  QWEN3_235B moonep ERROR_OOM 182587 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1
3
+ QWEN3_30B moonep ERROR_OOM 182625 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1
4
+ DSV2_LITE_L24 moonep SELECTED none 0 157151 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1
5
+ QWEN2_57B moonep SELECTED none 0 141553 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.json ADDED
@@ -0,0 +1,454 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "arm": "natural",
4
+ "aux_loss_coeff": "1e-3",
5
+ "backend": "moonep",
6
+ "branch": "moonep_backend_token_routing_analysis",
7
+ "capacity": "none",
8
+ "capacity_mode": "none",
9
+ "cold_expert_fraction_max": 0.578125,
10
+ "cold_expert_fraction_mean": 0.15625,
11
+ "cold_expert_fraction_min": 0.0,
12
+ "cold_expert_fraction_n": 230,
13
+ "commit": "35ae6e7f",
14
+ "config": "DSV2_LITE_L24",
15
+ "cuda_graph": "0",
16
+ "dense_first_layers": "1",
17
+ "experts_per_rank": "8",
18
+ "force_balance": "0",
19
+ "global_batch_size": "8",
20
+ "global_snapshots": 230,
21
+ "hottest_expert_token_fraction_max": 0.999969482421875,
22
+ "hottest_expert_token_fraction_mean": 0.7797577567722487,
23
+ "hottest_expert_token_fraction_min": 0.162750244140625,
24
+ "hottest_expert_token_fraction_n": 230,
25
+ "incomplete_snapshots": 0,
26
+ "malformed_records": 0,
27
+ "moe_layers": "23",
28
+ "moonep_bytes_global_snapshots": 230,
29
+ "moonep_bytes_incomplete_snapshots": 0,
30
+ "moonep_bytes_malformed_records": 0,
31
+ "moonep_global_combine_payload_bytes_max": 2592899072.0,
32
+ "moonep_global_combine_payload_bytes_mean": 2282338571.130435,
33
+ "moonep_global_combine_payload_bytes_min": 2026799104.0,
34
+ "moonep_global_combine_payload_bytes_n": 230,
35
+ "moonep_global_dispatch_payload_bytes_max": 2592899072.0,
36
+ "moonep_global_dispatch_payload_bytes_mean": 2282338571.130435,
37
+ "moonep_global_dispatch_payload_bytes_min": 2026799104.0,
38
+ "moonep_global_dispatch_payload_bytes_n": 230,
39
+ "moonep_global_prefetch_bytes_max": 346030080.0,
40
+ "moonep_global_prefetch_bytes_mean": 252601958.4,
41
+ "moonep_global_prefetch_bytes_min": 242221056.0,
42
+ "moonep_global_prefetch_bytes_n": 230,
43
+ "moonep_global_reduce_bytes_max": 346030080.0,
44
+ "moonep_global_reduce_bytes_mean": 252601958.4,
45
+ "moonep_global_reduce_bytes_min": 242221056.0,
46
+ "moonep_global_reduce_bytes_n": 230,
47
+ "moonep_global_token_bytes_max": 5188482536.0,
48
+ "moonep_global_token_bytes_mean": 4567339392.869565,
49
+ "moonep_global_token_bytes_min": 4056064752.0,
50
+ "moonep_global_token_bytes_n": 230,
51
+ "moonep_global_total_nvlink_bytes_max": 5672924648.0,
52
+ "moonep_global_total_nvlink_bytes_mean": 5072543309.669565,
53
+ "moonep_global_total_nvlink_bytes_min": 4540506864.0,
54
+ "moonep_global_total_nvlink_bytes_n": 230,
55
+ "moonep_global_weight_movement_bytes_max": 692060160.0,
56
+ "moonep_global_weight_movement_bytes_mean": 505203916.8,
57
+ "moonep_global_weight_movement_bytes_min": 484442112.0,
58
+ "moonep_global_weight_movement_bytes_n": 230,
59
+ "moonep_global_weight_to_token_ratio_max": 0.14851639788299403,
60
+ "moonep_global_weight_to_token_ratio_mean": 0.11096021713514241,
61
+ "moonep_global_weight_to_token_ratio_min": 0.09336874676530664,
62
+ "moonep_global_weight_to_token_ratio_n": 230,
63
+ "moonep_remote_demand_max": 3.0,
64
+ "moonep_remote_demand_mean": 0.9125,
65
+ "moonep_remote_demand_min": 0.0,
66
+ "moonep_remote_demand_n": 1840,
67
+ "moonep_slot_utilization_max": 0.375,
68
+ "moonep_slot_utilization_mean": 0.1140625,
69
+ "moonep_slot_utilization_min": 0.0,
70
+ "moonep_slot_utilization_n": 1840,
71
+ "moonep_unused_prefetch_slots_max": 16.0,
72
+ "moonep_unused_prefetch_slots_mean": 14.175,
73
+ "moonep_unused_prefetch_slots_min": 10.0,
74
+ "moonep_unused_prefetch_slots_n": 1840,
75
+ "num_experts": "64",
76
+ "num_layers": "24",
77
+ "post_physical_rank_cv_max": 0.24644338469771185,
78
+ "post_physical_rank_cv_mean": 0.0866314272142518,
79
+ "post_physical_rank_cv_min": 0.016159622971429973,
80
+ "post_physical_rank_cv_n": 230,
81
+ "post_physical_rank_imax_max": 1.2001146488570558,
82
+ "post_physical_rank_imax_mean": 1.1121655106296537,
83
+ "post_physical_rank_imax_min": 1.0188653892666146,
84
+ "post_physical_rank_imax_n": 230,
85
+ "pre_home_rank_cv_max": 1.3626457501059894,
86
+ "pre_home_rank_cv_mean": 0.6592086328754161,
87
+ "pre_home_rank_cv_min": 0.06474057713157837,
88
+ "pre_home_rank_cv_n": 230,
89
+ "pre_home_rank_imax_max": 4.328043619791667,
90
+ "pre_home_rank_imax_mean": 2.2370494897814766,
91
+ "pre_home_rank_imax_min": 1.1056925455729167,
92
+ "pre_home_rank_imax_n": 230,
93
+ "raw_expert_cv_max": 3.0868156557140782,
94
+ "raw_expert_cv_mean": 1.9632045302354437,
95
+ "raw_expert_cv_min": 0.37852206422070017,
96
+ "raw_expert_cv_n": 230,
97
+ "raw_expert_imax_ceiling": 10.666666666666666,
98
+ "raw_expert_imax_hit_ceiling": 1,
99
+ "raw_expert_imax_max": 10.666341145833334,
100
+ "raw_expert_imax_mean": 8.317416072237318,
101
+ "raw_expert_imax_min": 1.7360026041666667,
102
+ "raw_expert_imax_n": 230,
103
+ "records": {
104
+ "moonep_bytes": 1840,
105
+ "moonep_routing": 1840,
106
+ "moonep_summary": 80,
107
+ "moonep_weights": 1840
108
+ },
109
+ "router_balancing": "seq_aux_loss",
110
+ "run_dir": "DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1",
111
+ "seq_length": "8192",
112
+ "shared_expert_overlap": "0",
113
+ "topk": "6"
114
+ },
115
+ {
116
+ "arm": "forced",
117
+ "aux_loss_coeff": "1e-3",
118
+ "backend": "moonep",
119
+ "branch": "moonep_backend_token_routing_analysis",
120
+ "capacity": "none",
121
+ "capacity_mode": "none",
122
+ "cold_expert_fraction_max": 0.0,
123
+ "cold_expert_fraction_mean": 0.0,
124
+ "cold_expert_fraction_min": 0.0,
125
+ "cold_expert_fraction_n": 230,
126
+ "commit": "35ae6e7f",
127
+ "config": "DSV2_LITE_L24",
128
+ "cuda_graph": "0",
129
+ "dense_first_layers": "1",
130
+ "experts_per_rank": "8",
131
+ "force_balance": "1",
132
+ "global_batch_size": "8",
133
+ "global_snapshots": 230,
134
+ "hottest_expert_token_fraction_max": 0.0982818603515625,
135
+ "hottest_expert_token_fraction_mean": 0.09643501613451087,
136
+ "hottest_expert_token_fraction_min": 0.09539794921875,
137
+ "hottest_expert_token_fraction_n": 230,
138
+ "incomplete_snapshots": 0,
139
+ "malformed_records": 0,
140
+ "moe_layers": "23",
141
+ "moonep_bytes_global_snapshots": 230,
142
+ "moonep_bytes_incomplete_snapshots": 0,
143
+ "moonep_bytes_malformed_records": 0,
144
+ "moonep_global_combine_payload_bytes_max": 2133762048.0,
145
+ "moonep_global_combine_payload_bytes_mean": 2129465023.4434783,
146
+ "moonep_global_combine_payload_bytes_min": 2124537856.0,
147
+ "moonep_global_combine_payload_bytes_n": 230,
148
+ "moonep_global_dispatch_payload_bytes_max": 2133762048.0,
149
+ "moonep_global_dispatch_payload_bytes_mean": 2129465023.4434783,
150
+ "moonep_global_dispatch_payload_bytes_min": 2124537856.0,
151
+ "moonep_global_dispatch_payload_bytes_n": 230,
152
+ "moonep_global_prefetch_bytes_max": 242221056.0,
153
+ "moonep_global_prefetch_bytes_mean": 238309411.61739132,
154
+ "moonep_global_prefetch_bytes_min": 207618048.0,
155
+ "moonep_global_prefetch_bytes_n": 230,
156
+ "moonep_global_reduce_bytes_max": 242221056.0,
157
+ "moonep_global_reduce_bytes_mean": 238309411.61739132,
158
+ "moonep_global_reduce_bytes_min": 207618048.0,
159
+ "moonep_global_reduce_bytes_n": 230,
160
+ "moonep_global_token_bytes_max": 4270279408.0,
161
+ "moonep_global_token_bytes_mean": 4261681018.2608695,
162
+ "moonep_global_token_bytes_min": 4251824656.0,
163
+ "moonep_global_token_bytes_n": 230,
164
+ "moonep_global_total_nvlink_bytes_max": 4754721520.0,
165
+ "moonep_global_total_nvlink_bytes_mean": 4738299841.495652,
166
+ "moonep_global_total_nvlink_bytes_min": 4667434440.0,
167
+ "moonep_global_total_nvlink_bytes_n": 230,
168
+ "moonep_global_weight_movement_bytes_max": 484442112.0,
169
+ "moonep_global_weight_movement_bytes_mean": 476618823.23478264,
170
+ "moonep_global_weight_movement_bytes_min": 415236096.0,
171
+ "moonep_global_weight_movement_bytes_n": 230,
172
+ "moonep_global_weight_to_token_ratio_max": 0.11393746242954192,
173
+ "moonep_global_weight_to_token_ratio_mean": 0.11183841293995254,
174
+ "moonep_global_weight_to_token_ratio_min": 0.09728537714366937,
175
+ "moonep_global_weight_to_token_ratio_n": 230,
176
+ "moonep_remote_demand_max": 1.0,
177
+ "moonep_remote_demand_mean": 0.8608695652173913,
178
+ "moonep_remote_demand_min": 0.0,
179
+ "moonep_remote_demand_n": 1840,
180
+ "moonep_slot_utilization_max": 0.125,
181
+ "moonep_slot_utilization_mean": 0.10760869565217392,
182
+ "moonep_slot_utilization_min": 0.0,
183
+ "moonep_slot_utilization_n": 1840,
184
+ "moonep_unused_prefetch_slots_max": 16.0,
185
+ "moonep_unused_prefetch_slots_mean": 14.278260869565218,
186
+ "moonep_unused_prefetch_slots_min": 14.0,
187
+ "moonep_unused_prefetch_slots_n": 1840,
188
+ "num_experts": "64",
189
+ "num_layers": "24",
190
+ "post_physical_rank_cv_max": 0.0031667036922078573,
191
+ "post_physical_rank_cv_mean": 0.0016501228109707747,
192
+ "post_physical_rank_cv_min": 0.0006791931370339787,
193
+ "post_physical_rank_cv_n": 230,
194
+ "post_physical_rank_imax_max": 1.006201201251668,
195
+ "post_physical_rank_imax_mean": 1.0026117364992109,
196
+ "post_physical_rank_imax_min": 1.0009196904719393,
197
+ "post_physical_rank_imax_n": 230,
198
+ "pre_home_rank_cv_max": 0.006673533732490181,
199
+ "pre_home_rank_cv_mean": 0.003923942735502633,
200
+ "pre_home_rank_cv_min": 0.0014708009315538319,
201
+ "pre_home_rank_cv_n": 230,
202
+ "pre_home_rank_imax_max": 1.0122884114583333,
203
+ "pre_home_rank_imax_mean": 1.0061054616734602,
204
+ "pre_home_rank_imax_min": 1.00213623046875,
205
+ "pre_home_rank_imax_n": 230,
206
+ "raw_expert_cv_max": 0.015123587213954862,
207
+ "raw_expert_cv_mean": 0.012068912140234289,
208
+ "raw_expert_cv_min": 0.008171875787738593,
209
+ "raw_expert_cv_n": 230,
210
+ "raw_expert_imax_ceiling": 10.666666666666666,
211
+ "raw_expert_imax_hit_ceiling": 0,
212
+ "raw_expert_imax_max": 1.04833984375,
213
+ "raw_expert_imax_mean": 1.0286401721014493,
214
+ "raw_expert_imax_min": 1.017578125,
215
+ "raw_expert_imax_n": 230,
216
+ "records": {
217
+ "moonep_bytes": 1840,
218
+ "moonep_routing": 1840,
219
+ "moonep_summary": 80,
220
+ "moonep_weights": 1840
221
+ },
222
+ "router_balancing": "seq_aux_loss",
223
+ "run_dir": "DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1",
224
+ "seq_length": "8192",
225
+ "shared_expert_overlap": "0",
226
+ "topk": "6"
227
+ },
228
+ {
229
+ "arm": "natural",
230
+ "aux_loss_coeff": "1e-2",
231
+ "backend": "moonep",
232
+ "branch": "moonep_backend_token_routing_analysis",
233
+ "capacity": "none",
234
+ "capacity_mode": "none",
235
+ "cold_expert_fraction_max": 0.796875,
236
+ "cold_expert_fraction_mean": 0.39234375,
237
+ "cold_expert_fraction_min": 0.0,
238
+ "cold_expert_fraction_n": 100,
239
+ "commit": "35ae6e7f",
240
+ "config": "QWEN2_57B",
241
+ "cuda_graph": "0",
242
+ "dense_first_layers": "0",
243
+ "experts_per_rank": "8",
244
+ "force_balance": "0",
245
+ "global_batch_size": "8",
246
+ "global_snapshots": 100,
247
+ "hottest_expert_token_fraction_max": 1.0,
248
+ "hottest_expert_token_fraction_mean": 0.9268069458007813,
249
+ "hottest_expert_token_fraction_min": 0.2415771484375,
250
+ "hottest_expert_token_fraction_n": 100,
251
+ "incomplete_snapshots": 0,
252
+ "malformed_records": 0,
253
+ "moe_layers": "10",
254
+ "moonep_bytes_global_snapshots": 100,
255
+ "moonep_bytes_incomplete_snapshots": 0,
256
+ "moonep_bytes_malformed_records": 0,
257
+ "moonep_global_combine_payload_bytes_max": 3287445504.0,
258
+ "moonep_global_combine_payload_bytes_mean": 2789546475.52,
259
+ "moonep_global_combine_payload_bytes_min": 2180663296.0,
260
+ "moonep_global_combine_payload_bytes_n": 100,
261
+ "moonep_global_dispatch_payload_bytes_max": 3287445504.0,
262
+ "moonep_global_dispatch_payload_bytes_mean": 2789546475.52,
263
+ "moonep_global_dispatch_payload_bytes_min": 2180663296.0,
264
+ "moonep_global_dispatch_payload_bytes_n": 100,
265
+ "moonep_global_prefetch_bytes_max": 990904320.0,
266
+ "moonep_global_prefetch_bytes_mean": 823551590.4,
267
+ "moonep_global_prefetch_bytes_min": 660602880.0,
268
+ "moonep_global_prefetch_bytes_n": 100,
269
+ "moonep_global_reduce_bytes_max": 990904320.0,
270
+ "moonep_global_reduce_bytes_mean": 823551590.4,
271
+ "moonep_global_reduce_bytes_min": 660602880.0,
272
+ "moonep_global_reduce_bytes_n": 100,
273
+ "moonep_global_token_bytes_max": 6576725968.0,
274
+ "moonep_global_token_bytes_mean": 5580893210.72,
275
+ "moonep_global_token_bytes_min": 4363119160.0,
276
+ "moonep_global_token_bytes_n": 100,
277
+ "moonep_global_total_nvlink_bytes_max": 8328785856.0,
278
+ "moonep_global_total_nvlink_bytes_mean": 7227996391.52,
279
+ "moonep_global_total_nvlink_bytes_min": 5904525880.0,
280
+ "moonep_global_total_nvlink_bytes_n": 100,
281
+ "moonep_global_weight_movement_bytes_max": 1981808640.0,
282
+ "moonep_global_weight_movement_bytes_mean": 1647103180.8,
283
+ "moonep_global_weight_movement_bytes_min": 1321205760.0,
284
+ "moonep_global_weight_movement_bytes_n": 100,
285
+ "moonep_global_weight_to_token_ratio_max": 0.4380461607026548,
286
+ "moonep_global_weight_to_token_ratio_mean": 0.2995223715370738,
287
+ "moonep_global_weight_to_token_ratio_min": 0.20089110697762314,
288
+ "moonep_global_weight_to_token_ratio_n": 100,
289
+ "moonep_remote_demand_max": 2.0,
290
+ "moonep_remote_demand_mean": 0.935,
291
+ "moonep_remote_demand_min": 0.0,
292
+ "moonep_remote_demand_n": 800,
293
+ "moonep_slot_utilization_max": 0.25,
294
+ "moonep_slot_utilization_mean": 0.116875,
295
+ "moonep_slot_utilization_min": 0.0,
296
+ "moonep_slot_utilization_n": 800,
297
+ "moonep_unused_prefetch_slots_max": 16.0,
298
+ "moonep_unused_prefetch_slots_mean": 14.13,
299
+ "moonep_unused_prefetch_slots_min": 12.0,
300
+ "moonep_unused_prefetch_slots_n": 800,
301
+ "num_experts": "64",
302
+ "num_layers": "10",
303
+ "post_physical_rank_cv_max": 0.20167894084637375,
304
+ "post_physical_rank_cv_mean": 0.08583132480611003,
305
+ "post_physical_rank_cv_min": 0.00019943885935289461,
306
+ "post_physical_rank_cv_n": 100,
307
+ "post_physical_rank_imax_max": 1.2419115350424064,
308
+ "post_physical_rank_imax_mean": 1.1036399766472182,
309
+ "post_physical_rank_imax_min": 1.0002518334236372,
310
+ "post_physical_rank_imax_n": 100,
311
+ "pre_home_rank_cv_max": 1.4500421285803669,
312
+ "pre_home_rank_cv_mean": 0.7161558574760766,
313
+ "pre_home_rank_cv_min": 0.06359546419183554,
314
+ "pre_home_rank_cv_n": 100,
315
+ "pre_home_rank_imax_max": 4.147735595703125,
316
+ "pre_home_rank_imax_mean": 2.322430419921875,
317
+ "pre_home_rank_imax_min": 1.11138916015625,
318
+ "pre_home_rank_imax_n": 100,
319
+ "raw_expert_cv_max": 2.6452554407770648,
320
+ "raw_expert_cv_mean": 2.184735306615502,
321
+ "raw_expert_cv_min": 0.4993102636020994,
322
+ "raw_expert_cv_n": 100,
323
+ "raw_expert_imax_ceiling": 8.0,
324
+ "raw_expert_imax_hit_ceiling": 1,
325
+ "raw_expert_imax_max": 8.0,
326
+ "raw_expert_imax_mean": 7.41445556640625,
327
+ "raw_expert_imax_min": 1.9326171875,
328
+ "raw_expert_imax_n": 100,
329
+ "records": {
330
+ "moonep_bytes": 800,
331
+ "moonep_routing": 800,
332
+ "moonep_summary": 80,
333
+ "moonep_weights": 800
334
+ },
335
+ "router_balancing": "aux_loss",
336
+ "run_dir": "QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1",
337
+ "seq_length": "4096",
338
+ "shared_expert_overlap": "0",
339
+ "topk": "8"
340
+ },
341
+ {
342
+ "arm": "forced",
343
+ "aux_loss_coeff": "1e-2",
344
+ "backend": "moonep",
345
+ "branch": "moonep_backend_token_routing_analysis",
346
+ "capacity": "none",
347
+ "capacity_mode": "none",
348
+ "cold_expert_fraction_max": 0.0,
349
+ "cold_expert_fraction_mean": 0.0,
350
+ "cold_expert_fraction_min": 0.0,
351
+ "cold_expert_fraction_n": 100,
352
+ "commit": "35ae6e7f",
353
+ "config": "QWEN2_57B",
354
+ "cuda_graph": "0",
355
+ "dense_first_layers": "0",
356
+ "experts_per_rank": "8",
357
+ "force_balance": "1",
358
+ "global_batch_size": "8",
359
+ "global_snapshots": 100,
360
+ "hottest_expert_token_fraction_max": 0.1317138671875,
361
+ "hottest_expert_token_fraction_mean": 0.12946014404296874,
362
+ "hottest_expert_token_fraction_min": 0.128021240234375,
363
+ "hottest_expert_token_fraction_n": 100,
364
+ "incomplete_snapshots": 0,
365
+ "malformed_records": 0,
366
+ "moe_layers": "10",
367
+ "moonep_bytes_global_snapshots": 100,
368
+ "moonep_bytes_incomplete_snapshots": 0,
369
+ "moonep_bytes_malformed_records": 0,
370
+ "moonep_global_combine_payload_bytes_max": 2237204480.0,
371
+ "moonep_global_combine_payload_bytes_mean": 2231120138.24,
372
+ "moonep_global_combine_payload_bytes_min": 2224459776.0,
373
+ "moonep_global_combine_payload_bytes_n": 100,
374
+ "moonep_global_dispatch_payload_bytes_max": 2237204480.0,
375
+ "moonep_global_dispatch_payload_bytes_mean": 2231120138.24,
376
+ "moonep_global_dispatch_payload_bytes_min": 2224459776.0,
377
+ "moonep_global_dispatch_payload_bytes_n": 100,
378
+ "moonep_global_prefetch_bytes_max": 770703360.0,
379
+ "moonep_global_prefetch_bytes_mean": 762996326.4,
380
+ "moonep_global_prefetch_bytes_min": 660602880.0,
381
+ "moonep_global_prefetch_bytes_n": 100,
382
+ "moonep_global_reduce_bytes_max": 770703360.0,
383
+ "moonep_global_reduce_bytes_mean": 762996326.4,
384
+ "moonep_global_reduce_bytes_min": 660602880.0,
385
+ "moonep_global_reduce_bytes_n": 100,
386
+ "moonep_global_token_bytes_max": 4476243024.0,
387
+ "moonep_global_token_bytes_mean": 4464073623.92,
388
+ "moonep_global_token_bytes_min": 4450749096.0,
389
+ "moonep_global_token_bytes_n": 100,
390
+ "moonep_global_total_nvlink_bytes_max": 6017649744.0,
391
+ "moonep_global_total_nvlink_bytes_mean": 5990066276.72,
392
+ "moonep_global_total_nvlink_bytes_min": 5771954856.0,
393
+ "moonep_global_total_nvlink_bytes_n": 100,
394
+ "moonep_global_weight_movement_bytes_max": 1541406720.0,
395
+ "moonep_global_weight_movement_bytes_mean": 1525992652.8,
396
+ "moonep_global_weight_movement_bytes_min": 1321205760.0,
397
+ "moonep_global_weight_movement_bytes_n": 100,
398
+ "moonep_global_weight_to_token_ratio_max": 0.3460953541488461,
399
+ "moonep_global_weight_to_token_ratio_mean": 0.34183623349354486,
400
+ "moonep_global_weight_to_token_ratio_min": 0.2956613023580652,
401
+ "moonep_global_weight_to_token_ratio_n": 100,
402
+ "moonep_remote_demand_max": 1.0,
403
+ "moonep_remote_demand_mean": 0.86625,
404
+ "moonep_remote_demand_min": 0.0,
405
+ "moonep_remote_demand_n": 800,
406
+ "moonep_slot_utilization_max": 0.125,
407
+ "moonep_slot_utilization_mean": 0.10828125,
408
+ "moonep_slot_utilization_min": 0.0,
409
+ "moonep_slot_utilization_n": 800,
410
+ "moonep_unused_prefetch_slots_max": 16.0,
411
+ "moonep_unused_prefetch_slots_mean": 14.2675,
412
+ "moonep_unused_prefetch_slots_min": 14.0,
413
+ "moonep_unused_prefetch_slots_n": 800,
414
+ "num_experts": "64",
415
+ "num_layers": "10",
416
+ "post_physical_rank_cv_max": 0.004173107469722948,
417
+ "post_physical_rank_cv_mean": 0.0021866389418670955,
418
+ "post_physical_rank_cv_min": 0.0008634926254374552,
419
+ "post_physical_rank_cv_n": 100,
420
+ "post_physical_rank_imax_max": 1.0063246661981728,
421
+ "post_physical_rank_imax_mean": 1.0033137485288246,
422
+ "post_physical_rank_imax_min": 1.001134997246789,
423
+ "post_physical_rank_imax_n": 100,
424
+ "pre_home_rank_cv_max": 0.00995108219738984,
425
+ "pre_home_rank_cv_mean": 0.0057066499140920315,
426
+ "pre_home_rank_cv_min": 0.0010965114882337452,
427
+ "pre_home_rank_cv_n": 100,
428
+ "pre_home_rank_imax_max": 1.014678955078125,
429
+ "pre_home_rank_imax_mean": 1.009002685546875,
430
+ "pre_home_rank_imax_min": 1.001373291015625,
431
+ "pre_home_rank_imax_n": 100,
432
+ "raw_expert_cv_max": 0.0179363414576349,
433
+ "raw_expert_cv_mean": 0.014900499877977456,
434
+ "raw_expert_cv_min": 0.011926166451983852,
435
+ "raw_expert_cv_n": 100,
436
+ "raw_expert_imax_ceiling": 8.0,
437
+ "raw_expert_imax_hit_ceiling": 0,
438
+ "raw_expert_imax_max": 1.0537109375,
439
+ "raw_expert_imax_mean": 1.03568115234375,
440
+ "raw_expert_imax_min": 1.024169921875,
441
+ "raw_expert_imax_n": 100,
442
+ "records": {
443
+ "moonep_bytes": 800,
444
+ "moonep_routing": 800,
445
+ "moonep_summary": 80,
446
+ "moonep_weights": 800
447
+ },
448
+ "router_balancing": "aux_loss",
449
+ "run_dir": "QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1",
450
+ "seq_length": "4096",
451
+ "shared_expert_overlap": "0",
452
+ "topk": "8"
453
+ }
454
+ ]
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ run_dir arm config backend num_experts topk experts_per_rank num_layers moe_layers dense_first_layers seq_length global_batch_size capacity_mode capacity force_balance cuda_graph branch commit router_balancing aux_loss_coeff shared_expert_overlap cold_expert_fraction_max cold_expert_fraction_mean cold_expert_fraction_min cold_expert_fraction_n global_snapshots hottest_expert_token_fraction_max hottest_expert_token_fraction_mean hottest_expert_token_fraction_min hottest_expert_token_fraction_n incomplete_snapshots malformed_records moonep_bytes_global_snapshots moonep_bytes_incomplete_snapshots moonep_bytes_malformed_records moonep_global_combine_payload_bytes_max moonep_global_combine_payload_bytes_mean moonep_global_combine_payload_bytes_min moonep_global_combine_payload_bytes_n moonep_global_dispatch_payload_bytes_max moonep_global_dispatch_payload_bytes_mean moonep_global_dispatch_payload_bytes_min moonep_global_dispatch_payload_bytes_n moonep_global_prefetch_bytes_max moonep_global_prefetch_bytes_mean moonep_global_prefetch_bytes_min moonep_global_prefetch_bytes_n moonep_global_reduce_bytes_max moonep_global_reduce_bytes_mean moonep_global_reduce_bytes_min moonep_global_reduce_bytes_n moonep_global_token_bytes_max moonep_global_token_bytes_mean moonep_global_token_bytes_min moonep_global_token_bytes_n moonep_global_total_nvlink_bytes_max moonep_global_total_nvlink_bytes_mean moonep_global_total_nvlink_bytes_min moonep_global_total_nvlink_bytes_n moonep_global_weight_movement_bytes_max moonep_global_weight_movement_bytes_mean moonep_global_weight_movement_bytes_min moonep_global_weight_movement_bytes_n moonep_global_weight_to_token_ratio_max moonep_global_weight_to_token_ratio_mean moonep_global_weight_to_token_ratio_min moonep_global_weight_to_token_ratio_n moonep_remote_demand_max moonep_remote_demand_mean moonep_remote_demand_min moonep_remote_demand_n moonep_slot_utilization_max moonep_slot_utilization_mean moonep_slot_utilization_min moonep_slot_utilization_n moonep_unused_prefetch_slots_max moonep_unused_prefetch_slots_mean moonep_unused_prefetch_slots_min moonep_unused_prefetch_slots_n post_physical_rank_cv_max post_physical_rank_cv_mean post_physical_rank_cv_min post_physical_rank_cv_n post_physical_rank_imax_max post_physical_rank_imax_mean post_physical_rank_imax_min post_physical_rank_imax_n pre_home_rank_cv_max pre_home_rank_cv_mean pre_home_rank_cv_min pre_home_rank_cv_n pre_home_rank_imax_max pre_home_rank_imax_mean pre_home_rank_imax_min pre_home_rank_imax_n raw_expert_cv_max raw_expert_cv_mean raw_expert_cv_min raw_expert_cv_n raw_expert_imax_ceiling raw_expert_imax_hit_ceiling raw_expert_imax_max raw_expert_imax_mean raw_expert_imax_min raw_expert_imax_n
2
+ DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1 natural DSV2_LITE_L24 moonep 64 6 8 24 23 1 8192 8 none none 0 0 moonep_backend_token_routing_analysis 35ae6e7f seq_aux_loss 1e-3 0 0.578125 0.15625 0.0 230 230 0.999969482421875 0.7797577567722487 0.162750244140625 230 0 0 230 0 0 2592899072.0 2282338571.130435 2026799104.0 230 2592899072.0 2282338571.130435 2026799104.0 230 346030080.0 252601958.4 242221056.0 230 346030080.0 252601958.4 242221056.0 230 5188482536.0 4567339392.869565 4056064752.0 230 5672924648.0 5072543309.669565 4540506864.0 230 692060160.0 505203916.8 484442112.0 230 0.14851639788299403 0.11096021713514241 0.09336874676530664 230 3.0 0.9125 0.0 1840 0.375 0.1140625 0.0 1840 16.0 14.175 10.0 1840 0.24644338469771185 0.0866314272142518 0.016159622971429973 230 1.2001146488570558 1.1121655106296537 1.0188653892666146 230 1.3626457501059894 0.6592086328754161 0.06474057713157837 230 4.328043619791667 2.2370494897814766 1.1056925455729167 230 3.0868156557140782 1.9632045302354437 0.37852206422070017 230 10.666666666666666 1 10.666341145833334 8.317416072237318 1.7360026041666667 230
3
+ DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1 forced DSV2_LITE_L24 moonep 64 6 8 24 23 1 8192 8 none none 1 0 moonep_backend_token_routing_analysis 35ae6e7f seq_aux_loss 1e-3 0 0.0 0.0 0.0 230 230 0.0982818603515625 0.09643501613451087 0.09539794921875 230 0 0 230 0 0 2133762048.0 2129465023.4434783 2124537856.0 230 2133762048.0 2129465023.4434783 2124537856.0 230 242221056.0 238309411.61739132 207618048.0 230 242221056.0 238309411.61739132 207618048.0 230 4270279408.0 4261681018.2608695 4251824656.0 230 4754721520.0 4738299841.495652 4667434440.0 230 484442112.0 476618823.23478264 415236096.0 230 0.11393746242954192 0.11183841293995254 0.09728537714366937 230 1.0 0.8608695652173913 0.0 1840 0.125 0.10760869565217392 0.0 1840 16.0 14.278260869565218 14.0 1840 0.0031667036922078573 0.0016501228109707747 0.0006791931370339787 230 1.006201201251668 1.0026117364992109 1.0009196904719393 230 0.006673533732490181 0.003923942735502633 0.0014708009315538319 230 1.0122884114583333 1.0061054616734602 1.00213623046875 230 0.015123587213954862 0.012068912140234289 0.008171875787738593 230 10.666666666666666 0 1.04833984375 1.0286401721014493 1.017578125 230
4
+ QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1 natural QWEN2_57B moonep 64 8 8 10 10 0 4096 8 none none 0 0 moonep_backend_token_routing_analysis 35ae6e7f aux_loss 1e-2 0 0.796875 0.39234375 0.0 100 100 1.0 0.9268069458007813 0.2415771484375 100 0 0 100 0 0 3287445504.0 2789546475.52 2180663296.0 100 3287445504.0 2789546475.52 2180663296.0 100 990904320.0 823551590.4 660602880.0 100 990904320.0 823551590.4 660602880.0 100 6576725968.0 5580893210.72 4363119160.0 100 8328785856.0 7227996391.52 5904525880.0 100 1981808640.0 1647103180.8 1321205760.0 100 0.4380461607026548 0.2995223715370738 0.20089110697762314 100 2.0 0.935 0.0 800 0.25 0.116875 0.0 800 16.0 14.13 12.0 800 0.20167894084637375 0.08583132480611003 0.00019943885935289461 100 1.2419115350424064 1.1036399766472182 1.0002518334236372 100 1.4500421285803669 0.7161558574760766 0.06359546419183554 100 4.147735595703125 2.322430419921875 1.11138916015625 100 2.6452554407770648 2.184735306615502 0.4993102636020994 100 8.0 1 8.0 7.41445556640625 1.9326171875 100
5
+ QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1 forced QWEN2_57B moonep 64 8 8 10 10 0 4096 8 none none 1 0 moonep_backend_token_routing_analysis 35ae6e7f aux_loss 1e-2 0 0.0 0.0 0.0 100 100 0.1317138671875 0.12946014404296874 0.128021240234375 100 0 0 100 0 0 2237204480.0 2231120138.24 2224459776.0 100 2237204480.0 2231120138.24 2224459776.0 100 770703360.0 762996326.4 660602880.0 100 770703360.0 762996326.4 660602880.0 100 4476243024.0 4464073623.92 4450749096.0 100 6017649744.0 5990066276.72 5771954856.0 100 1541406720.0 1525992652.8 1321205760.0 100 0.3460953541488461 0.34183623349354486 0.2956613023580652 100 1.0 0.86625 0.0 800 0.125 0.10828125 0.0 800 16.0 14.2675 14.0 800 0.004173107469722948 0.0021866389418670955 0.0008634926254374552 100 1.0063246661981728 1.0033137485288246 1.001134997246789 100 0.00995108219738984 0.0057066499140920315 0.0010965114882337452 100 1.014678955078125 1.009002685546875 1.001373291015625 100 0.0179363414576349 0.014900499877977456 0.011926166451983852 100 8.0 0 1.0537109375 1.03568115234375 1.024169921875 100
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/results.tsv CHANGED
@@ -1,2 +1,7 @@
1
  config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
2
  QWEN3_235B moonep matched 16 8192 8 none none 1 OOM NA NA 182587 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1
 
 
 
 
 
 
1
  config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
2
  QWEN3_235B moonep matched 16 8192 8 none none 1 OOM NA NA 182587 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1
3
+ QWEN3_30B moonep matched 16 4096 8 none none 1 OOM NA NA 182625 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1
4
+ DSV2_LITE_L24 moonep matched 8 8192 8 none none 1 OK 848.31 791.80 157151 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1
5
+ DSV2_LITE_L24 moonep matched 8 8192 8 none none+forcebal 1 OK 791.39 788.20 157151 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1
6
+ QWEN2_57B moonep matched 8 4096 8 none none 1 OK 366.93 362.10 141553 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1
7
+ QWEN2_57B moonep matched 8 4096 8 none none+forcebal 1 OK 407.76 358.70 141551 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1