granularity results tierb_study_20260826_144644/moonep/tierbprobe @ 2026-08-26T15:12:53Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0
Browse files- tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/mem_samples.csv +224 -0
- tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/run.env +61 -0
- tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/train.log +0 -0
- tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/mem_samples.csv +224 -0
- tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/run.env +61 -0
- tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/train.log +0 -0
- tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv +192 -0
- tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/run.env +61 -0
- tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log +0 -0
- tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/mem_samples.csv +192 -0
- tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/run.env +61 -0
- tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/train.log +0 -0
- tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv +224 -0
- tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/run.env +61 -0
- tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log +0 -0
- tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv +5 -0
- tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv +3 -0
- tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.json +454 -0
- tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv +5 -0
- tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/results.tsv +5 -0
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/mem_samples.csv
ADDED
|
@@ -0,0 +1,224 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
0, 0
|
| 2 |
+
1, 0
|
| 3 |
+
2, 0
|
| 4 |
+
3, 0
|
| 5 |
+
4, 0
|
| 6 |
+
5, 0
|
| 7 |
+
6, 0
|
| 8 |
+
7, 0
|
| 9 |
+
0, 0
|
| 10 |
+
1, 0
|
| 11 |
+
2, 0
|
| 12 |
+
3, 0
|
| 13 |
+
4, 0
|
| 14 |
+
5, 0
|
| 15 |
+
6, 0
|
| 16 |
+
7, 0
|
| 17 |
+
0, 1
|
| 18 |
+
1, 1
|
| 19 |
+
2, 1
|
| 20 |
+
3, 1
|
| 21 |
+
4, 1
|
| 22 |
+
5, 1
|
| 23 |
+
6, 1
|
| 24 |
+
7, 1
|
| 25 |
+
0, 4
|
| 26 |
+
1, 4
|
| 27 |
+
2, 4
|
| 28 |
+
3, 4
|
| 29 |
+
4, 1
|
| 30 |
+
5, 1
|
| 31 |
+
6, 1
|
| 32 |
+
7, 1
|
| 33 |
+
0, 4
|
| 34 |
+
1, 4
|
| 35 |
+
2, 4
|
| 36 |
+
3, 4
|
| 37 |
+
4, 4
|
| 38 |
+
5, 4
|
| 39 |
+
6, 4
|
| 40 |
+
7, 4
|
| 41 |
+
0, 4
|
| 42 |
+
1, 629
|
| 43 |
+
2, 629
|
| 44 |
+
3, 629
|
| 45 |
+
4, 629
|
| 46 |
+
5, 629
|
| 47 |
+
6, 629
|
| 48 |
+
7, 629
|
| 49 |
+
0, 891
|
| 50 |
+
1, 901
|
| 51 |
+
2, 887
|
| 52 |
+
3, 889
|
| 53 |
+
4, 899
|
| 54 |
+
5, 885
|
| 55 |
+
6, 897
|
| 56 |
+
7, 901
|
| 57 |
+
0, 1955
|
| 58 |
+
1, 1955
|
| 59 |
+
2, 1955
|
| 60 |
+
3, 1955
|
| 61 |
+
4, 1955
|
| 62 |
+
5, 1955
|
| 63 |
+
6, 1955
|
| 64 |
+
7, 1955
|
| 65 |
+
0, 3925
|
| 66 |
+
1, 3937
|
| 67 |
+
2, 3903
|
| 68 |
+
3, 3925
|
| 69 |
+
4, 3941
|
| 70 |
+
5, 3909
|
| 71 |
+
6, 3921
|
| 72 |
+
7, 3895
|
| 73 |
+
0, 22631
|
| 74 |
+
1, 22631
|
| 75 |
+
2, 22631
|
| 76 |
+
3, 22631
|
| 77 |
+
4, 22631
|
| 78 |
+
5, 22631
|
| 79 |
+
6, 22631
|
| 80 |
+
7, 22631
|
| 81 |
+
0, 35805
|
| 82 |
+
1, 35959
|
| 83 |
+
2, 35811
|
| 84 |
+
3, 35851
|
| 85 |
+
4, 35751
|
| 86 |
+
5, 35885
|
| 87 |
+
6, 35829
|
| 88 |
+
7, 35893
|
| 89 |
+
0, 45895
|
| 90 |
+
1, 45969
|
| 91 |
+
2, 45845
|
| 92 |
+
3, 45907
|
| 93 |
+
4, 45879
|
| 94 |
+
5, 45919
|
| 95 |
+
6, 45893
|
| 96 |
+
7, 45985
|
| 97 |
+
0, 46289
|
| 98 |
+
1, 46409
|
| 99 |
+
2, 46283
|
| 100 |
+
3, 46343
|
| 101 |
+
4, 46295
|
| 102 |
+
5, 46339
|
| 103 |
+
6, 46331
|
| 104 |
+
7, 46409
|
| 105 |
+
0, 46291
|
| 106 |
+
1, 46411
|
| 107 |
+
2, 46285
|
| 108 |
+
3, 46345
|
| 109 |
+
4, 46297
|
| 110 |
+
5, 46341
|
| 111 |
+
6, 46333
|
| 112 |
+
7, 46411
|
| 113 |
+
0, 69799
|
| 114 |
+
1, 69857
|
| 115 |
+
2, 69793
|
| 116 |
+
3, 69789
|
| 117 |
+
4, 69869
|
| 118 |
+
5, 69851
|
| 119 |
+
6, 69905
|
| 120 |
+
7, 71969
|
| 121 |
+
0, 138955
|
| 122 |
+
1, 138949
|
| 123 |
+
2, 138923
|
| 124 |
+
3, 138927
|
| 125 |
+
4, 138937
|
| 126 |
+
5, 138947
|
| 127 |
+
6, 139737
|
| 128 |
+
7, 138937
|
| 129 |
+
0, 139647
|
| 130 |
+
1, 139651
|
| 131 |
+
2, 139669
|
| 132 |
+
3, 139659
|
| 133 |
+
4, 139641
|
| 134 |
+
5, 139623
|
| 135 |
+
6, 140457
|
| 136 |
+
7, 139645
|
| 137 |
+
0, 139801
|
| 138 |
+
1, 139793
|
| 139 |
+
2, 139795
|
| 140 |
+
3, 139791
|
| 141 |
+
4, 139807
|
| 142 |
+
5, 139787
|
| 143 |
+
6, 140565
|
| 144 |
+
7, 139793
|
| 145 |
+
0, 142039
|
| 146 |
+
1, 142059
|
| 147 |
+
2, 142053
|
| 148 |
+
3, 142049
|
| 149 |
+
4, 142031
|
| 150 |
+
5, 142003
|
| 151 |
+
6, 142783
|
| 152 |
+
7, 142047
|
| 153 |
+
0, 143185
|
| 154 |
+
1, 143209
|
| 155 |
+
2, 143177
|
| 156 |
+
3, 143173
|
| 157 |
+
4, 143199
|
| 158 |
+
5, 143113
|
| 159 |
+
6, 143935
|
| 160 |
+
7, 143135
|
| 161 |
+
0, 157149
|
| 162 |
+
1, 157133
|
| 163 |
+
2, 157143
|
| 164 |
+
3, 157135
|
| 165 |
+
4, 157151
|
| 166 |
+
5, 157127
|
| 167 |
+
6, 156339
|
| 168 |
+
7, 157133
|
| 169 |
+
0, 157149
|
| 170 |
+
1, 157133
|
| 171 |
+
2, 157143
|
| 172 |
+
3, 157135
|
| 173 |
+
4, 157151
|
| 174 |
+
5, 157127
|
| 175 |
+
6, 156339
|
| 176 |
+
7, 157133
|
| 177 |
+
0, 157149
|
| 178 |
+
1, 157133
|
| 179 |
+
2, 157143
|
| 180 |
+
3, 157135
|
| 181 |
+
4, 157151
|
| 182 |
+
5, 157127
|
| 183 |
+
6, 156339
|
| 184 |
+
7, 157133
|
| 185 |
+
0, 157149
|
| 186 |
+
1, 157133
|
| 187 |
+
2, 157143
|
| 188 |
+
3, 157135
|
| 189 |
+
4, 157151
|
| 190 |
+
5, 157127
|
| 191 |
+
6, 156339
|
| 192 |
+
7, 157133
|
| 193 |
+
0, 157149
|
| 194 |
+
1, 157133
|
| 195 |
+
2, 157143
|
| 196 |
+
3, 157135
|
| 197 |
+
4, 157151
|
| 198 |
+
5, 157127
|
| 199 |
+
6, 156339
|
| 200 |
+
7, 157133
|
| 201 |
+
0, 154621
|
| 202 |
+
1, 155057
|
| 203 |
+
2, 155261
|
| 204 |
+
3, 155315
|
| 205 |
+
4, 154623
|
| 206 |
+
5, 154839
|
| 207 |
+
6, 154845
|
| 208 |
+
7, 155313
|
| 209 |
+
0, 152791
|
| 210 |
+
1, 154217
|
| 211 |
+
2, 153639
|
| 212 |
+
3, 153573
|
| 213 |
+
4, 152875
|
| 214 |
+
5, 152983
|
| 215 |
+
6, 152777
|
| 216 |
+
7, 153459
|
| 217 |
+
0, 9111
|
| 218 |
+
1, 9112
|
| 219 |
+
2, 9112
|
| 220 |
+
3, 9112
|
| 221 |
+
4, 9112
|
| 222 |
+
5, 9112
|
| 223 |
+
6, 152103
|
| 224 |
+
7, 9112
|
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/run.env
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=moonep
|
| 2 |
+
config=DSV2_LITE_L24
|
| 3 |
+
family=tierB
|
| 4 |
+
repo=/workspace/Megatron-MoE-Echo
|
| 5 |
+
branch=moonep_backend_token_routing_analysis
|
| 6 |
+
commit=35ae6e7f
|
| 7 |
+
num_layers=24
|
| 8 |
+
hidden=2048
|
| 9 |
+
ffn_hidden=10944
|
| 10 |
+
moe_ffn_hidden=1408
|
| 11 |
+
num_experts=64
|
| 12 |
+
topk=6
|
| 13 |
+
experts_per_rank=8
|
| 14 |
+
expert_params=8650752
|
| 15 |
+
heads=16
|
| 16 |
+
kv_groups=0
|
| 17 |
+
kv_channels=128
|
| 18 |
+
qk_layernorm=1
|
| 19 |
+
shared_ffn=2816
|
| 20 |
+
shared_expert_overlap=0
|
| 21 |
+
mla=1
|
| 22 |
+
dense_first_layers=1
|
| 23 |
+
moe_layers=23
|
| 24 |
+
rotary_base=10000
|
| 25 |
+
rotary_scaling=40
|
| 26 |
+
qkv_bias=0
|
| 27 |
+
router_balancing=seq_aux_loss
|
| 28 |
+
aux_loss_coeff=1e-3
|
| 29 |
+
router_pre_softmax=1
|
| 30 |
+
router_fusion=0
|
| 31 |
+
router_dtype=fp32
|
| 32 |
+
device_initiated_grouped_gemm=1
|
| 33 |
+
norm_epsilon=1e-6
|
| 34 |
+
seq_length=8192
|
| 35 |
+
micro_batch_size=1
|
| 36 |
+
global_batch_size=8
|
| 37 |
+
num_microbatches=1
|
| 38 |
+
assignments_per_rank=49152
|
| 39 |
+
mean_tokens_per_expert=6144
|
| 40 |
+
slot_policy=matched
|
| 41 |
+
slots_per_rank=8
|
| 42 |
+
capacity_mode=none
|
| 43 |
+
capacity=none
|
| 44 |
+
cuda_graph=0
|
| 45 |
+
probe_mode=1
|
| 46 |
+
async_grad_reduce=0
|
| 47 |
+
branch_role=probe
|
| 48 |
+
memory_dump_mode=none
|
| 49 |
+
memory_snapshot_path=
|
| 50 |
+
cuda_graph_warmup_steps=3
|
| 51 |
+
recompute=none
|
| 52 |
+
force_balance=0
|
| 53 |
+
train_iters=12
|
| 54 |
+
ep_size=8
|
| 55 |
+
echo_dispatch_overlap=1
|
| 56 |
+
echo_dispatch_recompute=0
|
| 57 |
+
ultraep_quota_min_tokens=
|
| 58 |
+
ultraep_weight_sync_join=
|
| 59 |
+
profile_enable=0
|
| 60 |
+
kernel_trace=0
|
| 61 |
+
fp8_recipe=mxfp8
|
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/train.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/mem_samples.csv
ADDED
|
@@ -0,0 +1,224 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
0, 0
|
| 2 |
+
1, 0
|
| 3 |
+
2, 0
|
| 4 |
+
3, 0
|
| 5 |
+
4, 0
|
| 6 |
+
5, 0
|
| 7 |
+
6, 0
|
| 8 |
+
7, 0
|
| 9 |
+
0, 0
|
| 10 |
+
1, 0
|
| 11 |
+
2, 0
|
| 12 |
+
3, 0
|
| 13 |
+
4, 0
|
| 14 |
+
5, 0
|
| 15 |
+
6, 0
|
| 16 |
+
7, 0
|
| 17 |
+
0, 4
|
| 18 |
+
1, 1
|
| 19 |
+
2, 1
|
| 20 |
+
3, 1
|
| 21 |
+
4, 1
|
| 22 |
+
5, 1
|
| 23 |
+
6, 1
|
| 24 |
+
7, 1
|
| 25 |
+
0, 4
|
| 26 |
+
1, 4
|
| 27 |
+
2, 4
|
| 28 |
+
3, 4
|
| 29 |
+
4, 4
|
| 30 |
+
5, 1
|
| 31 |
+
6, 1
|
| 32 |
+
7, 1
|
| 33 |
+
0, 4
|
| 34 |
+
1, 4
|
| 35 |
+
2, 4
|
| 36 |
+
3, 4
|
| 37 |
+
4, 4
|
| 38 |
+
5, 4
|
| 39 |
+
6, 4
|
| 40 |
+
7, 4
|
| 41 |
+
0, 629
|
| 42 |
+
1, 725
|
| 43 |
+
2, 725
|
| 44 |
+
3, 725
|
| 45 |
+
4, 725
|
| 46 |
+
5, 725
|
| 47 |
+
6, 725
|
| 48 |
+
7, 725
|
| 49 |
+
0, 969
|
| 50 |
+
1, 983
|
| 51 |
+
2, 975
|
| 52 |
+
3, 997
|
| 53 |
+
4, 967
|
| 54 |
+
5, 1007
|
| 55 |
+
6, 975
|
| 56 |
+
7, 963
|
| 57 |
+
0, 1955
|
| 58 |
+
1, 1955
|
| 59 |
+
2, 1955
|
| 60 |
+
3, 1955
|
| 61 |
+
4, 1955
|
| 62 |
+
5, 1955
|
| 63 |
+
6, 1955
|
| 64 |
+
7, 1955
|
| 65 |
+
0, 3875
|
| 66 |
+
1, 3881
|
| 67 |
+
2, 3897
|
| 68 |
+
3, 3855
|
| 69 |
+
4, 3869
|
| 70 |
+
5, 3873
|
| 71 |
+
6, 3895
|
| 72 |
+
7, 3883
|
| 73 |
+
0, 31783
|
| 74 |
+
1, 31827
|
| 75 |
+
2, 31849
|
| 76 |
+
3, 31805
|
| 77 |
+
4, 31849
|
| 78 |
+
5, 31871
|
| 79 |
+
6, 31871
|
| 80 |
+
7, 31893
|
| 81 |
+
0, 35867
|
| 82 |
+
1, 35991
|
| 83 |
+
2, 35861
|
| 84 |
+
3, 35921
|
| 85 |
+
4, 35873
|
| 86 |
+
5, 35921
|
| 87 |
+
6, 35909
|
| 88 |
+
7, 35991
|
| 89 |
+
0, 45843
|
| 90 |
+
1, 46017
|
| 91 |
+
2, 45833
|
| 92 |
+
3, 45925
|
| 93 |
+
4, 45857
|
| 94 |
+
5, 45869
|
| 95 |
+
6, 45869
|
| 96 |
+
7, 45963
|
| 97 |
+
0, 46289
|
| 98 |
+
1, 46409
|
| 99 |
+
2, 46283
|
| 100 |
+
3, 46343
|
| 101 |
+
4, 46295
|
| 102 |
+
5, 46339
|
| 103 |
+
6, 46331
|
| 104 |
+
7, 46409
|
| 105 |
+
0, 46293
|
| 106 |
+
1, 46411
|
| 107 |
+
2, 46287
|
| 108 |
+
3, 46347
|
| 109 |
+
4, 46299
|
| 110 |
+
5, 46345
|
| 111 |
+
6, 46333
|
| 112 |
+
7, 46415
|
| 113 |
+
0, 111863
|
| 114 |
+
1, 111853
|
| 115 |
+
2, 111859
|
| 116 |
+
3, 111859
|
| 117 |
+
4, 111869
|
| 118 |
+
5, 111853
|
| 119 |
+
6, 111841
|
| 120 |
+
7, 111857
|
| 121 |
+
0, 139413
|
| 122 |
+
1, 139405
|
| 123 |
+
2, 139407
|
| 124 |
+
3, 139401
|
| 125 |
+
4, 139417
|
| 126 |
+
5, 139399
|
| 127 |
+
6, 140177
|
| 128 |
+
7, 139405
|
| 129 |
+
0, 139679
|
| 130 |
+
1, 139691
|
| 131 |
+
2, 139657
|
| 132 |
+
3, 139655
|
| 133 |
+
4, 139695
|
| 134 |
+
5, 139655
|
| 135 |
+
6, 140417
|
| 136 |
+
7, 139705
|
| 137 |
+
0, 139801
|
| 138 |
+
1, 139793
|
| 139 |
+
2, 139795
|
| 140 |
+
3, 139791
|
| 141 |
+
4, 139807
|
| 142 |
+
5, 139787
|
| 143 |
+
6, 140565
|
| 144 |
+
7, 139793
|
| 145 |
+
0, 142111
|
| 146 |
+
1, 142103
|
| 147 |
+
2, 142105
|
| 148 |
+
3, 142101
|
| 149 |
+
4, 142117
|
| 150 |
+
5, 142097
|
| 151 |
+
6, 142875
|
| 152 |
+
7, 142103
|
| 153 |
+
0, 143115
|
| 154 |
+
1, 143077
|
| 155 |
+
2, 143095
|
| 156 |
+
3, 143075
|
| 157 |
+
4, 143125
|
| 158 |
+
5, 143105
|
| 159 |
+
6, 143863
|
| 160 |
+
7, 143101
|
| 161 |
+
0, 157149
|
| 162 |
+
1, 157133
|
| 163 |
+
2, 157143
|
| 164 |
+
3, 157135
|
| 165 |
+
4, 157151
|
| 166 |
+
5, 157127
|
| 167 |
+
6, 156339
|
| 168 |
+
7, 157133
|
| 169 |
+
0, 157149
|
| 170 |
+
1, 157133
|
| 171 |
+
2, 157143
|
| 172 |
+
3, 157135
|
| 173 |
+
4, 157151
|
| 174 |
+
5, 157127
|
| 175 |
+
6, 156339
|
| 176 |
+
7, 157133
|
| 177 |
+
0, 157149
|
| 178 |
+
1, 157133
|
| 179 |
+
2, 157143
|
| 180 |
+
3, 157135
|
| 181 |
+
4, 157151
|
| 182 |
+
5, 157127
|
| 183 |
+
6, 156339
|
| 184 |
+
7, 157133
|
| 185 |
+
0, 157149
|
| 186 |
+
1, 157133
|
| 187 |
+
2, 157143
|
| 188 |
+
3, 157135
|
| 189 |
+
4, 157151
|
| 190 |
+
5, 157127
|
| 191 |
+
6, 156339
|
| 192 |
+
7, 157133
|
| 193 |
+
0, 156503
|
| 194 |
+
1, 156551
|
| 195 |
+
2, 157143
|
| 196 |
+
3, 156809
|
| 197 |
+
4, 156505
|
| 198 |
+
5, 156545
|
| 199 |
+
6, 156339
|
| 200 |
+
7, 156807
|
| 201 |
+
0, 154621
|
| 202 |
+
1, 154669
|
| 203 |
+
2, 155261
|
| 204 |
+
3, 154539
|
| 205 |
+
4, 154623
|
| 206 |
+
5, 155051
|
| 207 |
+
6, 154457
|
| 208 |
+
7, 154925
|
| 209 |
+
0, 152157
|
| 210 |
+
1, 152925
|
| 211 |
+
2, 152935
|
| 212 |
+
3, 152143
|
| 213 |
+
4, 152649
|
| 214 |
+
5, 153155
|
| 215 |
+
6, 152321
|
| 216 |
+
7, 152925
|
| 217 |
+
0, 1
|
| 218 |
+
1, 4
|
| 219 |
+
2, 4
|
| 220 |
+
3, 4
|
| 221 |
+
4, 4
|
| 222 |
+
5, 4
|
| 223 |
+
6, 142237
|
| 224 |
+
7, 4
|
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/run.env
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=moonep
|
| 2 |
+
config=DSV2_LITE_L24
|
| 3 |
+
family=tierB
|
| 4 |
+
repo=/workspace/Megatron-MoE-Echo
|
| 5 |
+
branch=moonep_backend_token_routing_analysis
|
| 6 |
+
commit=35ae6e7f
|
| 7 |
+
num_layers=24
|
| 8 |
+
hidden=2048
|
| 9 |
+
ffn_hidden=10944
|
| 10 |
+
moe_ffn_hidden=1408
|
| 11 |
+
num_experts=64
|
| 12 |
+
topk=6
|
| 13 |
+
experts_per_rank=8
|
| 14 |
+
expert_params=8650752
|
| 15 |
+
heads=16
|
| 16 |
+
kv_groups=0
|
| 17 |
+
kv_channels=128
|
| 18 |
+
qk_layernorm=1
|
| 19 |
+
shared_ffn=2816
|
| 20 |
+
shared_expert_overlap=0
|
| 21 |
+
mla=1
|
| 22 |
+
dense_first_layers=1
|
| 23 |
+
moe_layers=23
|
| 24 |
+
rotary_base=10000
|
| 25 |
+
rotary_scaling=40
|
| 26 |
+
qkv_bias=0
|
| 27 |
+
router_balancing=seq_aux_loss
|
| 28 |
+
aux_loss_coeff=1e-3
|
| 29 |
+
router_pre_softmax=1
|
| 30 |
+
router_fusion=0
|
| 31 |
+
router_dtype=fp32
|
| 32 |
+
device_initiated_grouped_gemm=1
|
| 33 |
+
norm_epsilon=1e-6
|
| 34 |
+
seq_length=8192
|
| 35 |
+
micro_batch_size=1
|
| 36 |
+
global_batch_size=8
|
| 37 |
+
num_microbatches=1
|
| 38 |
+
assignments_per_rank=49152
|
| 39 |
+
mean_tokens_per_expert=6144
|
| 40 |
+
slot_policy=matched
|
| 41 |
+
slots_per_rank=8
|
| 42 |
+
capacity_mode=none
|
| 43 |
+
capacity=none
|
| 44 |
+
cuda_graph=0
|
| 45 |
+
probe_mode=1
|
| 46 |
+
async_grad_reduce=0
|
| 47 |
+
branch_role=probe
|
| 48 |
+
memory_dump_mode=none
|
| 49 |
+
memory_snapshot_path=
|
| 50 |
+
cuda_graph_warmup_steps=3
|
| 51 |
+
recompute=none
|
| 52 |
+
force_balance=1
|
| 53 |
+
train_iters=12
|
| 54 |
+
ep_size=8
|
| 55 |
+
echo_dispatch_overlap=1
|
| 56 |
+
echo_dispatch_recompute=0
|
| 57 |
+
ultraep_quota_min_tokens=
|
| 58 |
+
ultraep_weight_sync_join=
|
| 59 |
+
profile_enable=0
|
| 60 |
+
kernel_trace=0
|
| 61 |
+
fp8_recipe=mxfp8
|
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/train.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
0, 0
|
| 2 |
+
1, 0
|
| 3 |
+
2, 0
|
| 4 |
+
3, 0
|
| 5 |
+
4, 0
|
| 6 |
+
5, 0
|
| 7 |
+
6, 0
|
| 8 |
+
7, 0
|
| 9 |
+
0, 0
|
| 10 |
+
1, 0
|
| 11 |
+
2, 0
|
| 12 |
+
3, 0
|
| 13 |
+
4, 0
|
| 14 |
+
5, 0
|
| 15 |
+
6, 0
|
| 16 |
+
7, 0
|
| 17 |
+
0, 1
|
| 18 |
+
1, 1
|
| 19 |
+
2, 1
|
| 20 |
+
3, 1
|
| 21 |
+
4, 1
|
| 22 |
+
5, 1
|
| 23 |
+
6, 1
|
| 24 |
+
7, 1
|
| 25 |
+
0, 4
|
| 26 |
+
1, 4
|
| 27 |
+
2, 4
|
| 28 |
+
3, 4
|
| 29 |
+
4, 1
|
| 30 |
+
5, 1
|
| 31 |
+
6, 1
|
| 32 |
+
7, 1
|
| 33 |
+
0, 4
|
| 34 |
+
1, 4
|
| 35 |
+
2, 4
|
| 36 |
+
3, 4
|
| 37 |
+
4, 4
|
| 38 |
+
5, 4
|
| 39 |
+
6, 4
|
| 40 |
+
7, 4
|
| 41 |
+
0, 4
|
| 42 |
+
1, 629
|
| 43 |
+
2, 629
|
| 44 |
+
3, 629
|
| 45 |
+
4, 629
|
| 46 |
+
5, 629
|
| 47 |
+
6, 629
|
| 48 |
+
7, 629
|
| 49 |
+
0, 975
|
| 50 |
+
1, 971
|
| 51 |
+
2, 981
|
| 52 |
+
3, 987
|
| 53 |
+
4, 973
|
| 54 |
+
5, 965
|
| 55 |
+
6, 957
|
| 56 |
+
7, 953
|
| 57 |
+
0, 1803
|
| 58 |
+
1, 1803
|
| 59 |
+
2, 1803
|
| 60 |
+
3, 1803
|
| 61 |
+
4, 1803
|
| 62 |
+
5, 1803
|
| 63 |
+
6, 1803
|
| 64 |
+
7, 1803
|
| 65 |
+
0, 6689
|
| 66 |
+
1, 6699
|
| 67 |
+
2, 6695
|
| 68 |
+
3, 6701
|
| 69 |
+
4, 6683
|
| 70 |
+
5, 6687
|
| 71 |
+
6, 6693
|
| 72 |
+
7, 6695
|
| 73 |
+
0, 58449
|
| 74 |
+
1, 57903
|
| 75 |
+
2, 57859
|
| 76 |
+
3, 58599
|
| 77 |
+
4, 58603
|
| 78 |
+
5, 58577
|
| 79 |
+
6, 58575
|
| 80 |
+
7, 58567
|
| 81 |
+
0, 59447
|
| 82 |
+
1, 58893
|
| 83 |
+
2, 58873
|
| 84 |
+
3, 59577
|
| 85 |
+
4, 59577
|
| 86 |
+
5, 59575
|
| 87 |
+
6, 59581
|
| 88 |
+
7, 59587
|
| 89 |
+
0, 59719
|
| 90 |
+
1, 59149
|
| 91 |
+
2, 59149
|
| 92 |
+
3, 59849
|
| 93 |
+
4, 59849
|
| 94 |
+
5, 59849
|
| 95 |
+
6, 59849
|
| 96 |
+
7, 59849
|
| 97 |
+
0, 60419
|
| 98 |
+
1, 59851
|
| 99 |
+
2, 59851
|
| 100 |
+
3, 60551
|
| 101 |
+
4, 60551
|
| 102 |
+
5, 60551
|
| 103 |
+
6, 60551
|
| 104 |
+
7, 60551
|
| 105 |
+
0, 67691
|
| 106 |
+
1, 67695
|
| 107 |
+
2, 68433
|
| 108 |
+
3, 69157
|
| 109 |
+
4, 68407
|
| 110 |
+
5, 76049
|
| 111 |
+
6, 75091
|
| 112 |
+
7, 69115
|
| 113 |
+
0, 112523
|
| 114 |
+
1, 113183
|
| 115 |
+
2, 114755
|
| 116 |
+
3, 113143
|
| 117 |
+
4, 113013
|
| 118 |
+
5, 117111
|
| 119 |
+
6, 117111
|
| 120 |
+
7, 113573
|
| 121 |
+
0, 118643
|
| 122 |
+
1, 118685
|
| 123 |
+
2, 118655
|
| 124 |
+
3, 118669
|
| 125 |
+
4, 118675
|
| 126 |
+
5, 118659
|
| 127 |
+
6, 118667
|
| 128 |
+
7, 118707
|
| 129 |
+
0, 118941
|
| 130 |
+
1, 118987
|
| 131 |
+
2, 118967
|
| 132 |
+
3, 119001
|
| 133 |
+
4, 119001
|
| 134 |
+
5, 119001
|
| 135 |
+
6, 119001
|
| 136 |
+
7, 119001
|
| 137 |
+
0, 119103
|
| 138 |
+
1, 119113
|
| 139 |
+
2, 119139
|
| 140 |
+
3, 119151
|
| 141 |
+
4, 119127
|
| 142 |
+
5, 119135
|
| 143 |
+
6, 119151
|
| 144 |
+
7, 119129
|
| 145 |
+
0, 128615
|
| 146 |
+
1, 128353
|
| 147 |
+
2, 128475
|
| 148 |
+
3, 128499
|
| 149 |
+
4, 128445
|
| 150 |
+
5, 128485
|
| 151 |
+
6, 128443
|
| 152 |
+
7, 128461
|
| 153 |
+
0, 141323
|
| 154 |
+
1, 141553
|
| 155 |
+
2, 141527
|
| 156 |
+
3, 141353
|
| 157 |
+
4, 141353
|
| 158 |
+
5, 141353
|
| 159 |
+
6, 141353
|
| 160 |
+
7, 141353
|
| 161 |
+
0, 141323
|
| 162 |
+
1, 141553
|
| 163 |
+
2, 141527
|
| 164 |
+
3, 141353
|
| 165 |
+
4, 141353
|
| 166 |
+
5, 141353
|
| 167 |
+
6, 141353
|
| 168 |
+
7, 141353
|
| 169 |
+
0, 140677
|
| 170 |
+
1, 140581
|
| 171 |
+
2, 140299
|
| 172 |
+
3, 140707
|
| 173 |
+
4, 140707
|
| 174 |
+
5, 140707
|
| 175 |
+
6, 140707
|
| 176 |
+
7, 140707
|
| 177 |
+
0, 139463
|
| 178 |
+
1, 139437
|
| 179 |
+
2, 139239
|
| 180 |
+
3, 139477
|
| 181 |
+
4, 139819
|
| 182 |
+
5, 139557
|
| 183 |
+
6, 139819
|
| 184 |
+
7, 139431
|
| 185 |
+
0, 13561
|
| 186 |
+
1, 12604
|
| 187 |
+
2, 12604
|
| 188 |
+
3, 12604
|
| 189 |
+
4, 12604
|
| 190 |
+
5, 12604
|
| 191 |
+
6, 12796
|
| 192 |
+
7, 139431
|
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/run.env
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=moonep
|
| 2 |
+
config=QWEN2_57B
|
| 3 |
+
family=tierB
|
| 4 |
+
repo=/workspace/Megatron-MoE-Echo
|
| 5 |
+
branch=moonep_backend_token_routing_analysis
|
| 6 |
+
commit=35ae6e7f
|
| 7 |
+
num_layers=10
|
| 8 |
+
hidden=3584
|
| 9 |
+
ffn_hidden=2560
|
| 10 |
+
moe_ffn_hidden=2560
|
| 11 |
+
num_experts=64
|
| 12 |
+
topk=8
|
| 13 |
+
experts_per_rank=8
|
| 14 |
+
expert_params=27525120
|
| 15 |
+
heads=28
|
| 16 |
+
kv_groups=4
|
| 17 |
+
kv_channels=
|
| 18 |
+
qk_layernorm=0
|
| 19 |
+
shared_ffn=20480
|
| 20 |
+
shared_expert_overlap=0
|
| 21 |
+
mla=0
|
| 22 |
+
dense_first_layers=0
|
| 23 |
+
moe_layers=10
|
| 24 |
+
rotary_base=1000000
|
| 25 |
+
rotary_scaling=
|
| 26 |
+
qkv_bias=1
|
| 27 |
+
router_balancing=aux_loss
|
| 28 |
+
aux_loss_coeff=1e-2
|
| 29 |
+
router_pre_softmax=1
|
| 30 |
+
router_fusion=0
|
| 31 |
+
router_dtype=
|
| 32 |
+
device_initiated_grouped_gemm=1
|
| 33 |
+
norm_epsilon=1e-6
|
| 34 |
+
seq_length=4096
|
| 35 |
+
micro_batch_size=1
|
| 36 |
+
global_batch_size=8
|
| 37 |
+
num_microbatches=1
|
| 38 |
+
assignments_per_rank=32768
|
| 39 |
+
mean_tokens_per_expert=4096
|
| 40 |
+
slot_policy=matched
|
| 41 |
+
slots_per_rank=8
|
| 42 |
+
capacity_mode=none
|
| 43 |
+
capacity=none
|
| 44 |
+
cuda_graph=0
|
| 45 |
+
probe_mode=1
|
| 46 |
+
async_grad_reduce=0
|
| 47 |
+
branch_role=probe
|
| 48 |
+
memory_dump_mode=none
|
| 49 |
+
memory_snapshot_path=
|
| 50 |
+
cuda_graph_warmup_steps=3
|
| 51 |
+
recompute=none
|
| 52 |
+
force_balance=0
|
| 53 |
+
train_iters=12
|
| 54 |
+
ep_size=8
|
| 55 |
+
echo_dispatch_overlap=1
|
| 56 |
+
echo_dispatch_recompute=0
|
| 57 |
+
ultraep_quota_min_tokens=
|
| 58 |
+
ultraep_weight_sync_join=
|
| 59 |
+
profile_enable=0
|
| 60 |
+
kernel_trace=0
|
| 61 |
+
fp8_recipe=mxfp8
|
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/mem_samples.csv
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
0, 0
|
| 2 |
+
1, 0
|
| 3 |
+
2, 0
|
| 4 |
+
3, 0
|
| 5 |
+
4, 0
|
| 6 |
+
5, 0
|
| 7 |
+
6, 0
|
| 8 |
+
7, 0
|
| 9 |
+
0, 0
|
| 10 |
+
1, 0
|
| 11 |
+
2, 0
|
| 12 |
+
3, 0
|
| 13 |
+
4, 0
|
| 14 |
+
5, 0
|
| 15 |
+
6, 0
|
| 16 |
+
7, 0
|
| 17 |
+
0, 1
|
| 18 |
+
1, 1
|
| 19 |
+
2, 1
|
| 20 |
+
3, 1
|
| 21 |
+
4, 1
|
| 22 |
+
5, 1
|
| 23 |
+
6, 1
|
| 24 |
+
7, 1
|
| 25 |
+
0, 4
|
| 26 |
+
1, 4
|
| 27 |
+
2, 4
|
| 28 |
+
3, 4
|
| 29 |
+
4, 1
|
| 30 |
+
5, 1
|
| 31 |
+
6, 1
|
| 32 |
+
7, 1
|
| 33 |
+
0, 4
|
| 34 |
+
1, 4
|
| 35 |
+
2, 4
|
| 36 |
+
3, 4
|
| 37 |
+
4, 4
|
| 38 |
+
5, 4
|
| 39 |
+
6, 4
|
| 40 |
+
7, 4
|
| 41 |
+
0, 725
|
| 42 |
+
1, 725
|
| 43 |
+
2, 725
|
| 44 |
+
3, 725
|
| 45 |
+
4, 725
|
| 46 |
+
5, 725
|
| 47 |
+
6, 725
|
| 48 |
+
7, 725
|
| 49 |
+
0, 953
|
| 50 |
+
1, 945
|
| 51 |
+
2, 959
|
| 52 |
+
3, 943
|
| 53 |
+
4, 953
|
| 54 |
+
5, 955
|
| 55 |
+
6, 923
|
| 56 |
+
7, 945
|
| 57 |
+
0, 1803
|
| 58 |
+
1, 1803
|
| 59 |
+
2, 1803
|
| 60 |
+
3, 1803
|
| 61 |
+
4, 1803
|
| 62 |
+
5, 1803
|
| 63 |
+
6, 1803
|
| 64 |
+
7, 1803
|
| 65 |
+
0, 6793
|
| 66 |
+
1, 6799
|
| 67 |
+
2, 6801
|
| 68 |
+
3, 6819
|
| 69 |
+
4, 6785
|
| 70 |
+
5, 6795
|
| 71 |
+
6, 6803
|
| 72 |
+
7, 6801
|
| 73 |
+
0, 58463
|
| 74 |
+
1, 57881
|
| 75 |
+
2, 57909
|
| 76 |
+
3, 58565
|
| 77 |
+
4, 58555
|
| 78 |
+
5, 58609
|
| 79 |
+
6, 58607
|
| 80 |
+
7, 58573
|
| 81 |
+
0, 59479
|
| 82 |
+
1, 58911
|
| 83 |
+
2, 58911
|
| 84 |
+
3, 59611
|
| 85 |
+
4, 59611
|
| 86 |
+
5, 59611
|
| 87 |
+
6, 59611
|
| 88 |
+
7, 59611
|
| 89 |
+
0, 59719
|
| 90 |
+
1, 59151
|
| 91 |
+
2, 59151
|
| 92 |
+
3, 59851
|
| 93 |
+
4, 59851
|
| 94 |
+
5, 59851
|
| 95 |
+
6, 59851
|
| 96 |
+
7, 59851
|
| 97 |
+
0, 60419
|
| 98 |
+
1, 59851
|
| 99 |
+
2, 59851
|
| 100 |
+
3, 60551
|
| 101 |
+
4, 60551
|
| 102 |
+
5, 60551
|
| 103 |
+
6, 60551
|
| 104 |
+
7, 60551
|
| 105 |
+
0, 86691
|
| 106 |
+
1, 86427
|
| 107 |
+
2, 86883
|
| 108 |
+
3, 85141
|
| 109 |
+
4, 86917
|
| 110 |
+
5, 86117
|
| 111 |
+
6, 86711
|
| 112 |
+
7, 89399
|
| 113 |
+
0, 117915
|
| 114 |
+
1, 117957
|
| 115 |
+
2, 117943
|
| 116 |
+
3, 117989
|
| 117 |
+
4, 117977
|
| 118 |
+
5, 117985
|
| 119 |
+
6, 117987
|
| 120 |
+
7, 117993
|
| 121 |
+
0, 118939
|
| 122 |
+
1, 118983
|
| 123 |
+
2, 118967
|
| 124 |
+
3, 119001
|
| 125 |
+
4, 119001
|
| 126 |
+
5, 119001
|
| 127 |
+
6, 119001
|
| 128 |
+
7, 119001
|
| 129 |
+
0, 118939
|
| 130 |
+
1, 118983
|
| 131 |
+
2, 118967
|
| 132 |
+
3, 119001
|
| 133 |
+
4, 119001
|
| 134 |
+
5, 119001
|
| 135 |
+
6, 119001
|
| 136 |
+
7, 119001
|
| 137 |
+
0, 119537
|
| 138 |
+
1, 119599
|
| 139 |
+
2, 119569
|
| 140 |
+
3, 119615
|
| 141 |
+
4, 119607
|
| 142 |
+
5, 119617
|
| 143 |
+
6, 119609
|
| 144 |
+
7, 119607
|
| 145 |
+
0, 128707
|
| 146 |
+
1, 128371
|
| 147 |
+
2, 128547
|
| 148 |
+
3, 128579
|
| 149 |
+
4, 128525
|
| 150 |
+
5, 128523
|
| 151 |
+
6, 128521
|
| 152 |
+
7, 128469
|
| 153 |
+
0, 141321
|
| 154 |
+
1, 141551
|
| 155 |
+
2, 141525
|
| 156 |
+
3, 141351
|
| 157 |
+
4, 141351
|
| 158 |
+
5, 141351
|
| 159 |
+
6, 141351
|
| 160 |
+
7, 141351
|
| 161 |
+
0, 141321
|
| 162 |
+
1, 141551
|
| 163 |
+
2, 141525
|
| 164 |
+
3, 141351
|
| 165 |
+
4, 141351
|
| 166 |
+
5, 141351
|
| 167 |
+
6, 141351
|
| 168 |
+
7, 141351
|
| 169 |
+
0, 140675
|
| 170 |
+
1, 140905
|
| 171 |
+
2, 140879
|
| 172 |
+
3, 140705
|
| 173 |
+
4, 140705
|
| 174 |
+
5, 140705
|
| 175 |
+
6, 140705
|
| 176 |
+
7, 140705
|
| 177 |
+
0, 139401
|
| 178 |
+
1, 139305
|
| 179 |
+
2, 138959
|
| 180 |
+
3, 138785
|
| 181 |
+
4, 138785
|
| 182 |
+
5, 138785
|
| 183 |
+
6, 138785
|
| 184 |
+
7, 138849
|
| 185 |
+
0, 12601
|
| 186 |
+
1, 12604
|
| 187 |
+
2, 12732
|
| 188 |
+
3, 138783
|
| 189 |
+
4, 13244
|
| 190 |
+
5, 12604
|
| 191 |
+
6, 12604
|
| 192 |
+
7, 12604
|
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/run.env
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=moonep
|
| 2 |
+
config=QWEN2_57B
|
| 3 |
+
family=tierB
|
| 4 |
+
repo=/workspace/Megatron-MoE-Echo
|
| 5 |
+
branch=moonep_backend_token_routing_analysis
|
| 6 |
+
commit=35ae6e7f
|
| 7 |
+
num_layers=10
|
| 8 |
+
hidden=3584
|
| 9 |
+
ffn_hidden=2560
|
| 10 |
+
moe_ffn_hidden=2560
|
| 11 |
+
num_experts=64
|
| 12 |
+
topk=8
|
| 13 |
+
experts_per_rank=8
|
| 14 |
+
expert_params=27525120
|
| 15 |
+
heads=28
|
| 16 |
+
kv_groups=4
|
| 17 |
+
kv_channels=
|
| 18 |
+
qk_layernorm=0
|
| 19 |
+
shared_ffn=20480
|
| 20 |
+
shared_expert_overlap=0
|
| 21 |
+
mla=0
|
| 22 |
+
dense_first_layers=0
|
| 23 |
+
moe_layers=10
|
| 24 |
+
rotary_base=1000000
|
| 25 |
+
rotary_scaling=
|
| 26 |
+
qkv_bias=1
|
| 27 |
+
router_balancing=aux_loss
|
| 28 |
+
aux_loss_coeff=1e-2
|
| 29 |
+
router_pre_softmax=1
|
| 30 |
+
router_fusion=0
|
| 31 |
+
router_dtype=
|
| 32 |
+
device_initiated_grouped_gemm=1
|
| 33 |
+
norm_epsilon=1e-6
|
| 34 |
+
seq_length=4096
|
| 35 |
+
micro_batch_size=1
|
| 36 |
+
global_batch_size=8
|
| 37 |
+
num_microbatches=1
|
| 38 |
+
assignments_per_rank=32768
|
| 39 |
+
mean_tokens_per_expert=4096
|
| 40 |
+
slot_policy=matched
|
| 41 |
+
slots_per_rank=8
|
| 42 |
+
capacity_mode=none
|
| 43 |
+
capacity=none
|
| 44 |
+
cuda_graph=0
|
| 45 |
+
probe_mode=1
|
| 46 |
+
async_grad_reduce=0
|
| 47 |
+
branch_role=probe
|
| 48 |
+
memory_dump_mode=none
|
| 49 |
+
memory_snapshot_path=
|
| 50 |
+
cuda_graph_warmup_steps=3
|
| 51 |
+
recompute=none
|
| 52 |
+
force_balance=1
|
| 53 |
+
train_iters=12
|
| 54 |
+
ep_size=8
|
| 55 |
+
echo_dispatch_overlap=1
|
| 56 |
+
echo_dispatch_recompute=0
|
| 57 |
+
ultraep_quota_min_tokens=
|
| 58 |
+
ultraep_weight_sync_join=
|
| 59 |
+
profile_enable=0
|
| 60 |
+
kernel_trace=0
|
| 61 |
+
fp8_recipe=mxfp8
|
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/train.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv
ADDED
|
@@ -0,0 +1,224 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
0, 0
|
| 2 |
+
1, 0
|
| 3 |
+
2, 0
|
| 4 |
+
3, 0
|
| 5 |
+
4, 0
|
| 6 |
+
5, 0
|
| 7 |
+
6, 0
|
| 8 |
+
7, 0
|
| 9 |
+
0, 0
|
| 10 |
+
1, 0
|
| 11 |
+
2, 0
|
| 12 |
+
3, 0
|
| 13 |
+
4, 0
|
| 14 |
+
5, 0
|
| 15 |
+
6, 0
|
| 16 |
+
7, 0
|
| 17 |
+
0, 4
|
| 18 |
+
1, 1
|
| 19 |
+
2, 1
|
| 20 |
+
3, 1
|
| 21 |
+
4, 1
|
| 22 |
+
5, 1
|
| 23 |
+
6, 1
|
| 24 |
+
7, 1
|
| 25 |
+
0, 4
|
| 26 |
+
1, 4
|
| 27 |
+
2, 4
|
| 28 |
+
3, 4
|
| 29 |
+
4, 1
|
| 30 |
+
5, 1
|
| 31 |
+
6, 1
|
| 32 |
+
7, 1
|
| 33 |
+
0, 4
|
| 34 |
+
1, 4
|
| 35 |
+
2, 4
|
| 36 |
+
3, 4
|
| 37 |
+
4, 4
|
| 38 |
+
5, 4
|
| 39 |
+
6, 4
|
| 40 |
+
7, 4
|
| 41 |
+
0, 4
|
| 42 |
+
1, 4
|
| 43 |
+
2, 4
|
| 44 |
+
3, 4
|
| 45 |
+
4, 4
|
| 46 |
+
5, 4
|
| 47 |
+
6, 4
|
| 48 |
+
7, 4
|
| 49 |
+
0, 821
|
| 50 |
+
1, 821
|
| 51 |
+
2, 821
|
| 52 |
+
3, 821
|
| 53 |
+
4, 821
|
| 54 |
+
5, 821
|
| 55 |
+
6, 821
|
| 56 |
+
7, 821
|
| 57 |
+
0, 1831
|
| 58 |
+
1, 1831
|
| 59 |
+
2, 1831
|
| 60 |
+
3, 1831
|
| 61 |
+
4, 1831
|
| 62 |
+
5, 1831
|
| 63 |
+
6, 1831
|
| 64 |
+
7, 1831
|
| 65 |
+
0, 3735
|
| 66 |
+
1, 3745
|
| 67 |
+
2, 3723
|
| 68 |
+
3, 3727
|
| 69 |
+
4, 3737
|
| 70 |
+
5, 3743
|
| 71 |
+
6, 3733
|
| 72 |
+
7, 3725
|
| 73 |
+
0, 15371
|
| 74 |
+
1, 15373
|
| 75 |
+
2, 15373
|
| 76 |
+
3, 15373
|
| 77 |
+
4, 15373
|
| 78 |
+
5, 15373
|
| 79 |
+
6, 15373
|
| 80 |
+
7, 15373
|
| 81 |
+
0, 55341
|
| 82 |
+
1, 55343
|
| 83 |
+
2, 55343
|
| 84 |
+
3, 55343
|
| 85 |
+
4, 55343
|
| 86 |
+
5, 55343
|
| 87 |
+
6, 55343
|
| 88 |
+
7, 55343
|
| 89 |
+
0, 67357
|
| 90 |
+
1, 67409
|
| 91 |
+
2, 67367
|
| 92 |
+
3, 67315
|
| 93 |
+
4, 67353
|
| 94 |
+
5, 67413
|
| 95 |
+
6, 67383
|
| 96 |
+
7, 67383
|
| 97 |
+
0, 68527
|
| 98 |
+
1, 68565
|
| 99 |
+
2, 68543
|
| 100 |
+
3, 68493
|
| 101 |
+
4, 68529
|
| 102 |
+
5, 68543
|
| 103 |
+
6, 68565
|
| 104 |
+
7, 68543
|
| 105 |
+
0, 68673
|
| 106 |
+
1, 68711
|
| 107 |
+
2, 68689
|
| 108 |
+
3, 68639
|
| 109 |
+
4, 68675
|
| 110 |
+
5, 68689
|
| 111 |
+
6, 68711
|
| 112 |
+
7, 68689
|
| 113 |
+
0, 68675
|
| 114 |
+
1, 68713
|
| 115 |
+
2, 68691
|
| 116 |
+
3, 68641
|
| 117 |
+
4, 68677
|
| 118 |
+
5, 68691
|
| 119 |
+
6, 68713
|
| 120 |
+
7, 68691
|
| 121 |
+
0, 71871
|
| 122 |
+
1, 71625
|
| 123 |
+
2, 71653
|
| 124 |
+
3, 71883
|
| 125 |
+
4, 71605
|
| 126 |
+
5, 72187
|
| 127 |
+
6, 71755
|
| 128 |
+
7, 71857
|
| 129 |
+
0, 117205
|
| 130 |
+
1, 114877
|
| 131 |
+
2, 116895
|
| 132 |
+
3, 117519
|
| 133 |
+
4, 114367
|
| 134 |
+
5, 117101
|
| 135 |
+
6, 116825
|
| 136 |
+
7, 116873
|
| 137 |
+
0, 152069
|
| 138 |
+
1, 149561
|
| 139 |
+
2, 150911
|
| 140 |
+
3, 152077
|
| 141 |
+
4, 148449
|
| 142 |
+
5, 150911
|
| 143 |
+
6, 150223
|
| 144 |
+
7, 150475
|
| 145 |
+
0, 156087
|
| 146 |
+
1, 156099
|
| 147 |
+
2, 156085
|
| 148 |
+
3, 156093
|
| 149 |
+
4, 156089
|
| 150 |
+
5, 156085
|
| 151 |
+
6, 156101
|
| 152 |
+
7, 156085
|
| 153 |
+
0, 156777
|
| 154 |
+
1, 156735
|
| 155 |
+
2, 156791
|
| 156 |
+
3, 156803
|
| 157 |
+
4, 156741
|
| 158 |
+
5, 156731
|
| 159 |
+
6, 156711
|
| 160 |
+
7, 156763
|
| 161 |
+
0, 156877
|
| 162 |
+
1, 156889
|
| 163 |
+
2, 156875
|
| 164 |
+
3, 156883
|
| 165 |
+
4, 156879
|
| 166 |
+
5, 156875
|
| 167 |
+
6, 156891
|
| 168 |
+
7, 156875
|
| 169 |
+
0, 157525
|
| 170 |
+
1, 157537
|
| 171 |
+
2, 157523
|
| 172 |
+
3, 157531
|
| 173 |
+
4, 157527
|
| 174 |
+
5, 157523
|
| 175 |
+
6, 157539
|
| 176 |
+
7, 157523
|
| 177 |
+
0, 167421
|
| 178 |
+
1, 167335
|
| 179 |
+
2, 167449
|
| 180 |
+
3, 167299
|
| 181 |
+
4, 167619
|
| 182 |
+
5, 167435
|
| 183 |
+
6, 167361
|
| 184 |
+
7, 167425
|
| 185 |
+
0, 182487
|
| 186 |
+
1, 182571
|
| 187 |
+
2, 182505
|
| 188 |
+
3, 182525
|
| 189 |
+
4, 182511
|
| 190 |
+
5, 182505
|
| 191 |
+
6, 182625
|
| 192 |
+
7, 182505
|
| 193 |
+
0, 182487
|
| 194 |
+
1, 182571
|
| 195 |
+
2, 182505
|
| 196 |
+
3, 182525
|
| 197 |
+
4, 182511
|
| 198 |
+
5, 182505
|
| 199 |
+
6, 182625
|
| 200 |
+
7, 182505
|
| 201 |
+
0, 182487
|
| 202 |
+
1, 182571
|
| 203 |
+
2, 182505
|
| 204 |
+
3, 182525
|
| 205 |
+
4, 182511
|
| 206 |
+
5, 182505
|
| 207 |
+
6, 182625
|
| 208 |
+
7, 182505
|
| 209 |
+
0, 182487
|
| 210 |
+
1, 182571
|
| 211 |
+
2, 182505
|
| 212 |
+
3, 182525
|
| 213 |
+
4, 182511
|
| 214 |
+
5, 182505
|
| 215 |
+
6, 182625
|
| 216 |
+
7, 182505
|
| 217 |
+
0, 182487
|
| 218 |
+
1, 182571
|
| 219 |
+
2, 182505
|
| 220 |
+
3, 182525
|
| 221 |
+
4, 182511
|
| 222 |
+
5, 182505
|
| 223 |
+
6, 182625
|
| 224 |
+
7, 182505
|
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/run.env
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=moonep
|
| 2 |
+
config=QWEN3_30B
|
| 3 |
+
family=tierB
|
| 4 |
+
repo=/workspace/Megatron-MoE-Echo
|
| 5 |
+
branch=moonep_backend_token_routing_analysis
|
| 6 |
+
commit=35ae6e7f
|
| 7 |
+
num_layers=48
|
| 8 |
+
hidden=2048
|
| 9 |
+
ffn_hidden=6144
|
| 10 |
+
moe_ffn_hidden=768
|
| 11 |
+
num_experts=128
|
| 12 |
+
topk=8
|
| 13 |
+
experts_per_rank=16
|
| 14 |
+
expert_params=4718592
|
| 15 |
+
heads=32
|
| 16 |
+
kv_groups=4
|
| 17 |
+
kv_channels=128
|
| 18 |
+
qk_layernorm=1
|
| 19 |
+
shared_ffn=0
|
| 20 |
+
shared_expert_overlap=0
|
| 21 |
+
mla=0
|
| 22 |
+
dense_first_layers=0
|
| 23 |
+
moe_layers=48
|
| 24 |
+
rotary_base=1000000
|
| 25 |
+
rotary_scaling=
|
| 26 |
+
qkv_bias=0
|
| 27 |
+
router_balancing=aux_loss
|
| 28 |
+
aux_loss_coeff=1e-3
|
| 29 |
+
router_pre_softmax=0
|
| 30 |
+
router_fusion=0
|
| 31 |
+
router_dtype=fp32
|
| 32 |
+
device_initiated_grouped_gemm=1
|
| 33 |
+
norm_epsilon=1e-6
|
| 34 |
+
seq_length=4096
|
| 35 |
+
micro_batch_size=1
|
| 36 |
+
global_batch_size=8
|
| 37 |
+
num_microbatches=1
|
| 38 |
+
assignments_per_rank=32768
|
| 39 |
+
mean_tokens_per_expert=2048
|
| 40 |
+
slot_policy=matched
|
| 41 |
+
slots_per_rank=16
|
| 42 |
+
capacity_mode=none
|
| 43 |
+
capacity=none
|
| 44 |
+
cuda_graph=0
|
| 45 |
+
probe_mode=1
|
| 46 |
+
async_grad_reduce=0
|
| 47 |
+
branch_role=probe
|
| 48 |
+
memory_dump_mode=none
|
| 49 |
+
memory_snapshot_path=
|
| 50 |
+
cuda_graph_warmup_steps=3
|
| 51 |
+
recompute=none
|
| 52 |
+
force_balance=0
|
| 53 |
+
train_iters=12
|
| 54 |
+
ep_size=8
|
| 55 |
+
echo_dispatch_overlap=1
|
| 56 |
+
echo_dispatch_recompute=0
|
| 57 |
+
ultraep_quota_min_tokens=
|
| 58 |
+
ultraep_weight_sync_join=
|
| 59 |
+
profile_enable=0
|
| 60 |
+
kernel_trace=0
|
| 61 |
+
fp8_recipe=mxfp8
|
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv
CHANGED
|
@@ -1,2 +1,7 @@
|
|
| 1 |
model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir
|
| 2 |
QWEN3_235B moonep natural none OOM 182587 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir
|
| 2 |
QWEN3_235B moonep natural none OOM 182587 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1
|
| 3 |
+
QWEN3_30B moonep natural none OOM 182625 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1
|
| 4 |
+
DSV2_LITE_L24 moonep natural none OK 157151 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1
|
| 5 |
+
DSV2_LITE_L24 moonep forced none OK 157151 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1
|
| 6 |
+
QWEN2_57B moonep natural none OK 141553 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1
|
| 7 |
+
QWEN2_57B moonep forced none OK 141551 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1
|
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv
CHANGED
|
@@ -1,2 +1,5 @@
|
|
| 1 |
model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir
|
| 2 |
QWEN3_235B moonep ERROR_OOM 182587 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir
|
| 2 |
QWEN3_235B moonep ERROR_OOM 182587 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1
|
| 3 |
+
QWEN3_30B moonep ERROR_OOM 182625 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1
|
| 4 |
+
DSV2_LITE_L24 moonep SELECTED none 0 157151 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1
|
| 5 |
+
QWEN2_57B moonep SELECTED none 0 141553 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1
|
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.json
ADDED
|
@@ -0,0 +1,454 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"arm": "natural",
|
| 4 |
+
"aux_loss_coeff": "1e-3",
|
| 5 |
+
"backend": "moonep",
|
| 6 |
+
"branch": "moonep_backend_token_routing_analysis",
|
| 7 |
+
"capacity": "none",
|
| 8 |
+
"capacity_mode": "none",
|
| 9 |
+
"cold_expert_fraction_max": 0.578125,
|
| 10 |
+
"cold_expert_fraction_mean": 0.15625,
|
| 11 |
+
"cold_expert_fraction_min": 0.0,
|
| 12 |
+
"cold_expert_fraction_n": 230,
|
| 13 |
+
"commit": "35ae6e7f",
|
| 14 |
+
"config": "DSV2_LITE_L24",
|
| 15 |
+
"cuda_graph": "0",
|
| 16 |
+
"dense_first_layers": "1",
|
| 17 |
+
"experts_per_rank": "8",
|
| 18 |
+
"force_balance": "0",
|
| 19 |
+
"global_batch_size": "8",
|
| 20 |
+
"global_snapshots": 230,
|
| 21 |
+
"hottest_expert_token_fraction_max": 0.999969482421875,
|
| 22 |
+
"hottest_expert_token_fraction_mean": 0.7797577567722487,
|
| 23 |
+
"hottest_expert_token_fraction_min": 0.162750244140625,
|
| 24 |
+
"hottest_expert_token_fraction_n": 230,
|
| 25 |
+
"incomplete_snapshots": 0,
|
| 26 |
+
"malformed_records": 0,
|
| 27 |
+
"moe_layers": "23",
|
| 28 |
+
"moonep_bytes_global_snapshots": 230,
|
| 29 |
+
"moonep_bytes_incomplete_snapshots": 0,
|
| 30 |
+
"moonep_bytes_malformed_records": 0,
|
| 31 |
+
"moonep_global_combine_payload_bytes_max": 2592899072.0,
|
| 32 |
+
"moonep_global_combine_payload_bytes_mean": 2282338571.130435,
|
| 33 |
+
"moonep_global_combine_payload_bytes_min": 2026799104.0,
|
| 34 |
+
"moonep_global_combine_payload_bytes_n": 230,
|
| 35 |
+
"moonep_global_dispatch_payload_bytes_max": 2592899072.0,
|
| 36 |
+
"moonep_global_dispatch_payload_bytes_mean": 2282338571.130435,
|
| 37 |
+
"moonep_global_dispatch_payload_bytes_min": 2026799104.0,
|
| 38 |
+
"moonep_global_dispatch_payload_bytes_n": 230,
|
| 39 |
+
"moonep_global_prefetch_bytes_max": 346030080.0,
|
| 40 |
+
"moonep_global_prefetch_bytes_mean": 252601958.4,
|
| 41 |
+
"moonep_global_prefetch_bytes_min": 242221056.0,
|
| 42 |
+
"moonep_global_prefetch_bytes_n": 230,
|
| 43 |
+
"moonep_global_reduce_bytes_max": 346030080.0,
|
| 44 |
+
"moonep_global_reduce_bytes_mean": 252601958.4,
|
| 45 |
+
"moonep_global_reduce_bytes_min": 242221056.0,
|
| 46 |
+
"moonep_global_reduce_bytes_n": 230,
|
| 47 |
+
"moonep_global_token_bytes_max": 5188482536.0,
|
| 48 |
+
"moonep_global_token_bytes_mean": 4567339392.869565,
|
| 49 |
+
"moonep_global_token_bytes_min": 4056064752.0,
|
| 50 |
+
"moonep_global_token_bytes_n": 230,
|
| 51 |
+
"moonep_global_total_nvlink_bytes_max": 5672924648.0,
|
| 52 |
+
"moonep_global_total_nvlink_bytes_mean": 5072543309.669565,
|
| 53 |
+
"moonep_global_total_nvlink_bytes_min": 4540506864.0,
|
| 54 |
+
"moonep_global_total_nvlink_bytes_n": 230,
|
| 55 |
+
"moonep_global_weight_movement_bytes_max": 692060160.0,
|
| 56 |
+
"moonep_global_weight_movement_bytes_mean": 505203916.8,
|
| 57 |
+
"moonep_global_weight_movement_bytes_min": 484442112.0,
|
| 58 |
+
"moonep_global_weight_movement_bytes_n": 230,
|
| 59 |
+
"moonep_global_weight_to_token_ratio_max": 0.14851639788299403,
|
| 60 |
+
"moonep_global_weight_to_token_ratio_mean": 0.11096021713514241,
|
| 61 |
+
"moonep_global_weight_to_token_ratio_min": 0.09336874676530664,
|
| 62 |
+
"moonep_global_weight_to_token_ratio_n": 230,
|
| 63 |
+
"moonep_remote_demand_max": 3.0,
|
| 64 |
+
"moonep_remote_demand_mean": 0.9125,
|
| 65 |
+
"moonep_remote_demand_min": 0.0,
|
| 66 |
+
"moonep_remote_demand_n": 1840,
|
| 67 |
+
"moonep_slot_utilization_max": 0.375,
|
| 68 |
+
"moonep_slot_utilization_mean": 0.1140625,
|
| 69 |
+
"moonep_slot_utilization_min": 0.0,
|
| 70 |
+
"moonep_slot_utilization_n": 1840,
|
| 71 |
+
"moonep_unused_prefetch_slots_max": 16.0,
|
| 72 |
+
"moonep_unused_prefetch_slots_mean": 14.175,
|
| 73 |
+
"moonep_unused_prefetch_slots_min": 10.0,
|
| 74 |
+
"moonep_unused_prefetch_slots_n": 1840,
|
| 75 |
+
"num_experts": "64",
|
| 76 |
+
"num_layers": "24",
|
| 77 |
+
"post_physical_rank_cv_max": 0.24644338469771185,
|
| 78 |
+
"post_physical_rank_cv_mean": 0.0866314272142518,
|
| 79 |
+
"post_physical_rank_cv_min": 0.016159622971429973,
|
| 80 |
+
"post_physical_rank_cv_n": 230,
|
| 81 |
+
"post_physical_rank_imax_max": 1.2001146488570558,
|
| 82 |
+
"post_physical_rank_imax_mean": 1.1121655106296537,
|
| 83 |
+
"post_physical_rank_imax_min": 1.0188653892666146,
|
| 84 |
+
"post_physical_rank_imax_n": 230,
|
| 85 |
+
"pre_home_rank_cv_max": 1.3626457501059894,
|
| 86 |
+
"pre_home_rank_cv_mean": 0.6592086328754161,
|
| 87 |
+
"pre_home_rank_cv_min": 0.06474057713157837,
|
| 88 |
+
"pre_home_rank_cv_n": 230,
|
| 89 |
+
"pre_home_rank_imax_max": 4.328043619791667,
|
| 90 |
+
"pre_home_rank_imax_mean": 2.2370494897814766,
|
| 91 |
+
"pre_home_rank_imax_min": 1.1056925455729167,
|
| 92 |
+
"pre_home_rank_imax_n": 230,
|
| 93 |
+
"raw_expert_cv_max": 3.0868156557140782,
|
| 94 |
+
"raw_expert_cv_mean": 1.9632045302354437,
|
| 95 |
+
"raw_expert_cv_min": 0.37852206422070017,
|
| 96 |
+
"raw_expert_cv_n": 230,
|
| 97 |
+
"raw_expert_imax_ceiling": 10.666666666666666,
|
| 98 |
+
"raw_expert_imax_hit_ceiling": 1,
|
| 99 |
+
"raw_expert_imax_max": 10.666341145833334,
|
| 100 |
+
"raw_expert_imax_mean": 8.317416072237318,
|
| 101 |
+
"raw_expert_imax_min": 1.7360026041666667,
|
| 102 |
+
"raw_expert_imax_n": 230,
|
| 103 |
+
"records": {
|
| 104 |
+
"moonep_bytes": 1840,
|
| 105 |
+
"moonep_routing": 1840,
|
| 106 |
+
"moonep_summary": 80,
|
| 107 |
+
"moonep_weights": 1840
|
| 108 |
+
},
|
| 109 |
+
"router_balancing": "seq_aux_loss",
|
| 110 |
+
"run_dir": "DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1",
|
| 111 |
+
"seq_length": "8192",
|
| 112 |
+
"shared_expert_overlap": "0",
|
| 113 |
+
"topk": "6"
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"arm": "forced",
|
| 117 |
+
"aux_loss_coeff": "1e-3",
|
| 118 |
+
"backend": "moonep",
|
| 119 |
+
"branch": "moonep_backend_token_routing_analysis",
|
| 120 |
+
"capacity": "none",
|
| 121 |
+
"capacity_mode": "none",
|
| 122 |
+
"cold_expert_fraction_max": 0.0,
|
| 123 |
+
"cold_expert_fraction_mean": 0.0,
|
| 124 |
+
"cold_expert_fraction_min": 0.0,
|
| 125 |
+
"cold_expert_fraction_n": 230,
|
| 126 |
+
"commit": "35ae6e7f",
|
| 127 |
+
"config": "DSV2_LITE_L24",
|
| 128 |
+
"cuda_graph": "0",
|
| 129 |
+
"dense_first_layers": "1",
|
| 130 |
+
"experts_per_rank": "8",
|
| 131 |
+
"force_balance": "1",
|
| 132 |
+
"global_batch_size": "8",
|
| 133 |
+
"global_snapshots": 230,
|
| 134 |
+
"hottest_expert_token_fraction_max": 0.0982818603515625,
|
| 135 |
+
"hottest_expert_token_fraction_mean": 0.09643501613451087,
|
| 136 |
+
"hottest_expert_token_fraction_min": 0.09539794921875,
|
| 137 |
+
"hottest_expert_token_fraction_n": 230,
|
| 138 |
+
"incomplete_snapshots": 0,
|
| 139 |
+
"malformed_records": 0,
|
| 140 |
+
"moe_layers": "23",
|
| 141 |
+
"moonep_bytes_global_snapshots": 230,
|
| 142 |
+
"moonep_bytes_incomplete_snapshots": 0,
|
| 143 |
+
"moonep_bytes_malformed_records": 0,
|
| 144 |
+
"moonep_global_combine_payload_bytes_max": 2133762048.0,
|
| 145 |
+
"moonep_global_combine_payload_bytes_mean": 2129465023.4434783,
|
| 146 |
+
"moonep_global_combine_payload_bytes_min": 2124537856.0,
|
| 147 |
+
"moonep_global_combine_payload_bytes_n": 230,
|
| 148 |
+
"moonep_global_dispatch_payload_bytes_max": 2133762048.0,
|
| 149 |
+
"moonep_global_dispatch_payload_bytes_mean": 2129465023.4434783,
|
| 150 |
+
"moonep_global_dispatch_payload_bytes_min": 2124537856.0,
|
| 151 |
+
"moonep_global_dispatch_payload_bytes_n": 230,
|
| 152 |
+
"moonep_global_prefetch_bytes_max": 242221056.0,
|
| 153 |
+
"moonep_global_prefetch_bytes_mean": 238309411.61739132,
|
| 154 |
+
"moonep_global_prefetch_bytes_min": 207618048.0,
|
| 155 |
+
"moonep_global_prefetch_bytes_n": 230,
|
| 156 |
+
"moonep_global_reduce_bytes_max": 242221056.0,
|
| 157 |
+
"moonep_global_reduce_bytes_mean": 238309411.61739132,
|
| 158 |
+
"moonep_global_reduce_bytes_min": 207618048.0,
|
| 159 |
+
"moonep_global_reduce_bytes_n": 230,
|
| 160 |
+
"moonep_global_token_bytes_max": 4270279408.0,
|
| 161 |
+
"moonep_global_token_bytes_mean": 4261681018.2608695,
|
| 162 |
+
"moonep_global_token_bytes_min": 4251824656.0,
|
| 163 |
+
"moonep_global_token_bytes_n": 230,
|
| 164 |
+
"moonep_global_total_nvlink_bytes_max": 4754721520.0,
|
| 165 |
+
"moonep_global_total_nvlink_bytes_mean": 4738299841.495652,
|
| 166 |
+
"moonep_global_total_nvlink_bytes_min": 4667434440.0,
|
| 167 |
+
"moonep_global_total_nvlink_bytes_n": 230,
|
| 168 |
+
"moonep_global_weight_movement_bytes_max": 484442112.0,
|
| 169 |
+
"moonep_global_weight_movement_bytes_mean": 476618823.23478264,
|
| 170 |
+
"moonep_global_weight_movement_bytes_min": 415236096.0,
|
| 171 |
+
"moonep_global_weight_movement_bytes_n": 230,
|
| 172 |
+
"moonep_global_weight_to_token_ratio_max": 0.11393746242954192,
|
| 173 |
+
"moonep_global_weight_to_token_ratio_mean": 0.11183841293995254,
|
| 174 |
+
"moonep_global_weight_to_token_ratio_min": 0.09728537714366937,
|
| 175 |
+
"moonep_global_weight_to_token_ratio_n": 230,
|
| 176 |
+
"moonep_remote_demand_max": 1.0,
|
| 177 |
+
"moonep_remote_demand_mean": 0.8608695652173913,
|
| 178 |
+
"moonep_remote_demand_min": 0.0,
|
| 179 |
+
"moonep_remote_demand_n": 1840,
|
| 180 |
+
"moonep_slot_utilization_max": 0.125,
|
| 181 |
+
"moonep_slot_utilization_mean": 0.10760869565217392,
|
| 182 |
+
"moonep_slot_utilization_min": 0.0,
|
| 183 |
+
"moonep_slot_utilization_n": 1840,
|
| 184 |
+
"moonep_unused_prefetch_slots_max": 16.0,
|
| 185 |
+
"moonep_unused_prefetch_slots_mean": 14.278260869565218,
|
| 186 |
+
"moonep_unused_prefetch_slots_min": 14.0,
|
| 187 |
+
"moonep_unused_prefetch_slots_n": 1840,
|
| 188 |
+
"num_experts": "64",
|
| 189 |
+
"num_layers": "24",
|
| 190 |
+
"post_physical_rank_cv_max": 0.0031667036922078573,
|
| 191 |
+
"post_physical_rank_cv_mean": 0.0016501228109707747,
|
| 192 |
+
"post_physical_rank_cv_min": 0.0006791931370339787,
|
| 193 |
+
"post_physical_rank_cv_n": 230,
|
| 194 |
+
"post_physical_rank_imax_max": 1.006201201251668,
|
| 195 |
+
"post_physical_rank_imax_mean": 1.0026117364992109,
|
| 196 |
+
"post_physical_rank_imax_min": 1.0009196904719393,
|
| 197 |
+
"post_physical_rank_imax_n": 230,
|
| 198 |
+
"pre_home_rank_cv_max": 0.006673533732490181,
|
| 199 |
+
"pre_home_rank_cv_mean": 0.003923942735502633,
|
| 200 |
+
"pre_home_rank_cv_min": 0.0014708009315538319,
|
| 201 |
+
"pre_home_rank_cv_n": 230,
|
| 202 |
+
"pre_home_rank_imax_max": 1.0122884114583333,
|
| 203 |
+
"pre_home_rank_imax_mean": 1.0061054616734602,
|
| 204 |
+
"pre_home_rank_imax_min": 1.00213623046875,
|
| 205 |
+
"pre_home_rank_imax_n": 230,
|
| 206 |
+
"raw_expert_cv_max": 0.015123587213954862,
|
| 207 |
+
"raw_expert_cv_mean": 0.012068912140234289,
|
| 208 |
+
"raw_expert_cv_min": 0.008171875787738593,
|
| 209 |
+
"raw_expert_cv_n": 230,
|
| 210 |
+
"raw_expert_imax_ceiling": 10.666666666666666,
|
| 211 |
+
"raw_expert_imax_hit_ceiling": 0,
|
| 212 |
+
"raw_expert_imax_max": 1.04833984375,
|
| 213 |
+
"raw_expert_imax_mean": 1.0286401721014493,
|
| 214 |
+
"raw_expert_imax_min": 1.017578125,
|
| 215 |
+
"raw_expert_imax_n": 230,
|
| 216 |
+
"records": {
|
| 217 |
+
"moonep_bytes": 1840,
|
| 218 |
+
"moonep_routing": 1840,
|
| 219 |
+
"moonep_summary": 80,
|
| 220 |
+
"moonep_weights": 1840
|
| 221 |
+
},
|
| 222 |
+
"router_balancing": "seq_aux_loss",
|
| 223 |
+
"run_dir": "DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1",
|
| 224 |
+
"seq_length": "8192",
|
| 225 |
+
"shared_expert_overlap": "0",
|
| 226 |
+
"topk": "6"
|
| 227 |
+
},
|
| 228 |
+
{
|
| 229 |
+
"arm": "natural",
|
| 230 |
+
"aux_loss_coeff": "1e-2",
|
| 231 |
+
"backend": "moonep",
|
| 232 |
+
"branch": "moonep_backend_token_routing_analysis",
|
| 233 |
+
"capacity": "none",
|
| 234 |
+
"capacity_mode": "none",
|
| 235 |
+
"cold_expert_fraction_max": 0.796875,
|
| 236 |
+
"cold_expert_fraction_mean": 0.39234375,
|
| 237 |
+
"cold_expert_fraction_min": 0.0,
|
| 238 |
+
"cold_expert_fraction_n": 100,
|
| 239 |
+
"commit": "35ae6e7f",
|
| 240 |
+
"config": "QWEN2_57B",
|
| 241 |
+
"cuda_graph": "0",
|
| 242 |
+
"dense_first_layers": "0",
|
| 243 |
+
"experts_per_rank": "8",
|
| 244 |
+
"force_balance": "0",
|
| 245 |
+
"global_batch_size": "8",
|
| 246 |
+
"global_snapshots": 100,
|
| 247 |
+
"hottest_expert_token_fraction_max": 1.0,
|
| 248 |
+
"hottest_expert_token_fraction_mean": 0.9268069458007813,
|
| 249 |
+
"hottest_expert_token_fraction_min": 0.2415771484375,
|
| 250 |
+
"hottest_expert_token_fraction_n": 100,
|
| 251 |
+
"incomplete_snapshots": 0,
|
| 252 |
+
"malformed_records": 0,
|
| 253 |
+
"moe_layers": "10",
|
| 254 |
+
"moonep_bytes_global_snapshots": 100,
|
| 255 |
+
"moonep_bytes_incomplete_snapshots": 0,
|
| 256 |
+
"moonep_bytes_malformed_records": 0,
|
| 257 |
+
"moonep_global_combine_payload_bytes_max": 3287445504.0,
|
| 258 |
+
"moonep_global_combine_payload_bytes_mean": 2789546475.52,
|
| 259 |
+
"moonep_global_combine_payload_bytes_min": 2180663296.0,
|
| 260 |
+
"moonep_global_combine_payload_bytes_n": 100,
|
| 261 |
+
"moonep_global_dispatch_payload_bytes_max": 3287445504.0,
|
| 262 |
+
"moonep_global_dispatch_payload_bytes_mean": 2789546475.52,
|
| 263 |
+
"moonep_global_dispatch_payload_bytes_min": 2180663296.0,
|
| 264 |
+
"moonep_global_dispatch_payload_bytes_n": 100,
|
| 265 |
+
"moonep_global_prefetch_bytes_max": 990904320.0,
|
| 266 |
+
"moonep_global_prefetch_bytes_mean": 823551590.4,
|
| 267 |
+
"moonep_global_prefetch_bytes_min": 660602880.0,
|
| 268 |
+
"moonep_global_prefetch_bytes_n": 100,
|
| 269 |
+
"moonep_global_reduce_bytes_max": 990904320.0,
|
| 270 |
+
"moonep_global_reduce_bytes_mean": 823551590.4,
|
| 271 |
+
"moonep_global_reduce_bytes_min": 660602880.0,
|
| 272 |
+
"moonep_global_reduce_bytes_n": 100,
|
| 273 |
+
"moonep_global_token_bytes_max": 6576725968.0,
|
| 274 |
+
"moonep_global_token_bytes_mean": 5580893210.72,
|
| 275 |
+
"moonep_global_token_bytes_min": 4363119160.0,
|
| 276 |
+
"moonep_global_token_bytes_n": 100,
|
| 277 |
+
"moonep_global_total_nvlink_bytes_max": 8328785856.0,
|
| 278 |
+
"moonep_global_total_nvlink_bytes_mean": 7227996391.52,
|
| 279 |
+
"moonep_global_total_nvlink_bytes_min": 5904525880.0,
|
| 280 |
+
"moonep_global_total_nvlink_bytes_n": 100,
|
| 281 |
+
"moonep_global_weight_movement_bytes_max": 1981808640.0,
|
| 282 |
+
"moonep_global_weight_movement_bytes_mean": 1647103180.8,
|
| 283 |
+
"moonep_global_weight_movement_bytes_min": 1321205760.0,
|
| 284 |
+
"moonep_global_weight_movement_bytes_n": 100,
|
| 285 |
+
"moonep_global_weight_to_token_ratio_max": 0.4380461607026548,
|
| 286 |
+
"moonep_global_weight_to_token_ratio_mean": 0.2995223715370738,
|
| 287 |
+
"moonep_global_weight_to_token_ratio_min": 0.20089110697762314,
|
| 288 |
+
"moonep_global_weight_to_token_ratio_n": 100,
|
| 289 |
+
"moonep_remote_demand_max": 2.0,
|
| 290 |
+
"moonep_remote_demand_mean": 0.935,
|
| 291 |
+
"moonep_remote_demand_min": 0.0,
|
| 292 |
+
"moonep_remote_demand_n": 800,
|
| 293 |
+
"moonep_slot_utilization_max": 0.25,
|
| 294 |
+
"moonep_slot_utilization_mean": 0.116875,
|
| 295 |
+
"moonep_slot_utilization_min": 0.0,
|
| 296 |
+
"moonep_slot_utilization_n": 800,
|
| 297 |
+
"moonep_unused_prefetch_slots_max": 16.0,
|
| 298 |
+
"moonep_unused_prefetch_slots_mean": 14.13,
|
| 299 |
+
"moonep_unused_prefetch_slots_min": 12.0,
|
| 300 |
+
"moonep_unused_prefetch_slots_n": 800,
|
| 301 |
+
"num_experts": "64",
|
| 302 |
+
"num_layers": "10",
|
| 303 |
+
"post_physical_rank_cv_max": 0.20167894084637375,
|
| 304 |
+
"post_physical_rank_cv_mean": 0.08583132480611003,
|
| 305 |
+
"post_physical_rank_cv_min": 0.00019943885935289461,
|
| 306 |
+
"post_physical_rank_cv_n": 100,
|
| 307 |
+
"post_physical_rank_imax_max": 1.2419115350424064,
|
| 308 |
+
"post_physical_rank_imax_mean": 1.1036399766472182,
|
| 309 |
+
"post_physical_rank_imax_min": 1.0002518334236372,
|
| 310 |
+
"post_physical_rank_imax_n": 100,
|
| 311 |
+
"pre_home_rank_cv_max": 1.4500421285803669,
|
| 312 |
+
"pre_home_rank_cv_mean": 0.7161558574760766,
|
| 313 |
+
"pre_home_rank_cv_min": 0.06359546419183554,
|
| 314 |
+
"pre_home_rank_cv_n": 100,
|
| 315 |
+
"pre_home_rank_imax_max": 4.147735595703125,
|
| 316 |
+
"pre_home_rank_imax_mean": 2.322430419921875,
|
| 317 |
+
"pre_home_rank_imax_min": 1.11138916015625,
|
| 318 |
+
"pre_home_rank_imax_n": 100,
|
| 319 |
+
"raw_expert_cv_max": 2.6452554407770648,
|
| 320 |
+
"raw_expert_cv_mean": 2.184735306615502,
|
| 321 |
+
"raw_expert_cv_min": 0.4993102636020994,
|
| 322 |
+
"raw_expert_cv_n": 100,
|
| 323 |
+
"raw_expert_imax_ceiling": 8.0,
|
| 324 |
+
"raw_expert_imax_hit_ceiling": 1,
|
| 325 |
+
"raw_expert_imax_max": 8.0,
|
| 326 |
+
"raw_expert_imax_mean": 7.41445556640625,
|
| 327 |
+
"raw_expert_imax_min": 1.9326171875,
|
| 328 |
+
"raw_expert_imax_n": 100,
|
| 329 |
+
"records": {
|
| 330 |
+
"moonep_bytes": 800,
|
| 331 |
+
"moonep_routing": 800,
|
| 332 |
+
"moonep_summary": 80,
|
| 333 |
+
"moonep_weights": 800
|
| 334 |
+
},
|
| 335 |
+
"router_balancing": "aux_loss",
|
| 336 |
+
"run_dir": "QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1",
|
| 337 |
+
"seq_length": "4096",
|
| 338 |
+
"shared_expert_overlap": "0",
|
| 339 |
+
"topk": "8"
|
| 340 |
+
},
|
| 341 |
+
{
|
| 342 |
+
"arm": "forced",
|
| 343 |
+
"aux_loss_coeff": "1e-2",
|
| 344 |
+
"backend": "moonep",
|
| 345 |
+
"branch": "moonep_backend_token_routing_analysis",
|
| 346 |
+
"capacity": "none",
|
| 347 |
+
"capacity_mode": "none",
|
| 348 |
+
"cold_expert_fraction_max": 0.0,
|
| 349 |
+
"cold_expert_fraction_mean": 0.0,
|
| 350 |
+
"cold_expert_fraction_min": 0.0,
|
| 351 |
+
"cold_expert_fraction_n": 100,
|
| 352 |
+
"commit": "35ae6e7f",
|
| 353 |
+
"config": "QWEN2_57B",
|
| 354 |
+
"cuda_graph": "0",
|
| 355 |
+
"dense_first_layers": "0",
|
| 356 |
+
"experts_per_rank": "8",
|
| 357 |
+
"force_balance": "1",
|
| 358 |
+
"global_batch_size": "8",
|
| 359 |
+
"global_snapshots": 100,
|
| 360 |
+
"hottest_expert_token_fraction_max": 0.1317138671875,
|
| 361 |
+
"hottest_expert_token_fraction_mean": 0.12946014404296874,
|
| 362 |
+
"hottest_expert_token_fraction_min": 0.128021240234375,
|
| 363 |
+
"hottest_expert_token_fraction_n": 100,
|
| 364 |
+
"incomplete_snapshots": 0,
|
| 365 |
+
"malformed_records": 0,
|
| 366 |
+
"moe_layers": "10",
|
| 367 |
+
"moonep_bytes_global_snapshots": 100,
|
| 368 |
+
"moonep_bytes_incomplete_snapshots": 0,
|
| 369 |
+
"moonep_bytes_malformed_records": 0,
|
| 370 |
+
"moonep_global_combine_payload_bytes_max": 2237204480.0,
|
| 371 |
+
"moonep_global_combine_payload_bytes_mean": 2231120138.24,
|
| 372 |
+
"moonep_global_combine_payload_bytes_min": 2224459776.0,
|
| 373 |
+
"moonep_global_combine_payload_bytes_n": 100,
|
| 374 |
+
"moonep_global_dispatch_payload_bytes_max": 2237204480.0,
|
| 375 |
+
"moonep_global_dispatch_payload_bytes_mean": 2231120138.24,
|
| 376 |
+
"moonep_global_dispatch_payload_bytes_min": 2224459776.0,
|
| 377 |
+
"moonep_global_dispatch_payload_bytes_n": 100,
|
| 378 |
+
"moonep_global_prefetch_bytes_max": 770703360.0,
|
| 379 |
+
"moonep_global_prefetch_bytes_mean": 762996326.4,
|
| 380 |
+
"moonep_global_prefetch_bytes_min": 660602880.0,
|
| 381 |
+
"moonep_global_prefetch_bytes_n": 100,
|
| 382 |
+
"moonep_global_reduce_bytes_max": 770703360.0,
|
| 383 |
+
"moonep_global_reduce_bytes_mean": 762996326.4,
|
| 384 |
+
"moonep_global_reduce_bytes_min": 660602880.0,
|
| 385 |
+
"moonep_global_reduce_bytes_n": 100,
|
| 386 |
+
"moonep_global_token_bytes_max": 4476243024.0,
|
| 387 |
+
"moonep_global_token_bytes_mean": 4464073623.92,
|
| 388 |
+
"moonep_global_token_bytes_min": 4450749096.0,
|
| 389 |
+
"moonep_global_token_bytes_n": 100,
|
| 390 |
+
"moonep_global_total_nvlink_bytes_max": 6017649744.0,
|
| 391 |
+
"moonep_global_total_nvlink_bytes_mean": 5990066276.72,
|
| 392 |
+
"moonep_global_total_nvlink_bytes_min": 5771954856.0,
|
| 393 |
+
"moonep_global_total_nvlink_bytes_n": 100,
|
| 394 |
+
"moonep_global_weight_movement_bytes_max": 1541406720.0,
|
| 395 |
+
"moonep_global_weight_movement_bytes_mean": 1525992652.8,
|
| 396 |
+
"moonep_global_weight_movement_bytes_min": 1321205760.0,
|
| 397 |
+
"moonep_global_weight_movement_bytes_n": 100,
|
| 398 |
+
"moonep_global_weight_to_token_ratio_max": 0.3460953541488461,
|
| 399 |
+
"moonep_global_weight_to_token_ratio_mean": 0.34183623349354486,
|
| 400 |
+
"moonep_global_weight_to_token_ratio_min": 0.2956613023580652,
|
| 401 |
+
"moonep_global_weight_to_token_ratio_n": 100,
|
| 402 |
+
"moonep_remote_demand_max": 1.0,
|
| 403 |
+
"moonep_remote_demand_mean": 0.86625,
|
| 404 |
+
"moonep_remote_demand_min": 0.0,
|
| 405 |
+
"moonep_remote_demand_n": 800,
|
| 406 |
+
"moonep_slot_utilization_max": 0.125,
|
| 407 |
+
"moonep_slot_utilization_mean": 0.10828125,
|
| 408 |
+
"moonep_slot_utilization_min": 0.0,
|
| 409 |
+
"moonep_slot_utilization_n": 800,
|
| 410 |
+
"moonep_unused_prefetch_slots_max": 16.0,
|
| 411 |
+
"moonep_unused_prefetch_slots_mean": 14.2675,
|
| 412 |
+
"moonep_unused_prefetch_slots_min": 14.0,
|
| 413 |
+
"moonep_unused_prefetch_slots_n": 800,
|
| 414 |
+
"num_experts": "64",
|
| 415 |
+
"num_layers": "10",
|
| 416 |
+
"post_physical_rank_cv_max": 0.004173107469722948,
|
| 417 |
+
"post_physical_rank_cv_mean": 0.0021866389418670955,
|
| 418 |
+
"post_physical_rank_cv_min": 0.0008634926254374552,
|
| 419 |
+
"post_physical_rank_cv_n": 100,
|
| 420 |
+
"post_physical_rank_imax_max": 1.0063246661981728,
|
| 421 |
+
"post_physical_rank_imax_mean": 1.0033137485288246,
|
| 422 |
+
"post_physical_rank_imax_min": 1.001134997246789,
|
| 423 |
+
"post_physical_rank_imax_n": 100,
|
| 424 |
+
"pre_home_rank_cv_max": 0.00995108219738984,
|
| 425 |
+
"pre_home_rank_cv_mean": 0.0057066499140920315,
|
| 426 |
+
"pre_home_rank_cv_min": 0.0010965114882337452,
|
| 427 |
+
"pre_home_rank_cv_n": 100,
|
| 428 |
+
"pre_home_rank_imax_max": 1.014678955078125,
|
| 429 |
+
"pre_home_rank_imax_mean": 1.009002685546875,
|
| 430 |
+
"pre_home_rank_imax_min": 1.001373291015625,
|
| 431 |
+
"pre_home_rank_imax_n": 100,
|
| 432 |
+
"raw_expert_cv_max": 0.0179363414576349,
|
| 433 |
+
"raw_expert_cv_mean": 0.014900499877977456,
|
| 434 |
+
"raw_expert_cv_min": 0.011926166451983852,
|
| 435 |
+
"raw_expert_cv_n": 100,
|
| 436 |
+
"raw_expert_imax_ceiling": 8.0,
|
| 437 |
+
"raw_expert_imax_hit_ceiling": 0,
|
| 438 |
+
"raw_expert_imax_max": 1.0537109375,
|
| 439 |
+
"raw_expert_imax_mean": 1.03568115234375,
|
| 440 |
+
"raw_expert_imax_min": 1.024169921875,
|
| 441 |
+
"raw_expert_imax_n": 100,
|
| 442 |
+
"records": {
|
| 443 |
+
"moonep_bytes": 800,
|
| 444 |
+
"moonep_routing": 800,
|
| 445 |
+
"moonep_summary": 80,
|
| 446 |
+
"moonep_weights": 800
|
| 447 |
+
},
|
| 448 |
+
"router_balancing": "aux_loss",
|
| 449 |
+
"run_dir": "QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1",
|
| 450 |
+
"seq_length": "4096",
|
| 451 |
+
"shared_expert_overlap": "0",
|
| 452 |
+
"topk": "8"
|
| 453 |
+
}
|
| 454 |
+
]
|
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
run_dir arm config backend num_experts topk experts_per_rank num_layers moe_layers dense_first_layers seq_length global_batch_size capacity_mode capacity force_balance cuda_graph branch commit router_balancing aux_loss_coeff shared_expert_overlap cold_expert_fraction_max cold_expert_fraction_mean cold_expert_fraction_min cold_expert_fraction_n global_snapshots hottest_expert_token_fraction_max hottest_expert_token_fraction_mean hottest_expert_token_fraction_min hottest_expert_token_fraction_n incomplete_snapshots malformed_records moonep_bytes_global_snapshots moonep_bytes_incomplete_snapshots moonep_bytes_malformed_records moonep_global_combine_payload_bytes_max moonep_global_combine_payload_bytes_mean moonep_global_combine_payload_bytes_min moonep_global_combine_payload_bytes_n moonep_global_dispatch_payload_bytes_max moonep_global_dispatch_payload_bytes_mean moonep_global_dispatch_payload_bytes_min moonep_global_dispatch_payload_bytes_n moonep_global_prefetch_bytes_max moonep_global_prefetch_bytes_mean moonep_global_prefetch_bytes_min moonep_global_prefetch_bytes_n moonep_global_reduce_bytes_max moonep_global_reduce_bytes_mean moonep_global_reduce_bytes_min moonep_global_reduce_bytes_n moonep_global_token_bytes_max moonep_global_token_bytes_mean moonep_global_token_bytes_min moonep_global_token_bytes_n moonep_global_total_nvlink_bytes_max moonep_global_total_nvlink_bytes_mean moonep_global_total_nvlink_bytes_min moonep_global_total_nvlink_bytes_n moonep_global_weight_movement_bytes_max moonep_global_weight_movement_bytes_mean moonep_global_weight_movement_bytes_min moonep_global_weight_movement_bytes_n moonep_global_weight_to_token_ratio_max moonep_global_weight_to_token_ratio_mean moonep_global_weight_to_token_ratio_min moonep_global_weight_to_token_ratio_n moonep_remote_demand_max moonep_remote_demand_mean moonep_remote_demand_min moonep_remote_demand_n moonep_slot_utilization_max moonep_slot_utilization_mean moonep_slot_utilization_min moonep_slot_utilization_n moonep_unused_prefetch_slots_max moonep_unused_prefetch_slots_mean moonep_unused_prefetch_slots_min moonep_unused_prefetch_slots_n post_physical_rank_cv_max post_physical_rank_cv_mean post_physical_rank_cv_min post_physical_rank_cv_n post_physical_rank_imax_max post_physical_rank_imax_mean post_physical_rank_imax_min post_physical_rank_imax_n pre_home_rank_cv_max pre_home_rank_cv_mean pre_home_rank_cv_min pre_home_rank_cv_n pre_home_rank_imax_max pre_home_rank_imax_mean pre_home_rank_imax_min pre_home_rank_imax_n raw_expert_cv_max raw_expert_cv_mean raw_expert_cv_min raw_expert_cv_n raw_expert_imax_ceiling raw_expert_imax_hit_ceiling raw_expert_imax_max raw_expert_imax_mean raw_expert_imax_min raw_expert_imax_n
|
| 2 |
+
DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1 natural DSV2_LITE_L24 moonep 64 6 8 24 23 1 8192 8 none none 0 0 moonep_backend_token_routing_analysis 35ae6e7f seq_aux_loss 1e-3 0 0.578125 0.15625 0.0 230 230 0.999969482421875 0.7797577567722487 0.162750244140625 230 0 0 230 0 0 2592899072.0 2282338571.130435 2026799104.0 230 2592899072.0 2282338571.130435 2026799104.0 230 346030080.0 252601958.4 242221056.0 230 346030080.0 252601958.4 242221056.0 230 5188482536.0 4567339392.869565 4056064752.0 230 5672924648.0 5072543309.669565 4540506864.0 230 692060160.0 505203916.8 484442112.0 230 0.14851639788299403 0.11096021713514241 0.09336874676530664 230 3.0 0.9125 0.0 1840 0.375 0.1140625 0.0 1840 16.0 14.175 10.0 1840 0.24644338469771185 0.0866314272142518 0.016159622971429973 230 1.2001146488570558 1.1121655106296537 1.0188653892666146 230 1.3626457501059894 0.6592086328754161 0.06474057713157837 230 4.328043619791667 2.2370494897814766 1.1056925455729167 230 3.0868156557140782 1.9632045302354437 0.37852206422070017 230 10.666666666666666 1 10.666341145833334 8.317416072237318 1.7360026041666667 230
|
| 3 |
+
DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1 forced DSV2_LITE_L24 moonep 64 6 8 24 23 1 8192 8 none none 1 0 moonep_backend_token_routing_analysis 35ae6e7f seq_aux_loss 1e-3 0 0.0 0.0 0.0 230 230 0.0982818603515625 0.09643501613451087 0.09539794921875 230 0 0 230 0 0 2133762048.0 2129465023.4434783 2124537856.0 230 2133762048.0 2129465023.4434783 2124537856.0 230 242221056.0 238309411.61739132 207618048.0 230 242221056.0 238309411.61739132 207618048.0 230 4270279408.0 4261681018.2608695 4251824656.0 230 4754721520.0 4738299841.495652 4667434440.0 230 484442112.0 476618823.23478264 415236096.0 230 0.11393746242954192 0.11183841293995254 0.09728537714366937 230 1.0 0.8608695652173913 0.0 1840 0.125 0.10760869565217392 0.0 1840 16.0 14.278260869565218 14.0 1840 0.0031667036922078573 0.0016501228109707747 0.0006791931370339787 230 1.006201201251668 1.0026117364992109 1.0009196904719393 230 0.006673533732490181 0.003923942735502633 0.0014708009315538319 230 1.0122884114583333 1.0061054616734602 1.00213623046875 230 0.015123587213954862 0.012068912140234289 0.008171875787738593 230 10.666666666666666 0 1.04833984375 1.0286401721014493 1.017578125 230
|
| 4 |
+
QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1 natural QWEN2_57B moonep 64 8 8 10 10 0 4096 8 none none 0 0 moonep_backend_token_routing_analysis 35ae6e7f aux_loss 1e-2 0 0.796875 0.39234375 0.0 100 100 1.0 0.9268069458007813 0.2415771484375 100 0 0 100 0 0 3287445504.0 2789546475.52 2180663296.0 100 3287445504.0 2789546475.52 2180663296.0 100 990904320.0 823551590.4 660602880.0 100 990904320.0 823551590.4 660602880.0 100 6576725968.0 5580893210.72 4363119160.0 100 8328785856.0 7227996391.52 5904525880.0 100 1981808640.0 1647103180.8 1321205760.0 100 0.4380461607026548 0.2995223715370738 0.20089110697762314 100 2.0 0.935 0.0 800 0.25 0.116875 0.0 800 16.0 14.13 12.0 800 0.20167894084637375 0.08583132480611003 0.00019943885935289461 100 1.2419115350424064 1.1036399766472182 1.0002518334236372 100 1.4500421285803669 0.7161558574760766 0.06359546419183554 100 4.147735595703125 2.322430419921875 1.11138916015625 100 2.6452554407770648 2.184735306615502 0.4993102636020994 100 8.0 1 8.0 7.41445556640625 1.9326171875 100
|
| 5 |
+
QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1 forced QWEN2_57B moonep 64 8 8 10 10 0 4096 8 none none 1 0 moonep_backend_token_routing_analysis 35ae6e7f aux_loss 1e-2 0 0.0 0.0 0.0 100 100 0.1317138671875 0.12946014404296874 0.128021240234375 100 0 0 100 0 0 2237204480.0 2231120138.24 2224459776.0 100 2237204480.0 2231120138.24 2224459776.0 100 770703360.0 762996326.4 660602880.0 100 770703360.0 762996326.4 660602880.0 100 4476243024.0 4464073623.92 4450749096.0 100 6017649744.0 5990066276.72 5771954856.0 100 1541406720.0 1525992652.8 1321205760.0 100 0.3460953541488461 0.34183623349354486 0.2956613023580652 100 1.0 0.86625 0.0 800 0.125 0.10828125 0.0 800 16.0 14.2675 14.0 800 0.004173107469722948 0.0021866389418670955 0.0008634926254374552 100 1.0063246661981728 1.0033137485288246 1.001134997246789 100 0.00995108219738984 0.0057066499140920315 0.0010965114882337452 100 1.014678955078125 1.009002685546875 1.001373291015625 100 0.0179363414576349 0.014900499877977456 0.011926166451983852 100 8.0 0 1.0537109375 1.03568115234375 1.024169921875 100
|
tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/results.tsv
CHANGED
|
@@ -1,2 +1,7 @@
|
|
| 1 |
config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
|
| 2 |
QWEN3_235B moonep matched 16 8192 8 none none 1 OOM NA NA 182587 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
|
| 2 |
QWEN3_235B moonep matched 16 8192 8 none none 1 OOM NA NA 182587 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1
|
| 3 |
+
QWEN3_30B moonep matched 16 4096 8 none none 1 OOM NA NA 182625 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1
|
| 4 |
+
DSV2_LITE_L24 moonep matched 8 8192 8 none none 1 OK 848.31 791.80 157151 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1
|
| 5 |
+
DSV2_LITE_L24 moonep matched 8 8192 8 none none+forcebal 1 OK 791.39 788.20 157151 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1
|
| 6 |
+
QWEN2_57B moonep matched 8 4096 8 none none 1 OK 366.93 362.10 141553 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1
|
| 7 |
+
QWEN2_57B moonep matched 8 4096 8 none none+forcebal 1 OK 407.76 358.70 141551 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1
|