granularity results tiera-rerun-fixed-moonep @ 2026-08-30T05:20:52Z from sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0
Browse files- tiera-rerun-fixed-moonep/STATUS.txt +3 -2
- tiera-rerun-fixed-moonep/collected_all.tsv +22 -0
- tiera-rerun-fixed-moonep/hf_watcher.log +2 -0
- tiera-rerun-fixed-moonep/probe_summary_all.json +1729 -0
- tiera-rerun-fixed-moonep/probe_summary_all.tsv +22 -0
- tiera-rerun-fixed-moonep/steps.tsv +1 -0
- tiera-rerun-fixed-moonep/study.log +181 -0
tiera-rerun-fixed-moonep/STATUS.txt
CHANGED
|
@@ -1,9 +1,10 @@
|
|
| 1 |
study_root: /workspace/tiera-rerun-fixed-moonep
|
| 2 |
gbs: 16
|
| 3 |
-
updated: 2026-08-30T05:
|
| 4 |
-
current:
|
| 5 |
|
| 6 |
completed steps:
|
| 7 |
|
| 8 |
failed steps:
|
| 9 |
FAIL echo tierbprobe 16 2026-08-30T04:52:38+00:00 2026-08-30T05:06:38+00:00 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe
|
|
|
|
|
|
| 1 |
study_root: /workspace/tiera-rerun-fixed-moonep
|
| 2 |
gbs: 16
|
| 3 |
+
updated: 2026-08-30T05:20:43+00:00
|
| 4 |
+
current: idle
|
| 5 |
|
| 6 |
completed steps:
|
| 7 |
|
| 8 |
failed steps:
|
| 9 |
FAIL echo tierbprobe 16 2026-08-30T04:52:38+00:00 2026-08-30T05:06:38+00:00 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe
|
| 10 |
+
FAIL ultraep tierbprobe 16 2026-08-30T05:06:44+00:00 2026-08-30T05:20:40+00:00 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe
|
tiera-rerun-fixed-moonep/collected_all.tsv
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
run_dir rep phase is_trace async config backend slot_policy slots_per_rank num_experts topk moe_ffn_hidden experts_per_rank expert_params num_layers seq_length global_batch_size num_microbatches assignments_per_rank mean_tokens_per_expert capacity capacity_mode recompute force_balance cuda_graph memory_dump_mode memory_snapshot_path ultraep_quota_min_tokens shared_expert_overlap branch commit dense_first_layers moe_layers router_balancing aux_loss_coeff router_pre_softmax router_fusion rotary_base qkv_bias status iters_seen iter_ms_mean iter_ms_min iter_ms_stdev tflops peak_mib_sampled megatron_max_reserved_mib drop_warning
|
| 2 |
+
echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1 rep1 tierbprobe 0 0 G1 echo matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.00 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 671.53 666.90 4.51 496.5 125477 94730
|
| 3 |
+
echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1 rep1 tierbprobe 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 660.43 642.00 14.73 505.0 119267 88146
|
| 4 |
+
echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1 rep1 tierbprobe 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 1 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 628.07 616.10 9.43 530.9 119267 88146
|
| 5 |
+
echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap2p00/rep1 rep1 tierbprobe 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 2.00 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 675.77 669.20 6.13 493.4 130289 99098
|
| 6 |
+
echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625/rep1 rep1 tierbprobe 0 0 G3 echo matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 1.072265625 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 681.37 670.20 12.74 489.4 122853 91544
|
| 7 |
+
echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625__forcebal/rep1 rep1 tierbprobe 0 0 G3 echo matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 1.072265625 static none 1 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 664.27 659.10 5.62 501.9 122899 91652
|
| 8 |
+
echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap2p00/rep1 rep1 tierbprobe 0 0 G3 echo matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 2.00 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 721.76 713.60 7.90 461.9 136111 103816
|
| 9 |
+
echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375/rep1 rep1 tierbprobe 0 0 G4 echo matched 16 128 32 896 16 11010048 16 4096 16 2 131072 8192 1.068359375 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 775.73 738.90 77.17 432.9 129621 96956
|
| 10 |
+
echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375__forcebal/rep1 rep1 tierbprobe 0 0 G4 echo matched 16 128 32 896 16 11010048 16 4096 16 2 131072 8192 1.068359375 static none 1 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 768.04 733.40 78.16 437.3 129579 96956
|
| 11 |
+
echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap2p00/rep1 rep1 tierbprobe 0 0 G4 echo matched 16 128 32 896 16 11010048 16 4096 16 2 131072 8192 2.00 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 829.14 790.80 78.61 404.7 146037 114976
|
| 12 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1 rep1 tierbprobe 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 1.25 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 893.49 886.90 5.95 373.1 108731 80984 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=283999 total_dropped_tok
|
| 13 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1 rep1 tierbprobe 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 1.8125 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 914.84 903.60 9.13 364.4 113767 84920 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=284540 total_dropped_tok
|
| 14 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 rep1 tierbprobe 0 0 G2 ultraep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.0625 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 551.94 527.00 15.90 604.4 107987 75842 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=1048800 total_dropped_to
|
| 15 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1 rep1 tierbprobe 0 0 G2 ultraep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.25 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 558.91 532.70 19.50 597.1 110343 78094 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=1048800 total_dropped_to
|
| 16 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1 rep1 tierbprobe 0 0 G2 ultraep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.25 static none 1 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 426.26 422.10 3.34 782.2 110343 78094 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=32 total_expected_recv=1047984 total_dropped_to
|
| 17 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 rep1 tierbprobe 0 0 G3 ultraep matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 1.0625 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 528.17 522.80 4.70 631.3 111567 79350 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=32 total_expected_recv=2101251 total_dropped_to
|
| 18 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1 rep1 tierbprobe 0 0 G3 ultraep matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 1.25 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 536.43 529.00 4.61 621.5 114319 82188 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=2089759 total_dropped_to
|
| 19 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1 rep1 tierbprobe 0 0 G3 ultraep matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 1.25 static none 1 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 472.34 470.60 1.41 705.8 114267 82222 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=2098254 total_dropped_to
|
| 20 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1 rep1 tierbprobe 0 0 G4 ultraep matched 16 128 32 896 16 11010048 16 4096 16 2 131072 8192 1.095703125 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 594.23 563.90 67.44 566.2 118667 85006 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=32 total_expected_recv=4197711 total_dropped_to
|
| 21 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1 rep1 tierbprobe 0 0 G4 ultraep matched 16 128 32 896 16 11010048 16 4096 16 2 131072 8192 1.25 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 610.06 573.90 80.80 553.1 120487 87986 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=32 total_expected_recv=4196979 total_dropped_to
|
| 22 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1 rep1 tierbprobe 0 0 G4 ultraep matched 16 128 32 896 16 11010048 16 4096 16 2 131072 8192 1.25 static none 1 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 596.29 545.60 85.07 567.8 120487 87986 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=4190539 total_dropped_to
|
tiera-rerun-fixed-moonep/hf_watcher.log
CHANGED
|
@@ -7,3 +7,5 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
|
|
| 7 |
uploaded: /workspace/tiera-rerun-fixed-moonep -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tiera-rerun-fixed-moonep
|
| 8 |
https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
|
| 9 |
[2026-08-30T05:12:45+00:00] backup pass
|
|
|
|
|
|
|
|
|
| 7 |
uploaded: /workspace/tiera-rerun-fixed-moonep -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tiera-rerun-fixed-moonep
|
| 8 |
https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
|
| 9 |
[2026-08-30T05:12:45+00:00] backup pass
|
| 10 |
+
uploaded: /workspace/tiera-rerun-fixed-moonep -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tiera-rerun-fixed-moonep
|
| 11 |
+
https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
|
tiera-rerun-fixed-moonep/probe_summary_all.json
ADDED
|
@@ -0,0 +1,1729 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"arm": "natural",
|
| 4 |
+
"assignment_sum_mismatches": 0,
|
| 5 |
+
"aux_loss_coeff": "1e-2",
|
| 6 |
+
"backend": "echo",
|
| 7 |
+
"branch": "token-drop-analysis",
|
| 8 |
+
"capacity": "2.00",
|
| 9 |
+
"capacity_mode": "static",
|
| 10 |
+
"cold_expert_fraction_max": 0.75,
|
| 11 |
+
"cold_expert_fraction_mean": 0.18587239583333334,
|
| 12 |
+
"cold_expert_fraction_min": 0.0,
|
| 13 |
+
"cold_expert_fraction_n": 384,
|
| 14 |
+
"commit": "b099fd84",
|
| 15 |
+
"config": "G1",
|
| 16 |
+
"cuda_graph": "0",
|
| 17 |
+
"dense_first_layers": "0",
|
| 18 |
+
"drop_accounting_source": "dispatch",
|
| 19 |
+
"drop_expected_assignments": 25165824.0,
|
| 20 |
+
"drop_rate_pct_overall": 0.0,
|
| 21 |
+
"echo_logged_load_reduction_valid": 0,
|
| 22 |
+
"echo_logged_pre_rank_metric_valid": 0,
|
| 23 |
+
"echo_moved_assignment_fraction_max": 0.6113739013671875,
|
| 24 |
+
"echo_moved_assignment_fraction_mean": 0.5011123021443685,
|
| 25 |
+
"echo_moved_assignment_fraction_min": 0.4014892578125,
|
| 26 |
+
"echo_moved_assignment_fraction_n": 384,
|
| 27 |
+
"experts_per_rank": "1",
|
| 28 |
+
"force_balance": "0",
|
| 29 |
+
"global_batch_size": "16",
|
| 30 |
+
"global_snapshots": 384,
|
| 31 |
+
"hottest_expert_token_fraction_max": 1.0,
|
| 32 |
+
"hottest_expert_token_fraction_mean": 0.9049501419067383,
|
| 33 |
+
"hottest_expert_token_fraction_min": 0.374786376953125,
|
| 34 |
+
"hottest_expert_token_fraction_n": 384,
|
| 35 |
+
"incomplete_snapshots": 0,
|
| 36 |
+
"malformed_records": 0,
|
| 37 |
+
"moe_layers": "16",
|
| 38 |
+
"num_experts": "8",
|
| 39 |
+
"num_layers": "16",
|
| 40 |
+
"observed_dropped_tokens": 0,
|
| 41 |
+
"post_physical_rank_cv_max": 0.3858312238475252,
|
| 42 |
+
"post_physical_rank_cv_mean": 0.04369442624774788,
|
| 43 |
+
"post_physical_rank_cv_min": 0.0,
|
| 44 |
+
"post_physical_rank_cv_n": 384,
|
| 45 |
+
"post_physical_rank_imax_max": 1.8909912109375,
|
| 46 |
+
"post_physical_rank_imax_mean": 1.0827395121256511,
|
| 47 |
+
"post_physical_rank_imax_min": 1.0,
|
| 48 |
+
"post_physical_rank_imax_n": 384,
|
| 49 |
+
"pre_home_rank_cv_max": 1.7320508075688772,
|
| 50 |
+
"pre_home_rank_cv_mean": 1.4536835274449291,
|
| 51 |
+
"pre_home_rank_cv_min": 0.2541278385706527,
|
| 52 |
+
"pre_home_rank_cv_n": 384,
|
| 53 |
+
"pre_home_rank_imax_max": 4.0,
|
| 54 |
+
"pre_home_rank_imax_mean": 3.619800567626953,
|
| 55 |
+
"pre_home_rank_imax_min": 1.4991455078125,
|
| 56 |
+
"pre_home_rank_imax_n": 384,
|
| 57 |
+
"raw_expert_cv_max": 1.7320508075688772,
|
| 58 |
+
"raw_expert_cv_mean": 1.4536835274449291,
|
| 59 |
+
"raw_expert_cv_min": 0.2541278385706527,
|
| 60 |
+
"raw_expert_cv_n": 384,
|
| 61 |
+
"raw_expert_imax_ceiling": 4.0,
|
| 62 |
+
"raw_expert_imax_hit_ceiling": 1,
|
| 63 |
+
"raw_expert_imax_max": 4.0,
|
| 64 |
+
"raw_expert_imax_mean": 3.619800567626953,
|
| 65 |
+
"raw_expert_imax_min": 1.4991455078125,
|
| 66 |
+
"raw_expert_imax_n": 384,
|
| 67 |
+
"records": {
|
| 68 |
+
"drop_dispatch": 3072,
|
| 69 |
+
"drop_experts": 3072,
|
| 70 |
+
"drop_setup": 3072,
|
| 71 |
+
"echo_post": 3072,
|
| 72 |
+
"echo_pre": 3072
|
| 73 |
+
},
|
| 74 |
+
"required_capacity_max": 1.90625,
|
| 75 |
+
"required_capacity_mean": 1.0099385579427083,
|
| 76 |
+
"required_capacity_min": 0.53125,
|
| 77 |
+
"required_capacity_n": 3072,
|
| 78 |
+
"router_balancing": "aux_loss",
|
| 79 |
+
"run_dir": "echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1",
|
| 80 |
+
"saturated_dispatch_rows": 0,
|
| 81 |
+
"seq_length": "4096",
|
| 82 |
+
"shared_expert_overlap": "0",
|
| 83 |
+
"topk": "2",
|
| 84 |
+
"valid_max_rank_reduction_max": 0.75,
|
| 85 |
+
"valid_max_rank_reduction_mean": 0.6772174606036838,
|
| 86 |
+
"valid_max_rank_reduction_min": 0.11090858104318564,
|
| 87 |
+
"valid_max_rank_reduction_n": 384,
|
| 88 |
+
"zero_drop": 1
|
| 89 |
+
},
|
| 90 |
+
{
|
| 91 |
+
"arm": "natural",
|
| 92 |
+
"assignment_sum_mismatches": 0,
|
| 93 |
+
"aux_loss_coeff": "1e-2",
|
| 94 |
+
"backend": "echo",
|
| 95 |
+
"branch": "token-drop-analysis",
|
| 96 |
+
"capacity": "1.08203125",
|
| 97 |
+
"capacity_mode": "static",
|
| 98 |
+
"cold_expert_fraction_max": 0.71875,
|
| 99 |
+
"cold_expert_fraction_mean": 0.159912109375,
|
| 100 |
+
"cold_expert_fraction_min": 0.0,
|
| 101 |
+
"cold_expert_fraction_n": 384,
|
| 102 |
+
"commit": "b099fd84",
|
| 103 |
+
"config": "G2",
|
| 104 |
+
"cuda_graph": "0",
|
| 105 |
+
"dense_first_layers": "0",
|
| 106 |
+
"drop_accounting_source": "dispatch",
|
| 107 |
+
"drop_expected_assignments": 100663296.0,
|
| 108 |
+
"drop_rate_pct_overall": 0.0,
|
| 109 |
+
"echo_logged_load_reduction_valid": 0,
|
| 110 |
+
"echo_logged_pre_rank_metric_valid": 0,
|
| 111 |
+
"echo_moved_assignment_fraction_max": 0.5000267028808594,
|
| 112 |
+
"echo_moved_assignment_fraction_mean": 0.499999205271403,
|
| 113 |
+
"echo_moved_assignment_fraction_min": 0.49983978271484375,
|
| 114 |
+
"echo_moved_assignment_fraction_n": 384,
|
| 115 |
+
"experts_per_rank": "4",
|
| 116 |
+
"force_balance": "0",
|
| 117 |
+
"global_batch_size": "16",
|
| 118 |
+
"global_snapshots": 384,
|
| 119 |
+
"hottest_expert_token_fraction_max": 1.0,
|
| 120 |
+
"hottest_expert_token_fraction_mean": 0.9680010477701823,
|
| 121 |
+
"hottest_expert_token_fraction_min": 0.613189697265625,
|
| 122 |
+
"hottest_expert_token_fraction_n": 384,
|
| 123 |
+
"incomplete_snapshots": 0,
|
| 124 |
+
"malformed_records": 0,
|
| 125 |
+
"moe_layers": "16",
|
| 126 |
+
"num_experts": "32",
|
| 127 |
+
"num_layers": "16",
|
| 128 |
+
"observed_dropped_tokens": 0,
|
| 129 |
+
"post_physical_rank_cv_max": 0.00213623046875,
|
| 130 |
+
"post_physical_rank_cv_mean": 1.5938873823057422e-05,
|
| 131 |
+
"post_physical_rank_cv_min": 0.0,
|
| 132 |
+
"post_physical_rank_cv_n": 384,
|
| 133 |
+
"post_physical_rank_imax_max": 1.0042724609375,
|
| 134 |
+
"post_physical_rank_imax_mean": 1.0000306765238445,
|
| 135 |
+
"post_physical_rank_imax_min": 1.0,
|
| 136 |
+
"post_physical_rank_imax_n": 384,
|
| 137 |
+
"pre_home_rank_cv_max": 1.3175703221722546,
|
| 138 |
+
"pre_home_rank_cv_mean": 0.6974670284251169,
|
| 139 |
+
"pre_home_rank_cv_min": 0.0007166759378550646,
|
| 140 |
+
"pre_home_rank_cv_n": 384,
|
| 141 |
+
"pre_home_rank_imax_max": 3.998382568359375,
|
| 142 |
+
"pre_home_rank_imax_mean": 2.1703339417775473,
|
| 143 |
+
"pre_home_rank_imax_min": 1.001617431640625,
|
| 144 |
+
"pre_home_rank_imax_n": 384,
|
| 145 |
+
"raw_expert_cv_max": 1.7320419981493513,
|
| 146 |
+
"raw_expert_cv_mean": 1.5150670645228896,
|
| 147 |
+
"raw_expert_cv_min": 0.6923179293416859,
|
| 148 |
+
"raw_expert_cv_n": 384,
|
| 149 |
+
"raw_expert_imax_ceiling": 4.0,
|
| 150 |
+
"raw_expert_imax_hit_ceiling": 1,
|
| 151 |
+
"raw_expert_imax_max": 4.0,
|
| 152 |
+
"raw_expert_imax_mean": 3.872004191080729,
|
| 153 |
+
"raw_expert_imax_min": 2.4527587890625,
|
| 154 |
+
"raw_expert_imax_n": 384,
|
| 155 |
+
"records": {
|
| 156 |
+
"drop_dispatch": 3072,
|
| 157 |
+
"drop_experts": 3072,
|
| 158 |
+
"drop_setup": 3072,
|
| 159 |
+
"echo_post": 3072,
|
| 160 |
+
"echo_pre": 3072
|
| 161 |
+
},
|
| 162 |
+
"required_capacity_max": 1.02734375,
|
| 163 |
+
"required_capacity_mean": 1.0103823343912761,
|
| 164 |
+
"required_capacity_min": 1.0,
|
| 165 |
+
"required_capacity_n": 3072,
|
| 166 |
+
"router_balancing": "aux_loss",
|
| 167 |
+
"run_dir": "echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1",
|
| 168 |
+
"saturated_dispatch_rows": 0,
|
| 169 |
+
"seq_length": "4096",
|
| 170 |
+
"shared_expert_overlap": "0",
|
| 171 |
+
"topk": "8",
|
| 172 |
+
"valid_max_rank_reduction_max": 0.7498988696295957,
|
| 173 |
+
"valid_max_rank_reduction_mean": 0.5145680576214848,
|
| 174 |
+
"valid_max_rank_reduction_min": 0.0016148197800188904,
|
| 175 |
+
"valid_max_rank_reduction_n": 384,
|
| 176 |
+
"zero_drop": 1
|
| 177 |
+
},
|
| 178 |
+
{
|
| 179 |
+
"arm": "forced",
|
| 180 |
+
"assignment_sum_mismatches": 0,
|
| 181 |
+
"aux_loss_coeff": "1e-2",
|
| 182 |
+
"backend": "echo",
|
| 183 |
+
"branch": "token-drop-analysis",
|
| 184 |
+
"capacity": "1.08203125",
|
| 185 |
+
"capacity_mode": "static",
|
| 186 |
+
"cold_expert_fraction_max": 0.0,
|
| 187 |
+
"cold_expert_fraction_mean": 0.0,
|
| 188 |
+
"cold_expert_fraction_min": 0.0,
|
| 189 |
+
"cold_expert_fraction_n": 384,
|
| 190 |
+
"commit": "b099fd84",
|
| 191 |
+
"config": "G2",
|
| 192 |
+
"cuda_graph": "0",
|
| 193 |
+
"dense_first_layers": "0",
|
| 194 |
+
"drop_accounting_source": "dispatch",
|
| 195 |
+
"drop_expected_assignments": 100663296.0,
|
| 196 |
+
"drop_rate_pct_overall": 0.0,
|
| 197 |
+
"echo_logged_load_reduction_valid": 0,
|
| 198 |
+
"echo_logged_pre_rank_metric_valid": 0,
|
| 199 |
+
"echo_moved_assignment_fraction_max": 0.5,
|
| 200 |
+
"echo_moved_assignment_fraction_mean": 0.4999997019767761,
|
| 201 |
+
"echo_moved_assignment_fraction_min": 0.49993896484375,
|
| 202 |
+
"echo_moved_assignment_fraction_n": 384,
|
| 203 |
+
"experts_per_rank": "4",
|
| 204 |
+
"force_balance": "1",
|
| 205 |
+
"global_batch_size": "16",
|
| 206 |
+
"global_snapshots": 384,
|
| 207 |
+
"hottest_expert_token_fraction_max": 0.2593994140625,
|
| 208 |
+
"hottest_expert_token_fraction_mean": 0.25501465797424316,
|
| 209 |
+
"hottest_expert_token_fraction_min": 0.252777099609375,
|
| 210 |
+
"hottest_expert_token_fraction_n": 384,
|
| 211 |
+
"incomplete_snapshots": 0,
|
| 212 |
+
"malformed_records": 0,
|
| 213 |
+
"moe_layers": "16",
|
| 214 |
+
"num_experts": "32",
|
| 215 |
+
"num_layers": "16",
|
| 216 |
+
"observed_dropped_tokens": 0,
|
| 217 |
+
"post_physical_rank_cv_max": 0.000421485208646641,
|
| 218 |
+
"post_physical_rank_cv_mean": 2.2897106263584403e-06,
|
| 219 |
+
"post_physical_rank_cv_min": 0.0,
|
| 220 |
+
"post_physical_rank_cv_n": 384,
|
| 221 |
+
"post_physical_rank_imax_max": 1.00067138671875,
|
| 222 |
+
"post_physical_rank_imax_mean": 1.0000041325887044,
|
| 223 |
+
"post_physical_rank_imax_min": 1.0,
|
| 224 |
+
"post_physical_rank_imax_n": 384,
|
| 225 |
+
"pre_home_rank_cv_max": 0.007852555942790041,
|
| 226 |
+
"pre_home_rank_cv_mean": 0.004394259674654735,
|
| 227 |
+
"pre_home_rank_cv_min": 0.0016880800477701613,
|
| 228 |
+
"pre_home_rank_cv_n": 384,
|
| 229 |
+
"pre_home_rank_imax_max": 1.01715087890625,
|
| 230 |
+
"pre_home_rank_imax_mean": 1.006885051727295,
|
| 231 |
+
"pre_home_rank_imax_min": 1.002349853515625,
|
| 232 |
+
"pre_home_rank_imax_n": 384,
|
| 233 |
+
"raw_expert_cv_max": 0.013095170649570529,
|
| 234 |
+
"raw_expert_cv_mean": 0.009386188814570484,
|
| 235 |
+
"raw_expert_cv_min": 0.006191008040561732,
|
| 236 |
+
"raw_expert_cv_n": 384,
|
| 237 |
+
"raw_expert_imax_ceiling": 4.0,
|
| 238 |
+
"raw_expert_imax_hit_ceiling": 0,
|
| 239 |
+
"raw_expert_imax_max": 1.03759765625,
|
| 240 |
+
"raw_expert_imax_mean": 1.0200586318969727,
|
| 241 |
+
"raw_expert_imax_min": 1.0111083984375,
|
| 242 |
+
"raw_expert_imax_n": 384,
|
| 243 |
+
"records": {
|
| 244 |
+
"drop_dispatch": 3072,
|
| 245 |
+
"drop_experts": 3072,
|
| 246 |
+
"drop_setup": 3072,
|
| 247 |
+
"echo_post": 3072,
|
| 248 |
+
"echo_pre": 3072
|
| 249 |
+
},
|
| 250 |
+
"required_capacity_max": 1.0234375,
|
| 251 |
+
"required_capacity_mean": 1.010022481282552,
|
| 252 |
+
"required_capacity_min": 1.00390625,
|
| 253 |
+
"required_capacity_n": 3072,
|
| 254 |
+
"router_balancing": "aux_loss",
|
| 255 |
+
"run_dir": "echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1",
|
| 256 |
+
"saturated_dispatch_rows": 0,
|
| 257 |
+
"seq_length": "4096",
|
| 258 |
+
"shared_expert_overlap": "0",
|
| 259 |
+
"topk": "8",
|
| 260 |
+
"valid_max_rank_reduction_max": 0.01686168616861686,
|
| 261 |
+
"valid_max_rank_reduction_mean": 0.006827796124259797,
|
| 262 |
+
"valid_max_rank_reduction_min": 0.0023443446491094536,
|
| 263 |
+
"valid_max_rank_reduction_n": 384,
|
| 264 |
+
"zero_drop": 1
|
| 265 |
+
},
|
| 266 |
+
{
|
| 267 |
+
"arm": "natural",
|
| 268 |
+
"assignment_sum_mismatches": 0,
|
| 269 |
+
"aux_loss_coeff": "1e-2",
|
| 270 |
+
"backend": "echo",
|
| 271 |
+
"branch": "token-drop-analysis",
|
| 272 |
+
"capacity": "2.00",
|
| 273 |
+
"capacity_mode": "static",
|
| 274 |
+
"cold_expert_fraction_max": 0.71875,
|
| 275 |
+
"cold_expert_fraction_mean": 0.15087890625,
|
| 276 |
+
"cold_expert_fraction_min": 0.0,
|
| 277 |
+
"cold_expert_fraction_n": 384,
|
| 278 |
+
"commit": "b099fd84",
|
| 279 |
+
"config": "G2",
|
| 280 |
+
"cuda_graph": "0",
|
| 281 |
+
"dense_first_layers": "0",
|
| 282 |
+
"drop_accounting_source": "dispatch",
|
| 283 |
+
"drop_expected_assignments": 100663296.0,
|
| 284 |
+
"drop_rate_pct_overall": 0.0,
|
| 285 |
+
"echo_logged_load_reduction_valid": 0,
|
| 286 |
+
"echo_logged_pre_rank_metric_valid": 0,
|
| 287 |
+
"echo_moved_assignment_fraction_max": 0.500030517578125,
|
| 288 |
+
"echo_moved_assignment_fraction_mean": 0.4999993046124776,
|
| 289 |
+
"echo_moved_assignment_fraction_min": 0.4998779296875,
|
| 290 |
+
"echo_moved_assignment_fraction_n": 384,
|
| 291 |
+
"experts_per_rank": "4",
|
| 292 |
+
"force_balance": "0",
|
| 293 |
+
"global_batch_size": "16",
|
| 294 |
+
"global_snapshots": 384,
|
| 295 |
+
"hottest_expert_token_fraction_max": 1.0,
|
| 296 |
+
"hottest_expert_token_fraction_mean": 0.9681024551391602,
|
| 297 |
+
"hottest_expert_token_fraction_min": 0.620849609375,
|
| 298 |
+
"hottest_expert_token_fraction_n": 384,
|
| 299 |
+
"incomplete_snapshots": 0,
|
| 300 |
+
"malformed_records": 0,
|
| 301 |
+
"moe_layers": "16",
|
| 302 |
+
"num_experts": "32",
|
| 303 |
+
"num_layers": "16",
|
| 304 |
+
"observed_dropped_tokens": 0,
|
| 305 |
+
"post_physical_rank_cv_max": 0.0021209716796875,
|
| 306 |
+
"post_physical_rank_cv_mean": 1.466968415560366e-05,
|
| 307 |
+
"post_physical_rank_cv_min": 0.0,
|
| 308 |
+
"post_physical_rank_cv_n": 384,
|
| 309 |
+
"post_physical_rank_imax_max": 1.004241943359375,
|
| 310 |
+
"post_physical_rank_imax_mean": 1.0000279744466145,
|
| 311 |
+
"post_physical_rank_imax_min": 1.0,
|
| 312 |
+
"post_physical_rank_imax_n": 384,
|
| 313 |
+
"pre_home_rank_cv_max": 1.317620465314074,
|
| 314 |
+
"pre_home_rank_cv_mean": 0.6968048692022525,
|
| 315 |
+
"pre_home_rank_cv_min": 0.0012016720234495348,
|
| 316 |
+
"pre_home_rank_cv_n": 384,
|
| 317 |
+
"pre_home_rank_imax_max": 3.998260498046875,
|
| 318 |
+
"pre_home_rank_imax_mean": 2.1875499884287515,
|
| 319 |
+
"pre_home_rank_imax_min": 1.00238037109375,
|
| 320 |
+
"pre_home_rank_imax_n": 384,
|
| 321 |
+
"raw_expert_cv_max": 1.7320331892227239,
|
| 322 |
+
"raw_expert_cv_mean": 1.5175145186167975,
|
| 323 |
+
"raw_expert_cv_min": 0.6923179293416859,
|
| 324 |
+
"raw_expert_cv_n": 384,
|
| 325 |
+
"raw_expert_imax_ceiling": 4.0,
|
| 326 |
+
"raw_expert_imax_hit_ceiling": 1,
|
| 327 |
+
"raw_expert_imax_max": 4.0,
|
| 328 |
+
"raw_expert_imax_mean": 3.8724098205566406,
|
| 329 |
+
"raw_expert_imax_min": 2.4833984375,
|
| 330 |
+
"raw_expert_imax_n": 384,
|
| 331 |
+
"records": {
|
| 332 |
+
"drop_dispatch": 3072,
|
| 333 |
+
"drop_experts": 3072,
|
| 334 |
+
"drop_setup": 3072,
|
| 335 |
+
"echo_post": 3072,
|
| 336 |
+
"echo_pre": 3072
|
| 337 |
+
},
|
| 338 |
+
"required_capacity_max": 1.02734375,
|
| 339 |
+
"required_capacity_mean": 1.0105133056640625,
|
| 340 |
+
"required_capacity_min": 1.0,
|
| 341 |
+
"required_capacity_n": 3072,
|
| 342 |
+
"router_balancing": "aux_loss",
|
| 343 |
+
"run_dir": "echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap2p00/rep1",
|
| 344 |
+
"saturated_dispatch_rows": 0,
|
| 345 |
+
"seq_length": "4096",
|
| 346 |
+
"shared_expert_overlap": "0",
|
| 347 |
+
"topk": "8",
|
| 348 |
+
"valid_max_rank_reduction_max": 0.7498912338281876,
|
| 349 |
+
"valid_max_rank_reduction_mean": 0.5172585999990112,
|
| 350 |
+
"valid_max_rank_reduction_min": 0.0023747183827558913,
|
| 351 |
+
"valid_max_rank_reduction_n": 384,
|
| 352 |
+
"zero_drop": 1
|
| 353 |
+
},
|
| 354 |
+
{
|
| 355 |
+
"arm": "natural",
|
| 356 |
+
"assignment_sum_mismatches": 0,
|
| 357 |
+
"aux_loss_coeff": "1e-2",
|
| 358 |
+
"backend": "echo",
|
| 359 |
+
"branch": "token-drop-analysis",
|
| 360 |
+
"capacity": "1.072265625",
|
| 361 |
+
"capacity_mode": "static",
|
| 362 |
+
"cold_expert_fraction_max": 0.6875,
|
| 363 |
+
"cold_expert_fraction_mean": 0.1546630859375,
|
| 364 |
+
"cold_expert_fraction_min": 0.0,
|
| 365 |
+
"cold_expert_fraction_n": 384,
|
| 366 |
+
"commit": "b099fd84",
|
| 367 |
+
"config": "G3",
|
| 368 |
+
"cuda_graph": "0",
|
| 369 |
+
"dense_first_layers": "0",
|
| 370 |
+
"drop_accounting_source": "dispatch",
|
| 371 |
+
"drop_expected_assignments": 201326592.0,
|
| 372 |
+
"drop_rate_pct_overall": 0.0,
|
| 373 |
+
"echo_logged_load_reduction_valid": 0,
|
| 374 |
+
"echo_logged_pre_rank_metric_valid": 0,
|
| 375 |
+
"echo_moved_assignment_fraction_max": 0.5,
|
| 376 |
+
"echo_moved_assignment_fraction_mean": 0.5,
|
| 377 |
+
"echo_moved_assignment_fraction_min": 0.5,
|
| 378 |
+
"echo_moved_assignment_fraction_n": 384,
|
| 379 |
+
"experts_per_rank": "8",
|
| 380 |
+
"force_balance": "0",
|
| 381 |
+
"global_batch_size": "16",
|
| 382 |
+
"global_snapshots": 384,
|
| 383 |
+
"hottest_expert_token_fraction_max": 1.0,
|
| 384 |
+
"hottest_expert_token_fraction_mean": 0.9773949782053629,
|
| 385 |
+
"hottest_expert_token_fraction_min": 0.634124755859375,
|
| 386 |
+
"hottest_expert_token_fraction_n": 384,
|
| 387 |
+
"incomplete_snapshots": 0,
|
| 388 |
+
"malformed_records": 0,
|
| 389 |
+
"moe_layers": "16",
|
| 390 |
+
"num_experts": "64",
|
| 391 |
+
"num_layers": "16",
|
| 392 |
+
"observed_dropped_tokens": 0,
|
| 393 |
+
"post_physical_rank_cv_max": 0.0,
|
| 394 |
+
"post_physical_rank_cv_mean": 0.0,
|
| 395 |
+
"post_physical_rank_cv_min": 0.0,
|
| 396 |
+
"post_physical_rank_cv_n": 384,
|
| 397 |
+
"post_physical_rank_imax_max": 1.0,
|
| 398 |
+
"post_physical_rank_imax_mean": 1.0,
|
| 399 |
+
"post_physical_rank_imax_min": 1.0,
|
| 400 |
+
"post_physical_rank_imax_n": 384,
|
| 401 |
+
"pre_home_rank_cv_max": 0.8292601101580809,
|
| 402 |
+
"pre_home_rank_cv_mean": 0.47371518867838985,
|
| 403 |
+
"pre_home_rank_cv_min": 0.1091963461644949,
|
| 404 |
+
"pre_home_rank_cv_n": 384,
|
| 405 |
+
"pre_home_rank_imax_max": 2.9946746826171875,
|
| 406 |
+
"pre_home_rank_imax_mean": 1.7838420073191326,
|
| 407 |
+
"pre_home_rank_imax_min": 1.153167724609375,
|
| 408 |
+
"pre_home_rank_imax_n": 384,
|
| 409 |
+
"raw_expert_cv_max": 1.7316460620806386,
|
| 410 |
+
"raw_expert_cv_mean": 1.525739653918128,
|
| 411 |
+
"raw_expert_cv_min": 0.6814247311375694,
|
| 412 |
+
"raw_expert_cv_n": 384,
|
| 413 |
+
"raw_expert_imax_ceiling": 4.0,
|
| 414 |
+
"raw_expert_imax_hit_ceiling": 1,
|
| 415 |
+
"raw_expert_imax_max": 4.0,
|
| 416 |
+
"raw_expert_imax_mean": 3.9095799128214517,
|
| 417 |
+
"raw_expert_imax_min": 2.5364990234375,
|
| 418 |
+
"raw_expert_imax_n": 384,
|
| 419 |
+
"records": {
|
| 420 |
+
"drop_dispatch": 3072,
|
| 421 |
+
"drop_experts": 3072,
|
| 422 |
+
"drop_setup": 3072,
|
| 423 |
+
"echo_post": 3072,
|
| 424 |
+
"echo_pre": 3072
|
| 425 |
+
},
|
| 426 |
+
"required_capacity_max": 1.01953125,
|
| 427 |
+
"required_capacity_mean": 1.0093224843343098,
|
| 428 |
+
"required_capacity_min": 1.0,
|
| 429 |
+
"required_capacity_n": 3072,
|
| 430 |
+
"router_balancing": "aux_loss",
|
| 431 |
+
"run_dir": "echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625/rep1",
|
| 432 |
+
"saturated_dispatch_rows": 0,
|
| 433 |
+
"seq_length": "4096",
|
| 434 |
+
"shared_expert_overlap": "0",
|
| 435 |
+
"topk": "16",
|
| 436 |
+
"valid_max_rank_reduction_max": 0.6660739125339475,
|
| 437 |
+
"valid_max_rank_reduction_mean": 0.41846288670458703,
|
| 438 |
+
"valid_max_rank_reduction_min": 0.1328234578029481,
|
| 439 |
+
"valid_max_rank_reduction_n": 384,
|
| 440 |
+
"zero_drop": 1
|
| 441 |
+
},
|
| 442 |
+
{
|
| 443 |
+
"arm": "forced",
|
| 444 |
+
"assignment_sum_mismatches": 0,
|
| 445 |
+
"aux_loss_coeff": "1e-2",
|
| 446 |
+
"backend": "echo",
|
| 447 |
+
"branch": "token-drop-analysis",
|
| 448 |
+
"capacity": "1.072265625",
|
| 449 |
+
"capacity_mode": "static",
|
| 450 |
+
"cold_expert_fraction_max": 0.0,
|
| 451 |
+
"cold_expert_fraction_mean": 0.0,
|
| 452 |
+
"cold_expert_fraction_min": 0.0,
|
| 453 |
+
"cold_expert_fraction_n": 384,
|
| 454 |
+
"commit": "b099fd84",
|
| 455 |
+
"config": "G3",
|
| 456 |
+
"cuda_graph": "0",
|
| 457 |
+
"dense_first_layers": "0",
|
| 458 |
+
"drop_accounting_source": "dispatch",
|
| 459 |
+
"drop_expected_assignments": 201326592.0,
|
| 460 |
+
"drop_rate_pct_overall": 0.0,
|
| 461 |
+
"echo_logged_load_reduction_valid": 0,
|
| 462 |
+
"echo_logged_pre_rank_metric_valid": 0,
|
| 463 |
+
"echo_moved_assignment_fraction_max": 0.5,
|
| 464 |
+
"echo_moved_assignment_fraction_mean": 0.5,
|
| 465 |
+
"echo_moved_assignment_fraction_min": 0.5,
|
| 466 |
+
"echo_moved_assignment_fraction_n": 384,
|
| 467 |
+
"experts_per_rank": "8",
|
| 468 |
+
"force_balance": "1",
|
| 469 |
+
"global_batch_size": "16",
|
| 470 |
+
"global_snapshots": 384,
|
| 471 |
+
"hottest_expert_token_fraction_max": 0.26123046875,
|
| 472 |
+
"hottest_expert_token_fraction_mean": 0.2557719548543294,
|
| 473 |
+
"hottest_expert_token_fraction_min": 0.253570556640625,
|
| 474 |
+
"hottest_expert_token_fraction_n": 384,
|
| 475 |
+
"incomplete_snapshots": 0,
|
| 476 |
+
"malformed_records": 0,
|
| 477 |
+
"moe_layers": "16",
|
| 478 |
+
"num_experts": "64",
|
| 479 |
+
"num_layers": "16",
|
| 480 |
+
"observed_dropped_tokens": 0,
|
| 481 |
+
"post_physical_rank_cv_max": 0.0,
|
| 482 |
+
"post_physical_rank_cv_mean": 0.0,
|
| 483 |
+
"post_physical_rank_cv_min": 0.0,
|
| 484 |
+
"post_physical_rank_cv_n": 384,
|
| 485 |
+
"post_physical_rank_imax_max": 1.0,
|
| 486 |
+
"post_physical_rank_imax_mean": 1.0,
|
| 487 |
+
"post_physical_rank_imax_min": 1.0,
|
| 488 |
+
"post_physical_rank_imax_n": 384,
|
| 489 |
+
"pre_home_rank_cv_max": 0.00610409733861204,
|
| 490 |
+
"pre_home_rank_cv_mean": 0.0030369541814070297,
|
| 491 |
+
"pre_home_rank_cv_min": 0.0013949279180933694,
|
| 492 |
+
"pre_home_rank_cv_n": 384,
|
| 493 |
+
"pre_home_rank_imax_max": 1.0114288330078125,
|
| 494 |
+
"pre_home_rank_imax_mean": 1.0047780275344849,
|
| 495 |
+
"pre_home_rank_imax_min": 1.001708984375,
|
| 496 |
+
"pre_home_rank_imax_n": 384,
|
| 497 |
+
"raw_expert_cv_max": 0.011936138368762292,
|
| 498 |
+
"raw_expert_cv_mean": 0.009561000998578461,
|
| 499 |
+
"raw_expert_cv_min": 0.006960450128604076,
|
| 500 |
+
"raw_expert_cv_n": 384,
|
| 501 |
+
"raw_expert_imax_ceiling": 4.0,
|
| 502 |
+
"raw_expert_imax_hit_ceiling": 0,
|
| 503 |
+
"raw_expert_imax_max": 1.044921875,
|
| 504 |
+
"raw_expert_imax_mean": 1.0230878194173176,
|
| 505 |
+
"raw_expert_imax_min": 1.0142822265625,
|
| 506 |
+
"raw_expert_imax_n": 384,
|
| 507 |
+
"records": {
|
| 508 |
+
"drop_dispatch": 3072,
|
| 509 |
+
"drop_experts": 3072,
|
| 510 |
+
"drop_setup": 3072,
|
| 511 |
+
"echo_post": 3072,
|
| 512 |
+
"echo_pre": 3072
|
| 513 |
+
},
|
| 514 |
+
"required_capacity_max": 1.017578125,
|
| 515 |
+
"required_capacity_mean": 1.0088431040445964,
|
| 516 |
+
"required_capacity_min": 1.00390625,
|
| 517 |
+
"required_capacity_n": 3072,
|
| 518 |
+
"router_balancing": "aux_loss",
|
| 519 |
+
"run_dir": "echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625__forcebal/rep1",
|
| 520 |
+
"saturated_dispatch_rows": 0,
|
| 521 |
+
"seq_length": "4096",
|
| 522 |
+
"shared_expert_overlap": "0",
|
| 523 |
+
"topk": "16",
|
| 524 |
+
"valid_max_rank_reduction_max": 0.011299690729425963,
|
| 525 |
+
"valid_max_rank_reduction_mean": 0.004752627833504172,
|
| 526 |
+
"valid_max_rank_reduction_min": 0.0017060687301974165,
|
| 527 |
+
"valid_max_rank_reduction_n": 384,
|
| 528 |
+
"zero_drop": 1
|
| 529 |
+
},
|
| 530 |
+
{
|
| 531 |
+
"arm": "natural",
|
| 532 |
+
"assignment_sum_mismatches": 0,
|
| 533 |
+
"aux_loss_coeff": "1e-2",
|
| 534 |
+
"backend": "echo",
|
| 535 |
+
"branch": "token-drop-analysis",
|
| 536 |
+
"capacity": "2.00",
|
| 537 |
+
"capacity_mode": "static",
|
| 538 |
+
"cold_expert_fraction_max": 0.6875,
|
| 539 |
+
"cold_expert_fraction_mean": 0.15531412760416666,
|
| 540 |
+
"cold_expert_fraction_min": 0.0,
|
| 541 |
+
"cold_expert_fraction_n": 384,
|
| 542 |
+
"commit": "b099fd84",
|
| 543 |
+
"config": "G3",
|
| 544 |
+
"cuda_graph": "0",
|
| 545 |
+
"dense_first_layers": "0",
|
| 546 |
+
"drop_accounting_source": "dispatch",
|
| 547 |
+
"drop_expected_assignments": 201326592.0,
|
| 548 |
+
"drop_rate_pct_overall": 0.0,
|
| 549 |
+
"echo_logged_load_reduction_valid": 0,
|
| 550 |
+
"echo_logged_pre_rank_metric_valid": 0,
|
| 551 |
+
"echo_moved_assignment_fraction_max": 0.5,
|
| 552 |
+
"echo_moved_assignment_fraction_mean": 0.5,
|
| 553 |
+
"echo_moved_assignment_fraction_min": 0.5,
|
| 554 |
+
"echo_moved_assignment_fraction_n": 384,
|
| 555 |
+
"experts_per_rank": "8",
|
| 556 |
+
"force_balance": "0",
|
| 557 |
+
"global_batch_size": "16",
|
| 558 |
+
"global_snapshots": 384,
|
| 559 |
+
"hottest_expert_token_fraction_max": 1.0,
|
| 560 |
+
"hottest_expert_token_fraction_mean": 0.9773902098337809,
|
| 561 |
+
"hottest_expert_token_fraction_min": 0.638397216796875,
|
| 562 |
+
"hottest_expert_token_fraction_n": 384,
|
| 563 |
+
"incomplete_snapshots": 0,
|
| 564 |
+
"malformed_records": 0,
|
| 565 |
+
"moe_layers": "16",
|
| 566 |
+
"num_experts": "64",
|
| 567 |
+
"num_layers": "16",
|
| 568 |
+
"observed_dropped_tokens": 0,
|
| 569 |
+
"post_physical_rank_cv_max": 0.0,
|
| 570 |
+
"post_physical_rank_cv_mean": 0.0,
|
| 571 |
+
"post_physical_rank_cv_min": 0.0,
|
| 572 |
+
"post_physical_rank_cv_n": 384,
|
| 573 |
+
"post_physical_rank_imax_max": 1.0,
|
| 574 |
+
"post_physical_rank_imax_mean": 1.0,
|
| 575 |
+
"post_physical_rank_imax_min": 1.0,
|
| 576 |
+
"post_physical_rank_imax_n": 384,
|
| 577 |
+
"pre_home_rank_cv_max": 0.8292092994255315,
|
| 578 |
+
"pre_home_rank_cv_mean": 0.47732093430435846,
|
| 579 |
+
"pre_home_rank_cv_min": 0.1091963461644949,
|
| 580 |
+
"pre_home_rank_cv_n": 384,
|
| 581 |
+
"pre_home_rank_imax_max": 2.9329681396484375,
|
| 582 |
+
"pre_home_rank_imax_mean": 1.7915768225987752,
|
| 583 |
+
"pre_home_rank_imax_min": 1.1547698974609375,
|
| 584 |
+
"pre_home_rank_imax_n": 384,
|
| 585 |
+
"raw_expert_cv_max": 1.7316636929706255,
|
| 586 |
+
"raw_expert_cv_mean": 1.5249635478133088,
|
| 587 |
+
"raw_expert_cv_min": 0.6839438720259622,
|
| 588 |
+
"raw_expert_cv_n": 384,
|
| 589 |
+
"raw_expert_imax_ceiling": 4.0,
|
| 590 |
+
"raw_expert_imax_hit_ceiling": 1,
|
| 591 |
+
"raw_expert_imax_max": 4.0,
|
| 592 |
+
"raw_expert_imax_mean": 3.9095608393351235,
|
| 593 |
+
"raw_expert_imax_min": 2.5535888671875,
|
| 594 |
+
"raw_expert_imax_n": 384,
|
| 595 |
+
"records": {
|
| 596 |
+
"drop_dispatch": 3072,
|
| 597 |
+
"drop_experts": 3072,
|
| 598 |
+
"drop_setup": 3072,
|
| 599 |
+
"echo_post": 3072,
|
| 600 |
+
"echo_pre": 3072
|
| 601 |
+
},
|
| 602 |
+
"required_capacity_max": 1.01953125,
|
| 603 |
+
"required_capacity_mean": 1.0093262990315754,
|
| 604 |
+
"required_capacity_min": 1.0,
|
| 605 |
+
"required_capacity_n": 3072,
|
| 606 |
+
"router_balancing": "aux_loss",
|
| 607 |
+
"run_dir": "echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap2p00/rep1",
|
| 608 |
+
"saturated_dispatch_rows": 0,
|
| 609 |
+
"seq_length": "4096",
|
| 610 |
+
"shared_expert_overlap": "0",
|
| 611 |
+
"topk": "16",
|
| 612 |
+
"valid_max_rank_reduction_max": 0.6590484613583747,
|
| 613 |
+
"valid_max_rank_reduction_mean": 0.4215810507718854,
|
| 614 |
+
"valid_max_rank_reduction_min": 0.13402661240238375,
|
| 615 |
+
"valid_max_rank_reduction_n": 384,
|
| 616 |
+
"zero_drop": 1
|
| 617 |
+
},
|
| 618 |
+
{
|
| 619 |
+
"arm": "natural",
|
| 620 |
+
"assignment_sum_mismatches": 0,
|
| 621 |
+
"aux_loss_coeff": "1e-2",
|
| 622 |
+
"backend": "echo",
|
| 623 |
+
"branch": "token-drop-analysis",
|
| 624 |
+
"capacity": "1.068359375",
|
| 625 |
+
"capacity_mode": "static",
|
| 626 |
+
"cold_expert_fraction_max": 0.6796875,
|
| 627 |
+
"cold_expert_fraction_mean": 0.16324869791666666,
|
| 628 |
+
"cold_expert_fraction_min": 0.0,
|
| 629 |
+
"cold_expert_fraction_n": 384,
|
| 630 |
+
"commit": "b099fd84",
|
| 631 |
+
"config": "G4",
|
| 632 |
+
"cuda_graph": "0",
|
| 633 |
+
"dense_first_layers": "0",
|
| 634 |
+
"drop_accounting_source": "dispatch",
|
| 635 |
+
"drop_expected_assignments": 402653184.0,
|
| 636 |
+
"drop_rate_pct_overall": 0.0,
|
| 637 |
+
"echo_logged_load_reduction_valid": 0,
|
| 638 |
+
"echo_logged_pre_rank_metric_valid": 0,
|
| 639 |
+
"echo_moved_assignment_fraction_max": 0.5,
|
| 640 |
+
"echo_moved_assignment_fraction_mean": 0.5,
|
| 641 |
+
"echo_moved_assignment_fraction_min": 0.5,
|
| 642 |
+
"echo_moved_assignment_fraction_n": 384,
|
| 643 |
+
"experts_per_rank": "16",
|
| 644 |
+
"force_balance": "0",
|
| 645 |
+
"global_batch_size": "16",
|
| 646 |
+
"global_snapshots": 384,
|
| 647 |
+
"hottest_expert_token_fraction_max": 1.0,
|
| 648 |
+
"hottest_expert_token_fraction_mean": 0.9825874964396158,
|
| 649 |
+
"hottest_expert_token_fraction_min": 0.5750732421875,
|
| 650 |
+
"hottest_expert_token_fraction_n": 384,
|
| 651 |
+
"incomplete_snapshots": 0,
|
| 652 |
+
"malformed_records": 0,
|
| 653 |
+
"moe_layers": "16",
|
| 654 |
+
"num_experts": "128",
|
| 655 |
+
"num_layers": "16",
|
| 656 |
+
"observed_dropped_tokens": 0,
|
| 657 |
+
"post_physical_rank_cv_max": 0.0,
|
| 658 |
+
"post_physical_rank_cv_mean": 0.0,
|
| 659 |
+
"post_physical_rank_cv_min": 0.0,
|
| 660 |
+
"post_physical_rank_cv_n": 384,
|
| 661 |
+
"post_physical_rank_imax_max": 1.0,
|
| 662 |
+
"post_physical_rank_imax_mean": 1.0,
|
| 663 |
+
"post_physical_rank_imax_min": 1.0,
|
| 664 |
+
"post_physical_rank_imax_n": 384,
|
| 665 |
+
"pre_home_rank_cv_max": 0.6262571920835467,
|
| 666 |
+
"pre_home_rank_cv_mean": 0.33797223191114484,
|
| 667 |
+
"pre_home_rank_cv_min": 0.08738761375151358,
|
| 668 |
+
"pre_home_rank_cv_n": 384,
|
| 669 |
+
"pre_home_rank_imax_max": 2.2536544799804688,
|
| 670 |
+
"pre_home_rank_imax_mean": 1.550683856010437,
|
| 671 |
+
"pre_home_rank_imax_min": 1.1217041015625,
|
| 672 |
+
"pre_home_rank_imax_n": 384,
|
| 673 |
+
"raw_expert_cv_max": 1.7303400196303844,
|
| 674 |
+
"raw_expert_cv_mean": 1.5202854761326712,
|
| 675 |
+
"raw_expert_cv_min": 0.6197223679268041,
|
| 676 |
+
"raw_expert_cv_n": 384,
|
| 677 |
+
"raw_expert_imax_ceiling": 4.0,
|
| 678 |
+
"raw_expert_imax_hit_ceiling": 1,
|
| 679 |
+
"raw_expert_imax_max": 4.0,
|
| 680 |
+
"raw_expert_imax_mean": 3.9303499857584634,
|
| 681 |
+
"raw_expert_imax_min": 2.30029296875,
|
| 682 |
+
"raw_expert_imax_n": 384,
|
| 683 |
+
"records": {
|
| 684 |
+
"drop_dispatch": 3072,
|
| 685 |
+
"drop_experts": 3072,
|
| 686 |
+
"drop_setup": 3072,
|
| 687 |
+
"echo_post": 3072,
|
| 688 |
+
"echo_pre": 3072
|
| 689 |
+
},
|
| 690 |
+
"required_capacity_max": 1.0166015625,
|
| 691 |
+
"required_capacity_mean": 1.0086215337117512,
|
| 692 |
+
"required_capacity_min": 1.0009765625,
|
| 693 |
+
"required_capacity_n": 3072,
|
| 694 |
+
"router_balancing": "aux_loss",
|
| 695 |
+
"run_dir": "echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375/rep1",
|
| 696 |
+
"saturated_dispatch_rows": 0,
|
| 697 |
+
"seq_length": "4096",
|
| 698 |
+
"shared_expert_overlap": "0",
|
| 699 |
+
"topk": "32",
|
| 700 |
+
"valid_max_rank_reduction_max": 0.5562762575704744,
|
| 701 |
+
"valid_max_rank_reduction_mean": 0.3421405808049787,
|
| 702 |
+
"valid_max_rank_reduction_min": 0.10849929263249537,
|
| 703 |
+
"valid_max_rank_reduction_n": 384,
|
| 704 |
+
"zero_drop": 1
|
| 705 |
+
},
|
| 706 |
+
{
|
| 707 |
+
"arm": "forced",
|
| 708 |
+
"assignment_sum_mismatches": 0,
|
| 709 |
+
"aux_loss_coeff": "1e-2",
|
| 710 |
+
"backend": "echo",
|
| 711 |
+
"branch": "token-drop-analysis",
|
| 712 |
+
"capacity": "1.068359375",
|
| 713 |
+
"capacity_mode": "static",
|
| 714 |
+
"cold_expert_fraction_max": 0.0,
|
| 715 |
+
"cold_expert_fraction_mean": 0.0,
|
| 716 |
+
"cold_expert_fraction_min": 0.0,
|
| 717 |
+
"cold_expert_fraction_n": 384,
|
| 718 |
+
"commit": "b099fd84",
|
| 719 |
+
"config": "G4",
|
| 720 |
+
"cuda_graph": "0",
|
| 721 |
+
"dense_first_layers": "0",
|
| 722 |
+
"drop_accounting_source": "dispatch",
|
| 723 |
+
"drop_expected_assignments": 402653184.0,
|
| 724 |
+
"drop_rate_pct_overall": 0.0,
|
| 725 |
+
"echo_logged_load_reduction_valid": 0,
|
| 726 |
+
"echo_logged_pre_rank_metric_valid": 0,
|
| 727 |
+
"echo_moved_assignment_fraction_max": 0.5,
|
| 728 |
+
"echo_moved_assignment_fraction_mean": 0.5,
|
| 729 |
+
"echo_moved_assignment_fraction_min": 0.5,
|
| 730 |
+
"echo_moved_assignment_fraction_n": 384,
|
| 731 |
+
"experts_per_rank": "16",
|
| 732 |
+
"force_balance": "1",
|
| 733 |
+
"global_batch_size": "16",
|
| 734 |
+
"global_snapshots": 384,
|
| 735 |
+
"hottest_expert_token_fraction_max": 0.259674072265625,
|
| 736 |
+
"hottest_expert_token_fraction_mean": 0.25621263186136883,
|
| 737 |
+
"hottest_expert_token_fraction_min": 0.253997802734375,
|
| 738 |
+
"hottest_expert_token_fraction_n": 384,
|
| 739 |
+
"incomplete_snapshots": 0,
|
| 740 |
+
"malformed_records": 0,
|
| 741 |
+
"moe_layers": "16",
|
| 742 |
+
"num_experts": "128",
|
| 743 |
+
"num_layers": "16",
|
| 744 |
+
"observed_dropped_tokens": 0,
|
| 745 |
+
"post_physical_rank_cv_max": 0.0,
|
| 746 |
+
"post_physical_rank_cv_mean": 0.0,
|
| 747 |
+
"post_physical_rank_cv_min": 0.0,
|
| 748 |
+
"post_physical_rank_cv_n": 384,
|
| 749 |
+
"post_physical_rank_imax_max": 1.0,
|
| 750 |
+
"post_physical_rank_imax_mean": 1.0,
|
| 751 |
+
"post_physical_rank_imax_min": 1.0,
|
| 752 |
+
"post_physical_rank_imax_n": 384,
|
| 753 |
+
"pre_home_rank_cv_max": 0.0036539583092664103,
|
| 754 |
+
"pre_home_rank_cv_mean": 0.002179247890018717,
|
| 755 |
+
"pre_home_rank_cv_min": 0.0006899225660317828,
|
| 756 |
+
"pre_home_rank_cv_n": 384,
|
| 757 |
+
"pre_home_rank_imax_max": 1.0071258544921875,
|
| 758 |
+
"pre_home_rank_imax_mean": 1.0034127434094746,
|
| 759 |
+
"pre_home_rank_imax_min": 1.0009918212890625,
|
| 760 |
+
"pre_home_rank_imax_n": 384,
|
| 761 |
+
"raw_expert_cv_max": 0.01168330115931115,
|
| 762 |
+
"raw_expert_cv_mean": 0.009563696224675585,
|
| 763 |
+
"raw_expert_cv_min": 0.008002983940271118,
|
| 764 |
+
"raw_expert_cv_n": 384,
|
| 765 |
+
"raw_expert_imax_ceiling": 4.0,
|
| 766 |
+
"raw_expert_imax_hit_ceiling": 0,
|
| 767 |
+
"raw_expert_imax_max": 1.0386962890625,
|
| 768 |
+
"raw_expert_imax_mean": 1.0248505274454753,
|
| 769 |
+
"raw_expert_imax_min": 1.0159912109375,
|
| 770 |
+
"raw_expert_imax_n": 384,
|
| 771 |
+
"records": {
|
| 772 |
+
"drop_dispatch": 3072,
|
| 773 |
+
"drop_experts": 3072,
|
| 774 |
+
"drop_setup": 3072,
|
| 775 |
+
"echo_post": 3072,
|
| 776 |
+
"echo_pre": 3072
|
| 777 |
+
},
|
| 778 |
+
"required_capacity_max": 1.013671875,
|
| 779 |
+
"required_capacity_mean": 1.0081939697265625,
|
| 780 |
+
"required_capacity_min": 1.00390625,
|
| 781 |
+
"required_capacity_n": 3072,
|
| 782 |
+
"router_balancing": "aux_loss",
|
| 783 |
+
"run_dir": "echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375__forcebal/rep1",
|
| 784 |
+
"saturated_dispatch_rows": 0,
|
| 785 |
+
"seq_length": "4096",
|
| 786 |
+
"shared_expert_overlap": "0",
|
| 787 |
+
"topk": "32",
|
| 788 |
+
"valid_max_rank_reduction_max": 0.007075435965031893,
|
| 789 |
+
"valid_max_rank_reduction_mean": 0.0033999001306743632,
|
| 790 |
+
"valid_max_rank_reduction_min": 0.000990838554290331,
|
| 791 |
+
"valid_max_rank_reduction_n": 384,
|
| 792 |
+
"zero_drop": 1
|
| 793 |
+
},
|
| 794 |
+
{
|
| 795 |
+
"arm": "natural",
|
| 796 |
+
"assignment_sum_mismatches": 0,
|
| 797 |
+
"aux_loss_coeff": "1e-2",
|
| 798 |
+
"backend": "echo",
|
| 799 |
+
"branch": "token-drop-analysis",
|
| 800 |
+
"capacity": "2.00",
|
| 801 |
+
"capacity_mode": "static",
|
| 802 |
+
"cold_expert_fraction_max": 0.671875,
|
| 803 |
+
"cold_expert_fraction_mean": 0.16269938151041666,
|
| 804 |
+
"cold_expert_fraction_min": 0.0,
|
| 805 |
+
"cold_expert_fraction_n": 384,
|
| 806 |
+
"commit": "b099fd84",
|
| 807 |
+
"config": "G4",
|
| 808 |
+
"cuda_graph": "0",
|
| 809 |
+
"dense_first_layers": "0",
|
| 810 |
+
"drop_accounting_source": "dispatch",
|
| 811 |
+
"drop_expected_assignments": 402653184.0,
|
| 812 |
+
"drop_rate_pct_overall": 0.0,
|
| 813 |
+
"echo_logged_load_reduction_valid": 0,
|
| 814 |
+
"echo_logged_pre_rank_metric_valid": 0,
|
| 815 |
+
"echo_moved_assignment_fraction_max": 0.5,
|
| 816 |
+
"echo_moved_assignment_fraction_mean": 0.5,
|
| 817 |
+
"echo_moved_assignment_fraction_min": 0.5,
|
| 818 |
+
"echo_moved_assignment_fraction_n": 384,
|
| 819 |
+
"experts_per_rank": "16",
|
| 820 |
+
"force_balance": "0",
|
| 821 |
+
"global_batch_size": "16",
|
| 822 |
+
"global_snapshots": 384,
|
| 823 |
+
"hottest_expert_token_fraction_max": 1.0,
|
| 824 |
+
"hottest_expert_token_fraction_mean": 0.9825185934702555,
|
| 825 |
+
"hottest_expert_token_fraction_min": 0.57318115234375,
|
| 826 |
+
"hottest_expert_token_fraction_n": 384,
|
| 827 |
+
"incomplete_snapshots": 0,
|
| 828 |
+
"malformed_records": 0,
|
| 829 |
+
"moe_layers": "16",
|
| 830 |
+
"num_experts": "128",
|
| 831 |
+
"num_layers": "16",
|
| 832 |
+
"observed_dropped_tokens": 0,
|
| 833 |
+
"post_physical_rank_cv_max": 0.0,
|
| 834 |
+
"post_physical_rank_cv_mean": 0.0,
|
| 835 |
+
"post_physical_rank_cv_min": 0.0,
|
| 836 |
+
"post_physical_rank_cv_n": 384,
|
| 837 |
+
"post_physical_rank_imax_max": 1.0,
|
| 838 |
+
"post_physical_rank_imax_mean": 1.0,
|
| 839 |
+
"post_physical_rank_imax_min": 1.0,
|
| 840 |
+
"post_physical_rank_imax_n": 384,
|
| 841 |
+
"pre_home_rank_cv_max": 0.6019828931462307,
|
| 842 |
+
"pre_home_rank_cv_mean": 0.3383858718614603,
|
| 843 |
+
"pre_home_rank_cv_min": 0.08738761375151358,
|
| 844 |
+
"pre_home_rank_cv_n": 384,
|
| 845 |
+
"pre_home_rank_imax_max": 2.255584716796875,
|
| 846 |
+
"pre_home_rank_imax_mean": 1.5542763471603394,
|
| 847 |
+
"pre_home_rank_imax_min": 1.1217041015625,
|
| 848 |
+
"pre_home_rank_imax_n": 384,
|
| 849 |
+
"raw_expert_cv_max": 1.7304659197188927,
|
| 850 |
+
"raw_expert_cv_mean": 1.521021036726468,
|
| 851 |
+
"raw_expert_cv_min": 0.6188791003597877,
|
| 852 |
+
"raw_expert_cv_n": 384,
|
| 853 |
+
"raw_expert_imax_ceiling": 4.0,
|
| 854 |
+
"raw_expert_imax_hit_ceiling": 1,
|
| 855 |
+
"raw_expert_imax_max": 4.0,
|
| 856 |
+
"raw_expert_imax_mean": 3.930074373881022,
|
| 857 |
+
"raw_expert_imax_min": 2.292724609375,
|
| 858 |
+
"raw_expert_imax_n": 384,
|
| 859 |
+
"records": {
|
| 860 |
+
"drop_dispatch": 3072,
|
| 861 |
+
"drop_experts": 3072,
|
| 862 |
+
"drop_setup": 3072,
|
| 863 |
+
"echo_post": 3072,
|
| 864 |
+
"echo_pre": 3072
|
| 865 |
+
},
|
| 866 |
+
"required_capacity_max": 1.0166015625,
|
| 867 |
+
"required_capacity_mean": 1.0086164474487305,
|
| 868 |
+
"required_capacity_min": 1.0009765625,
|
| 869 |
+
"required_capacity_n": 3072,
|
| 870 |
+
"router_balancing": "aux_loss",
|
| 871 |
+
"run_dir": "echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap2p00/rep1",
|
| 872 |
+
"saturated_dispatch_rows": 0,
|
| 873 |
+
"seq_length": "4096",
|
| 874 |
+
"shared_expert_overlap": "0",
|
| 875 |
+
"topk": "32",
|
| 876 |
+
"valid_max_rank_reduction_max": 0.5566559781358661,
|
| 877 |
+
"valid_max_rank_reduction_mean": 0.3437128068160493,
|
| 878 |
+
"valid_max_rank_reduction_min": 0.10849929263249537,
|
| 879 |
+
"valid_max_rank_reduction_n": 384,
|
| 880 |
+
"zero_drop": 1
|
| 881 |
+
},
|
| 882 |
+
{
|
| 883 |
+
"arm": "natural",
|
| 884 |
+
"assignment_sum_mismatches": 0,
|
| 885 |
+
"aux_loss_coeff": "1e-2",
|
| 886 |
+
"backend": "ultraep",
|
| 887 |
+
"branch": "ultraep-token-drop-analysis",
|
| 888 |
+
"capacity": "1.25",
|
| 889 |
+
"capacity_mode": "static",
|
| 890 |
+
"commit": "eaf829e7",
|
| 891 |
+
"config": "G1",
|
| 892 |
+
"cuda_graph": "0",
|
| 893 |
+
"dense_first_layers": "0",
|
| 894 |
+
"drop_expected_assignments": 25165824.0,
|
| 895 |
+
"drop_rate_pct_max": 2.891776,
|
| 896 |
+
"drop_rate_pct_mean": 0.607152,
|
| 897 |
+
"drop_rate_pct_min": 0.0,
|
| 898 |
+
"drop_rate_pct_n": 96,
|
| 899 |
+
"drop_rate_pct_overall": 0.6209174791971842,
|
| 900 |
+
"dropped_tokens_sum_all_records": 156259.0,
|
| 901 |
+
"duplicate_vector_mismatches": 0,
|
| 902 |
+
"experts_per_rank": "1",
|
| 903 |
+
"force_balance": "0",
|
| 904 |
+
"global_batch_size": "16",
|
| 905 |
+
"global_snapshots": 192,
|
| 906 |
+
"incomplete_snapshots": 0,
|
| 907 |
+
"malformed_records": 0,
|
| 908 |
+
"moe_layers": "16",
|
| 909 |
+
"num_experts": "8",
|
| 910 |
+
"num_layers": "16",
|
| 911 |
+
"observed_dropped_tokens": 156259.0,
|
| 912 |
+
"peak_demand_max": 14128.0,
|
| 913 |
+
"peak_demand_mean": 10918.34375,
|
| 914 |
+
"peak_demand_min": 8275.0,
|
| 915 |
+
"peak_demand_n": 96,
|
| 916 |
+
"peak_rank_overflow_max": 3888.0,
|
| 917 |
+
"peak_rank_overflow_mean": 1076.4583333333333,
|
| 918 |
+
"peak_rank_overflow_min": 0.0,
|
| 919 |
+
"peak_rank_overflow_n": 96,
|
| 920 |
+
"post_physical_rank_cv_max": 0.6189049804826644,
|
| 921 |
+
"post_physical_rank_cv_mean": 0.08780981215252336,
|
| 922 |
+
"post_physical_rank_cv_min": 0.015802566559489504,
|
| 923 |
+
"post_physical_rank_cv_n": 192,
|
| 924 |
+
"post_physical_rank_imax_max": 1.724609375,
|
| 925 |
+
"post_physical_rank_imax_mean": 1.063955307006836,
|
| 926 |
+
"post_physical_rank_imax_min": 1.010009765625,
|
| 927 |
+
"post_physical_rank_imax_n": 192,
|
| 928 |
+
"pre_home_rank_cv_max": 1.7320508075688772,
|
| 929 |
+
"pre_home_rank_cv_mean": 1.4485647787782712,
|
| 930 |
+
"pre_home_rank_cv_min": 0.3594758110042677,
|
| 931 |
+
"pre_home_rank_cv_n": 192,
|
| 932 |
+
"pre_home_rank_imax_max": 4.0,
|
| 933 |
+
"pre_home_rank_imax_mean": 3.6084569295247397,
|
| 934 |
+
"pre_home_rank_imax_min": 1.6087646484375,
|
| 935 |
+
"pre_home_rank_imax_n": 192,
|
| 936 |
+
"records": {
|
| 937 |
+
"drop_dispatch": 1536,
|
| 938 |
+
"drop_experts": 1536,
|
| 939 |
+
"drop_setup": 1536,
|
| 940 |
+
"ultraep_drop": 96,
|
| 941 |
+
"ultraep_post": 1536,
|
| 942 |
+
"ultraep_pre": 1536
|
| 943 |
+
},
|
| 944 |
+
"required_capacity_max": 1.724609,
|
| 945 |
+
"required_capacity_mean": 1.33280565625,
|
| 946 |
+
"required_capacity_min": 1.010132,
|
| 947 |
+
"required_capacity_n": 96,
|
| 948 |
+
"router_balancing": "aux_loss",
|
| 949 |
+
"run_dir": "ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1",
|
| 950 |
+
"seq_length": "4096",
|
| 951 |
+
"shared_expert_overlap": "0",
|
| 952 |
+
"topk": "2",
|
| 953 |
+
"valid_max_rank_reduction_max": 0.7474516266861991,
|
| 954 |
+
"valid_max_rank_reduction_mean": 0.6845405986286021,
|
| 955 |
+
"valid_max_rank_reduction_min": 0.27725927612110174,
|
| 956 |
+
"valid_max_rank_reduction_n": 192,
|
| 957 |
+
"zero_drop": 0
|
| 958 |
+
},
|
| 959 |
+
{
|
| 960 |
+
"arm": "natural",
|
| 961 |
+
"assignment_sum_mismatches": 0,
|
| 962 |
+
"aux_loss_coeff": "1e-2",
|
| 963 |
+
"backend": "ultraep",
|
| 964 |
+
"branch": "ultraep-token-drop-analysis",
|
| 965 |
+
"capacity": "1.8125",
|
| 966 |
+
"capacity_mode": "static",
|
| 967 |
+
"commit": "eaf829e7",
|
| 968 |
+
"config": "G1",
|
| 969 |
+
"cuda_graph": "0",
|
| 970 |
+
"dense_first_layers": "0",
|
| 971 |
+
"drop_expected_assignments": 25165824.0,
|
| 972 |
+
"drop_rate_pct_max": 1.545951,
|
| 973 |
+
"drop_rate_pct_mean": 0.03138885416666667,
|
| 974 |
+
"drop_rate_pct_min": 0.0,
|
| 975 |
+
"drop_rate_pct_n": 96,
|
| 976 |
+
"drop_rate_pct_overall": 0.032329559326171875,
|
| 977 |
+
"dropped_tokens_sum_all_records": 8136.0,
|
| 978 |
+
"duplicate_vector_mismatches": 0,
|
| 979 |
+
"experts_per_rank": "1",
|
| 980 |
+
"force_balance": "0",
|
| 981 |
+
"global_batch_size": "16",
|
| 982 |
+
"global_snapshots": 192,
|
| 983 |
+
"incomplete_snapshots": 0,
|
| 984 |
+
"malformed_records": 0,
|
| 985 |
+
"moe_layers": "16",
|
| 986 |
+
"num_experts": "8",
|
| 987 |
+
"num_layers": "16",
|
| 988 |
+
"observed_dropped_tokens": 8136.0,
|
| 989 |
+
"peak_demand_max": 18916.0,
|
| 990 |
+
"peak_demand_mean": 11069.770833333334,
|
| 991 |
+
"peak_demand_min": 8275.0,
|
| 992 |
+
"peak_demand_n": 96,
|
| 993 |
+
"peak_rank_overflow_max": 4068.0,
|
| 994 |
+
"peak_rank_overflow_mean": 84.75,
|
| 995 |
+
"peak_rank_overflow_min": 0.0,
|
| 996 |
+
"peak_rank_overflow_n": 96,
|
| 997 |
+
"post_physical_rank_cv_max": 1.0236043430060333,
|
| 998 |
+
"post_physical_rank_cv_mean": 0.09497773141059589,
|
| 999 |
+
"post_physical_rank_cv_min": 0.015224448662595366,
|
| 1000 |
+
"post_physical_rank_cv_n": 192,
|
| 1001 |
+
"post_physical_rank_imax_max": 2.30908203125,
|
| 1002 |
+
"post_physical_rank_imax_mean": 1.0747025807698567,
|
| 1003 |
+
"post_physical_rank_imax_min": 1.010009765625,
|
| 1004 |
+
"post_physical_rank_imax_n": 192,
|
| 1005 |
+
"pre_home_rank_cv_max": 1.7320508075688772,
|
| 1006 |
+
"pre_home_rank_cv_mean": 1.444809403446171,
|
| 1007 |
+
"pre_home_rank_cv_min": 0.3594758110042677,
|
| 1008 |
+
"pre_home_rank_cv_n": 192,
|
| 1009 |
+
"pre_home_rank_imax_max": 4.0,
|
| 1010 |
+
"pre_home_rank_imax_mean": 3.6151854197184243,
|
| 1011 |
+
"pre_home_rank_imax_min": 1.6087646484375,
|
| 1012 |
+
"pre_home_rank_imax_n": 192,
|
| 1013 |
+
"records": {
|
| 1014 |
+
"drop_dispatch": 1536,
|
| 1015 |
+
"drop_experts": 1536,
|
| 1016 |
+
"drop_setup": 1536,
|
| 1017 |
+
"ultraep_drop": 96,
|
| 1018 |
+
"ultraep_post": 1536,
|
| 1019 |
+
"ultraep_pre": 1536
|
| 1020 |
+
},
|
| 1021 |
+
"required_capacity_max": 2.309082,
|
| 1022 |
+
"required_capacity_mean": 1.3512904166666668,
|
| 1023 |
+
"required_capacity_min": 1.010132,
|
| 1024 |
+
"required_capacity_n": 96,
|
| 1025 |
+
"router_balancing": "aux_loss",
|
| 1026 |
+
"run_dir": "ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1",
|
| 1027 |
+
"seq_length": "4096",
|
| 1028 |
+
"shared_expert_overlap": "0",
|
| 1029 |
+
"topk": "2",
|
| 1030 |
+
"valid_max_rank_reduction_max": 0.7474516266861991,
|
| 1031 |
+
"valid_max_rank_reduction_mean": 0.6828229027074976,
|
| 1032 |
+
"valid_max_rank_reduction_min": 0.27725927612110174,
|
| 1033 |
+
"valid_max_rank_reduction_n": 192,
|
| 1034 |
+
"zero_drop": 0
|
| 1035 |
+
},
|
| 1036 |
+
{
|
| 1037 |
+
"arm": "natural",
|
| 1038 |
+
"assignment_sum_mismatches": 0,
|
| 1039 |
+
"aux_loss_coeff": "1e-2",
|
| 1040 |
+
"backend": "ultraep",
|
| 1041 |
+
"branch": "ultraep-token-drop-analysis",
|
| 1042 |
+
"capacity": "1.0625",
|
| 1043 |
+
"capacity_mode": "static",
|
| 1044 |
+
"commit": "eaf829e7",
|
| 1045 |
+
"config": "G2",
|
| 1046 |
+
"cuda_graph": "0",
|
| 1047 |
+
"dense_first_layers": "0",
|
| 1048 |
+
"drop_expected_assignments": 100663296.0,
|
| 1049 |
+
"drop_rate_pct_max": 0.0,
|
| 1050 |
+
"drop_rate_pct_mean": 0.0,
|
| 1051 |
+
"drop_rate_pct_min": 0.0,
|
| 1052 |
+
"drop_rate_pct_n": 96,
|
| 1053 |
+
"drop_rate_pct_overall": 0.0,
|
| 1054 |
+
"dropped_tokens_sum_all_records": 0.0,
|
| 1055 |
+
"duplicate_vector_mismatches": 0,
|
| 1056 |
+
"experts_per_rank": "4",
|
| 1057 |
+
"force_balance": "0",
|
| 1058 |
+
"global_batch_size": "16",
|
| 1059 |
+
"global_snapshots": 192,
|
| 1060 |
+
"incomplete_snapshots": 0,
|
| 1061 |
+
"malformed_records": 0,
|
| 1062 |
+
"moe_layers": "16",
|
| 1063 |
+
"num_experts": "32",
|
| 1064 |
+
"num_layers": "16",
|
| 1065 |
+
"observed_dropped_tokens": 0.0,
|
| 1066 |
+
"peak_demand_max": 33099.0,
|
| 1067 |
+
"peak_demand_mean": 33096.927083333336,
|
| 1068 |
+
"peak_demand_min": 33095.0,
|
| 1069 |
+
"peak_demand_n": 96,
|
| 1070 |
+
"peak_rank_overflow_max": 0.0,
|
| 1071 |
+
"peak_rank_overflow_mean": 0.0,
|
| 1072 |
+
"peak_rank_overflow_min": 0.0,
|
| 1073 |
+
"peak_rank_overflow_n": 96,
|
| 1074 |
+
"post_physical_rank_cv_max": 0.026494891363095952,
|
| 1075 |
+
"post_physical_rank_cv_mean": 0.013436746135527637,
|
| 1076 |
+
"post_physical_rank_cv_min": 0.004813851711632,
|
| 1077 |
+
"post_physical_rank_cv_n": 192,
|
| 1078 |
+
"post_physical_rank_imax_max": 1.010101318359375,
|
| 1079 |
+
"post_physical_rank_imax_mean": 1.0100231170654297,
|
| 1080 |
+
"post_physical_rank_imax_min": 1.010009765625,
|
| 1081 |
+
"post_physical_rank_imax_n": 192,
|
| 1082 |
+
"pre_home_rank_cv_max": 1.3185407730096428,
|
| 1083 |
+
"pre_home_rank_cv_mean": 0.7088423169085597,
|
| 1084 |
+
"pre_home_rank_cv_min": 0.07964958686803787,
|
| 1085 |
+
"pre_home_rank_cv_n": 192,
|
| 1086 |
+
"pre_home_rank_imax_max": 3.99853515625,
|
| 1087 |
+
"pre_home_rank_imax_mean": 2.2032313346862793,
|
| 1088 |
+
"pre_home_rank_imax_min": 1.17706298828125,
|
| 1089 |
+
"pre_home_rank_imax_n": 192,
|
| 1090 |
+
"records": {
|
| 1091 |
+
"drop_dispatch": 1536,
|
| 1092 |
+
"drop_experts": 1536,
|
| 1093 |
+
"drop_setup": 1536,
|
| 1094 |
+
"ultraep_drop": 96,
|
| 1095 |
+
"ultraep_post": 1536,
|
| 1096 |
+
"ultraep_pre": 1536
|
| 1097 |
+
},
|
| 1098 |
+
"required_capacity_max": 1.010101,
|
| 1099 |
+
"required_capacity_mean": 1.0100380104166666,
|
| 1100 |
+
"required_capacity_min": 1.009979,
|
| 1101 |
+
"required_capacity_n": 96,
|
| 1102 |
+
"router_balancing": "aux_loss",
|
| 1103 |
+
"run_dir": "ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1",
|
| 1104 |
+
"seq_length": "4096",
|
| 1105 |
+
"shared_expert_overlap": "0",
|
| 1106 |
+
"topk": "8",
|
| 1107 |
+
"valid_max_rank_reduction_max": 0.7474050555623397,
|
| 1108 |
+
"valid_max_rank_reduction_mean": 0.5179521802733736,
|
| 1109 |
+
"valid_max_rank_reduction_min": 0.14192377495462796,
|
| 1110 |
+
"valid_max_rank_reduction_n": 192,
|
| 1111 |
+
"zero_drop": 1
|
| 1112 |
+
},
|
| 1113 |
+
{
|
| 1114 |
+
"arm": "natural",
|
| 1115 |
+
"assignment_sum_mismatches": 0,
|
| 1116 |
+
"aux_loss_coeff": "1e-2",
|
| 1117 |
+
"backend": "ultraep",
|
| 1118 |
+
"branch": "ultraep-token-drop-analysis",
|
| 1119 |
+
"capacity": "1.25",
|
| 1120 |
+
"capacity_mode": "static",
|
| 1121 |
+
"commit": "eaf829e7",
|
| 1122 |
+
"config": "G2",
|
| 1123 |
+
"cuda_graph": "0",
|
| 1124 |
+
"dense_first_layers": "0",
|
| 1125 |
+
"drop_expected_assignments": 100663296.0,
|
| 1126 |
+
"drop_rate_pct_max": 0.0,
|
| 1127 |
+
"drop_rate_pct_mean": 0.0,
|
| 1128 |
+
"drop_rate_pct_min": 0.0,
|
| 1129 |
+
"drop_rate_pct_n": 96,
|
| 1130 |
+
"drop_rate_pct_overall": 0.0,
|
| 1131 |
+
"dropped_tokens_sum_all_records": 0.0,
|
| 1132 |
+
"duplicate_vector_mismatches": 0,
|
| 1133 |
+
"experts_per_rank": "4",
|
| 1134 |
+
"force_balance": "0",
|
| 1135 |
+
"global_batch_size": "16",
|
| 1136 |
+
"global_snapshots": 192,
|
| 1137 |
+
"incomplete_snapshots": 0,
|
| 1138 |
+
"malformed_records": 0,
|
| 1139 |
+
"moe_layers": "16",
|
| 1140 |
+
"num_experts": "32",
|
| 1141 |
+
"num_layers": "16",
|
| 1142 |
+
"observed_dropped_tokens": 0.0,
|
| 1143 |
+
"peak_demand_max": 33099.0,
|
| 1144 |
+
"peak_demand_mean": 33096.875,
|
| 1145 |
+
"peak_demand_min": 33096.0,
|
| 1146 |
+
"peak_demand_n": 96,
|
| 1147 |
+
"peak_rank_overflow_max": 0.0,
|
| 1148 |
+
"peak_rank_overflow_mean": 0.0,
|
| 1149 |
+
"peak_rank_overflow_min": 0.0,
|
| 1150 |
+
"peak_rank_overflow_n": 96,
|
| 1151 |
+
"post_physical_rank_cv_max": 0.026494891363095952,
|
| 1152 |
+
"post_physical_rank_cv_mean": 0.013604201092033112,
|
| 1153 |
+
"post_physical_rank_cv_min": 0.004841919368473151,
|
| 1154 |
+
"post_physical_rank_cv_n": 192,
|
| 1155 |
+
"post_physical_rank_imax_max": 1.01007080078125,
|
| 1156 |
+
"post_physical_rank_imax_mean": 1.0100212097167969,
|
| 1157 |
+
"post_physical_rank_imax_min": 1.010009765625,
|
| 1158 |
+
"post_physical_rank_imax_n": 192,
|
| 1159 |
+
"pre_home_rank_cv_max": 1.3181329746469168,
|
| 1160 |
+
"pre_home_rank_cv_mean": 0.7143701949255296,
|
| 1161 |
+
"pre_home_rank_cv_min": 0.22890444224298007,
|
| 1162 |
+
"pre_home_rank_cv_n": 192,
|
| 1163 |
+
"pre_home_rank_imax_max": 3.99853515625,
|
| 1164 |
+
"pre_home_rank_imax_mean": 2.2135567665100098,
|
| 1165 |
+
"pre_home_rank_imax_min": 1.33551025390625,
|
| 1166 |
+
"pre_home_rank_imax_n": 192,
|
| 1167 |
+
"records": {
|
| 1168 |
+
"drop_dispatch": 1536,
|
| 1169 |
+
"drop_experts": 1536,
|
| 1170 |
+
"drop_setup": 1536,
|
| 1171 |
+
"ultraep_drop": 96,
|
| 1172 |
+
"ultraep_post": 1536,
|
| 1173 |
+
"ultraep_pre": 1536
|
| 1174 |
+
},
|
| 1175 |
+
"required_capacity_max": 1.010101,
|
| 1176 |
+
"required_capacity_mean": 1.0100364479166666,
|
| 1177 |
+
"required_capacity_min": 1.01001,
|
| 1178 |
+
"required_capacity_n": 96,
|
| 1179 |
+
"router_balancing": "aux_loss",
|
| 1180 |
+
"run_dir": "ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1",
|
| 1181 |
+
"seq_length": "4096",
|
| 1182 |
+
"shared_expert_overlap": "0",
|
| 1183 |
+
"topk": "8",
|
| 1184 |
+
"valid_max_rank_reduction_max": 0.7474050555623397,
|
| 1185 |
+
"valid_max_rank_reduction_mean": 0.5202877868268106,
|
| 1186 |
+
"valid_max_rank_reduction_min": 0.24370458388556282,
|
| 1187 |
+
"valid_max_rank_reduction_n": 192,
|
| 1188 |
+
"zero_drop": 1
|
| 1189 |
+
},
|
| 1190 |
+
{
|
| 1191 |
+
"arm": "forced",
|
| 1192 |
+
"assignment_sum_mismatches": 0,
|
| 1193 |
+
"aux_loss_coeff": "1e-2",
|
| 1194 |
+
"backend": "ultraep",
|
| 1195 |
+
"branch": "ultraep-token-drop-analysis",
|
| 1196 |
+
"capacity": "1.25",
|
| 1197 |
+
"capacity_mode": "static",
|
| 1198 |
+
"commit": "eaf829e7",
|
| 1199 |
+
"config": "G2",
|
| 1200 |
+
"cuda_graph": "0",
|
| 1201 |
+
"dense_first_layers": "0",
|
| 1202 |
+
"drop_expected_assignments": 100663296.0,
|
| 1203 |
+
"drop_rate_pct_max": 0.0,
|
| 1204 |
+
"drop_rate_pct_mean": 0.0,
|
| 1205 |
+
"drop_rate_pct_min": 0.0,
|
| 1206 |
+
"drop_rate_pct_n": 96,
|
| 1207 |
+
"drop_rate_pct_overall": 0.0,
|
| 1208 |
+
"dropped_tokens_sum_all_records": 0.0,
|
| 1209 |
+
"duplicate_vector_mismatches": 0,
|
| 1210 |
+
"experts_per_rank": "4",
|
| 1211 |
+
"force_balance": "1",
|
| 1212 |
+
"global_batch_size": "16",
|
| 1213 |
+
"global_snapshots": 192,
|
| 1214 |
+
"incomplete_snapshots": 0,
|
| 1215 |
+
"malformed_records": 0,
|
| 1216 |
+
"moe_layers": "16",
|
| 1217 |
+
"num_experts": "32",
|
| 1218 |
+
"num_layers": "16",
|
| 1219 |
+
"observed_dropped_tokens": 0.0,
|
| 1220 |
+
"peak_demand_max": 33096.0,
|
| 1221 |
+
"peak_demand_mean": 33052.15625,
|
| 1222 |
+
"peak_demand_min": 32901.0,
|
| 1223 |
+
"peak_demand_n": 96,
|
| 1224 |
+
"peak_rank_overflow_max": 0.0,
|
| 1225 |
+
"peak_rank_overflow_mean": 0.0,
|
| 1226 |
+
"peak_rank_overflow_min": 0.0,
|
| 1227 |
+
"peak_rank_overflow_n": 96,
|
| 1228 |
+
"post_physical_rank_cv_max": 0.00717872106815807,
|
| 1229 |
+
"post_physical_rank_cv_mean": 0.004277032344446821,
|
| 1230 |
+
"post_physical_rank_cv_min": 0.0016880800477701613,
|
| 1231 |
+
"post_physical_rank_cv_n": 192,
|
| 1232 |
+
"post_physical_rank_imax_max": 1.010009765625,
|
| 1233 |
+
"post_physical_rank_imax_mean": 1.006682554880778,
|
| 1234 |
+
"post_physical_rank_imax_min": 1.002349853515625,
|
| 1235 |
+
"post_physical_rank_imax_n": 192,
|
| 1236 |
+
"pre_home_rank_cv_max": 0.007852555942790041,
|
| 1237 |
+
"pre_home_rank_cv_mean": 0.00436334506476056,
|
| 1238 |
+
"pre_home_rank_cv_min": 0.0016880800477701613,
|
| 1239 |
+
"pre_home_rank_cv_n": 192,
|
| 1240 |
+
"pre_home_rank_imax_max": 1.01544189453125,
|
| 1241 |
+
"pre_home_rank_imax_mean": 1.0068944295247395,
|
| 1242 |
+
"pre_home_rank_imax_min": 1.002349853515625,
|
| 1243 |
+
"pre_home_rank_imax_n": 192,
|
| 1244 |
+
"records": {
|
| 1245 |
+
"drop_dispatch": 1536,
|
| 1246 |
+
"drop_experts": 1536,
|
| 1247 |
+
"drop_setup": 1536,
|
| 1248 |
+
"ultraep_drop": 96,
|
| 1249 |
+
"ultraep_post": 1536,
|
| 1250 |
+
"ultraep_pre": 1536
|
| 1251 |
+
},
|
| 1252 |
+
"required_capacity_max": 1.01001,
|
| 1253 |
+
"required_capacity_mean": 1.0086718541666666,
|
| 1254 |
+
"required_capacity_min": 1.004059,
|
| 1255 |
+
"required_capacity_n": 96,
|
| 1256 |
+
"router_balancing": "aux_loss",
|
| 1257 |
+
"run_dir": "ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1",
|
| 1258 |
+
"seq_length": "4096",
|
| 1259 |
+
"shared_expert_overlap": "0",
|
| 1260 |
+
"topk": "8",
|
| 1261 |
+
"valid_max_rank_reduction_max": 0.005349522149425978,
|
| 1262 |
+
"valid_max_rank_reduction_mean": 0.0002090865394183532,
|
| 1263 |
+
"valid_max_rank_reduction_min": 0.0,
|
| 1264 |
+
"valid_max_rank_reduction_n": 192,
|
| 1265 |
+
"zero_drop": 1
|
| 1266 |
+
},
|
| 1267 |
+
{
|
| 1268 |
+
"arm": "natural",
|
| 1269 |
+
"assignment_sum_mismatches": 0,
|
| 1270 |
+
"aux_loss_coeff": "1e-2",
|
| 1271 |
+
"backend": "ultraep",
|
| 1272 |
+
"branch": "ultraep-token-drop-analysis",
|
| 1273 |
+
"capacity": "1.0625",
|
| 1274 |
+
"capacity_mode": "static",
|
| 1275 |
+
"commit": "eaf829e7",
|
| 1276 |
+
"config": "G3",
|
| 1277 |
+
"cuda_graph": "0",
|
| 1278 |
+
"dense_first_layers": "0",
|
| 1279 |
+
"drop_expected_assignments": 201326592.0,
|
| 1280 |
+
"drop_rate_pct_max": 0.0,
|
| 1281 |
+
"drop_rate_pct_mean": 0.0,
|
| 1282 |
+
"drop_rate_pct_min": 0.0,
|
| 1283 |
+
"drop_rate_pct_n": 96,
|
| 1284 |
+
"drop_rate_pct_overall": 0.0,
|
| 1285 |
+
"dropped_tokens_sum_all_records": 0.0,
|
| 1286 |
+
"duplicate_vector_mismatches": 0,
|
| 1287 |
+
"experts_per_rank": "8",
|
| 1288 |
+
"force_balance": "0",
|
| 1289 |
+
"global_batch_size": "16",
|
| 1290 |
+
"global_snapshots": 192,
|
| 1291 |
+
"incomplete_snapshots": 0,
|
| 1292 |
+
"malformed_records": 0,
|
| 1293 |
+
"moe_layers": "16",
|
| 1294 |
+
"num_experts": "64",
|
| 1295 |
+
"num_layers": "16",
|
| 1296 |
+
"observed_dropped_tokens": 0.0,
|
| 1297 |
+
"peak_demand_max": 66196.0,
|
| 1298 |
+
"peak_demand_mean": 66193.28125,
|
| 1299 |
+
"peak_demand_min": 66192.0,
|
| 1300 |
+
"peak_demand_n": 96,
|
| 1301 |
+
"peak_rank_overflow_max": 0.0,
|
| 1302 |
+
"peak_rank_overflow_mean": 0.0,
|
| 1303 |
+
"peak_rank_overflow_min": 0.0,
|
| 1304 |
+
"peak_rank_overflow_n": 96,
|
| 1305 |
+
"post_physical_rank_cv_max": 0.026489120474252784,
|
| 1306 |
+
"post_physical_rank_cv_mean": 0.014233627693615905,
|
| 1307 |
+
"post_physical_rank_cv_min": 0.004993448920418017,
|
| 1308 |
+
"post_physical_rank_cv_n": 192,
|
| 1309 |
+
"post_physical_rank_imax_max": 1.01007080078125,
|
| 1310 |
+
"post_physical_rank_imax_mean": 1.0100196202596028,
|
| 1311 |
+
"post_physical_rank_imax_min": 1.010009765625,
|
| 1312 |
+
"post_physical_rank_imax_n": 192,
|
| 1313 |
+
"pre_home_rank_cv_max": 0.8287584673770169,
|
| 1314 |
+
"pre_home_rank_cv_mean": 0.47726214620952834,
|
| 1315 |
+
"pre_home_rank_cv_min": 0.11019481148220499,
|
| 1316 |
+
"pre_home_rank_cv_n": 192,
|
| 1317 |
+
"pre_home_rank_imax_max": 2.6106719970703125,
|
| 1318 |
+
"pre_home_rank_imax_mean": 1.7842089335123699,
|
| 1319 |
+
"pre_home_rank_imax_min": 1.1877593994140625,
|
| 1320 |
+
"pre_home_rank_imax_n": 192,
|
| 1321 |
+
"records": {
|
| 1322 |
+
"drop_dispatch": 1536,
|
| 1323 |
+
"drop_experts": 1536,
|
| 1324 |
+
"drop_setup": 1536,
|
| 1325 |
+
"ultraep_drop": 96,
|
| 1326 |
+
"ultraep_post": 1536,
|
| 1327 |
+
"ultraep_pre": 1536
|
| 1328 |
+
},
|
| 1329 |
+
"required_capacity_max": 1.010071,
|
| 1330 |
+
"required_capacity_mean": 1.0100292499999999,
|
| 1331 |
+
"required_capacity_min": 1.01001,
|
| 1332 |
+
"required_capacity_n": 96,
|
| 1333 |
+
"router_balancing": "aux_loss",
|
| 1334 |
+
"run_dir": "ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1",
|
| 1335 |
+
"seq_length": "4096",
|
| 1336 |
+
"shared_expert_overlap": "0",
|
| 1337 |
+
"topk": "16",
|
| 1338 |
+
"valid_max_rank_reduction_max": 0.6131226876610966,
|
| 1339 |
+
"valid_max_rank_reduction_mean": 0.41320151666180793,
|
| 1340 |
+
"valid_max_rank_reduction_min": 0.14965121208617566,
|
| 1341 |
+
"valid_max_rank_reduction_n": 192,
|
| 1342 |
+
"zero_drop": 1
|
| 1343 |
+
},
|
| 1344 |
+
{
|
| 1345 |
+
"arm": "natural",
|
| 1346 |
+
"assignment_sum_mismatches": 0,
|
| 1347 |
+
"aux_loss_coeff": "1e-2",
|
| 1348 |
+
"backend": "ultraep",
|
| 1349 |
+
"branch": "ultraep-token-drop-analysis",
|
| 1350 |
+
"capacity": "1.25",
|
| 1351 |
+
"capacity_mode": "static",
|
| 1352 |
+
"commit": "eaf829e7",
|
| 1353 |
+
"config": "G3",
|
| 1354 |
+
"cuda_graph": "0",
|
| 1355 |
+
"dense_first_layers": "0",
|
| 1356 |
+
"drop_expected_assignments": 201326592.0,
|
| 1357 |
+
"drop_rate_pct_max": 0.0,
|
| 1358 |
+
"drop_rate_pct_mean": 0.0,
|
| 1359 |
+
"drop_rate_pct_min": 0.0,
|
| 1360 |
+
"drop_rate_pct_n": 96,
|
| 1361 |
+
"drop_rate_pct_overall": 0.0,
|
| 1362 |
+
"dropped_tokens_sum_all_records": 0.0,
|
| 1363 |
+
"duplicate_vector_mismatches": 0,
|
| 1364 |
+
"experts_per_rank": "8",
|
| 1365 |
+
"force_balance": "0",
|
| 1366 |
+
"global_batch_size": "16",
|
| 1367 |
+
"global_snapshots": 192,
|
| 1368 |
+
"incomplete_snapshots": 0,
|
| 1369 |
+
"malformed_records": 0,
|
| 1370 |
+
"moe_layers": "16",
|
| 1371 |
+
"num_experts": "64",
|
| 1372 |
+
"num_layers": "16",
|
| 1373 |
+
"observed_dropped_tokens": 0.0,
|
| 1374 |
+
"peak_demand_max": 66195.0,
|
| 1375 |
+
"peak_demand_mean": 66193.34375,
|
| 1376 |
+
"peak_demand_min": 66192.0,
|
| 1377 |
+
"peak_demand_n": 96,
|
| 1378 |
+
"peak_rank_overflow_max": 0.0,
|
| 1379 |
+
"peak_rank_overflow_mean": 0.0,
|
| 1380 |
+
"peak_rank_overflow_min": 0.0,
|
| 1381 |
+
"peak_rank_overflow_n": 96,
|
| 1382 |
+
"post_physical_rank_cv_max": 0.026483352723689336,
|
| 1383 |
+
"post_physical_rank_cv_mean": 0.014464507564652654,
|
| 1384 |
+
"post_physical_rank_cv_min": 0.00571681991985643,
|
| 1385 |
+
"post_physical_rank_cv_n": 192,
|
| 1386 |
+
"post_physical_rank_imax_max": 1.0100555419921875,
|
| 1387 |
+
"post_physical_rank_imax_mean": 1.0100204149882,
|
| 1388 |
+
"post_physical_rank_imax_min": 1.010009765625,
|
| 1389 |
+
"post_physical_rank_imax_n": 192,
|
| 1390 |
+
"pre_home_rank_cv_max": 0.8316092132265209,
|
| 1391 |
+
"pre_home_rank_cv_mean": 0.4753321564816673,
|
| 1392 |
+
"pre_home_rank_cv_min": 0.11019481148220499,
|
| 1393 |
+
"pre_home_rank_cv_n": 192,
|
| 1394 |
+
"pre_home_rank_imax_max": 2.8477020263671875,
|
| 1395 |
+
"pre_home_rank_imax_mean": 1.7751669883728027,
|
| 1396 |
+
"pre_home_rank_imax_min": 1.1877593994140625,
|
| 1397 |
+
"pre_home_rank_imax_n": 192,
|
| 1398 |
+
"records": {
|
| 1399 |
+
"drop_dispatch": 1536,
|
| 1400 |
+
"drop_experts": 1536,
|
| 1401 |
+
"drop_setup": 1536,
|
| 1402 |
+
"ultraep_drop": 96,
|
| 1403 |
+
"ultraep_post": 1536,
|
| 1404 |
+
"ultraep_pre": 1536
|
| 1405 |
+
},
|
| 1406 |
+
"required_capacity_max": 1.010056,
|
| 1407 |
+
"required_capacity_mean": 1.01003021875,
|
| 1408 |
+
"required_capacity_min": 1.01001,
|
| 1409 |
+
"required_capacity_n": 96,
|
| 1410 |
+
"router_balancing": "aux_loss",
|
| 1411 |
+
"run_dir": "ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1",
|
| 1412 |
+
"seq_length": "4096",
|
| 1413 |
+
"shared_expert_overlap": "0",
|
| 1414 |
+
"topk": "16",
|
| 1415 |
+
"valid_max_rank_reduction_max": 0.6453085566397145,
|
| 1416 |
+
"valid_max_rank_reduction_mean": 0.41069380212188733,
|
| 1417 |
+
"valid_max_rank_reduction_min": 0.14965121208617566,
|
| 1418 |
+
"valid_max_rank_reduction_n": 192,
|
| 1419 |
+
"zero_drop": 1
|
| 1420 |
+
},
|
| 1421 |
+
{
|
| 1422 |
+
"arm": "forced",
|
| 1423 |
+
"assignment_sum_mismatches": 0,
|
| 1424 |
+
"aux_loss_coeff": "1e-2",
|
| 1425 |
+
"backend": "ultraep",
|
| 1426 |
+
"branch": "ultraep-token-drop-analysis",
|
| 1427 |
+
"capacity": "1.25",
|
| 1428 |
+
"capacity_mode": "static",
|
| 1429 |
+
"commit": "eaf829e7",
|
| 1430 |
+
"config": "G3",
|
| 1431 |
+
"cuda_graph": "0",
|
| 1432 |
+
"dense_first_layers": "0",
|
| 1433 |
+
"drop_expected_assignments": 201326592.0,
|
| 1434 |
+
"drop_rate_pct_max": 0.0,
|
| 1435 |
+
"drop_rate_pct_mean": 0.0,
|
| 1436 |
+
"drop_rate_pct_min": 0.0,
|
| 1437 |
+
"drop_rate_pct_n": 96,
|
| 1438 |
+
"drop_rate_pct_overall": 0.0,
|
| 1439 |
+
"dropped_tokens_sum_all_records": 0.0,
|
| 1440 |
+
"duplicate_vector_mismatches": 0,
|
| 1441 |
+
"experts_per_rank": "8",
|
| 1442 |
+
"force_balance": "1",
|
| 1443 |
+
"global_batch_size": "16",
|
| 1444 |
+
"global_snapshots": 192,
|
| 1445 |
+
"incomplete_snapshots": 0,
|
| 1446 |
+
"malformed_records": 0,
|
| 1447 |
+
"moe_layers": "16",
|
| 1448 |
+
"num_experts": "64",
|
| 1449 |
+
"num_layers": "16",
|
| 1450 |
+
"observed_dropped_tokens": 0.0,
|
| 1451 |
+
"peak_demand_max": 66192.0,
|
| 1452 |
+
"peak_demand_mean": 65961.36458333333,
|
| 1453 |
+
"peak_demand_min": 65746.0,
|
| 1454 |
+
"peak_demand_n": 96,
|
| 1455 |
+
"peak_rank_overflow_max": 0.0,
|
| 1456 |
+
"peak_rank_overflow_mean": 0.0,
|
| 1457 |
+
"peak_rank_overflow_min": 0.0,
|
| 1458 |
+
"peak_rank_overflow_n": 96,
|
| 1459 |
+
"post_physical_rank_cv_max": 0.005936610286960278,
|
| 1460 |
+
"post_physical_rank_cv_mean": 0.0030208295858681123,
|
| 1461 |
+
"post_physical_rank_cv_min": 0.0014995787955522665,
|
| 1462 |
+
"post_physical_rank_cv_n": 192,
|
| 1463 |
+
"post_physical_rank_imax_max": 1.010009765625,
|
| 1464 |
+
"post_physical_rank_imax_mean": 1.004675070444743,
|
| 1465 |
+
"post_physical_rank_imax_min": 1.001800537109375,
|
| 1466 |
+
"post_physical_rank_imax_n": 192,
|
| 1467 |
+
"pre_home_rank_cv_max": 0.005936610286960278,
|
| 1468 |
+
"pre_home_rank_cv_mean": 0.0030232844076121573,
|
| 1469 |
+
"pre_home_rank_cv_min": 0.0014995787955522665,
|
| 1470 |
+
"pre_home_rank_cv_n": 192,
|
| 1471 |
+
"pre_home_rank_imax_max": 1.0113067626953125,
|
| 1472 |
+
"pre_home_rank_imax_mean": 1.0046818256378174,
|
| 1473 |
+
"pre_home_rank_imax_min": 1.001800537109375,
|
| 1474 |
+
"pre_home_rank_imax_n": 192,
|
| 1475 |
+
"records": {
|
| 1476 |
+
"drop_dispatch": 1536,
|
| 1477 |
+
"drop_experts": 1536,
|
| 1478 |
+
"drop_setup": 1536,
|
| 1479 |
+
"ultraep_drop": 96,
|
| 1480 |
+
"ultraep_post": 1536,
|
| 1481 |
+
"ultraep_pre": 1536
|
| 1482 |
+
},
|
| 1483 |
+
"required_capacity_max": 1.01001,
|
| 1484 |
+
"required_capacity_mean": 1.0064905625,
|
| 1485 |
+
"required_capacity_min": 1.003204,
|
| 1486 |
+
"required_capacity_n": 96,
|
| 1487 |
+
"router_balancing": "aux_loss",
|
| 1488 |
+
"run_dir": "ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1",
|
| 1489 |
+
"seq_length": "4096",
|
| 1490 |
+
"shared_expert_overlap": "0",
|
| 1491 |
+
"topk": "16",
|
| 1492 |
+
"valid_max_rank_reduction_max": 0.0012824961902319055,
|
| 1493 |
+
"valid_max_rank_reduction_mean": 6.679667657457841e-06,
|
| 1494 |
+
"valid_max_rank_reduction_min": 0.0,
|
| 1495 |
+
"valid_max_rank_reduction_n": 192,
|
| 1496 |
+
"zero_drop": 1
|
| 1497 |
+
},
|
| 1498 |
+
{
|
| 1499 |
+
"arm": "natural",
|
| 1500 |
+
"assignment_sum_mismatches": 0,
|
| 1501 |
+
"aux_loss_coeff": "1e-2",
|
| 1502 |
+
"backend": "ultraep",
|
| 1503 |
+
"branch": "ultraep-token-drop-analysis",
|
| 1504 |
+
"capacity": "1.095703125",
|
| 1505 |
+
"capacity_mode": "static",
|
| 1506 |
+
"commit": "eaf829e7",
|
| 1507 |
+
"config": "G4",
|
| 1508 |
+
"cuda_graph": "0",
|
| 1509 |
+
"dense_first_layers": "0",
|
| 1510 |
+
"drop_expected_assignments": 402653184.0,
|
| 1511 |
+
"drop_rate_pct_max": 0.0,
|
| 1512 |
+
"drop_rate_pct_mean": 0.0,
|
| 1513 |
+
"drop_rate_pct_min": 0.0,
|
| 1514 |
+
"drop_rate_pct_n": 96,
|
| 1515 |
+
"drop_rate_pct_overall": 0.0,
|
| 1516 |
+
"dropped_tokens_sum_all_records": 0.0,
|
| 1517 |
+
"duplicate_vector_mismatches": 0,
|
| 1518 |
+
"experts_per_rank": "16",
|
| 1519 |
+
"force_balance": "0",
|
| 1520 |
+
"global_batch_size": "16",
|
| 1521 |
+
"global_snapshots": 192,
|
| 1522 |
+
"incomplete_snapshots": 0,
|
| 1523 |
+
"malformed_records": 0,
|
| 1524 |
+
"moe_layers": "16",
|
| 1525 |
+
"num_experts": "128",
|
| 1526 |
+
"num_layers": "16",
|
| 1527 |
+
"observed_dropped_tokens": 0.0,
|
| 1528 |
+
"peak_demand_max": 136700.0,
|
| 1529 |
+
"peak_demand_mean": 132779.11458333334,
|
| 1530 |
+
"peak_demand_min": 132383.0,
|
| 1531 |
+
"peak_demand_n": 96,
|
| 1532 |
+
"peak_rank_overflow_max": 0.0,
|
| 1533 |
+
"peak_rank_overflow_mean": 0.0,
|
| 1534 |
+
"peak_rank_overflow_min": 0.0,
|
| 1535 |
+
"peak_rank_overflow_n": 96,
|
| 1536 |
+
"post_physical_rank_cv_max": 0.026463165595102558,
|
| 1537 |
+
"post_physical_rank_cv_mean": 0.014430625474517489,
|
| 1538 |
+
"post_physical_rank_cv_min": 0.005656712434847103,
|
| 1539 |
+
"post_physical_rank_cv_n": 192,
|
| 1540 |
+
"post_physical_rank_imax_max": 1.0100250244140625,
|
| 1541 |
+
"post_physical_rank_imax_mean": 1.0100077788035076,
|
| 1542 |
+
"post_physical_rank_imax_min": 1.0100021362304688,
|
| 1543 |
+
"post_physical_rank_imax_n": 192,
|
| 1544 |
+
"pre_home_rank_cv_max": 0.607811574382448,
|
| 1545 |
+
"pre_home_rank_cv_mean": 0.34190630383555115,
|
| 1546 |
+
"pre_home_rank_cv_min": 0.08731759963810155,
|
| 1547 |
+
"pre_home_rank_cv_n": 192,
|
| 1548 |
+
"pre_home_rank_imax_max": 2.2518386840820312,
|
| 1549 |
+
"pre_home_rank_imax_mean": 1.561224063237508,
|
| 1550 |
+
"pre_home_rank_imax_min": 1.121063232421875,
|
| 1551 |
+
"pre_home_rank_imax_n": 192,
|
| 1552 |
+
"records": {
|
| 1553 |
+
"drop_dispatch": 1536,
|
| 1554 |
+
"drop_experts": 1536,
|
| 1555 |
+
"drop_setup": 1536,
|
| 1556 |
+
"ultraep_drop": 96,
|
| 1557 |
+
"ultraep_post": 1536,
|
| 1558 |
+
"ultraep_pre": 1536
|
| 1559 |
+
},
|
| 1560 |
+
"required_capacity_max": 1.042938,
|
| 1561 |
+
"required_capacity_mean": 1.0130241979166665,
|
| 1562 |
+
"required_capacity_min": 1.010002,
|
| 1563 |
+
"required_capacity_n": 96,
|
| 1564 |
+
"router_balancing": "aux_loss",
|
| 1565 |
+
"run_dir": "ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1",
|
| 1566 |
+
"seq_length": "4096",
|
| 1567 |
+
"shared_expert_overlap": "0",
|
| 1568 |
+
"topk": "32",
|
| 1569 |
+
"valid_max_rank_reduction_max": 0.5514766917497027,
|
| 1570 |
+
"valid_max_rank_reduction_mean": 0.3391114043117762,
|
| 1571 |
+
"valid_max_rank_reduction_min": 0.099060841159657,
|
| 1572 |
+
"valid_max_rank_reduction_n": 192,
|
| 1573 |
+
"zero_drop": 1
|
| 1574 |
+
},
|
| 1575 |
+
{
|
| 1576 |
+
"arm": "natural",
|
| 1577 |
+
"assignment_sum_mismatches": 0,
|
| 1578 |
+
"aux_loss_coeff": "1e-2",
|
| 1579 |
+
"backend": "ultraep",
|
| 1580 |
+
"branch": "ultraep-token-drop-analysis",
|
| 1581 |
+
"capacity": "1.25",
|
| 1582 |
+
"capacity_mode": "static",
|
| 1583 |
+
"commit": "eaf829e7",
|
| 1584 |
+
"config": "G4",
|
| 1585 |
+
"cuda_graph": "0",
|
| 1586 |
+
"dense_first_layers": "0",
|
| 1587 |
+
"drop_expected_assignments": 402653184.0,
|
| 1588 |
+
"drop_rate_pct_max": 0.0,
|
| 1589 |
+
"drop_rate_pct_mean": 0.0,
|
| 1590 |
+
"drop_rate_pct_min": 0.0,
|
| 1591 |
+
"drop_rate_pct_n": 96,
|
| 1592 |
+
"drop_rate_pct_overall": 0.0,
|
| 1593 |
+
"dropped_tokens_sum_all_records": 0.0,
|
| 1594 |
+
"duplicate_vector_mismatches": 0,
|
| 1595 |
+
"experts_per_rank": "16",
|
| 1596 |
+
"force_balance": "0",
|
| 1597 |
+
"global_batch_size": "16",
|
| 1598 |
+
"global_snapshots": 192,
|
| 1599 |
+
"incomplete_snapshots": 0,
|
| 1600 |
+
"malformed_records": 0,
|
| 1601 |
+
"moe_layers": "16",
|
| 1602 |
+
"num_experts": "128",
|
| 1603 |
+
"num_layers": "16",
|
| 1604 |
+
"observed_dropped_tokens": 0.0,
|
| 1605 |
+
"peak_demand_max": 136700.0,
|
| 1606 |
+
"peak_demand_mean": 132713.30208333334,
|
| 1607 |
+
"peak_demand_min": 132383.0,
|
| 1608 |
+
"peak_demand_n": 96,
|
| 1609 |
+
"peak_rank_overflow_max": 0.0,
|
| 1610 |
+
"peak_rank_overflow_mean": 0.0,
|
| 1611 |
+
"peak_rank_overflow_min": 0.0,
|
| 1612 |
+
"peak_rank_overflow_n": 96,
|
| 1613 |
+
"post_physical_rank_cv_max": 0.02646316532015595,
|
| 1614 |
+
"post_physical_rank_cv_mean": 0.01471717424328191,
|
| 1615 |
+
"post_physical_rank_cv_min": 0.0057054839456758865,
|
| 1616 |
+
"post_physical_rank_cv_n": 192,
|
| 1617 |
+
"post_physical_rank_imax_max": 1.0100326538085938,
|
| 1618 |
+
"post_physical_rank_imax_mean": 1.0100078185399373,
|
| 1619 |
+
"post_physical_rank_imax_min": 1.0100021362304688,
|
| 1620 |
+
"post_physical_rank_imax_n": 192,
|
| 1621 |
+
"pre_home_rank_cv_max": 0.6043235604226133,
|
| 1622 |
+
"pre_home_rank_cv_mean": 0.3401136548111448,
|
| 1623 |
+
"pre_home_rank_cv_min": 0.08731759963810155,
|
| 1624 |
+
"pre_home_rank_cv_n": 192,
|
| 1625 |
+
"pre_home_rank_imax_max": 2.2515869140625,
|
| 1626 |
+
"pre_home_rank_imax_mean": 1.5504355827967327,
|
| 1627 |
+
"pre_home_rank_imax_min": 1.121063232421875,
|
| 1628 |
+
"pre_home_rank_imax_n": 192,
|
| 1629 |
+
"records": {
|
| 1630 |
+
"drop_dispatch": 1536,
|
| 1631 |
+
"drop_experts": 1536,
|
| 1632 |
+
"drop_setup": 1536,
|
| 1633 |
+
"ultraep_drop": 96,
|
| 1634 |
+
"ultraep_post": 1536,
|
| 1635 |
+
"ultraep_pre": 1536
|
| 1636 |
+
},
|
| 1637 |
+
"required_capacity_max": 1.042938,
|
| 1638 |
+
"required_capacity_mean": 1.0125220937500001,
|
| 1639 |
+
"required_capacity_min": 1.010002,
|
| 1640 |
+
"required_capacity_n": 96,
|
| 1641 |
+
"router_balancing": "aux_loss",
|
| 1642 |
+
"run_dir": "ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1",
|
| 1643 |
+
"seq_length": "4096",
|
| 1644 |
+
"shared_expert_overlap": "0",
|
| 1645 |
+
"topk": "32",
|
| 1646 |
+
"valid_max_rank_reduction_max": 0.5514197614529683,
|
| 1647 |
+
"valid_max_rank_reduction_mean": 0.3357737985870543,
|
| 1648 |
+
"valid_max_rank_reduction_min": 0.099060841159657,
|
| 1649 |
+
"valid_max_rank_reduction_n": 192,
|
| 1650 |
+
"zero_drop": 1
|
| 1651 |
+
},
|
| 1652 |
+
{
|
| 1653 |
+
"arm": "forced",
|
| 1654 |
+
"assignment_sum_mismatches": 0,
|
| 1655 |
+
"aux_loss_coeff": "1e-2",
|
| 1656 |
+
"backend": "ultraep",
|
| 1657 |
+
"branch": "ultraep-token-drop-analysis",
|
| 1658 |
+
"capacity": "1.25",
|
| 1659 |
+
"capacity_mode": "static",
|
| 1660 |
+
"commit": "eaf829e7",
|
| 1661 |
+
"config": "G4",
|
| 1662 |
+
"cuda_graph": "0",
|
| 1663 |
+
"dense_first_layers": "0",
|
| 1664 |
+
"drop_expected_assignments": 402653184.0,
|
| 1665 |
+
"drop_rate_pct_max": 0.0,
|
| 1666 |
+
"drop_rate_pct_mean": 0.0,
|
| 1667 |
+
"drop_rate_pct_min": 0.0,
|
| 1668 |
+
"drop_rate_pct_n": 96,
|
| 1669 |
+
"drop_rate_pct_overall": 0.0,
|
| 1670 |
+
"dropped_tokens_sum_all_records": 0.0,
|
| 1671 |
+
"duplicate_vector_mismatches": 0,
|
| 1672 |
+
"experts_per_rank": "16",
|
| 1673 |
+
"force_balance": "1",
|
| 1674 |
+
"global_batch_size": "16",
|
| 1675 |
+
"global_snapshots": 192,
|
| 1676 |
+
"incomplete_snapshots": 0,
|
| 1677 |
+
"malformed_records": 0,
|
| 1678 |
+
"moe_layers": "16",
|
| 1679 |
+
"num_experts": "128",
|
| 1680 |
+
"num_layers": "16",
|
| 1681 |
+
"observed_dropped_tokens": 0.0,
|
| 1682 |
+
"peak_demand_max": 132006.0,
|
| 1683 |
+
"peak_demand_mean": 131665.52083333334,
|
| 1684 |
+
"peak_demand_min": 131398.0,
|
| 1685 |
+
"peak_demand_n": 96,
|
| 1686 |
+
"peak_rank_overflow_max": 0.0,
|
| 1687 |
+
"peak_rank_overflow_mean": 0.0,
|
| 1688 |
+
"peak_rank_overflow_min": 0.0,
|
| 1689 |
+
"peak_rank_overflow_n": 96,
|
| 1690 |
+
"post_physical_rank_cv_max": 0.00357507965332711,
|
| 1691 |
+
"post_physical_rank_cv_mean": 0.0021567674165372404,
|
| 1692 |
+
"post_physical_rank_cv_min": 0.0006899225660317828,
|
| 1693 |
+
"post_physical_rank_cv_n": 192,
|
| 1694 |
+
"post_physical_rank_imax_max": 1.0064849853515625,
|
| 1695 |
+
"post_physical_rank_imax_mean": 1.0033422311147053,
|
| 1696 |
+
"post_physical_rank_imax_min": 1.0009918212890625,
|
| 1697 |
+
"post_physical_rank_imax_n": 192,
|
| 1698 |
+
"pre_home_rank_cv_max": 0.00357507965332711,
|
| 1699 |
+
"pre_home_rank_cv_mean": 0.0021567674165372404,
|
| 1700 |
+
"pre_home_rank_cv_min": 0.0006899225660317828,
|
| 1701 |
+
"pre_home_rank_cv_n": 192,
|
| 1702 |
+
"pre_home_rank_imax_max": 1.0064849853515625,
|
| 1703 |
+
"pre_home_rank_imax_mean": 1.0033422311147053,
|
| 1704 |
+
"pre_home_rank_imax_min": 1.0009918212890625,
|
| 1705 |
+
"pre_home_rank_imax_n": 192,
|
| 1706 |
+
"records": {
|
| 1707 |
+
"drop_dispatch": 1536,
|
| 1708 |
+
"drop_experts": 1536,
|
| 1709 |
+
"drop_setup": 1536,
|
| 1710 |
+
"ultraep_drop": 96,
|
| 1711 |
+
"ultraep_post": 1536,
|
| 1712 |
+
"ultraep_pre": 1536
|
| 1713 |
+
},
|
| 1714 |
+
"required_capacity_max": 1.007126,
|
| 1715 |
+
"required_capacity_mean": 1.0045282500000001,
|
| 1716 |
+
"required_capacity_min": 1.002487,
|
| 1717 |
+
"required_capacity_n": 96,
|
| 1718 |
+
"router_balancing": "aux_loss",
|
| 1719 |
+
"run_dir": "ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1",
|
| 1720 |
+
"seq_length": "4096",
|
| 1721 |
+
"shared_expert_overlap": "0",
|
| 1722 |
+
"topk": "32",
|
| 1723 |
+
"valid_max_rank_reduction_max": 0.0,
|
| 1724 |
+
"valid_max_rank_reduction_mean": 0.0,
|
| 1725 |
+
"valid_max_rank_reduction_min": 0.0,
|
| 1726 |
+
"valid_max_rank_reduction_n": 192,
|
| 1727 |
+
"zero_drop": 1
|
| 1728 |
+
}
|
| 1729 |
+
]
|
tiera-rerun-fixed-moonep/probe_summary_all.tsv
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
run_dir arm config backend num_experts topk experts_per_rank num_layers moe_layers dense_first_layers seq_length global_batch_size capacity_mode capacity force_balance cuda_graph branch commit router_balancing aux_loss_coeff shared_expert_overlap assignment_sum_mismatches cold_expert_fraction_max cold_expert_fraction_mean cold_expert_fraction_min cold_expert_fraction_n drop_accounting_source drop_expected_assignments drop_rate_pct_max drop_rate_pct_mean drop_rate_pct_min drop_rate_pct_n drop_rate_pct_overall dropped_tokens_sum_all_records duplicate_vector_mismatches echo_logged_load_reduction_valid echo_logged_pre_rank_metric_valid echo_moved_assignment_fraction_max echo_moved_assignment_fraction_mean echo_moved_assignment_fraction_min echo_moved_assignment_fraction_n global_snapshots hottest_expert_token_fraction_max hottest_expert_token_fraction_mean hottest_expert_token_fraction_min hottest_expert_token_fraction_n incomplete_snapshots malformed_records observed_dropped_tokens peak_demand_max peak_demand_mean peak_demand_min peak_demand_n peak_rank_overflow_max peak_rank_overflow_mean peak_rank_overflow_min peak_rank_overflow_n post_physical_rank_cv_max post_physical_rank_cv_mean post_physical_rank_cv_min post_physical_rank_cv_n post_physical_rank_imax_max post_physical_rank_imax_mean post_physical_rank_imax_min post_physical_rank_imax_n pre_home_rank_cv_max pre_home_rank_cv_mean pre_home_rank_cv_min pre_home_rank_cv_n pre_home_rank_imax_max pre_home_rank_imax_mean pre_home_rank_imax_min pre_home_rank_imax_n raw_expert_cv_max raw_expert_cv_mean raw_expert_cv_min raw_expert_cv_n raw_expert_imax_ceiling raw_expert_imax_hit_ceiling raw_expert_imax_max raw_expert_imax_mean raw_expert_imax_min raw_expert_imax_n required_capacity_max required_capacity_mean required_capacity_min required_capacity_n saturated_dispatch_rows valid_max_rank_reduction_max valid_max_rank_reduction_mean valid_max_rank_reduction_min valid_max_rank_reduction_n zero_drop
|
| 2 |
+
echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1 natural G1 echo 8 2 1 16 16 0 4096 16 static 2.00 0 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.75 0.18587239583333334 0.0 384 dispatch 25165824.0 0.0 0 0 0.6113739013671875 0.5011123021443685 0.4014892578125 384 384 1.0 0.9049501419067383 0.374786376953125 384 0 0 0 0.3858312238475252 0.04369442624774788 0.0 384 1.8909912109375 1.0827395121256511 1.0 384 1.7320508075688772 1.4536835274449291 0.2541278385706527 384 4.0 3.619800567626953 1.4991455078125 384 1.7320508075688772 1.4536835274449291 0.2541278385706527 384 4.0 1 4.0 3.619800567626953 1.4991455078125 384 1.90625 1.0099385579427083 0.53125 3072 0 0.75 0.6772174606036838 0.11090858104318564 384 1
|
| 3 |
+
echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1 natural G2 echo 32 8 4 16 16 0 4096 16 static 1.08203125 0 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.71875 0.159912109375 0.0 384 dispatch 100663296.0 0.0 0 0 0.5000267028808594 0.499999205271403 0.49983978271484375 384 384 1.0 0.9680010477701823 0.613189697265625 384 0 0 0 0.00213623046875 1.5938873823057422e-05 0.0 384 1.0042724609375 1.0000306765238445 1.0 384 1.3175703221722546 0.6974670284251169 0.0007166759378550646 384 3.998382568359375 2.1703339417775473 1.001617431640625 384 1.7320419981493513 1.5150670645228896 0.6923179293416859 384 4.0 1 4.0 3.872004191080729 2.4527587890625 384 1.02734375 1.0103823343912761 1.0 3072 0 0.7498988696295957 0.5145680576214848 0.0016148197800188904 384 1
|
| 4 |
+
echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1 forced G2 echo 32 8 4 16 16 0 4096 16 static 1.08203125 1 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.0 0.0 0.0 384 dispatch 100663296.0 0.0 0 0 0.5 0.4999997019767761 0.49993896484375 384 384 0.2593994140625 0.25501465797424316 0.252777099609375 384 0 0 0 0.000421485208646641 2.2897106263584403e-06 0.0 384 1.00067138671875 1.0000041325887044 1.0 384 0.007852555942790041 0.004394259674654735 0.0016880800477701613 384 1.01715087890625 1.006885051727295 1.002349853515625 384 0.013095170649570529 0.009386188814570484 0.006191008040561732 384 4.0 0 1.03759765625 1.0200586318969727 1.0111083984375 384 1.0234375 1.010022481282552 1.00390625 3072 0 0.01686168616861686 0.006827796124259797 0.0023443446491094536 384 1
|
| 5 |
+
echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap2p00/rep1 natural G2 echo 32 8 4 16 16 0 4096 16 static 2.00 0 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.71875 0.15087890625 0.0 384 dispatch 100663296.0 0.0 0 0 0.500030517578125 0.4999993046124776 0.4998779296875 384 384 1.0 0.9681024551391602 0.620849609375 384 0 0 0 0.0021209716796875 1.466968415560366e-05 0.0 384 1.004241943359375 1.0000279744466145 1.0 384 1.317620465314074 0.6968048692022525 0.0012016720234495348 384 3.998260498046875 2.1875499884287515 1.00238037109375 384 1.7320331892227239 1.5175145186167975 0.6923179293416859 384 4.0 1 4.0 3.8724098205566406 2.4833984375 384 1.02734375 1.0105133056640625 1.0 3072 0 0.7498912338281876 0.5172585999990112 0.0023747183827558913 384 1
|
| 6 |
+
echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625/rep1 natural G3 echo 64 16 8 16 16 0 4096 16 static 1.072265625 0 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.6875 0.1546630859375 0.0 384 dispatch 201326592.0 0.0 0 0 0.5 0.5 0.5 384 384 1.0 0.9773949782053629 0.634124755859375 384 0 0 0 0.0 0.0 0.0 384 1.0 1.0 1.0 384 0.8292601101580809 0.47371518867838985 0.1091963461644949 384 2.9946746826171875 1.7838420073191326 1.153167724609375 384 1.7316460620806386 1.525739653918128 0.6814247311375694 384 4.0 1 4.0 3.9095799128214517 2.5364990234375 384 1.01953125 1.0093224843343098 1.0 3072 0 0.6660739125339475 0.41846288670458703 0.1328234578029481 384 1
|
| 7 |
+
echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625__forcebal/rep1 forced G3 echo 64 16 8 16 16 0 4096 16 static 1.072265625 1 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.0 0.0 0.0 384 dispatch 201326592.0 0.0 0 0 0.5 0.5 0.5 384 384 0.26123046875 0.2557719548543294 0.253570556640625 384 0 0 0 0.0 0.0 0.0 384 1.0 1.0 1.0 384 0.00610409733861204 0.0030369541814070297 0.0013949279180933694 384 1.0114288330078125 1.0047780275344849 1.001708984375 384 0.011936138368762292 0.009561000998578461 0.006960450128604076 384 4.0 0 1.044921875 1.0230878194173176 1.0142822265625 384 1.017578125 1.0088431040445964 1.00390625 3072 0 0.011299690729425963 0.004752627833504172 0.0017060687301974165 384 1
|
| 8 |
+
echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap2p00/rep1 natural G3 echo 64 16 8 16 16 0 4096 16 static 2.00 0 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.6875 0.15531412760416666 0.0 384 dispatch 201326592.0 0.0 0 0 0.5 0.5 0.5 384 384 1.0 0.9773902098337809 0.638397216796875 384 0 0 0 0.0 0.0 0.0 384 1.0 1.0 1.0 384 0.8292092994255315 0.47732093430435846 0.1091963461644949 384 2.9329681396484375 1.7915768225987752 1.1547698974609375 384 1.7316636929706255 1.5249635478133088 0.6839438720259622 384 4.0 1 4.0 3.9095608393351235 2.5535888671875 384 1.01953125 1.0093262990315754 1.0 3072 0 0.6590484613583747 0.4215810507718854 0.13402661240238375 384 1
|
| 9 |
+
echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375/rep1 natural G4 echo 128 32 16 16 16 0 4096 16 static 1.068359375 0 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.6796875 0.16324869791666666 0.0 384 dispatch 402653184.0 0.0 0 0 0.5 0.5 0.5 384 384 1.0 0.9825874964396158 0.5750732421875 384 0 0 0 0.0 0.0 0.0 384 1.0 1.0 1.0 384 0.6262571920835467 0.33797223191114484 0.08738761375151358 384 2.2536544799804688 1.550683856010437 1.1217041015625 384 1.7303400196303844 1.5202854761326712 0.6197223679268041 384 4.0 1 4.0 3.9303499857584634 2.30029296875 384 1.0166015625 1.0086215337117512 1.0009765625 3072 0 0.5562762575704744 0.3421405808049787 0.10849929263249537 384 1
|
| 10 |
+
echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375__forcebal/rep1 forced G4 echo 128 32 16 16 16 0 4096 16 static 1.068359375 1 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.0 0.0 0.0 384 dispatch 402653184.0 0.0 0 0 0.5 0.5 0.5 384 384 0.259674072265625 0.25621263186136883 0.253997802734375 384 0 0 0 0.0 0.0 0.0 384 1.0 1.0 1.0 384 0.0036539583092664103 0.002179247890018717 0.0006899225660317828 384 1.0071258544921875 1.0034127434094746 1.0009918212890625 384 0.01168330115931115 0.009563696224675585 0.008002983940271118 384 4.0 0 1.0386962890625 1.0248505274454753 1.0159912109375 384 1.013671875 1.0081939697265625 1.00390625 3072 0 0.007075435965031893 0.0033999001306743632 0.000990838554290331 384 1
|
| 11 |
+
echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap2p00/rep1 natural G4 echo 128 32 16 16 16 0 4096 16 static 2.00 0 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.671875 0.16269938151041666 0.0 384 dispatch 402653184.0 0.0 0 0 0.5 0.5 0.5 384 384 1.0 0.9825185934702555 0.57318115234375 384 0 0 0 0.0 0.0 0.0 384 1.0 1.0 1.0 384 0.6019828931462307 0.3383858718614603 0.08738761375151358 384 2.255584716796875 1.5542763471603394 1.1217041015625 384 1.7304659197188927 1.521021036726468 0.6188791003597877 384 4.0 1 4.0 3.930074373881022 2.292724609375 384 1.0166015625 1.0086164474487305 1.0009765625 3072 0 0.5566559781358661 0.3437128068160493 0.10849929263249537 384 1
|
| 12 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1 natural G1 ultraep 8 2 1 16 16 0 4096 16 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 25165824.0 2.891776 0.607152 0.0 96 0.6209174791971842 156259.0 0 192 0 0 156259.0 14128.0 10918.34375 8275.0 96 3888.0 1076.4583333333333 0.0 96 0.6189049804826644 0.08780981215252336 0.015802566559489504 192 1.724609375 1.063955307006836 1.010009765625 192 1.7320508075688772 1.4485647787782712 0.3594758110042677 192 4.0 3.6084569295247397 1.6087646484375 192 1.724609 1.33280565625 1.010132 96 0.7474516266861991 0.6845405986286021 0.27725927612110174 192 0
|
| 13 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1 natural G1 ultraep 8 2 1 16 16 0 4096 16 static 1.8125 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 25165824.0 1.545951 0.03138885416666667 0.0 96 0.032329559326171875 8136.0 0 192 0 0 8136.0 18916.0 11069.770833333334 8275.0 96 4068.0 84.75 0.0 96 1.0236043430060333 0.09497773141059589 0.015224448662595366 192 2.30908203125 1.0747025807698567 1.010009765625 192 1.7320508075688772 1.444809403446171 0.3594758110042677 192 4.0 3.6151854197184243 1.6087646484375 192 2.309082 1.3512904166666668 1.010132 96 0.7474516266861991 0.6828229027074976 0.27725927612110174 192 0
|
| 14 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 natural G2 ultraep 32 8 4 16 16 0 4096 16 static 1.0625 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 100663296.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 33099.0 33096.927083333336 33095.0 96 0.0 0.0 0.0 96 0.026494891363095952 0.013436746135527637 0.004813851711632 192 1.010101318359375 1.0100231170654297 1.010009765625 192 1.3185407730096428 0.7088423169085597 0.07964958686803787 192 3.99853515625 2.2032313346862793 1.17706298828125 192 1.010101 1.0100380104166666 1.009979 96 0.7474050555623397 0.5179521802733736 0.14192377495462796 192 1
|
| 15 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1 natural G2 ultraep 32 8 4 16 16 0 4096 16 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 100663296.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 33099.0 33096.875 33096.0 96 0.0 0.0 0.0 96 0.026494891363095952 0.013604201092033112 0.004841919368473151 192 1.01007080078125 1.0100212097167969 1.010009765625 192 1.3181329746469168 0.7143701949255296 0.22890444224298007 192 3.99853515625 2.2135567665100098 1.33551025390625 192 1.010101 1.0100364479166666 1.01001 96 0.7474050555623397 0.5202877868268106 0.24370458388556282 192 1
|
| 16 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1 forced G2 ultraep 32 8 4 16 16 0 4096 16 static 1.25 1 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 100663296.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 33096.0 33052.15625 32901.0 96 0.0 0.0 0.0 96 0.00717872106815807 0.004277032344446821 0.0016880800477701613 192 1.010009765625 1.006682554880778 1.002349853515625 192 0.007852555942790041 0.00436334506476056 0.0016880800477701613 192 1.01544189453125 1.0068944295247395 1.002349853515625 192 1.01001 1.0086718541666666 1.004059 96 0.005349522149425978 0.0002090865394183532 0.0 192 1
|
| 17 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 natural G3 ultraep 64 16 8 16 16 0 4096 16 static 1.0625 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 201326592.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 66196.0 66193.28125 66192.0 96 0.0 0.0 0.0 96 0.026489120474252784 0.014233627693615905 0.004993448920418017 192 1.01007080078125 1.0100196202596028 1.010009765625 192 0.8287584673770169 0.47726214620952834 0.11019481148220499 192 2.6106719970703125 1.7842089335123699 1.1877593994140625 192 1.010071 1.0100292499999999 1.01001 96 0.6131226876610966 0.41320151666180793 0.14965121208617566 192 1
|
| 18 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1 natural G3 ultraep 64 16 8 16 16 0 4096 16 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 201326592.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 66195.0 66193.34375 66192.0 96 0.0 0.0 0.0 96 0.026483352723689336 0.014464507564652654 0.00571681991985643 192 1.0100555419921875 1.0100204149882 1.010009765625 192 0.8316092132265209 0.4753321564816673 0.11019481148220499 192 2.8477020263671875 1.7751669883728027 1.1877593994140625 192 1.010056 1.01003021875 1.01001 96 0.6453085566397145 0.41069380212188733 0.14965121208617566 192 1
|
| 19 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1 forced G3 ultraep 64 16 8 16 16 0 4096 16 static 1.25 1 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 201326592.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 66192.0 65961.36458333333 65746.0 96 0.0 0.0 0.0 96 0.005936610286960278 0.0030208295858681123 0.0014995787955522665 192 1.010009765625 1.004675070444743 1.001800537109375 192 0.005936610286960278 0.0030232844076121573 0.0014995787955522665 192 1.0113067626953125 1.0046818256378174 1.001800537109375 192 1.01001 1.0064905625 1.003204 96 0.0012824961902319055 6.679667657457841e-06 0.0 192 1
|
| 20 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1 natural G4 ultraep 128 32 16 16 16 0 4096 16 static 1.095703125 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 402653184.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 136700.0 132779.11458333334 132383.0 96 0.0 0.0 0.0 96 0.026463165595102558 0.014430625474517489 0.005656712434847103 192 1.0100250244140625 1.0100077788035076 1.0100021362304688 192 0.607811574382448 0.34190630383555115 0.08731759963810155 192 2.2518386840820312 1.561224063237508 1.121063232421875 192 1.042938 1.0130241979166665 1.010002 96 0.5514766917497027 0.3391114043117762 0.099060841159657 192 1
|
| 21 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1 natural G4 ultraep 128 32 16 16 16 0 4096 16 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 402653184.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 136700.0 132713.30208333334 132383.0 96 0.0 0.0 0.0 96 0.02646316532015595 0.01471717424328191 0.0057054839456758865 192 1.0100326538085938 1.0100078185399373 1.0100021362304688 192 0.6043235604226133 0.3401136548111448 0.08731759963810155 192 2.2515869140625 1.5504355827967327 1.121063232421875 192 1.042938 1.0125220937500001 1.010002 96 0.5514197614529683 0.3357737985870543 0.099060841159657 192 1
|
| 22 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1 forced G4 ultraep 128 32 16 16 16 0 4096 16 static 1.25 1 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 402653184.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 132006.0 131665.52083333334 131398.0 96 0.0 0.0 0.0 96 0.00357507965332711 0.0021567674165372404 0.0006899225660317828 192 1.0064849853515625 1.0033422311147053 1.0009918212890625 192 0.00357507965332711 0.0021567674165372404 0.0006899225660317828 192 1.0064849853515625 1.0033422311147053 1.0009918212890625 192 1.007126 1.0045282500000001 1.002487 96 0.0 0.0 0.0 192 1
|
tiera-rerun-fixed-moonep/steps.tsv
CHANGED
|
@@ -1,2 +1,3 @@
|
|
| 1 |
status campaign phase gbs started ended result_dir
|
| 2 |
FAIL echo tierbprobe 16 2026-08-30T04:52:38+00:00 2026-08-30T05:06:38+00:00 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe
|
|
|
|
|
|
| 1 |
status campaign phase gbs started ended result_dir
|
| 2 |
FAIL echo tierbprobe 16 2026-08-30T04:52:38+00:00 2026-08-30T05:06:38+00:00 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe
|
| 3 |
+
FAIL ultraep tierbprobe 16 2026-08-30T05:06:44+00:00 2026-08-30T05:20:40+00:00 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe
|
tiera-rerun-fixed-moonep/study.log
CHANGED
|
@@ -189,3 +189,184 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
|
|
| 189 |
[guard] tearing down process group (outcome=DONE)
|
| 190 |
--------------------------------------------------------------------------
|
| 191 |
[05:11:57] CELL G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal (slots=4)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 189 |
[guard] tearing down process group (outcome=DONE)
|
| 190 |
--------------------------------------------------------------------------
|
| 191 |
[05:11:57] CELL G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal (slots=4)
|
| 192 |
+
[guard] iter 10/12 (60s)
|
| 193 |
+
[guard] iter 12/12 (70s)
|
| 194 |
+
[guard] tearing down process group (outcome=DONE)
|
| 195 |
+
--------------------------------------------------------------------------
|
| 196 |
+
[05:13:09] CELL G3__ultraep__matched__seq4096__gbs16__cap1p25 (slots=8)
|
| 197 |
+
[guard] iter 3/12 (60s)
|
| 198 |
+
[guard] iter 12/12 (70s)
|
| 199 |
+
[guard] tearing down process group (outcome=DONE)
|
| 200 |
+
[05:14:24] G3 ultraep: validating C=1.0625 (safety=+5%)
|
| 201 |
+
--------------------------------------------------------------------------
|
| 202 |
+
[05:14:24] CELL G3__ultraep__matched__seq4096__gbs16__cap1p0625 (slots=8)
|
| 203 |
+
[guard] iter 3/12 (60s)
|
| 204 |
+
[guard] iter 12/12 (70s)
|
| 205 |
+
[guard] tearing down process group (outcome=DONE)
|
| 206 |
+
--------------------------------------------------------------------------
|
| 207 |
+
[05:15:39] CELL G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal (slots=8)
|
| 208 |
+
[guard] iter 7/12 (60s)
|
| 209 |
+
[guard] iter 12/12 (70s)
|
| 210 |
+
[guard] tearing down process group (outcome=DONE)
|
| 211 |
+
--------------------------------------------------------------------------
|
| 212 |
+
[05:16:53] CELL G4__ultraep__matched__seq4096__gbs16__cap1p25 (slots=16)
|
| 213 |
+
[guard] iter 4/12 (60s)
|
| 214 |
+
[guard] iter 12/12 (70s)
|
| 215 |
+
[guard] tearing down process group (outcome=DONE)
|
| 216 |
+
[05:18:08] G4 ultraep: validating C=1.095703125 (safety=+5%)
|
| 217 |
+
--------------------------------------------------------------------------
|
| 218 |
+
[05:18:08] CELL G4__ultraep__matched__seq4096__gbs16__cap1p095703125 (slots=16)
|
| 219 |
+
[guard] iter 3/12 (60s)
|
| 220 |
+
[guard] iter 12/12 (70s)
|
| 221 |
+
[guard] tearing down process group (outcome=DONE)
|
| 222 |
+
--------------------------------------------------------------------------
|
| 223 |
+
[05:19:24] CELL G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal (slots=16)
|
| 224 |
+
[guard] iter 6/12 (60s)
|
| 225 |
+
[guard] iter 12/12 (70s)
|
| 226 |
+
[guard] tearing down process group (outcome=DONE)
|
| 227 |
+
--------------------------------------------------------------------------
|
| 228 |
+
|
| 229 |
+
==============================================================
|
| 230 |
+
SWEEP COMPLETE ok=11 failed=0
|
| 231 |
+
==============================================================
|
| 232 |
+
config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
|
| 233 |
+
G1 ultraep matched 1 4096 16 1.25 none 1 OK 893.49 886.90 108731 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1
|
| 234 |
+
G1 ultraep matched 1 4096 16 1.8125 none 1 OK 914.84 903.60 113767 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1
|
| 235 |
+
G2 ultraep matched 4 4096 16 1.25 none 1 OK 558.91 532.70 110343 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1
|
| 236 |
+
G2 ultraep matched 4 4096 16 1.0625 none 1 OK 551.94 527.00 107987 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1
|
| 237 |
+
G2 ultraep matched 4 4096 16 1.25 none+forcebal 1 OK 426.26 422.10 110343 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1
|
| 238 |
+
G3 ultraep matched 8 4096 16 1.25 none 1 OK 536.43 529.00 114319 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1
|
| 239 |
+
G3 ultraep matched 8 4096 16 1.0625 none 1 OK 528.17 522.80 111567 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1
|
| 240 |
+
G3 ultraep matched 8 4096 16 1.25 none+forcebal 1 OK 472.34 470.60 114267 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1
|
| 241 |
+
G4 ultraep matched 16 4096 16 1.25 none 1 OK 610.06 573.90 120487 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1
|
| 242 |
+
G4 ultraep matched 16 4096 16 1.095703125 none 1 OK 594.23 563.90 118667 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1
|
| 243 |
+
G4 ultraep matched 16 4096 16 1.25 none+forcebal 1 OK 596.29 545.60 120487 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1
|
| 244 |
+
|
| 245 |
+
Results: /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/results.tsv
|
| 246 |
+
Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16
|
| 247 |
+
==============================================================
|
| 248 |
+
11 probe runs -> /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/probe_summary.tsv
|
| 249 |
+
run arm cap rawExpI preRankI postRankI drop% reqCap
|
| 250 |
+
--------------------------------------------------------------------------------------------------------------------
|
| 251 |
+
G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1 natural 1.25 3.6085 1.0640 2.8918 1.7246
|
| 252 |
+
G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1 natural 1.8125 3.6152 1.0747 1.5460 2.3091
|
| 253 |
+
G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 natural 1.0625 2.2032 1.0100 0.0000 1.0101
|
| 254 |
+
G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1 natural 1.25 2.2136 1.0100 0.0000 1.0101
|
| 255 |
+
G2__ultraep__matched__seq4096__gbs16__cap1p25__force forced 1.25 1.0069 1.0067 0.0000 1.0100
|
| 256 |
+
G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 natural 1.0625 1.7842 1.0100 0.0000 1.0101
|
| 257 |
+
G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1 natural 1.25 1.7752 1.0100 0.0000 1.0101
|
| 258 |
+
G3__ultraep__matched__seq4096__gbs16__cap1p25__force forced 1.25 1.0047 1.0047 0.0000 1.0100
|
| 259 |
+
G4__ultraep__matched__seq4096__gbs16__cap1p095703125 natural 1.095703125 1.5612 1.0100 0.0000 1.0429
|
| 260 |
+
G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1 natural 1.25 1.5504 1.0100 0.0000 1.0429
|
| 261 |
+
G4__ultraep__matched__seq4096__gbs16__cap1p25__force forced 1.25 1.0033 1.0033 0.0000 1.0071
|
| 262 |
+
|
| 263 |
+
ECHO note: echo_logged_pre_rank_metric_valid=0 and echo_logged_load_reduction_valid=0 are intentional.
|
| 264 |
+
The branch's built-in fields mix source-rank and destination-rank axes; this tool reconstructs both globally.
|
| 265 |
+
|
| 266 |
+
TIER B PROBE -- what these logs contain, per backend
|
| 267 |
+
----------------------------------------------------
|
| 268 |
+
Capacity matrix:
|
| 269 |
+
ECHO: start C=2.00; on OOM descend 1.75, 1.50, 1.25. Stop at
|
| 270 |
+
the highest fitting zero-drop capacity. If a fitting capacity drops,
|
| 271 |
+
smaller values cannot help and the model is blocked.
|
| 272 |
+
UltraEP: measure required_capacity at C=1.25, round it UP to the next
|
| 273 |
+
0.01, then validate that capacity once.
|
| 274 |
+
MoonEP: natural + forced, no capacity factor.
|
| 275 |
+
|
| 276 |
+
The forced arm runs once at the selected/high-headroom capacity. It validates
|
| 277 |
+
instrumentation and is not part of capacity selection.
|
| 278 |
+
|
| 279 |
+
Each backend instruments a different thing:
|
| 280 |
+
|
| 281 |
+
echo local raw-router histograms and local post-reroute placement.
|
| 282 |
+
reduce_probe.py aligns and sums all eight source ranks before
|
| 283 |
+
calculating raw_expert_*, pre_home_rank_*,
|
| 284 |
+
post_physical_rank_* and valid_max_rank_reduction_*.
|
| 285 |
+
|
| 286 |
+
ultraep global pre/post rank vectors plus explicit drop accounting:
|
| 287 |
+
drop_rate_pct and required_capacity -- the capacity factor the run
|
| 288 |
+
WOULD have needed. The vectors are duplicated identically on all
|
| 289 |
+
logging ranks and are deduplicated, not summed.
|
| 290 |
+
|
| 291 |
+
moonep local raw-router histograms, actual payload rows per physical
|
| 292 |
+
destination, and the bytes-copy side. The reducer globally sums
|
| 293 |
+
each (iteration, layer) snapshot first. ECHO reports no comparable
|
| 294 |
+
byte counters, so MoonEP bytes are a self-comparison across models,
|
| 295 |
+
never a cross-backend ranking.
|
| 296 |
+
|
| 297 |
+
Do NOT quote ECHO's logged pre_echo imbalance_ratio or
|
| 298 |
+
max_load_reduction_pct. The branch mixes source-rank and destination-rank
|
| 299 |
+
axes; the reducer marks both invalid and reconstructs the comparison globally.
|
| 300 |
+
|
| 301 |
+
Two boundary cases:
|
| 302 |
+
raw_expert_imax_max == E/K
|
| 303 |
+
-> at least one global snapshot had one expert in every
|
| 304 |
+
token's top-K set. Inspect the MEAN before generalising.
|
| 305 |
+
required_capacity or min-safe-capacity at the configured capacity
|
| 306 |
+
-> clipped by the buffer; it means "at or over", not
|
| 307 |
+
"fits comfortably".
|
| 308 |
+
|
| 309 |
+
tools/reduce_probe.py has already written probe_summary.tsv and .json here.
|
| 310 |
+
|
| 311 |
+
Campaign results: /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe
|
| 312 |
+
Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe --compare
|
| 313 |
+
Already logged in to Hugging Face as: AbhishekGhosh1998
|
| 314 |
+
uploaded: /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tiera-rerun-fixed-moonep/ultraep/tierbprobe
|
| 315 |
+
https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
|
| 316 |
+
|
| 317 |
+
=== ROLL-UP
|
| 318 |
+
21 runs ok=21 oom=0 -> /workspace/tiera-rerun-fixed-moonep/collected_all.tsv
|
| 319 |
+
|
| 320 |
+
WARNING: 11 run(s) mention capacity saturation. A run that drops assignments does less expert work and must not be quoted as a win:
|
| 321 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1: WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=283999 total_dropped_tok
|
| 322 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1: WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=284540 total_dropped_tok
|
| 323 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1: WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=1048800 total_dropped_to
|
| 324 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1: WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=1048800 total_dropped_to
|
| 325 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1: WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=32 total_expected_recv=1047984 total_dropped_to
|
| 326 |
+
|
| 327 |
+
phase config seq gbs policy rc cg asy backend iter_ms vs_best
|
| 328 |
+
-------------------------------------------------------------------------------------
|
| 329 |
+
tierbprobeG1 4096 16 matched none 0 0 echo 671.53 1.00x
|
| 330 |
+
ultraep 904.16 1.35x
|
| 331 |
+
|
| 332 |
+
tierbprobeG2 4096 16 matched none 0 0 ultraep 512.37 1.00x
|
| 333 |
+
echo 654.76 1.28x
|
| 334 |
+
|
| 335 |
+
tierbprobeG3 4096 16 matched none 0 0 ultraep 512.31 1.00x
|
| 336 |
+
echo 689.13 1.35x
|
| 337 |
+
|
| 338 |
+
tierbprobeG4 4096 16 matched none 0 0 ultraep 600.19 1.00x
|
| 339 |
+
echo 790.97 1.32x
|
| 340 |
+
|
| 341 |
+
21 probe runs -> /workspace/tiera-rerun-fixed-moonep/probe_summary_all.tsv
|
| 342 |
+
run arm cap rawExpI preRankI postRankI drop% reqCap
|
| 343 |
+
--------------------------------------------------------------------------------------------------------------------
|
| 344 |
+
echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__ natural 2.00 3.6198* 3.6198 1.0827 1.9062
|
| 345 |
+
echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__ natural 1.08203125 3.8720* 2.1703 1.0000 1.0273
|
| 346 |
+
echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__ forced 1.08203125 1.0201 1.0069 1.0000 1.0234
|
| 347 |
+
echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__ natural 2.00 3.8724* 2.1875 1.0000 1.0273
|
| 348 |
+
echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__ natural 1.072265625 3.9096* 1.7838 1.0000 1.0195
|
| 349 |
+
echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__ forced 1.072265625 1.0231 1.0048 1.0000 1.0176
|
| 350 |
+
echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__ natural 2.00 3.9096* 1.7916 1.0000 1.0195
|
| 351 |
+
echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__ natural 1.068359375 3.9303* 1.5507 1.0000 1.0166
|
| 352 |
+
echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__ forced 1.068359375 1.0249 1.0034 1.0000 1.0137
|
| 353 |
+
echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__ natural 2.00 3.9301* 1.5543 1.0000 1.0166
|
| 354 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__mat natural 1.25 3.6085 1.0640 2.8918 1.7246
|
| 355 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__mat natural 1.8125 3.6152 1.0747 1.5460 2.3091
|
| 356 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__mat natural 1.0625 2.2032 1.0100 0.0000 1.0101
|
| 357 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__mat natural 1.25 2.2136 1.0100 0.0000 1.0101
|
| 358 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__mat forced 1.25 1.0069 1.0067 0.0000 1.0100
|
| 359 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__mat natural 1.0625 1.7842 1.0100 0.0000 1.0101
|
| 360 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__mat natural 1.25 1.7752 1.0100 0.0000 1.0101
|
| 361 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__mat forced 1.25 1.0047 1.0047 0.0000 1.0100
|
| 362 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__mat natural 1.095703125 1.5612 1.0100 0.0000 1.0429
|
| 363 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__mat natural 1.25 1.5504 1.0100 0.0000 1.0429
|
| 364 |
+
ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__mat forced 1.25 1.0033 1.0033 0.0000 1.0071
|
| 365 |
+
|
| 366 |
+
* At least one globally aggregated snapshot reached the E/K expert-Imax ceiling.
|
| 367 |
+
Inspect raw_expert_imax_mean and hottest_expert_token_fraction_mean before generalising.
|
| 368 |
+
|
| 369 |
+
ECHO note: echo_logged_pre_rank_metric_valid=0 and echo_logged_load_reduction_valid=0 are intentional.
|
| 370 |
+
The branch's built-in fields mix source-rank and destination-rank axes; this tool reconstructs both globally.
|
| 371 |
+
no memory-dump runs found under /workspace/tiera-rerun-fixed-moonep
|
| 372 |
+
Already logged in to Hugging Face as: AbhishekGhosh1998
|