granularity results granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4 @ 2026-09-21T07:04:07Z from sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0
Browse files- granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/.completed/moonep__tierb__gbs8 +1 -0
- granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/STATUS.txt +3 -2
- granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/collected_all.tsv +18 -0
- granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/hf_watcher.log +2 -0
- granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/probe_summary_all.json +723 -0
- granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/probe_summary_all.tsv +9 -0
- granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/steps.tsv +1 -0
- granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/study.log +62 -0
granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/.completed/moonep__tierb__gbs8
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
2026-09-21T07:04:03+00:00
|
granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/STATUS.txt
CHANGED
|
@@ -1,11 +1,12 @@
|
|
| 1 |
study_root: /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4
|
| 2 |
gbs: 8
|
| 3 |
-
updated: 2026-09-21T07:
|
| 4 |
-
current:
|
| 5 |
|
| 6 |
completed steps:
|
| 7 |
echo__tierb__gbs8
|
| 8 |
echo__tierbprobe__gbs8
|
|
|
|
| 9 |
moonep__tierbprobe__gbs8
|
| 10 |
ultraep__tierb__gbs8
|
| 11 |
ultraep__tierbprobe__gbs8
|
|
|
|
| 1 |
study_root: /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4
|
| 2 |
gbs: 8
|
| 3 |
+
updated: 2026-09-21T07:04:06+00:00
|
| 4 |
+
current: idle
|
| 5 |
|
| 6 |
completed steps:
|
| 7 |
echo__tierb__gbs8
|
| 8 |
echo__tierbprobe__gbs8
|
| 9 |
+
moonep__tierb__gbs8
|
| 10 |
moonep__tierbprobe__gbs8
|
| 11 |
ultraep__tierb__gbs8
|
| 12 |
ultraep__tierbprobe__gbs8
|
granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/collected_all.tsv
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
run_dir rep phase is_trace async config backend slot_policy slots_per_rank num_experts topk moe_ffn_hidden experts_per_rank expert_params num_layers seq_length global_batch_size num_microbatches assignments_per_rank mean_tokens_per_expert capacity capacity_mode recompute force_balance cuda_graph memory_dump_mode memory_snapshot_path ultraep_quota_min_tokens shared_expert_overlap branch commit dense_first_layers moe_layers router_balancing aux_loss_coeff router_pre_softmax router_fusion rotary_base qkv_bias source_model standard_num_layers base_rung q_lora_rank kv_lora_rank qk_head_dim qk_pos_emb_head_dim v_head_dim rotary_percent rotary_scaling mscale mscale_all_dim router_groups router_group_topk router_score_function router_expert_bias router_bias_update_rate router_topk_scaling_factor shared_expert_gate apply_layernorm_1p attention_output_gate linear_attention_type linear_attention_freq linear_conv_kernel_dim linear_key_head_dim linear_value_head_dim linear_num_key_heads linear_num_value_heads mtp_num_layers mtp_loss_scaling_factor no_weight_decay_cond_type max_position_embeddings init_method_std status iters_seen iter_ms_mean iter_ms_min iter_ms_stdev tflops peak_mib_sampled megatron_max_reserved_mib drop_warning
|
| 2 |
+
echo/tierb/tierb_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap1p13671875/rep1 rep1 tierb 0 0 DEEPSEEK_V3_L4 echo matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 1.13671875 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 198.47 197.00 1.73 375.2 131075 84018
|
| 3 |
+
echo/tierb/tierb_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap1p13671875/rep2 rep2 tierb 0 0 DEEPSEEK_V3_L4 echo matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 1.13671875 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 197.13 195.60 0.89 377.7 131075 84018
|
| 4 |
+
echo/tierb/tierb_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap1p13671875/rep3 rep3 tierb 0 0 DEEPSEEK_V3_L4 echo matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 1.13671875 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 197.41 196.00 1.70 377.2 131075 84018
|
| 5 |
+
echo/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap1p13671875/rep1 rep1 tierbprobe 0 0 DEEPSEEK_V3_L4 echo matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 1.13671875 static none 0 0 none 0 token-drop-analysis b099fd84 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 222.87 222.20 0.76 334.1 121233 84058
|
| 6 |
+
echo/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap1p13671875__forcebal/rep1 rep1 tierbprobe 0 0 DEEPSEEK_V3_L4 echo matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 1.13671875 static none 1 0 none 0 token-drop-analysis b099fd84 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 223.39 221.10 3.79 333.4 121233 84058
|
| 7 |
+
echo/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap2p00/rep1 rep1 tierbprobe 0 0 DEEPSEEK_V3_L4 echo matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 2.00 static none 0 0 none 0 token-drop-analysis b099fd84 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 224.39 221.70 4.00 332.0 123091 85514
|
| 8 |
+
moonep/tierb/tierb_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__async/rep1 rep1 tierb 0 1 DEEPSEEK_V3_L4 moonep matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 192.67 192.30 0.42 386.5 97849 60652
|
| 9 |
+
moonep/tierb/tierb_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__async/rep2 rep2 tierb 0 1 DEEPSEEK_V3_L4 moonep matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 193.27 192.10 0.96 385.3 97849 60652
|
| 10 |
+
moonep/tierb/tierb_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__async/rep3 rep3 tierb 0 1 DEEPSEEK_V3_L4 moonep matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 192.76 192.30 0.40 386.3 97849 60652
|
| 11 |
+
moonep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__async/rep1 rep1 tierbprobe 0 1 DEEPSEEK_V3_L4 moonep matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 none none none 0 0 none 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 209.43 208.50 0.95 355.6 100169 60636
|
| 12 |
+
moonep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1 rep1 tierbprobe 0 1 DEEPSEEK_V3_L4 moonep matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 none none none 1 0 none 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 207.81 207.30 0.38 358.3 100169 60636
|
| 13 |
+
ultraep/tierb/tierb_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p13671875/rep1 rep1 tierb 0 0 DEEPSEEK_V3_L4 ultraep matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 1.13671875 static none 0 1 none 128 0 ultraep-bracing-regions-through-kernels 03d57ef7 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 195.76 194.20 1.94 380.4 101995 68070
|
| 14 |
+
ultraep/tierb/tierb_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p13671875/rep2 rep2 tierb 0 0 DEEPSEEK_V3_L4 ultraep matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 1.13671875 static none 0 1 none 128 0 ultraep-bracing-regions-through-kernels 03d57ef7 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 195.61 194.00 1.84 380.7 101995 68070
|
| 15 |
+
ultraep/tierb/tierb_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p13671875/rep3 rep3 tierb 0 0 DEEPSEEK_V3_L4 ultraep matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 1.13671875 static none 0 1 none 128 0 ultraep-bracing-regions-through-kernels 03d57ef7 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 195.60 194.10 1.82 380.7 101995 68070
|
| 16 |
+
ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p0625/rep1 rep1 tierbprobe 0 0 DEEPSEEK_V3_L4 ultraep matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 1.0625 static none 0 0 none 128 0 ultraep-token-drop-analysis eaf829e7 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 214.57 209.60 5.53 347.2 100665 68054 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=1 total_expected_recv=32094 total_dropped_token
|
| 17 |
+
ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p25/rep1 rep1 tierbprobe 0 0 DEEPSEEK_V3_L4 ultraep matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 1.25 static none 0 0 none 128 0 ultraep-token-drop-analysis eaf829e7 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 211.10 209.30 1.96 352.8 102365 68054 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=1 total_expected_recv=33097 total_dropped_token
|
| 18 |
+
ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1 rep1 tierbprobe 0 0 DEEPSEEK_V3_L4 ultraep matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 1.25 static none 1 0 none 128 0 ultraep-token-drop-analysis eaf829e7 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 204.41 202.60 1.50 364.3 102365 68054 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=1 total_expected_recv=33019 total_dropped_token
|
granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/hf_watcher.log
CHANGED
|
@@ -7,3 +7,5 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
|
|
| 7 |
uploaded: /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4
|
| 8 |
https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
|
| 9 |
[2026-09-21T07:02:25+00:00] backup pass
|
|
|
|
|
|
|
|
|
| 7 |
uploaded: /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4
|
| 8 |
https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
|
| 9 |
[2026-09-21T07:02:25+00:00] backup pass
|
| 10 |
+
uploaded: /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4
|
| 11 |
+
https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
|
granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/probe_summary_all.json
ADDED
|
@@ -0,0 +1,723 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"arm": "natural",
|
| 4 |
+
"assignment_sum_mismatches": 0,
|
| 5 |
+
"aux_loss_coeff": "1e-4",
|
| 6 |
+
"backend": "echo",
|
| 7 |
+
"branch": "token-drop-analysis",
|
| 8 |
+
"capacity": "1.13671875",
|
| 9 |
+
"capacity_mode": "static",
|
| 10 |
+
"cold_expert_fraction_max": 0.0,
|
| 11 |
+
"cold_expert_fraction_mean": 0.0,
|
| 12 |
+
"cold_expert_fraction_min": 0.0,
|
| 13 |
+
"cold_expert_fraction_n": 12,
|
| 14 |
+
"commit": "b099fd84",
|
| 15 |
+
"config": "DEEPSEEK_V3_L4",
|
| 16 |
+
"cuda_graph": "0",
|
| 17 |
+
"dense_first_layers": "3",
|
| 18 |
+
"drop_accounting_source": "dispatch",
|
| 19 |
+
"drop_expected_assignments": 3145728.0,
|
| 20 |
+
"drop_rate_pct_overall": 0.0,
|
| 21 |
+
"echo_logged_load_reduction_valid": 0,
|
| 22 |
+
"echo_logged_pre_rank_metric_valid": 0,
|
| 23 |
+
"echo_moved_assignment_fraction_max": 0.5,
|
| 24 |
+
"echo_moved_assignment_fraction_mean": 0.5,
|
| 25 |
+
"echo_moved_assignment_fraction_min": 0.5,
|
| 26 |
+
"echo_moved_assignment_fraction_n": 12,
|
| 27 |
+
"experts_per_rank": "32",
|
| 28 |
+
"force_balance": "0",
|
| 29 |
+
"global_batch_size": "8",
|
| 30 |
+
"global_snapshots": 12,
|
| 31 |
+
"hottest_expert_token_fraction_max": 0.279449462890625,
|
| 32 |
+
"hottest_expert_token_fraction_mean": 0.16472371419270834,
|
| 33 |
+
"hottest_expert_token_fraction_min": 0.09417724609375,
|
| 34 |
+
"hottest_expert_token_fraction_n": 12,
|
| 35 |
+
"incomplete_snapshots": 0,
|
| 36 |
+
"malformed_records": 0,
|
| 37 |
+
"moe_layers": "1",
|
| 38 |
+
"num_experts": "256",
|
| 39 |
+
"num_layers": "4",
|
| 40 |
+
"observed_dropped_tokens": 0,
|
| 41 |
+
"post_physical_rank_cv_max": 0.0,
|
| 42 |
+
"post_physical_rank_cv_mean": 0.0,
|
| 43 |
+
"post_physical_rank_cv_min": 0.0,
|
| 44 |
+
"post_physical_rank_cv_n": 12,
|
| 45 |
+
"post_physical_rank_imax_max": 1.0,
|
| 46 |
+
"post_physical_rank_imax_mean": 1.0,
|
| 47 |
+
"post_physical_rank_imax_min": 1.0,
|
| 48 |
+
"post_physical_rank_imax_n": 12,
|
| 49 |
+
"pre_home_rank_cv_max": 0.3949158995862076,
|
| 50 |
+
"pre_home_rank_cv_mean": 0.18817834176075468,
|
| 51 |
+
"pre_home_rank_cv_min": 0.07608345138535468,
|
| 52 |
+
"pre_home_rank_cv_n": 12,
|
| 53 |
+
"pre_home_rank_imax_max": 1.563079833984375,
|
| 54 |
+
"pre_home_rank_imax_mean": 1.291656494140625,
|
| 55 |
+
"pre_home_rank_imax_min": 1.098876953125,
|
| 56 |
+
"pre_home_rank_imax_n": 12,
|
| 57 |
+
"raw_expert_cv_max": 1.4608217820834073,
|
| 58 |
+
"raw_expert_cv_mean": 0.8516673799234088,
|
| 59 |
+
"raw_expert_cv_min": 0.5323456097372459,
|
| 60 |
+
"raw_expert_cv_n": 12,
|
| 61 |
+
"raw_expert_imax_ceiling": 32.0,
|
| 62 |
+
"raw_expert_imax_hit_ceiling": 0,
|
| 63 |
+
"raw_expert_imax_max": 8.9423828125,
|
| 64 |
+
"raw_expert_imax_mean": 5.271158854166667,
|
| 65 |
+
"raw_expert_imax_min": 3.013671875,
|
| 66 |
+
"raw_expert_imax_n": 12,
|
| 67 |
+
"records": {
|
| 68 |
+
"drop_dispatch": 96,
|
| 69 |
+
"drop_experts": 96,
|
| 70 |
+
"drop_setup": 96,
|
| 71 |
+
"echo_post": 96,
|
| 72 |
+
"echo_pre": 96
|
| 73 |
+
},
|
| 74 |
+
"required_capacity_max": 1.08203125,
|
| 75 |
+
"required_capacity_mean": 1.0650227864583333,
|
| 76 |
+
"required_capacity_min": 1.046875,
|
| 77 |
+
"required_capacity_n": 96,
|
| 78 |
+
"router_balancing": "seq_aux_loss",
|
| 79 |
+
"run_dir": "echo/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap1p13671875/rep1",
|
| 80 |
+
"saturated_dispatch_rows": 0,
|
| 81 |
+
"seq_length": "4096",
|
| 82 |
+
"shared_expert_overlap": "0",
|
| 83 |
+
"topk": "8",
|
| 84 |
+
"valid_max_rank_reduction_max": 0.3602374118979285,
|
| 85 |
+
"valid_max_rank_reduction_mean": 0.21588444472672905,
|
| 86 |
+
"valid_max_rank_reduction_min": 0.08998000444345701,
|
| 87 |
+
"valid_max_rank_reduction_n": 12,
|
| 88 |
+
"zero_drop": 1
|
| 89 |
+
},
|
| 90 |
+
{
|
| 91 |
+
"arm": "forced",
|
| 92 |
+
"assignment_sum_mismatches": 0,
|
| 93 |
+
"aux_loss_coeff": "1e-4",
|
| 94 |
+
"backend": "echo",
|
| 95 |
+
"branch": "token-drop-analysis",
|
| 96 |
+
"capacity": "1.13671875",
|
| 97 |
+
"capacity_mode": "static",
|
| 98 |
+
"cold_expert_fraction_max": 0.0,
|
| 99 |
+
"cold_expert_fraction_mean": 0.0,
|
| 100 |
+
"cold_expert_fraction_min": 0.0,
|
| 101 |
+
"cold_expert_fraction_n": 12,
|
| 102 |
+
"commit": "b099fd84",
|
| 103 |
+
"config": "DEEPSEEK_V3_L4",
|
| 104 |
+
"cuda_graph": "0",
|
| 105 |
+
"dense_first_layers": "3",
|
| 106 |
+
"drop_accounting_source": "dispatch",
|
| 107 |
+
"drop_expected_assignments": 3145728.0,
|
| 108 |
+
"drop_rate_pct_overall": 0.0,
|
| 109 |
+
"echo_logged_load_reduction_valid": 0,
|
| 110 |
+
"echo_logged_pre_rank_metric_valid": 0,
|
| 111 |
+
"echo_moved_assignment_fraction_max": 0.5,
|
| 112 |
+
"echo_moved_assignment_fraction_mean": 0.5,
|
| 113 |
+
"echo_moved_assignment_fraction_min": 0.5,
|
| 114 |
+
"echo_moved_assignment_fraction_n": 12,
|
| 115 |
+
"experts_per_rank": "32",
|
| 116 |
+
"force_balance": "1",
|
| 117 |
+
"global_batch_size": "8",
|
| 118 |
+
"global_snapshots": 12,
|
| 119 |
+
"hottest_expert_token_fraction_max": 0.03521728515625,
|
| 120 |
+
"hottest_expert_token_fraction_mean": 0.034426371256510414,
|
| 121 |
+
"hottest_expert_token_fraction_min": 0.033843994140625,
|
| 122 |
+
"hottest_expert_token_fraction_n": 12,
|
| 123 |
+
"incomplete_snapshots": 0,
|
| 124 |
+
"malformed_records": 0,
|
| 125 |
+
"moe_layers": "1",
|
| 126 |
+
"num_experts": "256",
|
| 127 |
+
"num_layers": "4",
|
| 128 |
+
"observed_dropped_tokens": 0,
|
| 129 |
+
"post_physical_rank_cv_max": 0.0,
|
| 130 |
+
"post_physical_rank_cv_mean": 0.0,
|
| 131 |
+
"post_physical_rank_cv_min": 0.0,
|
| 132 |
+
"post_physical_rank_cv_n": 12,
|
| 133 |
+
"post_physical_rank_imax_max": 1.0,
|
| 134 |
+
"post_physical_rank_imax_mean": 1.0,
|
| 135 |
+
"post_physical_rank_imax_min": 1.0,
|
| 136 |
+
"post_physical_rank_imax_n": 12,
|
| 137 |
+
"pre_home_rank_cv_max": 0.009139900015747392,
|
| 138 |
+
"pre_home_rank_cv_mean": 0.006980946768958179,
|
| 139 |
+
"pre_home_rank_cv_min": 0.0030538170542637444,
|
| 140 |
+
"pre_home_rank_cv_n": 12,
|
| 141 |
+
"pre_home_rank_imax_max": 1.014892578125,
|
| 142 |
+
"pre_home_rank_imax_mean": 1.0104904174804688,
|
| 143 |
+
"pre_home_rank_imax_min": 1.00408935546875,
|
| 144 |
+
"pre_home_rank_imax_n": 12,
|
| 145 |
+
"raw_expert_cv_max": 0.03876371556245609,
|
| 146 |
+
"raw_expert_cv_mean": 0.03570960046443938,
|
| 147 |
+
"raw_expert_cv_min": 0.029816782685728113,
|
| 148 |
+
"raw_expert_cv_n": 12,
|
| 149 |
+
"raw_expert_imax_ceiling": 32.0,
|
| 150 |
+
"raw_expert_imax_hit_ceiling": 0,
|
| 151 |
+
"raw_expert_imax_max": 1.126953125,
|
| 152 |
+
"raw_expert_imax_mean": 1.1016438802083333,
|
| 153 |
+
"raw_expert_imax_min": 1.0830078125,
|
| 154 |
+
"raw_expert_imax_n": 12,
|
| 155 |
+
"records": {
|
| 156 |
+
"drop_dispatch": 96,
|
| 157 |
+
"drop_experts": 96,
|
| 158 |
+
"drop_setup": 96,
|
| 159 |
+
"echo_post": 96,
|
| 160 |
+
"echo_pre": 96
|
| 161 |
+
},
|
| 162 |
+
"required_capacity_max": 1.08203125,
|
| 163 |
+
"required_capacity_mean": 1.0631510416666667,
|
| 164 |
+
"required_capacity_min": 1.04296875,
|
| 165 |
+
"required_capacity_n": 96,
|
| 166 |
+
"router_balancing": "seq_aux_loss",
|
| 167 |
+
"run_dir": "echo/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap1p13671875__forcebal/rep1",
|
| 168 |
+
"saturated_dispatch_rows": 0,
|
| 169 |
+
"seq_length": "4096",
|
| 170 |
+
"shared_expert_overlap": "0",
|
| 171 |
+
"topk": "8",
|
| 172 |
+
"valid_max_rank_reduction_max": 0.01467404378157325,
|
| 173 |
+
"valid_max_rank_reduction_mean": 0.01036908147130012,
|
| 174 |
+
"valid_max_rank_reduction_min": 0.004072700747674914,
|
| 175 |
+
"valid_max_rank_reduction_n": 12,
|
| 176 |
+
"zero_drop": 1
|
| 177 |
+
},
|
| 178 |
+
{
|
| 179 |
+
"arm": "natural",
|
| 180 |
+
"assignment_sum_mismatches": 0,
|
| 181 |
+
"aux_loss_coeff": "1e-4",
|
| 182 |
+
"backend": "echo",
|
| 183 |
+
"branch": "token-drop-analysis",
|
| 184 |
+
"capacity": "2.00",
|
| 185 |
+
"capacity_mode": "static",
|
| 186 |
+
"cold_expert_fraction_max": 0.0,
|
| 187 |
+
"cold_expert_fraction_mean": 0.0,
|
| 188 |
+
"cold_expert_fraction_min": 0.0,
|
| 189 |
+
"cold_expert_fraction_n": 12,
|
| 190 |
+
"commit": "b099fd84",
|
| 191 |
+
"config": "DEEPSEEK_V3_L4",
|
| 192 |
+
"cuda_graph": "0",
|
| 193 |
+
"dense_first_layers": "3",
|
| 194 |
+
"drop_accounting_source": "dispatch",
|
| 195 |
+
"drop_expected_assignments": 3145728.0,
|
| 196 |
+
"drop_rate_pct_overall": 0.0,
|
| 197 |
+
"echo_logged_load_reduction_valid": 0,
|
| 198 |
+
"echo_logged_pre_rank_metric_valid": 0,
|
| 199 |
+
"echo_moved_assignment_fraction_max": 0.5,
|
| 200 |
+
"echo_moved_assignment_fraction_mean": 0.5,
|
| 201 |
+
"echo_moved_assignment_fraction_min": 0.5,
|
| 202 |
+
"echo_moved_assignment_fraction_n": 12,
|
| 203 |
+
"experts_per_rank": "32",
|
| 204 |
+
"force_balance": "0",
|
| 205 |
+
"global_batch_size": "8",
|
| 206 |
+
"global_snapshots": 12,
|
| 207 |
+
"hottest_expert_token_fraction_max": 0.279449462890625,
|
| 208 |
+
"hottest_expert_token_fraction_mean": 0.16472371419270834,
|
| 209 |
+
"hottest_expert_token_fraction_min": 0.09417724609375,
|
| 210 |
+
"hottest_expert_token_fraction_n": 12,
|
| 211 |
+
"incomplete_snapshots": 0,
|
| 212 |
+
"malformed_records": 0,
|
| 213 |
+
"moe_layers": "1",
|
| 214 |
+
"num_experts": "256",
|
| 215 |
+
"num_layers": "4",
|
| 216 |
+
"observed_dropped_tokens": 0,
|
| 217 |
+
"post_physical_rank_cv_max": 0.0,
|
| 218 |
+
"post_physical_rank_cv_mean": 0.0,
|
| 219 |
+
"post_physical_rank_cv_min": 0.0,
|
| 220 |
+
"post_physical_rank_cv_n": 12,
|
| 221 |
+
"post_physical_rank_imax_max": 1.0,
|
| 222 |
+
"post_physical_rank_imax_mean": 1.0,
|
| 223 |
+
"post_physical_rank_imax_min": 1.0,
|
| 224 |
+
"post_physical_rank_imax_n": 12,
|
| 225 |
+
"pre_home_rank_cv_max": 0.3949158995862076,
|
| 226 |
+
"pre_home_rank_cv_mean": 0.18817834176075468,
|
| 227 |
+
"pre_home_rank_cv_min": 0.07608345138535468,
|
| 228 |
+
"pre_home_rank_cv_n": 12,
|
| 229 |
+
"pre_home_rank_imax_max": 1.563079833984375,
|
| 230 |
+
"pre_home_rank_imax_mean": 1.291656494140625,
|
| 231 |
+
"pre_home_rank_imax_min": 1.098876953125,
|
| 232 |
+
"pre_home_rank_imax_n": 12,
|
| 233 |
+
"raw_expert_cv_max": 1.4608217820834073,
|
| 234 |
+
"raw_expert_cv_mean": 0.8516673799234088,
|
| 235 |
+
"raw_expert_cv_min": 0.5323456097372459,
|
| 236 |
+
"raw_expert_cv_n": 12,
|
| 237 |
+
"raw_expert_imax_ceiling": 32.0,
|
| 238 |
+
"raw_expert_imax_hit_ceiling": 0,
|
| 239 |
+
"raw_expert_imax_max": 8.9423828125,
|
| 240 |
+
"raw_expert_imax_mean": 5.271158854166667,
|
| 241 |
+
"raw_expert_imax_min": 3.013671875,
|
| 242 |
+
"raw_expert_imax_n": 12,
|
| 243 |
+
"records": {
|
| 244 |
+
"drop_dispatch": 96,
|
| 245 |
+
"drop_experts": 96,
|
| 246 |
+
"drop_setup": 96,
|
| 247 |
+
"echo_post": 96,
|
| 248 |
+
"echo_pre": 96
|
| 249 |
+
},
|
| 250 |
+
"required_capacity_max": 1.08203125,
|
| 251 |
+
"required_capacity_mean": 1.0650227864583333,
|
| 252 |
+
"required_capacity_min": 1.046875,
|
| 253 |
+
"required_capacity_n": 96,
|
| 254 |
+
"router_balancing": "seq_aux_loss",
|
| 255 |
+
"run_dir": "echo/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap2p00/rep1",
|
| 256 |
+
"saturated_dispatch_rows": 0,
|
| 257 |
+
"seq_length": "4096",
|
| 258 |
+
"shared_expert_overlap": "0",
|
| 259 |
+
"topk": "8",
|
| 260 |
+
"valid_max_rank_reduction_max": 0.3602374118979285,
|
| 261 |
+
"valid_max_rank_reduction_mean": 0.21588444472672905,
|
| 262 |
+
"valid_max_rank_reduction_min": 0.08998000444345701,
|
| 263 |
+
"valid_max_rank_reduction_n": 12,
|
| 264 |
+
"zero_drop": 1
|
| 265 |
+
},
|
| 266 |
+
{
|
| 267 |
+
"arm": "natural",
|
| 268 |
+
"aux_loss_coeff": "1e-4",
|
| 269 |
+
"backend": "moonep",
|
| 270 |
+
"branch": "moonep_backend_async_grad_reduce_token_routing_analysis",
|
| 271 |
+
"capacity": "none",
|
| 272 |
+
"capacity_mode": "none",
|
| 273 |
+
"cold_expert_fraction_max": 0.0,
|
| 274 |
+
"cold_expert_fraction_mean": 0.0,
|
| 275 |
+
"cold_expert_fraction_min": 0.0,
|
| 276 |
+
"cold_expert_fraction_n": 10,
|
| 277 |
+
"commit": "a6a1a221",
|
| 278 |
+
"config": "DEEPSEEK_V3_L4",
|
| 279 |
+
"cuda_graph": "0",
|
| 280 |
+
"dense_first_layers": "3",
|
| 281 |
+
"experts_per_rank": "32",
|
| 282 |
+
"force_balance": "0",
|
| 283 |
+
"global_batch_size": "8",
|
| 284 |
+
"global_snapshots": 10,
|
| 285 |
+
"hottest_expert_token_fraction_max": 0.277313232421875,
|
| 286 |
+
"hottest_expert_token_fraction_mean": 0.1758575439453125,
|
| 287 |
+
"hottest_expert_token_fraction_min": 0.092315673828125,
|
| 288 |
+
"hottest_expert_token_fraction_n": 10,
|
| 289 |
+
"incomplete_snapshots": 0,
|
| 290 |
+
"malformed_records": 0,
|
| 291 |
+
"moe_layers": "1",
|
| 292 |
+
"moonep_bytes_global_snapshots": 10,
|
| 293 |
+
"moonep_bytes_incomplete_snapshots": 0,
|
| 294 |
+
"moonep_bytes_malformed_records": 0,
|
| 295 |
+
"moonep_global_combine_payload_bytes_max": 3822120960.0,
|
| 296 |
+
"moonep_global_combine_payload_bytes_mean": 3501221068.8,
|
| 297 |
+
"moonep_global_combine_payload_bytes_min": 3346653184.0,
|
| 298 |
+
"moonep_global_combine_payload_bytes_n": 10,
|
| 299 |
+
"moonep_global_dispatch_payload_bytes_max": 3822120960.0,
|
| 300 |
+
"moonep_global_dispatch_payload_bytes_mean": 3501221068.8,
|
| 301 |
+
"moonep_global_dispatch_payload_bytes_min": 3346653184.0,
|
| 302 |
+
"moonep_global_dispatch_payload_bytes_n": 10,
|
| 303 |
+
"moonep_global_prefetch_bytes_max": 2466250752.0,
|
| 304 |
+
"moonep_global_prefetch_bytes_mean": 1867304140.8,
|
| 305 |
+
"moonep_global_prefetch_bytes_min": 1409286144.0,
|
| 306 |
+
"moonep_global_prefetch_bytes_n": 10,
|
| 307 |
+
"moonep_global_reduce_bytes_max": 2466250752.0,
|
| 308 |
+
"moonep_global_reduce_bytes_mean": 1867304140.8,
|
| 309 |
+
"moonep_global_reduce_bytes_min": 1409286144.0,
|
| 310 |
+
"moonep_global_reduce_bytes_n": 10,
|
| 311 |
+
"moonep_global_token_bytes_max": 7646072992.0,
|
| 312 |
+
"moonep_global_token_bytes_mean": 7004270806.4,
|
| 313 |
+
"moonep_global_token_bytes_min": 6695137352.0,
|
| 314 |
+
"moonep_global_token_bytes_n": 10,
|
| 315 |
+
"moonep_global_total_nvlink_bytes_max": 12578574496.0,
|
| 316 |
+
"moonep_global_total_nvlink_bytes_mean": 10738879088.0,
|
| 317 |
+
"moonep_global_total_nvlink_bytes_min": 9546567752.0,
|
| 318 |
+
"moonep_global_total_nvlink_bytes_n": 10,
|
| 319 |
+
"moonep_global_weight_movement_bytes_max": 4932501504.0,
|
| 320 |
+
"moonep_global_weight_movement_bytes_mean": 3734608281.6,
|
| 321 |
+
"moonep_global_weight_movement_bytes_min": 2818572288.0,
|
| 322 |
+
"moonep_global_weight_movement_bytes_n": 10,
|
| 323 |
+
"moonep_global_weight_to_token_ratio_max": 0.6762953708910178,
|
| 324 |
+
"moonep_global_weight_to_token_ratio_mean": 0.5305300760379464,
|
| 325 |
+
"moonep_global_weight_to_token_ratio_min": 0.4096318958168878,
|
| 326 |
+
"moonep_global_weight_to_token_ratio_n": 10,
|
| 327 |
+
"moonep_remote_demand_max": 4.0,
|
| 328 |
+
"moonep_remote_demand_mean": 1.325,
|
| 329 |
+
"moonep_remote_demand_min": 0.0,
|
| 330 |
+
"moonep_remote_demand_n": 80,
|
| 331 |
+
"moonep_slot_utilization_max": 0.125,
|
| 332 |
+
"moonep_slot_utilization_mean": 0.04140625,
|
| 333 |
+
"moonep_slot_utilization_min": 0.0,
|
| 334 |
+
"moonep_slot_utilization_n": 80,
|
| 335 |
+
"moonep_unused_prefetch_slots_max": 64.0,
|
| 336 |
+
"moonep_unused_prefetch_slots_mean": 61.35,
|
| 337 |
+
"moonep_unused_prefetch_slots_min": 56.0,
|
| 338 |
+
"moonep_unused_prefetch_slots_n": 80,
|
| 339 |
+
"num_experts": "256",
|
| 340 |
+
"num_layers": "4",
|
| 341 |
+
"post_physical_rank_cv_max": 0.054662301330415754,
|
| 342 |
+
"post_physical_rank_cv_mean": 0.03173931531099595,
|
| 343 |
+
"post_physical_rank_cv_min": 0.019783416729893442,
|
| 344 |
+
"post_physical_rank_cv_n": 10,
|
| 345 |
+
"post_physical_rank_imax_max": 1.0904317633276406,
|
| 346 |
+
"post_physical_rank_imax_mean": 1.051604154450151,
|
| 347 |
+
"post_physical_rank_imax_min": 1.0214255000518957,
|
| 348 |
+
"post_physical_rank_imax_n": 10,
|
| 349 |
+
"pre_home_rank_cv_max": 0.39536184425362714,
|
| 350 |
+
"pre_home_rank_cv_mean": 0.19281063246441604,
|
| 351 |
+
"pre_home_rank_cv_min": 0.08594336137506545,
|
| 352 |
+
"pre_home_rank_cv_n": 10,
|
| 353 |
+
"pre_home_rank_imax_max": 1.55706787109375,
|
| 354 |
+
"pre_home_rank_imax_mean": 1.3085418701171876,
|
| 355 |
+
"pre_home_rank_imax_min": 1.113128662109375,
|
| 356 |
+
"pre_home_rank_imax_n": 10,
|
| 357 |
+
"raw_expert_cv_max": 1.4500281317834722,
|
| 358 |
+
"raw_expert_cv_mean": 0.9091278053879644,
|
| 359 |
+
"raw_expert_cv_min": 0.5606721744656151,
|
| 360 |
+
"raw_expert_cv_n": 10,
|
| 361 |
+
"raw_expert_imax_ceiling": 32.0,
|
| 362 |
+
"raw_expert_imax_hit_ceiling": 0,
|
| 363 |
+
"raw_expert_imax_max": 8.8740234375,
|
| 364 |
+
"raw_expert_imax_mean": 5.62744140625,
|
| 365 |
+
"raw_expert_imax_min": 2.9541015625,
|
| 366 |
+
"raw_expert_imax_n": 10,
|
| 367 |
+
"records": {
|
| 368 |
+
"moonep_bytes": 80,
|
| 369 |
+
"moonep_routing": 80,
|
| 370 |
+
"moonep_summary": 80,
|
| 371 |
+
"moonep_weights": 80
|
| 372 |
+
},
|
| 373 |
+
"router_balancing": "seq_aux_loss",
|
| 374 |
+
"run_dir": "moonep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__async/rep1",
|
| 375 |
+
"seq_length": "4096",
|
| 376 |
+
"shared_expert_overlap": "0",
|
| 377 |
+
"topk": "8"
|
| 378 |
+
},
|
| 379 |
+
{
|
| 380 |
+
"arm": "forced",
|
| 381 |
+
"aux_loss_coeff": "1e-4",
|
| 382 |
+
"backend": "moonep",
|
| 383 |
+
"branch": "moonep_backend_async_grad_reduce_token_routing_analysis",
|
| 384 |
+
"capacity": "none",
|
| 385 |
+
"capacity_mode": "none",
|
| 386 |
+
"cold_expert_fraction_max": 0.0,
|
| 387 |
+
"cold_expert_fraction_mean": 0.0,
|
| 388 |
+
"cold_expert_fraction_min": 0.0,
|
| 389 |
+
"cold_expert_fraction_n": 10,
|
| 390 |
+
"commit": "a6a1a221",
|
| 391 |
+
"config": "DEEPSEEK_V3_L4",
|
| 392 |
+
"cuda_graph": "0",
|
| 393 |
+
"dense_first_layers": "3",
|
| 394 |
+
"experts_per_rank": "32",
|
| 395 |
+
"force_balance": "1",
|
| 396 |
+
"global_batch_size": "8",
|
| 397 |
+
"global_snapshots": 10,
|
| 398 |
+
"hottest_expert_token_fraction_max": 0.03497314453125,
|
| 399 |
+
"hottest_expert_token_fraction_mean": 0.034405517578125,
|
| 400 |
+
"hottest_expert_token_fraction_min": 0.03399658203125,
|
| 401 |
+
"hottest_expert_token_fraction_n": 10,
|
| 402 |
+
"incomplete_snapshots": 0,
|
| 403 |
+
"malformed_records": 0,
|
| 404 |
+
"moe_layers": "1",
|
| 405 |
+
"moonep_bytes_global_snapshots": 10,
|
| 406 |
+
"moonep_bytes_incomplete_snapshots": 0,
|
| 407 |
+
"moonep_bytes_malformed_records": 0,
|
| 408 |
+
"moonep_global_combine_payload_bytes_max": 3276406784.0,
|
| 409 |
+
"moonep_global_combine_payload_bytes_mean": 3268117708.8,
|
| 410 |
+
"moonep_global_combine_payload_bytes_min": 3260407808.0,
|
| 411 |
+
"moonep_global_combine_payload_bytes_n": 10,
|
| 412 |
+
"moonep_global_dispatch_payload_bytes_max": 3276406784.0,
|
| 413 |
+
"moonep_global_dispatch_payload_bytes_mean": 3268117708.8,
|
| 414 |
+
"moonep_global_dispatch_payload_bytes_min": 3260407808.0,
|
| 415 |
+
"moonep_global_dispatch_payload_bytes_n": 10,
|
| 416 |
+
"moonep_global_prefetch_bytes_max": 1233125376.0,
|
| 417 |
+
"moonep_global_prefetch_bytes_mean": 1233125376.0,
|
| 418 |
+
"moonep_global_prefetch_bytes_min": 1233125376.0,
|
| 419 |
+
"moonep_global_prefetch_bytes_n": 10,
|
| 420 |
+
"moonep_global_reduce_bytes_max": 1233125376.0,
|
| 421 |
+
"moonep_global_reduce_bytes_mean": 1233125376.0,
|
| 422 |
+
"moonep_global_reduce_bytes_min": 1233125376.0,
|
| 423 |
+
"moonep_global_reduce_bytes_n": 10,
|
| 424 |
+
"moonep_global_token_bytes_max": 6554650816.0,
|
| 425 |
+
"moonep_global_token_bytes_mean": 6538068851.2,
|
| 426 |
+
"moonep_global_token_bytes_min": 6522648928.0,
|
| 427 |
+
"moonep_global_token_bytes_n": 10,
|
| 428 |
+
"moonep_global_total_nvlink_bytes_max": 9020901568.0,
|
| 429 |
+
"moonep_global_total_nvlink_bytes_mean": 9004319603.2,
|
| 430 |
+
"moonep_global_total_nvlink_bytes_min": 8988899680.0,
|
| 431 |
+
"moonep_global_total_nvlink_bytes_n": 10,
|
| 432 |
+
"moonep_global_weight_movement_bytes_max": 2466250752.0,
|
| 433 |
+
"moonep_global_weight_movement_bytes_mean": 2466250752.0,
|
| 434 |
+
"moonep_global_weight_movement_bytes_min": 2466250752.0,
|
| 435 |
+
"moonep_global_weight_movement_bytes_n": 10,
|
| 436 |
+
"moonep_global_weight_to_token_ratio_max": 0.37810570202744476,
|
| 437 |
+
"moonep_global_weight_to_token_ratio_mean": 0.37721470246811023,
|
| 438 |
+
"moonep_global_weight_to_token_ratio_min": 0.3762596698484449,
|
| 439 |
+
"moonep_global_weight_to_token_ratio_n": 10,
|
| 440 |
+
"moonep_remote_demand_max": 1.0,
|
| 441 |
+
"moonep_remote_demand_mean": 0.875,
|
| 442 |
+
"moonep_remote_demand_min": 0.0,
|
| 443 |
+
"moonep_remote_demand_n": 80,
|
| 444 |
+
"moonep_slot_utilization_max": 0.03125,
|
| 445 |
+
"moonep_slot_utilization_mean": 0.02734375,
|
| 446 |
+
"moonep_slot_utilization_min": 0.0,
|
| 447 |
+
"moonep_slot_utilization_n": 80,
|
| 448 |
+
"moonep_unused_prefetch_slots_max": 64.0,
|
| 449 |
+
"moonep_unused_prefetch_slots_mean": 62.25,
|
| 450 |
+
"moonep_unused_prefetch_slots_min": 62.0,
|
| 451 |
+
"moonep_unused_prefetch_slots_n": 80,
|
| 452 |
+
"num_experts": "256",
|
| 453 |
+
"num_layers": "4",
|
| 454 |
+
"post_physical_rank_cv_max": 0.00471813998380015,
|
| 455 |
+
"post_physical_rank_cv_mean": 0.003239235393271,
|
| 456 |
+
"post_physical_rank_cv_min": 0.0017429035970738013,
|
| 457 |
+
"post_physical_rank_cv_n": 10,
|
| 458 |
+
"post_physical_rank_imax_max": 1.0078812597749087,
|
| 459 |
+
"post_physical_rank_imax_mean": 1.0051071175236657,
|
| 460 |
+
"post_physical_rank_imax_min": 1.002130562069864,
|
| 461 |
+
"post_physical_rank_imax_n": 10,
|
| 462 |
+
"pre_home_rank_cv_max": 0.009139900015747392,
|
| 463 |
+
"pre_home_rank_cv_mean": 0.007327007135739279,
|
| 464 |
+
"pre_home_rank_cv_min": 0.0030538170542637444,
|
| 465 |
+
"pre_home_rank_cv_n": 10,
|
| 466 |
+
"pre_home_rank_imax_max": 1.014892578125,
|
| 467 |
+
"pre_home_rank_imax_mean": 1.0111572265625,
|
| 468 |
+
"pre_home_rank_imax_min": 1.00408935546875,
|
| 469 |
+
"pre_home_rank_imax_n": 10,
|
| 470 |
+
"raw_expert_cv_max": 0.03876371556245609,
|
| 471 |
+
"raw_expert_cv_mean": 0.0363651698149183,
|
| 472 |
+
"raw_expert_cv_min": 0.0339868829612938,
|
| 473 |
+
"raw_expert_cv_n": 10,
|
| 474 |
+
"raw_expert_imax_ceiling": 32.0,
|
| 475 |
+
"raw_expert_imax_hit_ceiling": 0,
|
| 476 |
+
"raw_expert_imax_max": 1.119140625,
|
| 477 |
+
"raw_expert_imax_mean": 1.1009765625,
|
| 478 |
+
"raw_expert_imax_min": 1.087890625,
|
| 479 |
+
"raw_expert_imax_n": 10,
|
| 480 |
+
"records": {
|
| 481 |
+
"moonep_bytes": 80,
|
| 482 |
+
"moonep_routing": 80,
|
| 483 |
+
"moonep_summary": 80,
|
| 484 |
+
"moonep_weights": 80
|
| 485 |
+
},
|
| 486 |
+
"router_balancing": "seq_aux_loss",
|
| 487 |
+
"run_dir": "moonep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1",
|
| 488 |
+
"seq_length": "4096",
|
| 489 |
+
"shared_expert_overlap": "0",
|
| 490 |
+
"topk": "8"
|
| 491 |
+
},
|
| 492 |
+
{
|
| 493 |
+
"arm": "natural",
|
| 494 |
+
"assignment_sum_mismatches": 0,
|
| 495 |
+
"aux_loss_coeff": "1e-4",
|
| 496 |
+
"backend": "ultraep",
|
| 497 |
+
"branch": "ultraep-token-drop-analysis",
|
| 498 |
+
"capacity": "1.0625",
|
| 499 |
+
"capacity_mode": "static",
|
| 500 |
+
"commit": "eaf829e7",
|
| 501 |
+
"config": "DEEPSEEK_V3_L4",
|
| 502 |
+
"cuda_graph": "0",
|
| 503 |
+
"dense_first_layers": "3",
|
| 504 |
+
"drop_expected_assignments": 3145728.0,
|
| 505 |
+
"drop_rate_pct_max": 0.0,
|
| 506 |
+
"drop_rate_pct_mean": 0.0,
|
| 507 |
+
"drop_rate_pct_min": 0.0,
|
| 508 |
+
"drop_rate_pct_n": 96,
|
| 509 |
+
"drop_rate_pct_overall": 0.0,
|
| 510 |
+
"dropped_tokens_sum_all_records": 0.0,
|
| 511 |
+
"duplicate_vector_mismatches": 0,
|
| 512 |
+
"experts_per_rank": "32",
|
| 513 |
+
"force_balance": "0",
|
| 514 |
+
"global_batch_size": "8",
|
| 515 |
+
"global_snapshots": 12,
|
| 516 |
+
"incomplete_snapshots": 0,
|
| 517 |
+
"malformed_records": 0,
|
| 518 |
+
"moe_layers": "1",
|
| 519 |
+
"num_experts": "256",
|
| 520 |
+
"num_layers": "4",
|
| 521 |
+
"observed_dropped_tokens": 0.0,
|
| 522 |
+
"peak_demand_max": 33097.0,
|
| 523 |
+
"peak_demand_mean": 32768.0,
|
| 524 |
+
"peak_demand_min": 31546.0,
|
| 525 |
+
"peak_demand_n": 96,
|
| 526 |
+
"peak_rank_overflow_max": 0.0,
|
| 527 |
+
"peak_rank_overflow_mean": 0.0,
|
| 528 |
+
"peak_rank_overflow_min": 0.0,
|
| 529 |
+
"peak_rank_overflow_n": 96,
|
| 530 |
+
"post_physical_rank_cv_max": 0.017713692879015048,
|
| 531 |
+
"post_physical_rank_cv_mean": 0.014273621042617956,
|
| 532 |
+
"post_physical_rank_cv_min": 0.010671796788645344,
|
| 533 |
+
"post_physical_rank_cv_n": 12,
|
| 534 |
+
"post_physical_rank_imax_max": 1.010040283203125,
|
| 535 |
+
"post_physical_rank_imax_mean": 1.010022481282552,
|
| 536 |
+
"post_physical_rank_imax_min": 1.010009765625,
|
| 537 |
+
"post_physical_rank_imax_n": 12,
|
| 538 |
+
"pre_home_rank_cv_max": 0.39595445620995434,
|
| 539 |
+
"pre_home_rank_cv_mean": 0.18964812495168137,
|
| 540 |
+
"pre_home_rank_cv_min": 0.08292629783502173,
|
| 541 |
+
"pre_home_rank_cv_n": 12,
|
| 542 |
+
"pre_home_rank_imax_max": 1.5654296875,
|
| 543 |
+
"pre_home_rank_imax_mean": 1.2918370564778645,
|
| 544 |
+
"pre_home_rank_imax_min": 1.105194091796875,
|
| 545 |
+
"pre_home_rank_imax_n": 12,
|
| 546 |
+
"records": {
|
| 547 |
+
"drop_dispatch": 96,
|
| 548 |
+
"drop_experts": 96,
|
| 549 |
+
"drop_setup": 96,
|
| 550 |
+
"ultraep_drop": 96,
|
| 551 |
+
"ultraep_post": 96,
|
| 552 |
+
"ultraep_pre": 96
|
| 553 |
+
},
|
| 554 |
+
"required_capacity_max": 1.01004,
|
| 555 |
+
"required_capacity_mean": 1.0000000312500001,
|
| 556 |
+
"required_capacity_min": 0.962708,
|
| 557 |
+
"required_capacity_n": 96,
|
| 558 |
+
"router_balancing": "seq_aux_loss",
|
| 559 |
+
"run_dir": "ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p0625/rep1",
|
| 560 |
+
"seq_length": "4096",
|
| 561 |
+
"shared_expert_overlap": "0",
|
| 562 |
+
"topk": "8",
|
| 563 |
+
"valid_max_rank_reduction_max": 0.3548034934497817,
|
| 564 |
+
"valid_max_rank_reduction_mean": 0.20816004531911061,
|
| 565 |
+
"valid_max_rank_reduction_min": 0.08612453403285932,
|
| 566 |
+
"valid_max_rank_reduction_n": 12,
|
| 567 |
+
"zero_drop": 1
|
| 568 |
+
},
|
| 569 |
+
{
|
| 570 |
+
"arm": "natural",
|
| 571 |
+
"assignment_sum_mismatches": 0,
|
| 572 |
+
"aux_loss_coeff": "1e-4",
|
| 573 |
+
"backend": "ultraep",
|
| 574 |
+
"branch": "ultraep-token-drop-analysis",
|
| 575 |
+
"capacity": "1.25",
|
| 576 |
+
"capacity_mode": "static",
|
| 577 |
+
"commit": "eaf829e7",
|
| 578 |
+
"config": "DEEPSEEK_V3_L4",
|
| 579 |
+
"cuda_graph": "0",
|
| 580 |
+
"dense_first_layers": "3",
|
| 581 |
+
"drop_expected_assignments": 3145728.0,
|
| 582 |
+
"drop_rate_pct_max": 0.0,
|
| 583 |
+
"drop_rate_pct_mean": 0.0,
|
| 584 |
+
"drop_rate_pct_min": 0.0,
|
| 585 |
+
"drop_rate_pct_n": 96,
|
| 586 |
+
"drop_rate_pct_overall": 0.0,
|
| 587 |
+
"dropped_tokens_sum_all_records": 0.0,
|
| 588 |
+
"duplicate_vector_mismatches": 0,
|
| 589 |
+
"experts_per_rank": "32",
|
| 590 |
+
"force_balance": "0",
|
| 591 |
+
"global_batch_size": "8",
|
| 592 |
+
"global_snapshots": 12,
|
| 593 |
+
"incomplete_snapshots": 0,
|
| 594 |
+
"malformed_records": 0,
|
| 595 |
+
"moe_layers": "1",
|
| 596 |
+
"num_experts": "256",
|
| 597 |
+
"num_layers": "4",
|
| 598 |
+
"observed_dropped_tokens": 0.0,
|
| 599 |
+
"peak_demand_max": 33097.0,
|
| 600 |
+
"peak_demand_mean": 32768.0,
|
| 601 |
+
"peak_demand_min": 31280.0,
|
| 602 |
+
"peak_demand_n": 96,
|
| 603 |
+
"peak_rank_overflow_max": 0.0,
|
| 604 |
+
"peak_rank_overflow_mean": 0.0,
|
| 605 |
+
"peak_rank_overflow_min": 0.0,
|
| 606 |
+
"peak_rank_overflow_n": 96,
|
| 607 |
+
"post_physical_rank_cv_max": 0.018971492434756838,
|
| 608 |
+
"post_physical_rank_cv_mean": 0.014175223070035277,
|
| 609 |
+
"post_physical_rank_cv_min": 0.010671796788645344,
|
| 610 |
+
"post_physical_rank_cv_n": 12,
|
| 611 |
+
"post_physical_rank_imax_max": 1.010040283203125,
|
| 612 |
+
"post_physical_rank_imax_mean": 1.0100301106770833,
|
| 613 |
+
"post_physical_rank_imax_min": 1.010009765625,
|
| 614 |
+
"post_physical_rank_imax_n": 12,
|
| 615 |
+
"pre_home_rank_cv_max": 0.3967451453024771,
|
| 616 |
+
"pre_home_rank_cv_mean": 0.18882615919733423,
|
| 617 |
+
"pre_home_rank_cv_min": 0.08482749505976411,
|
| 618 |
+
"pre_home_rank_cv_n": 12,
|
| 619 |
+
"pre_home_rank_imax_max": 1.568817138671875,
|
| 620 |
+
"pre_home_rank_imax_mean": 1.2910486857096355,
|
| 621 |
+
"pre_home_rank_imax_min": 1.114654541015625,
|
| 622 |
+
"pre_home_rank_imax_n": 12,
|
| 623 |
+
"records": {
|
| 624 |
+
"drop_dispatch": 96,
|
| 625 |
+
"drop_experts": 96,
|
| 626 |
+
"drop_setup": 96,
|
| 627 |
+
"ultraep_drop": 96,
|
| 628 |
+
"ultraep_post": 96,
|
| 629 |
+
"ultraep_pre": 96
|
| 630 |
+
},
|
| 631 |
+
"required_capacity_max": 1.01004,
|
| 632 |
+
"required_capacity_mean": 1.0000000312500001,
|
| 633 |
+
"required_capacity_min": 0.95459,
|
| 634 |
+
"required_capacity_n": 96,
|
| 635 |
+
"router_balancing": "seq_aux_loss",
|
| 636 |
+
"run_dir": "ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p25/rep1",
|
| 637 |
+
"seq_length": "4096",
|
| 638 |
+
"shared_expert_overlap": "0",
|
| 639 |
+
"topk": "8",
|
| 640 |
+
"valid_max_rank_reduction_max": 0.35619662691851306,
|
| 641 |
+
"valid_max_rank_reduction_mean": 0.2084693322073339,
|
| 642 |
+
"valid_max_rank_reduction_min": 0.09388090349075975,
|
| 643 |
+
"valid_max_rank_reduction_n": 12,
|
| 644 |
+
"zero_drop": 1
|
| 645 |
+
},
|
| 646 |
+
{
|
| 647 |
+
"arm": "forced",
|
| 648 |
+
"assignment_sum_mismatches": 0,
|
| 649 |
+
"aux_loss_coeff": "1e-4",
|
| 650 |
+
"backend": "ultraep",
|
| 651 |
+
"branch": "ultraep-token-drop-analysis",
|
| 652 |
+
"capacity": "1.25",
|
| 653 |
+
"capacity_mode": "static",
|
| 654 |
+
"commit": "eaf829e7",
|
| 655 |
+
"config": "DEEPSEEK_V3_L4",
|
| 656 |
+
"cuda_graph": "0",
|
| 657 |
+
"dense_first_layers": "3",
|
| 658 |
+
"drop_expected_assignments": 3145728.0,
|
| 659 |
+
"drop_rate_pct_max": 0.0,
|
| 660 |
+
"drop_rate_pct_mean": 0.0,
|
| 661 |
+
"drop_rate_pct_min": 0.0,
|
| 662 |
+
"drop_rate_pct_n": 96,
|
| 663 |
+
"drop_rate_pct_overall": 0.0,
|
| 664 |
+
"dropped_tokens_sum_all_records": 0.0,
|
| 665 |
+
"duplicate_vector_mismatches": 0,
|
| 666 |
+
"experts_per_rank": "32",
|
| 667 |
+
"force_balance": "1",
|
| 668 |
+
"global_batch_size": "8",
|
| 669 |
+
"global_snapshots": 12,
|
| 670 |
+
"incomplete_snapshots": 0,
|
| 671 |
+
"malformed_records": 0,
|
| 672 |
+
"moe_layers": "1",
|
| 673 |
+
"num_experts": "256",
|
| 674 |
+
"num_layers": "4",
|
| 675 |
+
"observed_dropped_tokens": 0.0,
|
| 676 |
+
"peak_demand_max": 33097.0,
|
| 677 |
+
"peak_demand_mean": 32768.0,
|
| 678 |
+
"peak_demand_min": 32330.0,
|
| 679 |
+
"peak_demand_n": 96,
|
| 680 |
+
"peak_rank_overflow_max": 0.0,
|
| 681 |
+
"peak_rank_overflow_mean": 0.0,
|
| 682 |
+
"peak_rank_overflow_min": 0.0,
|
| 683 |
+
"peak_rank_overflow_n": 96,
|
| 684 |
+
"post_physical_rank_cv_max": 0.007872884560844515,
|
| 685 |
+
"post_physical_rank_cv_mean": 0.006243076586697642,
|
| 686 |
+
"post_physical_rank_cv_min": 0.0030538170542637444,
|
| 687 |
+
"post_physical_rank_cv_n": 12,
|
| 688 |
+
"post_physical_rank_imax_max": 1.010040283203125,
|
| 689 |
+
"post_physical_rank_imax_mean": 1.0086746215820312,
|
| 690 |
+
"post_physical_rank_imax_min": 1.00408935546875,
|
| 691 |
+
"post_physical_rank_imax_n": 12,
|
| 692 |
+
"pre_home_rank_cv_max": 0.009139900015747392,
|
| 693 |
+
"pre_home_rank_cv_mean": 0.006980946768958179,
|
| 694 |
+
"pre_home_rank_cv_min": 0.0030538170542637444,
|
| 695 |
+
"pre_home_rank_cv_n": 12,
|
| 696 |
+
"pre_home_rank_imax_max": 1.014892578125,
|
| 697 |
+
"pre_home_rank_imax_mean": 1.0104904174804688,
|
| 698 |
+
"pre_home_rank_imax_min": 1.00408935546875,
|
| 699 |
+
"pre_home_rank_imax_n": 12,
|
| 700 |
+
"records": {
|
| 701 |
+
"drop_dispatch": 96,
|
| 702 |
+
"drop_experts": 96,
|
| 703 |
+
"drop_setup": 96,
|
| 704 |
+
"ultraep_drop": 96,
|
| 705 |
+
"ultraep_post": 96,
|
| 706 |
+
"ultraep_pre": 96
|
| 707 |
+
},
|
| 708 |
+
"required_capacity_max": 1.01004,
|
| 709 |
+
"required_capacity_mean": 0.9999999791666667,
|
| 710 |
+
"required_capacity_min": 0.986633,
|
| 711 |
+
"required_capacity_n": 96,
|
| 712 |
+
"router_balancing": "seq_aux_loss",
|
| 713 |
+
"run_dir": "ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1",
|
| 714 |
+
"seq_length": "4096",
|
| 715 |
+
"shared_expert_overlap": "0",
|
| 716 |
+
"topk": "8",
|
| 717 |
+
"valid_max_rank_reduction_max": 0.004841231657445273,
|
| 718 |
+
"valid_max_rank_reduction_mean": 0.0017907127529931067,
|
| 719 |
+
"valid_max_rank_reduction_min": 0.0,
|
| 720 |
+
"valid_max_rank_reduction_n": 12,
|
| 721 |
+
"zero_drop": 1
|
| 722 |
+
}
|
| 723 |
+
]
|
granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/probe_summary_all.tsv
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
run_dir arm config backend num_experts topk experts_per_rank num_layers moe_layers dense_first_layers seq_length global_batch_size capacity_mode capacity force_balance cuda_graph branch commit router_balancing aux_loss_coeff shared_expert_overlap assignment_sum_mismatches cold_expert_fraction_max cold_expert_fraction_mean cold_expert_fraction_min cold_expert_fraction_n drop_accounting_source drop_expected_assignments drop_rate_pct_max drop_rate_pct_mean drop_rate_pct_min drop_rate_pct_n drop_rate_pct_overall dropped_tokens_sum_all_records duplicate_vector_mismatches echo_logged_load_reduction_valid echo_logged_pre_rank_metric_valid echo_moved_assignment_fraction_max echo_moved_assignment_fraction_mean echo_moved_assignment_fraction_min echo_moved_assignment_fraction_n global_snapshots hottest_expert_token_fraction_max hottest_expert_token_fraction_mean hottest_expert_token_fraction_min hottest_expert_token_fraction_n incomplete_snapshots malformed_records moonep_bytes_global_snapshots moonep_bytes_incomplete_snapshots moonep_bytes_malformed_records moonep_global_combine_payload_bytes_max moonep_global_combine_payload_bytes_mean moonep_global_combine_payload_bytes_min moonep_global_combine_payload_bytes_n moonep_global_dispatch_payload_bytes_max moonep_global_dispatch_payload_bytes_mean moonep_global_dispatch_payload_bytes_min moonep_global_dispatch_payload_bytes_n moonep_global_prefetch_bytes_max moonep_global_prefetch_bytes_mean moonep_global_prefetch_bytes_min moonep_global_prefetch_bytes_n moonep_global_reduce_bytes_max moonep_global_reduce_bytes_mean moonep_global_reduce_bytes_min moonep_global_reduce_bytes_n moonep_global_token_bytes_max moonep_global_token_bytes_mean moonep_global_token_bytes_min moonep_global_token_bytes_n moonep_global_total_nvlink_bytes_max moonep_global_total_nvlink_bytes_mean moonep_global_total_nvlink_bytes_min moonep_global_total_nvlink_bytes_n moonep_global_weight_movement_bytes_max moonep_global_weight_movement_bytes_mean moonep_global_weight_movement_bytes_min moonep_global_weight_movement_bytes_n moonep_global_weight_to_token_ratio_max moonep_global_weight_to_token_ratio_mean moonep_global_weight_to_token_ratio_min moonep_global_weight_to_token_ratio_n moonep_remote_demand_max moonep_remote_demand_mean moonep_remote_demand_min moonep_remote_demand_n moonep_slot_utilization_max moonep_slot_utilization_mean moonep_slot_utilization_min moonep_slot_utilization_n moonep_unused_prefetch_slots_max moonep_unused_prefetch_slots_mean moonep_unused_prefetch_slots_min moonep_unused_prefetch_slots_n observed_dropped_tokens peak_demand_max peak_demand_mean peak_demand_min peak_demand_n peak_rank_overflow_max peak_rank_overflow_mean peak_rank_overflow_min peak_rank_overflow_n post_physical_rank_cv_max post_physical_rank_cv_mean post_physical_rank_cv_min post_physical_rank_cv_n post_physical_rank_imax_max post_physical_rank_imax_mean post_physical_rank_imax_min post_physical_rank_imax_n pre_home_rank_cv_max pre_home_rank_cv_mean pre_home_rank_cv_min pre_home_rank_cv_n pre_home_rank_imax_max pre_home_rank_imax_mean pre_home_rank_imax_min pre_home_rank_imax_n raw_expert_cv_max raw_expert_cv_mean raw_expert_cv_min raw_expert_cv_n raw_expert_imax_ceiling raw_expert_imax_hit_ceiling raw_expert_imax_max raw_expert_imax_mean raw_expert_imax_min raw_expert_imax_n required_capacity_max required_capacity_mean required_capacity_min required_capacity_n saturated_dispatch_rows valid_max_rank_reduction_max valid_max_rank_reduction_mean valid_max_rank_reduction_min valid_max_rank_reduction_n zero_drop
|
| 2 |
+
echo/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap1p13671875/rep1 natural DEEPSEEK_V3_L4 echo 256 8 32 4 1 3 4096 8 static 1.13671875 0 0 token-drop-analysis b099fd84 seq_aux_loss 1e-4 0 0 0.0 0.0 0.0 12 dispatch 3145728.0 0.0 0 0 0.5 0.5 0.5 12 12 0.279449462890625 0.16472371419270834 0.09417724609375 12 0 0 0 0.0 0.0 0.0 12 1.0 1.0 1.0 12 0.3949158995862076 0.18817834176075468 0.07608345138535468 12 1.563079833984375 1.291656494140625 1.098876953125 12 1.4608217820834073 0.8516673799234088 0.5323456097372459 12 32.0 0 8.9423828125 5.271158854166667 3.013671875 12 1.08203125 1.0650227864583333 1.046875 96 0 0.3602374118979285 0.21588444472672905 0.08998000444345701 12 1
|
| 3 |
+
echo/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap1p13671875__forcebal/rep1 forced DEEPSEEK_V3_L4 echo 256 8 32 4 1 3 4096 8 static 1.13671875 1 0 token-drop-analysis b099fd84 seq_aux_loss 1e-4 0 0 0.0 0.0 0.0 12 dispatch 3145728.0 0.0 0 0 0.5 0.5 0.5 12 12 0.03521728515625 0.034426371256510414 0.033843994140625 12 0 0 0 0.0 0.0 0.0 12 1.0 1.0 1.0 12 0.009139900015747392 0.006980946768958179 0.0030538170542637444 12 1.014892578125 1.0104904174804688 1.00408935546875 12 0.03876371556245609 0.03570960046443938 0.029816782685728113 12 32.0 0 1.126953125 1.1016438802083333 1.0830078125 12 1.08203125 1.0631510416666667 1.04296875 96 0 0.01467404378157325 0.01036908147130012 0.004072700747674914 12 1
|
| 4 |
+
echo/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap2p00/rep1 natural DEEPSEEK_V3_L4 echo 256 8 32 4 1 3 4096 8 static 2.00 0 0 token-drop-analysis b099fd84 seq_aux_loss 1e-4 0 0 0.0 0.0 0.0 12 dispatch 3145728.0 0.0 0 0 0.5 0.5 0.5 12 12 0.279449462890625 0.16472371419270834 0.09417724609375 12 0 0 0 0.0 0.0 0.0 12 1.0 1.0 1.0 12 0.3949158995862076 0.18817834176075468 0.07608345138535468 12 1.563079833984375 1.291656494140625 1.098876953125 12 1.4608217820834073 0.8516673799234088 0.5323456097372459 12 32.0 0 8.9423828125 5.271158854166667 3.013671875 12 1.08203125 1.0650227864583333 1.046875 96 0 0.3602374118979285 0.21588444472672905 0.08998000444345701 12 1
|
| 5 |
+
moonep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__async/rep1 natural DEEPSEEK_V3_L4 moonep 256 8 32 4 1 3 4096 8 none none 0 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 seq_aux_loss 1e-4 0 0.0 0.0 0.0 10 10 0.277313232421875 0.1758575439453125 0.092315673828125 10 0 0 10 0 0 3822120960.0 3501221068.8 3346653184.0 10 3822120960.0 3501221068.8 3346653184.0 10 2466250752.0 1867304140.8 1409286144.0 10 2466250752.0 1867304140.8 1409286144.0 10 7646072992.0 7004270806.4 6695137352.0 10 12578574496.0 10738879088.0 9546567752.0 10 4932501504.0 3734608281.6 2818572288.0 10 0.6762953708910178 0.5305300760379464 0.4096318958168878 10 4.0 1.325 0.0 80 0.125 0.04140625 0.0 80 64.0 61.35 56.0 80 0.054662301330415754 0.03173931531099595 0.019783416729893442 10 1.0904317633276406 1.051604154450151 1.0214255000518957 10 0.39536184425362714 0.19281063246441604 0.08594336137506545 10 1.55706787109375 1.3085418701171876 1.113128662109375 10 1.4500281317834722 0.9091278053879644 0.5606721744656151 10 32.0 0 8.8740234375 5.62744140625 2.9541015625 10
|
| 6 |
+
moonep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1 forced DEEPSEEK_V3_L4 moonep 256 8 32 4 1 3 4096 8 none none 1 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 seq_aux_loss 1e-4 0 0.0 0.0 0.0 10 10 0.03497314453125 0.034405517578125 0.03399658203125 10 0 0 10 0 0 3276406784.0 3268117708.8 3260407808.0 10 3276406784.0 3268117708.8 3260407808.0 10 1233125376.0 1233125376.0 1233125376.0 10 1233125376.0 1233125376.0 1233125376.0 10 6554650816.0 6538068851.2 6522648928.0 10 9020901568.0 9004319603.2 8988899680.0 10 2466250752.0 2466250752.0 2466250752.0 10 0.37810570202744476 0.37721470246811023 0.3762596698484449 10 1.0 0.875 0.0 80 0.03125 0.02734375 0.0 80 64.0 62.25 62.0 80 0.00471813998380015 0.003239235393271 0.0017429035970738013 10 1.0078812597749087 1.0051071175236657 1.002130562069864 10 0.009139900015747392 0.007327007135739279 0.0030538170542637444 10 1.014892578125 1.0111572265625 1.00408935546875 10 0.03876371556245609 0.0363651698149183 0.0339868829612938 10 32.0 0 1.119140625 1.1009765625 1.087890625 10
|
| 7 |
+
ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p0625/rep1 natural DEEPSEEK_V3_L4 ultraep 256 8 32 4 1 3 4096 8 static 1.0625 0 0 ultraep-token-drop-analysis eaf829e7 seq_aux_loss 1e-4 0 0 3145728.0 0.0 0.0 0.0 96 0.0 0.0 0 12 0 0 0.0 33097.0 32768.0 31546.0 96 0.0 0.0 0.0 96 0.017713692879015048 0.014273621042617956 0.010671796788645344 12 1.010040283203125 1.010022481282552 1.010009765625 12 0.39595445620995434 0.18964812495168137 0.08292629783502173 12 1.5654296875 1.2918370564778645 1.105194091796875 12 1.01004 1.0000000312500001 0.962708 96 0.3548034934497817 0.20816004531911061 0.08612453403285932 12 1
|
| 8 |
+
ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p25/rep1 natural DEEPSEEK_V3_L4 ultraep 256 8 32 4 1 3 4096 8 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 seq_aux_loss 1e-4 0 0 3145728.0 0.0 0.0 0.0 96 0.0 0.0 0 12 0 0 0.0 33097.0 32768.0 31280.0 96 0.0 0.0 0.0 96 0.018971492434756838 0.014175223070035277 0.010671796788645344 12 1.010040283203125 1.0100301106770833 1.010009765625 12 0.3967451453024771 0.18882615919733423 0.08482749505976411 12 1.568817138671875 1.2910486857096355 1.114654541015625 12 1.01004 1.0000000312500001 0.95459 96 0.35619662691851306 0.2084693322073339 0.09388090349075975 12 1
|
| 9 |
+
ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1 forced DEEPSEEK_V3_L4 ultraep 256 8 32 4 1 3 4096 8 static 1.25 1 0 ultraep-token-drop-analysis eaf829e7 seq_aux_loss 1e-4 0 0 3145728.0 0.0 0.0 0.0 96 0.0 0.0 0 12 0 0 0.0 33097.0 32768.0 32330.0 96 0.0 0.0 0.0 96 0.007872884560844515 0.006243076586697642 0.0030538170542637444 12 1.010040283203125 1.0086746215820312 1.00408935546875 12 0.009139900015747392 0.006980946768958179 0.0030538170542637444 12 1.014892578125 1.0104904174804688 1.00408935546875 12 1.01004 0.9999999791666667 0.986633 96 0.004841231657445273 0.0017907127529931067 0.0 12 1
|
granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/steps.tsv
CHANGED
|
@@ -4,3 +4,4 @@ OK ultraep tierbprobe 8 2026-09-21T06:46:22+00:00 2026-09-21T06:49:53+00:00 /wor
|
|
| 4 |
OK moonep tierbprobe 8 2026-09-21T06:49:55+00:00 2026-09-21T06:52:36+00:00 /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/moonep/tierbprobe
|
| 5 |
OK echo tierb 8 2026-09-21T06:52:38+00:00 2026-09-21T06:56:36+00:00 /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/echo/tierb
|
| 6 |
OK ultraep tierb 8 2026-09-21T06:56:38+00:00 2026-09-21T07:00:08+00:00 /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/ultraep/tierb
|
|
|
|
|
|
| 4 |
OK moonep tierbprobe 8 2026-09-21T06:49:55+00:00 2026-09-21T06:52:36+00:00 /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/moonep/tierbprobe
|
| 5 |
OK echo tierb 8 2026-09-21T06:52:38+00:00 2026-09-21T06:56:36+00:00 /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/echo/tierb
|
| 6 |
OK ultraep tierb 8 2026-09-21T06:56:38+00:00 2026-09-21T07:00:08+00:00 /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/ultraep/tierb
|
| 7 |
+
OK moonep tierb 8 2026-09-21T07:00:11+00:00 2026-09-21T07:04:03+00:00 /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/moonep/tierb
|
granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/study.log
CHANGED
|
@@ -371,3 +371,65 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
|
|
| 371 |
[guard] iter 3/12 (60s)
|
| 372 |
[guard] iter 12/12 (70s)
|
| 373 |
[guard] tearing down process group (outcome=DONE)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 371 |
[guard] iter 3/12 (60s)
|
| 372 |
[guard] iter 12/12 (70s)
|
| 373 |
[guard] tearing down process group (outcome=DONE)
|
| 374 |
+
[guard] iter 3/12 (60s)
|
| 375 |
+
[guard] iter 12/12 (70s)
|
| 376 |
+
[guard] tearing down process group (outcome=DONE)
|
| 377 |
+
[guard] iter 1/12 (60s)
|
| 378 |
+
[guard] iter 12/12 (70s)
|
| 379 |
+
[guard] tearing down process group (outcome=DONE)
|
| 380 |
+
--------------------------------------------------------------------------
|
| 381 |
+
|
| 382 |
+
==============================================================
|
| 383 |
+
SWEEP COMPLETE ok=3 failed=0
|
| 384 |
+
==============================================================
|
| 385 |
+
config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
|
| 386 |
+
DEEPSEEK_V3_L4 moonep matched 32 4096 8 none none 1 OK 192.67 192.30 97849 /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/moonep/tierb/tierb_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__async/rep1
|
| 387 |
+
DEEPSEEK_V3_L4 moonep matched 32 4096 8 none none 2 OK 192.97 192.10 97849 /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/moonep/tierb/tierb_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__async/rep2
|
| 388 |
+
DEEPSEEK_V3_L4 moonep matched 32 4096 8 none none 3 OK 192.90 192.10 97849 /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/moonep/tierb/tierb_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__async/rep3
|
| 389 |
+
|
| 390 |
+
Results: /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/moonep/tierb/tierb_gbs8/results.tsv
|
| 391 |
+
Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/moonep/tierb/tierb_gbs8
|
| 392 |
+
==============================================================
|
| 393 |
+
|
| 394 |
+
Campaign results: /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/moonep/tierb
|
| 395 |
+
Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/moonep/tierb --compare
|
| 396 |
+
Already logged in to Hugging Face as: AbhishekGhosh1998
|
| 397 |
+
uploaded: /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/moonep/tierb -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/moonep/tierb
|
| 398 |
+
https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
|
| 399 |
+
|
| 400 |
+
=== ROLL-UP
|
| 401 |
+
17 runs ok=17 oom=0 -> /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/collected_all.tsv
|
| 402 |
+
|
| 403 |
+
WARNING: 3 run(s) mention capacity saturation. A run that drops assignments does less expert work and must not be quoted as a win:
|
| 404 |
+
ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p0625/rep1: WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=1 total_expected_recv=32094 total_dropped_token
|
| 405 |
+
ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p25/rep1: WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=1 total_expected_recv=33097 total_dropped_token
|
| 406 |
+
ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1: WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=1 total_expected_recv=33019 total_dropped_token
|
| 407 |
+
|
| 408 |
+
phase config seq gbs policy rc cg asy backend iter_ms vs_best
|
| 409 |
+
-------------------------------------------------------------------------------------
|
| 410 |
+
tierb DEEPSEEK_V3_L4 4096 8 matched none 1 0 ultraep 195.66 1.00x
|
| 411 |
+
echo 197.67 1.01x
|
| 412 |
+
|
| 413 |
+
tierb DEEPSEEK_V3_L4 4096 8 matched none 1 1 moonep 192.90 1.00x
|
| 414 |
+
|
| 415 |
+
tierbprobeDEEPSEEK_V3_L4 4096 8 matched none 0 0 ultraep 210.03 1.00x
|
| 416 |
+
echo 223.55 1.06x
|
| 417 |
+
|
| 418 |
+
tierbprobeDEEPSEEK_V3_L4 4096 8 matched none 0 1 moonep 208.62 1.00x
|
| 419 |
+
|
| 420 |
+
8 probe runs -> /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/probe_summary_all.tsv
|
| 421 |
+
run arm cap rawExpI preRankI postRankI drop% reqCap
|
| 422 |
+
--------------------------------------------------------------------------------------------------------------------
|
| 423 |
+
echo/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__echo natural 1.13671875 5.2712 1.2917 1.0000 1.0820
|
| 424 |
+
echo/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__echo forced 1.13671875 1.1016 1.0105 1.0000 1.0820
|
| 425 |
+
echo/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__echo natural 2.00 5.2712 1.2917 1.0000 1.0820
|
| 426 |
+
moonep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__mo natural none 5.6274 1.3085 1.0516
|
| 427 |
+
moonep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__mo forced none 1.1010 1.0112 1.0051
|
| 428 |
+
ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__u natural 1.0625 1.2918 1.0100 0.0000 1.0100
|
| 429 |
+
ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__u natural 1.25 1.2910 1.0100 0.0000 1.0100
|
| 430 |
+
ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__u forced 1.25 1.0105 1.0087 0.0000 1.0100
|
| 431 |
+
|
| 432 |
+
ECHO note: echo_logged_pre_rank_metric_valid=0 and echo_logged_load_reduction_valid=0 are intentional.
|
| 433 |
+
The branch's built-in fields mix source-rank and destination-rank axes; this tool reconstructs both globally.
|
| 434 |
+
no memory-dump runs found under /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4
|
| 435 |
+
Already logged in to Hugging Face as: AbhishekGhosh1998
|