AbhishekGhosh1998 commited on
Commit
ffca334
·
verified ·
1 Parent(s): a981508

granularity results granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4 @ 2026-09-21T07:04:07Z from sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0

Browse files
granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/.completed/moonep__tierb__gbs8 ADDED
@@ -0,0 +1 @@
 
 
1
+ 2026-09-21T07:04:03+00:00
granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/STATUS.txt CHANGED
@@ -1,11 +1,12 @@
1
  study_root: /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4
2
  gbs: 8
3
- updated: 2026-09-21T07:00:10+00:00
4
- current: moonep__tierb__gbs8
5
 
6
  completed steps:
7
  echo__tierb__gbs8
8
  echo__tierbprobe__gbs8
 
9
  moonep__tierbprobe__gbs8
10
  ultraep__tierb__gbs8
11
  ultraep__tierbprobe__gbs8
 
1
  study_root: /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4
2
  gbs: 8
3
+ updated: 2026-09-21T07:04:06+00:00
4
+ current: idle
5
 
6
  completed steps:
7
  echo__tierb__gbs8
8
  echo__tierbprobe__gbs8
9
+ moonep__tierb__gbs8
10
  moonep__tierbprobe__gbs8
11
  ultraep__tierb__gbs8
12
  ultraep__tierbprobe__gbs8
granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/collected_all.tsv ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ run_dir rep phase is_trace async config backend slot_policy slots_per_rank num_experts topk moe_ffn_hidden experts_per_rank expert_params num_layers seq_length global_batch_size num_microbatches assignments_per_rank mean_tokens_per_expert capacity capacity_mode recompute force_balance cuda_graph memory_dump_mode memory_snapshot_path ultraep_quota_min_tokens shared_expert_overlap branch commit dense_first_layers moe_layers router_balancing aux_loss_coeff router_pre_softmax router_fusion rotary_base qkv_bias source_model standard_num_layers base_rung q_lora_rank kv_lora_rank qk_head_dim qk_pos_emb_head_dim v_head_dim rotary_percent rotary_scaling mscale mscale_all_dim router_groups router_group_topk router_score_function router_expert_bias router_bias_update_rate router_topk_scaling_factor shared_expert_gate apply_layernorm_1p attention_output_gate linear_attention_type linear_attention_freq linear_conv_kernel_dim linear_key_head_dim linear_value_head_dim linear_num_key_heads linear_num_value_heads mtp_num_layers mtp_loss_scaling_factor no_weight_decay_cond_type max_position_embeddings init_method_std status iters_seen iter_ms_mean iter_ms_min iter_ms_stdev tflops peak_mib_sampled megatron_max_reserved_mib drop_warning
2
+ echo/tierb/tierb_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap1p13671875/rep1 rep1 tierb 0 0 DEEPSEEK_V3_L4 echo matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 1.13671875 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 198.47 197.00 1.73 375.2 131075 84018
3
+ echo/tierb/tierb_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap1p13671875/rep2 rep2 tierb 0 0 DEEPSEEK_V3_L4 echo matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 1.13671875 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 197.13 195.60 0.89 377.7 131075 84018
4
+ echo/tierb/tierb_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap1p13671875/rep3 rep3 tierb 0 0 DEEPSEEK_V3_L4 echo matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 1.13671875 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 197.41 196.00 1.70 377.2 131075 84018
5
+ echo/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap1p13671875/rep1 rep1 tierbprobe 0 0 DEEPSEEK_V3_L4 echo matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 1.13671875 static none 0 0 none 0 token-drop-analysis b099fd84 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 222.87 222.20 0.76 334.1 121233 84058
6
+ echo/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap1p13671875__forcebal/rep1 rep1 tierbprobe 0 0 DEEPSEEK_V3_L4 echo matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 1.13671875 static none 1 0 none 0 token-drop-analysis b099fd84 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 223.39 221.10 3.79 333.4 121233 84058
7
+ echo/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap2p00/rep1 rep1 tierbprobe 0 0 DEEPSEEK_V3_L4 echo matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 2.00 static none 0 0 none 0 token-drop-analysis b099fd84 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 224.39 221.70 4.00 332.0 123091 85514
8
+ moonep/tierb/tierb_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__async/rep1 rep1 tierb 0 1 DEEPSEEK_V3_L4 moonep matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 192.67 192.30 0.42 386.5 97849 60652
9
+ moonep/tierb/tierb_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__async/rep2 rep2 tierb 0 1 DEEPSEEK_V3_L4 moonep matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 193.27 192.10 0.96 385.3 97849 60652
10
+ moonep/tierb/tierb_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__async/rep3 rep3 tierb 0 1 DEEPSEEK_V3_L4 moonep matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 192.76 192.30 0.40 386.3 97849 60652
11
+ moonep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__async/rep1 rep1 tierbprobe 0 1 DEEPSEEK_V3_L4 moonep matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 none none none 0 0 none 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 209.43 208.50 0.95 355.6 100169 60636
12
+ moonep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1 rep1 tierbprobe 0 1 DEEPSEEK_V3_L4 moonep matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 none none none 1 0 none 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 207.81 207.30 0.38 358.3 100169 60636
13
+ ultraep/tierb/tierb_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p13671875/rep1 rep1 tierb 0 0 DEEPSEEK_V3_L4 ultraep matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 1.13671875 static none 0 1 none 128 0 ultraep-bracing-regions-through-kernels 03d57ef7 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 195.76 194.20 1.94 380.4 101995 68070
14
+ ultraep/tierb/tierb_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p13671875/rep2 rep2 tierb 0 0 DEEPSEEK_V3_L4 ultraep matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 1.13671875 static none 0 1 none 128 0 ultraep-bracing-regions-through-kernels 03d57ef7 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 195.61 194.00 1.84 380.7 101995 68070
15
+ ultraep/tierb/tierb_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p13671875/rep3 rep3 tierb 0 0 DEEPSEEK_V3_L4 ultraep matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 1.13671875 static none 0 1 none 128 0 ultraep-bracing-regions-through-kernels 03d57ef7 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 195.60 194.10 1.82 380.7 101995 68070
16
+ ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p0625/rep1 rep1 tierbprobe 0 0 DEEPSEEK_V3_L4 ultraep matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 1.0625 static none 0 0 none 128 0 ultraep-token-drop-analysis eaf829e7 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 214.57 209.60 5.53 347.2 100665 68054 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=1 total_expected_recv=32094 total_dropped_token
17
+ ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p25/rep1 rep1 tierbprobe 0 0 DEEPSEEK_V3_L4 ultraep matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 1.25 static none 0 0 none 128 0 ultraep-token-drop-analysis eaf829e7 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 211.10 209.30 1.96 352.8 102365 68054 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=1 total_expected_recv=33097 total_dropped_token
18
+ ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1 rep1 tierbprobe 0 0 DEEPSEEK_V3_L4 ultraep matched 32 256 8 2048 32 44040192 4 4096 8 1 32768 1024 1.25 static none 1 0 none 128 0 ultraep-token-drop-analysis eaf829e7 3 1 seq_aux_loss 1e-4 0 0 10000 0 DeepSeek-V3 61 1536 512 128 64 128 1.0 40 1.0 1.0 8 4 sigmoid 1 1e-3 2.5 0 0 0 0 0.1 4096 0.02 OK 12 204.41 202.60 1.50 364.3 102365 68054 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=1 total_expected_recv=33019 total_dropped_token
granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/hf_watcher.log CHANGED
@@ -7,3 +7,5 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
7
  uploaded: /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4
8
  https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
9
  [2026-09-21T07:02:25+00:00] backup pass
 
 
 
7
  uploaded: /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4
8
  https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
9
  [2026-09-21T07:02:25+00:00] backup pass
10
+ uploaded: /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4
11
+ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/probe_summary_all.json ADDED
@@ -0,0 +1,723 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "arm": "natural",
4
+ "assignment_sum_mismatches": 0,
5
+ "aux_loss_coeff": "1e-4",
6
+ "backend": "echo",
7
+ "branch": "token-drop-analysis",
8
+ "capacity": "1.13671875",
9
+ "capacity_mode": "static",
10
+ "cold_expert_fraction_max": 0.0,
11
+ "cold_expert_fraction_mean": 0.0,
12
+ "cold_expert_fraction_min": 0.0,
13
+ "cold_expert_fraction_n": 12,
14
+ "commit": "b099fd84",
15
+ "config": "DEEPSEEK_V3_L4",
16
+ "cuda_graph": "0",
17
+ "dense_first_layers": "3",
18
+ "drop_accounting_source": "dispatch",
19
+ "drop_expected_assignments": 3145728.0,
20
+ "drop_rate_pct_overall": 0.0,
21
+ "echo_logged_load_reduction_valid": 0,
22
+ "echo_logged_pre_rank_metric_valid": 0,
23
+ "echo_moved_assignment_fraction_max": 0.5,
24
+ "echo_moved_assignment_fraction_mean": 0.5,
25
+ "echo_moved_assignment_fraction_min": 0.5,
26
+ "echo_moved_assignment_fraction_n": 12,
27
+ "experts_per_rank": "32",
28
+ "force_balance": "0",
29
+ "global_batch_size": "8",
30
+ "global_snapshots": 12,
31
+ "hottest_expert_token_fraction_max": 0.279449462890625,
32
+ "hottest_expert_token_fraction_mean": 0.16472371419270834,
33
+ "hottest_expert_token_fraction_min": 0.09417724609375,
34
+ "hottest_expert_token_fraction_n": 12,
35
+ "incomplete_snapshots": 0,
36
+ "malformed_records": 0,
37
+ "moe_layers": "1",
38
+ "num_experts": "256",
39
+ "num_layers": "4",
40
+ "observed_dropped_tokens": 0,
41
+ "post_physical_rank_cv_max": 0.0,
42
+ "post_physical_rank_cv_mean": 0.0,
43
+ "post_physical_rank_cv_min": 0.0,
44
+ "post_physical_rank_cv_n": 12,
45
+ "post_physical_rank_imax_max": 1.0,
46
+ "post_physical_rank_imax_mean": 1.0,
47
+ "post_physical_rank_imax_min": 1.0,
48
+ "post_physical_rank_imax_n": 12,
49
+ "pre_home_rank_cv_max": 0.3949158995862076,
50
+ "pre_home_rank_cv_mean": 0.18817834176075468,
51
+ "pre_home_rank_cv_min": 0.07608345138535468,
52
+ "pre_home_rank_cv_n": 12,
53
+ "pre_home_rank_imax_max": 1.563079833984375,
54
+ "pre_home_rank_imax_mean": 1.291656494140625,
55
+ "pre_home_rank_imax_min": 1.098876953125,
56
+ "pre_home_rank_imax_n": 12,
57
+ "raw_expert_cv_max": 1.4608217820834073,
58
+ "raw_expert_cv_mean": 0.8516673799234088,
59
+ "raw_expert_cv_min": 0.5323456097372459,
60
+ "raw_expert_cv_n": 12,
61
+ "raw_expert_imax_ceiling": 32.0,
62
+ "raw_expert_imax_hit_ceiling": 0,
63
+ "raw_expert_imax_max": 8.9423828125,
64
+ "raw_expert_imax_mean": 5.271158854166667,
65
+ "raw_expert_imax_min": 3.013671875,
66
+ "raw_expert_imax_n": 12,
67
+ "records": {
68
+ "drop_dispatch": 96,
69
+ "drop_experts": 96,
70
+ "drop_setup": 96,
71
+ "echo_post": 96,
72
+ "echo_pre": 96
73
+ },
74
+ "required_capacity_max": 1.08203125,
75
+ "required_capacity_mean": 1.0650227864583333,
76
+ "required_capacity_min": 1.046875,
77
+ "required_capacity_n": 96,
78
+ "router_balancing": "seq_aux_loss",
79
+ "run_dir": "echo/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap1p13671875/rep1",
80
+ "saturated_dispatch_rows": 0,
81
+ "seq_length": "4096",
82
+ "shared_expert_overlap": "0",
83
+ "topk": "8",
84
+ "valid_max_rank_reduction_max": 0.3602374118979285,
85
+ "valid_max_rank_reduction_mean": 0.21588444472672905,
86
+ "valid_max_rank_reduction_min": 0.08998000444345701,
87
+ "valid_max_rank_reduction_n": 12,
88
+ "zero_drop": 1
89
+ },
90
+ {
91
+ "arm": "forced",
92
+ "assignment_sum_mismatches": 0,
93
+ "aux_loss_coeff": "1e-4",
94
+ "backend": "echo",
95
+ "branch": "token-drop-analysis",
96
+ "capacity": "1.13671875",
97
+ "capacity_mode": "static",
98
+ "cold_expert_fraction_max": 0.0,
99
+ "cold_expert_fraction_mean": 0.0,
100
+ "cold_expert_fraction_min": 0.0,
101
+ "cold_expert_fraction_n": 12,
102
+ "commit": "b099fd84",
103
+ "config": "DEEPSEEK_V3_L4",
104
+ "cuda_graph": "0",
105
+ "dense_first_layers": "3",
106
+ "drop_accounting_source": "dispatch",
107
+ "drop_expected_assignments": 3145728.0,
108
+ "drop_rate_pct_overall": 0.0,
109
+ "echo_logged_load_reduction_valid": 0,
110
+ "echo_logged_pre_rank_metric_valid": 0,
111
+ "echo_moved_assignment_fraction_max": 0.5,
112
+ "echo_moved_assignment_fraction_mean": 0.5,
113
+ "echo_moved_assignment_fraction_min": 0.5,
114
+ "echo_moved_assignment_fraction_n": 12,
115
+ "experts_per_rank": "32",
116
+ "force_balance": "1",
117
+ "global_batch_size": "8",
118
+ "global_snapshots": 12,
119
+ "hottest_expert_token_fraction_max": 0.03521728515625,
120
+ "hottest_expert_token_fraction_mean": 0.034426371256510414,
121
+ "hottest_expert_token_fraction_min": 0.033843994140625,
122
+ "hottest_expert_token_fraction_n": 12,
123
+ "incomplete_snapshots": 0,
124
+ "malformed_records": 0,
125
+ "moe_layers": "1",
126
+ "num_experts": "256",
127
+ "num_layers": "4",
128
+ "observed_dropped_tokens": 0,
129
+ "post_physical_rank_cv_max": 0.0,
130
+ "post_physical_rank_cv_mean": 0.0,
131
+ "post_physical_rank_cv_min": 0.0,
132
+ "post_physical_rank_cv_n": 12,
133
+ "post_physical_rank_imax_max": 1.0,
134
+ "post_physical_rank_imax_mean": 1.0,
135
+ "post_physical_rank_imax_min": 1.0,
136
+ "post_physical_rank_imax_n": 12,
137
+ "pre_home_rank_cv_max": 0.009139900015747392,
138
+ "pre_home_rank_cv_mean": 0.006980946768958179,
139
+ "pre_home_rank_cv_min": 0.0030538170542637444,
140
+ "pre_home_rank_cv_n": 12,
141
+ "pre_home_rank_imax_max": 1.014892578125,
142
+ "pre_home_rank_imax_mean": 1.0104904174804688,
143
+ "pre_home_rank_imax_min": 1.00408935546875,
144
+ "pre_home_rank_imax_n": 12,
145
+ "raw_expert_cv_max": 0.03876371556245609,
146
+ "raw_expert_cv_mean": 0.03570960046443938,
147
+ "raw_expert_cv_min": 0.029816782685728113,
148
+ "raw_expert_cv_n": 12,
149
+ "raw_expert_imax_ceiling": 32.0,
150
+ "raw_expert_imax_hit_ceiling": 0,
151
+ "raw_expert_imax_max": 1.126953125,
152
+ "raw_expert_imax_mean": 1.1016438802083333,
153
+ "raw_expert_imax_min": 1.0830078125,
154
+ "raw_expert_imax_n": 12,
155
+ "records": {
156
+ "drop_dispatch": 96,
157
+ "drop_experts": 96,
158
+ "drop_setup": 96,
159
+ "echo_post": 96,
160
+ "echo_pre": 96
161
+ },
162
+ "required_capacity_max": 1.08203125,
163
+ "required_capacity_mean": 1.0631510416666667,
164
+ "required_capacity_min": 1.04296875,
165
+ "required_capacity_n": 96,
166
+ "router_balancing": "seq_aux_loss",
167
+ "run_dir": "echo/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap1p13671875__forcebal/rep1",
168
+ "saturated_dispatch_rows": 0,
169
+ "seq_length": "4096",
170
+ "shared_expert_overlap": "0",
171
+ "topk": "8",
172
+ "valid_max_rank_reduction_max": 0.01467404378157325,
173
+ "valid_max_rank_reduction_mean": 0.01036908147130012,
174
+ "valid_max_rank_reduction_min": 0.004072700747674914,
175
+ "valid_max_rank_reduction_n": 12,
176
+ "zero_drop": 1
177
+ },
178
+ {
179
+ "arm": "natural",
180
+ "assignment_sum_mismatches": 0,
181
+ "aux_loss_coeff": "1e-4",
182
+ "backend": "echo",
183
+ "branch": "token-drop-analysis",
184
+ "capacity": "2.00",
185
+ "capacity_mode": "static",
186
+ "cold_expert_fraction_max": 0.0,
187
+ "cold_expert_fraction_mean": 0.0,
188
+ "cold_expert_fraction_min": 0.0,
189
+ "cold_expert_fraction_n": 12,
190
+ "commit": "b099fd84",
191
+ "config": "DEEPSEEK_V3_L4",
192
+ "cuda_graph": "0",
193
+ "dense_first_layers": "3",
194
+ "drop_accounting_source": "dispatch",
195
+ "drop_expected_assignments": 3145728.0,
196
+ "drop_rate_pct_overall": 0.0,
197
+ "echo_logged_load_reduction_valid": 0,
198
+ "echo_logged_pre_rank_metric_valid": 0,
199
+ "echo_moved_assignment_fraction_max": 0.5,
200
+ "echo_moved_assignment_fraction_mean": 0.5,
201
+ "echo_moved_assignment_fraction_min": 0.5,
202
+ "echo_moved_assignment_fraction_n": 12,
203
+ "experts_per_rank": "32",
204
+ "force_balance": "0",
205
+ "global_batch_size": "8",
206
+ "global_snapshots": 12,
207
+ "hottest_expert_token_fraction_max": 0.279449462890625,
208
+ "hottest_expert_token_fraction_mean": 0.16472371419270834,
209
+ "hottest_expert_token_fraction_min": 0.09417724609375,
210
+ "hottest_expert_token_fraction_n": 12,
211
+ "incomplete_snapshots": 0,
212
+ "malformed_records": 0,
213
+ "moe_layers": "1",
214
+ "num_experts": "256",
215
+ "num_layers": "4",
216
+ "observed_dropped_tokens": 0,
217
+ "post_physical_rank_cv_max": 0.0,
218
+ "post_physical_rank_cv_mean": 0.0,
219
+ "post_physical_rank_cv_min": 0.0,
220
+ "post_physical_rank_cv_n": 12,
221
+ "post_physical_rank_imax_max": 1.0,
222
+ "post_physical_rank_imax_mean": 1.0,
223
+ "post_physical_rank_imax_min": 1.0,
224
+ "post_physical_rank_imax_n": 12,
225
+ "pre_home_rank_cv_max": 0.3949158995862076,
226
+ "pre_home_rank_cv_mean": 0.18817834176075468,
227
+ "pre_home_rank_cv_min": 0.07608345138535468,
228
+ "pre_home_rank_cv_n": 12,
229
+ "pre_home_rank_imax_max": 1.563079833984375,
230
+ "pre_home_rank_imax_mean": 1.291656494140625,
231
+ "pre_home_rank_imax_min": 1.098876953125,
232
+ "pre_home_rank_imax_n": 12,
233
+ "raw_expert_cv_max": 1.4608217820834073,
234
+ "raw_expert_cv_mean": 0.8516673799234088,
235
+ "raw_expert_cv_min": 0.5323456097372459,
236
+ "raw_expert_cv_n": 12,
237
+ "raw_expert_imax_ceiling": 32.0,
238
+ "raw_expert_imax_hit_ceiling": 0,
239
+ "raw_expert_imax_max": 8.9423828125,
240
+ "raw_expert_imax_mean": 5.271158854166667,
241
+ "raw_expert_imax_min": 3.013671875,
242
+ "raw_expert_imax_n": 12,
243
+ "records": {
244
+ "drop_dispatch": 96,
245
+ "drop_experts": 96,
246
+ "drop_setup": 96,
247
+ "echo_post": 96,
248
+ "echo_pre": 96
249
+ },
250
+ "required_capacity_max": 1.08203125,
251
+ "required_capacity_mean": 1.0650227864583333,
252
+ "required_capacity_min": 1.046875,
253
+ "required_capacity_n": 96,
254
+ "router_balancing": "seq_aux_loss",
255
+ "run_dir": "echo/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap2p00/rep1",
256
+ "saturated_dispatch_rows": 0,
257
+ "seq_length": "4096",
258
+ "shared_expert_overlap": "0",
259
+ "topk": "8",
260
+ "valid_max_rank_reduction_max": 0.3602374118979285,
261
+ "valid_max_rank_reduction_mean": 0.21588444472672905,
262
+ "valid_max_rank_reduction_min": 0.08998000444345701,
263
+ "valid_max_rank_reduction_n": 12,
264
+ "zero_drop": 1
265
+ },
266
+ {
267
+ "arm": "natural",
268
+ "aux_loss_coeff": "1e-4",
269
+ "backend": "moonep",
270
+ "branch": "moonep_backend_async_grad_reduce_token_routing_analysis",
271
+ "capacity": "none",
272
+ "capacity_mode": "none",
273
+ "cold_expert_fraction_max": 0.0,
274
+ "cold_expert_fraction_mean": 0.0,
275
+ "cold_expert_fraction_min": 0.0,
276
+ "cold_expert_fraction_n": 10,
277
+ "commit": "a6a1a221",
278
+ "config": "DEEPSEEK_V3_L4",
279
+ "cuda_graph": "0",
280
+ "dense_first_layers": "3",
281
+ "experts_per_rank": "32",
282
+ "force_balance": "0",
283
+ "global_batch_size": "8",
284
+ "global_snapshots": 10,
285
+ "hottest_expert_token_fraction_max": 0.277313232421875,
286
+ "hottest_expert_token_fraction_mean": 0.1758575439453125,
287
+ "hottest_expert_token_fraction_min": 0.092315673828125,
288
+ "hottest_expert_token_fraction_n": 10,
289
+ "incomplete_snapshots": 0,
290
+ "malformed_records": 0,
291
+ "moe_layers": "1",
292
+ "moonep_bytes_global_snapshots": 10,
293
+ "moonep_bytes_incomplete_snapshots": 0,
294
+ "moonep_bytes_malformed_records": 0,
295
+ "moonep_global_combine_payload_bytes_max": 3822120960.0,
296
+ "moonep_global_combine_payload_bytes_mean": 3501221068.8,
297
+ "moonep_global_combine_payload_bytes_min": 3346653184.0,
298
+ "moonep_global_combine_payload_bytes_n": 10,
299
+ "moonep_global_dispatch_payload_bytes_max": 3822120960.0,
300
+ "moonep_global_dispatch_payload_bytes_mean": 3501221068.8,
301
+ "moonep_global_dispatch_payload_bytes_min": 3346653184.0,
302
+ "moonep_global_dispatch_payload_bytes_n": 10,
303
+ "moonep_global_prefetch_bytes_max": 2466250752.0,
304
+ "moonep_global_prefetch_bytes_mean": 1867304140.8,
305
+ "moonep_global_prefetch_bytes_min": 1409286144.0,
306
+ "moonep_global_prefetch_bytes_n": 10,
307
+ "moonep_global_reduce_bytes_max": 2466250752.0,
308
+ "moonep_global_reduce_bytes_mean": 1867304140.8,
309
+ "moonep_global_reduce_bytes_min": 1409286144.0,
310
+ "moonep_global_reduce_bytes_n": 10,
311
+ "moonep_global_token_bytes_max": 7646072992.0,
312
+ "moonep_global_token_bytes_mean": 7004270806.4,
313
+ "moonep_global_token_bytes_min": 6695137352.0,
314
+ "moonep_global_token_bytes_n": 10,
315
+ "moonep_global_total_nvlink_bytes_max": 12578574496.0,
316
+ "moonep_global_total_nvlink_bytes_mean": 10738879088.0,
317
+ "moonep_global_total_nvlink_bytes_min": 9546567752.0,
318
+ "moonep_global_total_nvlink_bytes_n": 10,
319
+ "moonep_global_weight_movement_bytes_max": 4932501504.0,
320
+ "moonep_global_weight_movement_bytes_mean": 3734608281.6,
321
+ "moonep_global_weight_movement_bytes_min": 2818572288.0,
322
+ "moonep_global_weight_movement_bytes_n": 10,
323
+ "moonep_global_weight_to_token_ratio_max": 0.6762953708910178,
324
+ "moonep_global_weight_to_token_ratio_mean": 0.5305300760379464,
325
+ "moonep_global_weight_to_token_ratio_min": 0.4096318958168878,
326
+ "moonep_global_weight_to_token_ratio_n": 10,
327
+ "moonep_remote_demand_max": 4.0,
328
+ "moonep_remote_demand_mean": 1.325,
329
+ "moonep_remote_demand_min": 0.0,
330
+ "moonep_remote_demand_n": 80,
331
+ "moonep_slot_utilization_max": 0.125,
332
+ "moonep_slot_utilization_mean": 0.04140625,
333
+ "moonep_slot_utilization_min": 0.0,
334
+ "moonep_slot_utilization_n": 80,
335
+ "moonep_unused_prefetch_slots_max": 64.0,
336
+ "moonep_unused_prefetch_slots_mean": 61.35,
337
+ "moonep_unused_prefetch_slots_min": 56.0,
338
+ "moonep_unused_prefetch_slots_n": 80,
339
+ "num_experts": "256",
340
+ "num_layers": "4",
341
+ "post_physical_rank_cv_max": 0.054662301330415754,
342
+ "post_physical_rank_cv_mean": 0.03173931531099595,
343
+ "post_physical_rank_cv_min": 0.019783416729893442,
344
+ "post_physical_rank_cv_n": 10,
345
+ "post_physical_rank_imax_max": 1.0904317633276406,
346
+ "post_physical_rank_imax_mean": 1.051604154450151,
347
+ "post_physical_rank_imax_min": 1.0214255000518957,
348
+ "post_physical_rank_imax_n": 10,
349
+ "pre_home_rank_cv_max": 0.39536184425362714,
350
+ "pre_home_rank_cv_mean": 0.19281063246441604,
351
+ "pre_home_rank_cv_min": 0.08594336137506545,
352
+ "pre_home_rank_cv_n": 10,
353
+ "pre_home_rank_imax_max": 1.55706787109375,
354
+ "pre_home_rank_imax_mean": 1.3085418701171876,
355
+ "pre_home_rank_imax_min": 1.113128662109375,
356
+ "pre_home_rank_imax_n": 10,
357
+ "raw_expert_cv_max": 1.4500281317834722,
358
+ "raw_expert_cv_mean": 0.9091278053879644,
359
+ "raw_expert_cv_min": 0.5606721744656151,
360
+ "raw_expert_cv_n": 10,
361
+ "raw_expert_imax_ceiling": 32.0,
362
+ "raw_expert_imax_hit_ceiling": 0,
363
+ "raw_expert_imax_max": 8.8740234375,
364
+ "raw_expert_imax_mean": 5.62744140625,
365
+ "raw_expert_imax_min": 2.9541015625,
366
+ "raw_expert_imax_n": 10,
367
+ "records": {
368
+ "moonep_bytes": 80,
369
+ "moonep_routing": 80,
370
+ "moonep_summary": 80,
371
+ "moonep_weights": 80
372
+ },
373
+ "router_balancing": "seq_aux_loss",
374
+ "run_dir": "moonep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__async/rep1",
375
+ "seq_length": "4096",
376
+ "shared_expert_overlap": "0",
377
+ "topk": "8"
378
+ },
379
+ {
380
+ "arm": "forced",
381
+ "aux_loss_coeff": "1e-4",
382
+ "backend": "moonep",
383
+ "branch": "moonep_backend_async_grad_reduce_token_routing_analysis",
384
+ "capacity": "none",
385
+ "capacity_mode": "none",
386
+ "cold_expert_fraction_max": 0.0,
387
+ "cold_expert_fraction_mean": 0.0,
388
+ "cold_expert_fraction_min": 0.0,
389
+ "cold_expert_fraction_n": 10,
390
+ "commit": "a6a1a221",
391
+ "config": "DEEPSEEK_V3_L4",
392
+ "cuda_graph": "0",
393
+ "dense_first_layers": "3",
394
+ "experts_per_rank": "32",
395
+ "force_balance": "1",
396
+ "global_batch_size": "8",
397
+ "global_snapshots": 10,
398
+ "hottest_expert_token_fraction_max": 0.03497314453125,
399
+ "hottest_expert_token_fraction_mean": 0.034405517578125,
400
+ "hottest_expert_token_fraction_min": 0.03399658203125,
401
+ "hottest_expert_token_fraction_n": 10,
402
+ "incomplete_snapshots": 0,
403
+ "malformed_records": 0,
404
+ "moe_layers": "1",
405
+ "moonep_bytes_global_snapshots": 10,
406
+ "moonep_bytes_incomplete_snapshots": 0,
407
+ "moonep_bytes_malformed_records": 0,
408
+ "moonep_global_combine_payload_bytes_max": 3276406784.0,
409
+ "moonep_global_combine_payload_bytes_mean": 3268117708.8,
410
+ "moonep_global_combine_payload_bytes_min": 3260407808.0,
411
+ "moonep_global_combine_payload_bytes_n": 10,
412
+ "moonep_global_dispatch_payload_bytes_max": 3276406784.0,
413
+ "moonep_global_dispatch_payload_bytes_mean": 3268117708.8,
414
+ "moonep_global_dispatch_payload_bytes_min": 3260407808.0,
415
+ "moonep_global_dispatch_payload_bytes_n": 10,
416
+ "moonep_global_prefetch_bytes_max": 1233125376.0,
417
+ "moonep_global_prefetch_bytes_mean": 1233125376.0,
418
+ "moonep_global_prefetch_bytes_min": 1233125376.0,
419
+ "moonep_global_prefetch_bytes_n": 10,
420
+ "moonep_global_reduce_bytes_max": 1233125376.0,
421
+ "moonep_global_reduce_bytes_mean": 1233125376.0,
422
+ "moonep_global_reduce_bytes_min": 1233125376.0,
423
+ "moonep_global_reduce_bytes_n": 10,
424
+ "moonep_global_token_bytes_max": 6554650816.0,
425
+ "moonep_global_token_bytes_mean": 6538068851.2,
426
+ "moonep_global_token_bytes_min": 6522648928.0,
427
+ "moonep_global_token_bytes_n": 10,
428
+ "moonep_global_total_nvlink_bytes_max": 9020901568.0,
429
+ "moonep_global_total_nvlink_bytes_mean": 9004319603.2,
430
+ "moonep_global_total_nvlink_bytes_min": 8988899680.0,
431
+ "moonep_global_total_nvlink_bytes_n": 10,
432
+ "moonep_global_weight_movement_bytes_max": 2466250752.0,
433
+ "moonep_global_weight_movement_bytes_mean": 2466250752.0,
434
+ "moonep_global_weight_movement_bytes_min": 2466250752.0,
435
+ "moonep_global_weight_movement_bytes_n": 10,
436
+ "moonep_global_weight_to_token_ratio_max": 0.37810570202744476,
437
+ "moonep_global_weight_to_token_ratio_mean": 0.37721470246811023,
438
+ "moonep_global_weight_to_token_ratio_min": 0.3762596698484449,
439
+ "moonep_global_weight_to_token_ratio_n": 10,
440
+ "moonep_remote_demand_max": 1.0,
441
+ "moonep_remote_demand_mean": 0.875,
442
+ "moonep_remote_demand_min": 0.0,
443
+ "moonep_remote_demand_n": 80,
444
+ "moonep_slot_utilization_max": 0.03125,
445
+ "moonep_slot_utilization_mean": 0.02734375,
446
+ "moonep_slot_utilization_min": 0.0,
447
+ "moonep_slot_utilization_n": 80,
448
+ "moonep_unused_prefetch_slots_max": 64.0,
449
+ "moonep_unused_prefetch_slots_mean": 62.25,
450
+ "moonep_unused_prefetch_slots_min": 62.0,
451
+ "moonep_unused_prefetch_slots_n": 80,
452
+ "num_experts": "256",
453
+ "num_layers": "4",
454
+ "post_physical_rank_cv_max": 0.00471813998380015,
455
+ "post_physical_rank_cv_mean": 0.003239235393271,
456
+ "post_physical_rank_cv_min": 0.0017429035970738013,
457
+ "post_physical_rank_cv_n": 10,
458
+ "post_physical_rank_imax_max": 1.0078812597749087,
459
+ "post_physical_rank_imax_mean": 1.0051071175236657,
460
+ "post_physical_rank_imax_min": 1.002130562069864,
461
+ "post_physical_rank_imax_n": 10,
462
+ "pre_home_rank_cv_max": 0.009139900015747392,
463
+ "pre_home_rank_cv_mean": 0.007327007135739279,
464
+ "pre_home_rank_cv_min": 0.0030538170542637444,
465
+ "pre_home_rank_cv_n": 10,
466
+ "pre_home_rank_imax_max": 1.014892578125,
467
+ "pre_home_rank_imax_mean": 1.0111572265625,
468
+ "pre_home_rank_imax_min": 1.00408935546875,
469
+ "pre_home_rank_imax_n": 10,
470
+ "raw_expert_cv_max": 0.03876371556245609,
471
+ "raw_expert_cv_mean": 0.0363651698149183,
472
+ "raw_expert_cv_min": 0.0339868829612938,
473
+ "raw_expert_cv_n": 10,
474
+ "raw_expert_imax_ceiling": 32.0,
475
+ "raw_expert_imax_hit_ceiling": 0,
476
+ "raw_expert_imax_max": 1.119140625,
477
+ "raw_expert_imax_mean": 1.1009765625,
478
+ "raw_expert_imax_min": 1.087890625,
479
+ "raw_expert_imax_n": 10,
480
+ "records": {
481
+ "moonep_bytes": 80,
482
+ "moonep_routing": 80,
483
+ "moonep_summary": 80,
484
+ "moonep_weights": 80
485
+ },
486
+ "router_balancing": "seq_aux_loss",
487
+ "run_dir": "moonep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1",
488
+ "seq_length": "4096",
489
+ "shared_expert_overlap": "0",
490
+ "topk": "8"
491
+ },
492
+ {
493
+ "arm": "natural",
494
+ "assignment_sum_mismatches": 0,
495
+ "aux_loss_coeff": "1e-4",
496
+ "backend": "ultraep",
497
+ "branch": "ultraep-token-drop-analysis",
498
+ "capacity": "1.0625",
499
+ "capacity_mode": "static",
500
+ "commit": "eaf829e7",
501
+ "config": "DEEPSEEK_V3_L4",
502
+ "cuda_graph": "0",
503
+ "dense_first_layers": "3",
504
+ "drop_expected_assignments": 3145728.0,
505
+ "drop_rate_pct_max": 0.0,
506
+ "drop_rate_pct_mean": 0.0,
507
+ "drop_rate_pct_min": 0.0,
508
+ "drop_rate_pct_n": 96,
509
+ "drop_rate_pct_overall": 0.0,
510
+ "dropped_tokens_sum_all_records": 0.0,
511
+ "duplicate_vector_mismatches": 0,
512
+ "experts_per_rank": "32",
513
+ "force_balance": "0",
514
+ "global_batch_size": "8",
515
+ "global_snapshots": 12,
516
+ "incomplete_snapshots": 0,
517
+ "malformed_records": 0,
518
+ "moe_layers": "1",
519
+ "num_experts": "256",
520
+ "num_layers": "4",
521
+ "observed_dropped_tokens": 0.0,
522
+ "peak_demand_max": 33097.0,
523
+ "peak_demand_mean": 32768.0,
524
+ "peak_demand_min": 31546.0,
525
+ "peak_demand_n": 96,
526
+ "peak_rank_overflow_max": 0.0,
527
+ "peak_rank_overflow_mean": 0.0,
528
+ "peak_rank_overflow_min": 0.0,
529
+ "peak_rank_overflow_n": 96,
530
+ "post_physical_rank_cv_max": 0.017713692879015048,
531
+ "post_physical_rank_cv_mean": 0.014273621042617956,
532
+ "post_physical_rank_cv_min": 0.010671796788645344,
533
+ "post_physical_rank_cv_n": 12,
534
+ "post_physical_rank_imax_max": 1.010040283203125,
535
+ "post_physical_rank_imax_mean": 1.010022481282552,
536
+ "post_physical_rank_imax_min": 1.010009765625,
537
+ "post_physical_rank_imax_n": 12,
538
+ "pre_home_rank_cv_max": 0.39595445620995434,
539
+ "pre_home_rank_cv_mean": 0.18964812495168137,
540
+ "pre_home_rank_cv_min": 0.08292629783502173,
541
+ "pre_home_rank_cv_n": 12,
542
+ "pre_home_rank_imax_max": 1.5654296875,
543
+ "pre_home_rank_imax_mean": 1.2918370564778645,
544
+ "pre_home_rank_imax_min": 1.105194091796875,
545
+ "pre_home_rank_imax_n": 12,
546
+ "records": {
547
+ "drop_dispatch": 96,
548
+ "drop_experts": 96,
549
+ "drop_setup": 96,
550
+ "ultraep_drop": 96,
551
+ "ultraep_post": 96,
552
+ "ultraep_pre": 96
553
+ },
554
+ "required_capacity_max": 1.01004,
555
+ "required_capacity_mean": 1.0000000312500001,
556
+ "required_capacity_min": 0.962708,
557
+ "required_capacity_n": 96,
558
+ "router_balancing": "seq_aux_loss",
559
+ "run_dir": "ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p0625/rep1",
560
+ "seq_length": "4096",
561
+ "shared_expert_overlap": "0",
562
+ "topk": "8",
563
+ "valid_max_rank_reduction_max": 0.3548034934497817,
564
+ "valid_max_rank_reduction_mean": 0.20816004531911061,
565
+ "valid_max_rank_reduction_min": 0.08612453403285932,
566
+ "valid_max_rank_reduction_n": 12,
567
+ "zero_drop": 1
568
+ },
569
+ {
570
+ "arm": "natural",
571
+ "assignment_sum_mismatches": 0,
572
+ "aux_loss_coeff": "1e-4",
573
+ "backend": "ultraep",
574
+ "branch": "ultraep-token-drop-analysis",
575
+ "capacity": "1.25",
576
+ "capacity_mode": "static",
577
+ "commit": "eaf829e7",
578
+ "config": "DEEPSEEK_V3_L4",
579
+ "cuda_graph": "0",
580
+ "dense_first_layers": "3",
581
+ "drop_expected_assignments": 3145728.0,
582
+ "drop_rate_pct_max": 0.0,
583
+ "drop_rate_pct_mean": 0.0,
584
+ "drop_rate_pct_min": 0.0,
585
+ "drop_rate_pct_n": 96,
586
+ "drop_rate_pct_overall": 0.0,
587
+ "dropped_tokens_sum_all_records": 0.0,
588
+ "duplicate_vector_mismatches": 0,
589
+ "experts_per_rank": "32",
590
+ "force_balance": "0",
591
+ "global_batch_size": "8",
592
+ "global_snapshots": 12,
593
+ "incomplete_snapshots": 0,
594
+ "malformed_records": 0,
595
+ "moe_layers": "1",
596
+ "num_experts": "256",
597
+ "num_layers": "4",
598
+ "observed_dropped_tokens": 0.0,
599
+ "peak_demand_max": 33097.0,
600
+ "peak_demand_mean": 32768.0,
601
+ "peak_demand_min": 31280.0,
602
+ "peak_demand_n": 96,
603
+ "peak_rank_overflow_max": 0.0,
604
+ "peak_rank_overflow_mean": 0.0,
605
+ "peak_rank_overflow_min": 0.0,
606
+ "peak_rank_overflow_n": 96,
607
+ "post_physical_rank_cv_max": 0.018971492434756838,
608
+ "post_physical_rank_cv_mean": 0.014175223070035277,
609
+ "post_physical_rank_cv_min": 0.010671796788645344,
610
+ "post_physical_rank_cv_n": 12,
611
+ "post_physical_rank_imax_max": 1.010040283203125,
612
+ "post_physical_rank_imax_mean": 1.0100301106770833,
613
+ "post_physical_rank_imax_min": 1.010009765625,
614
+ "post_physical_rank_imax_n": 12,
615
+ "pre_home_rank_cv_max": 0.3967451453024771,
616
+ "pre_home_rank_cv_mean": 0.18882615919733423,
617
+ "pre_home_rank_cv_min": 0.08482749505976411,
618
+ "pre_home_rank_cv_n": 12,
619
+ "pre_home_rank_imax_max": 1.568817138671875,
620
+ "pre_home_rank_imax_mean": 1.2910486857096355,
621
+ "pre_home_rank_imax_min": 1.114654541015625,
622
+ "pre_home_rank_imax_n": 12,
623
+ "records": {
624
+ "drop_dispatch": 96,
625
+ "drop_experts": 96,
626
+ "drop_setup": 96,
627
+ "ultraep_drop": 96,
628
+ "ultraep_post": 96,
629
+ "ultraep_pre": 96
630
+ },
631
+ "required_capacity_max": 1.01004,
632
+ "required_capacity_mean": 1.0000000312500001,
633
+ "required_capacity_min": 0.95459,
634
+ "required_capacity_n": 96,
635
+ "router_balancing": "seq_aux_loss",
636
+ "run_dir": "ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p25/rep1",
637
+ "seq_length": "4096",
638
+ "shared_expert_overlap": "0",
639
+ "topk": "8",
640
+ "valid_max_rank_reduction_max": 0.35619662691851306,
641
+ "valid_max_rank_reduction_mean": 0.2084693322073339,
642
+ "valid_max_rank_reduction_min": 0.09388090349075975,
643
+ "valid_max_rank_reduction_n": 12,
644
+ "zero_drop": 1
645
+ },
646
+ {
647
+ "arm": "forced",
648
+ "assignment_sum_mismatches": 0,
649
+ "aux_loss_coeff": "1e-4",
650
+ "backend": "ultraep",
651
+ "branch": "ultraep-token-drop-analysis",
652
+ "capacity": "1.25",
653
+ "capacity_mode": "static",
654
+ "commit": "eaf829e7",
655
+ "config": "DEEPSEEK_V3_L4",
656
+ "cuda_graph": "0",
657
+ "dense_first_layers": "3",
658
+ "drop_expected_assignments": 3145728.0,
659
+ "drop_rate_pct_max": 0.0,
660
+ "drop_rate_pct_mean": 0.0,
661
+ "drop_rate_pct_min": 0.0,
662
+ "drop_rate_pct_n": 96,
663
+ "drop_rate_pct_overall": 0.0,
664
+ "dropped_tokens_sum_all_records": 0.0,
665
+ "duplicate_vector_mismatches": 0,
666
+ "experts_per_rank": "32",
667
+ "force_balance": "1",
668
+ "global_batch_size": "8",
669
+ "global_snapshots": 12,
670
+ "incomplete_snapshots": 0,
671
+ "malformed_records": 0,
672
+ "moe_layers": "1",
673
+ "num_experts": "256",
674
+ "num_layers": "4",
675
+ "observed_dropped_tokens": 0.0,
676
+ "peak_demand_max": 33097.0,
677
+ "peak_demand_mean": 32768.0,
678
+ "peak_demand_min": 32330.0,
679
+ "peak_demand_n": 96,
680
+ "peak_rank_overflow_max": 0.0,
681
+ "peak_rank_overflow_mean": 0.0,
682
+ "peak_rank_overflow_min": 0.0,
683
+ "peak_rank_overflow_n": 96,
684
+ "post_physical_rank_cv_max": 0.007872884560844515,
685
+ "post_physical_rank_cv_mean": 0.006243076586697642,
686
+ "post_physical_rank_cv_min": 0.0030538170542637444,
687
+ "post_physical_rank_cv_n": 12,
688
+ "post_physical_rank_imax_max": 1.010040283203125,
689
+ "post_physical_rank_imax_mean": 1.0086746215820312,
690
+ "post_physical_rank_imax_min": 1.00408935546875,
691
+ "post_physical_rank_imax_n": 12,
692
+ "pre_home_rank_cv_max": 0.009139900015747392,
693
+ "pre_home_rank_cv_mean": 0.006980946768958179,
694
+ "pre_home_rank_cv_min": 0.0030538170542637444,
695
+ "pre_home_rank_cv_n": 12,
696
+ "pre_home_rank_imax_max": 1.014892578125,
697
+ "pre_home_rank_imax_mean": 1.0104904174804688,
698
+ "pre_home_rank_imax_min": 1.00408935546875,
699
+ "pre_home_rank_imax_n": 12,
700
+ "records": {
701
+ "drop_dispatch": 96,
702
+ "drop_experts": 96,
703
+ "drop_setup": 96,
704
+ "ultraep_drop": 96,
705
+ "ultraep_post": 96,
706
+ "ultraep_pre": 96
707
+ },
708
+ "required_capacity_max": 1.01004,
709
+ "required_capacity_mean": 0.9999999791666667,
710
+ "required_capacity_min": 0.986633,
711
+ "required_capacity_n": 96,
712
+ "router_balancing": "seq_aux_loss",
713
+ "run_dir": "ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1",
714
+ "seq_length": "4096",
715
+ "shared_expert_overlap": "0",
716
+ "topk": "8",
717
+ "valid_max_rank_reduction_max": 0.004841231657445273,
718
+ "valid_max_rank_reduction_mean": 0.0017907127529931067,
719
+ "valid_max_rank_reduction_min": 0.0,
720
+ "valid_max_rank_reduction_n": 12,
721
+ "zero_drop": 1
722
+ }
723
+ ]
granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/probe_summary_all.tsv ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ run_dir arm config backend num_experts topk experts_per_rank num_layers moe_layers dense_first_layers seq_length global_batch_size capacity_mode capacity force_balance cuda_graph branch commit router_balancing aux_loss_coeff shared_expert_overlap assignment_sum_mismatches cold_expert_fraction_max cold_expert_fraction_mean cold_expert_fraction_min cold_expert_fraction_n drop_accounting_source drop_expected_assignments drop_rate_pct_max drop_rate_pct_mean drop_rate_pct_min drop_rate_pct_n drop_rate_pct_overall dropped_tokens_sum_all_records duplicate_vector_mismatches echo_logged_load_reduction_valid echo_logged_pre_rank_metric_valid echo_moved_assignment_fraction_max echo_moved_assignment_fraction_mean echo_moved_assignment_fraction_min echo_moved_assignment_fraction_n global_snapshots hottest_expert_token_fraction_max hottest_expert_token_fraction_mean hottest_expert_token_fraction_min hottest_expert_token_fraction_n incomplete_snapshots malformed_records moonep_bytes_global_snapshots moonep_bytes_incomplete_snapshots moonep_bytes_malformed_records moonep_global_combine_payload_bytes_max moonep_global_combine_payload_bytes_mean moonep_global_combine_payload_bytes_min moonep_global_combine_payload_bytes_n moonep_global_dispatch_payload_bytes_max moonep_global_dispatch_payload_bytes_mean moonep_global_dispatch_payload_bytes_min moonep_global_dispatch_payload_bytes_n moonep_global_prefetch_bytes_max moonep_global_prefetch_bytes_mean moonep_global_prefetch_bytes_min moonep_global_prefetch_bytes_n moonep_global_reduce_bytes_max moonep_global_reduce_bytes_mean moonep_global_reduce_bytes_min moonep_global_reduce_bytes_n moonep_global_token_bytes_max moonep_global_token_bytes_mean moonep_global_token_bytes_min moonep_global_token_bytes_n moonep_global_total_nvlink_bytes_max moonep_global_total_nvlink_bytes_mean moonep_global_total_nvlink_bytes_min moonep_global_total_nvlink_bytes_n moonep_global_weight_movement_bytes_max moonep_global_weight_movement_bytes_mean moonep_global_weight_movement_bytes_min moonep_global_weight_movement_bytes_n moonep_global_weight_to_token_ratio_max moonep_global_weight_to_token_ratio_mean moonep_global_weight_to_token_ratio_min moonep_global_weight_to_token_ratio_n moonep_remote_demand_max moonep_remote_demand_mean moonep_remote_demand_min moonep_remote_demand_n moonep_slot_utilization_max moonep_slot_utilization_mean moonep_slot_utilization_min moonep_slot_utilization_n moonep_unused_prefetch_slots_max moonep_unused_prefetch_slots_mean moonep_unused_prefetch_slots_min moonep_unused_prefetch_slots_n observed_dropped_tokens peak_demand_max peak_demand_mean peak_demand_min peak_demand_n peak_rank_overflow_max peak_rank_overflow_mean peak_rank_overflow_min peak_rank_overflow_n post_physical_rank_cv_max post_physical_rank_cv_mean post_physical_rank_cv_min post_physical_rank_cv_n post_physical_rank_imax_max post_physical_rank_imax_mean post_physical_rank_imax_min post_physical_rank_imax_n pre_home_rank_cv_max pre_home_rank_cv_mean pre_home_rank_cv_min pre_home_rank_cv_n pre_home_rank_imax_max pre_home_rank_imax_mean pre_home_rank_imax_min pre_home_rank_imax_n raw_expert_cv_max raw_expert_cv_mean raw_expert_cv_min raw_expert_cv_n raw_expert_imax_ceiling raw_expert_imax_hit_ceiling raw_expert_imax_max raw_expert_imax_mean raw_expert_imax_min raw_expert_imax_n required_capacity_max required_capacity_mean required_capacity_min required_capacity_n saturated_dispatch_rows valid_max_rank_reduction_max valid_max_rank_reduction_mean valid_max_rank_reduction_min valid_max_rank_reduction_n zero_drop
2
+ echo/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap1p13671875/rep1 natural DEEPSEEK_V3_L4 echo 256 8 32 4 1 3 4096 8 static 1.13671875 0 0 token-drop-analysis b099fd84 seq_aux_loss 1e-4 0 0 0.0 0.0 0.0 12 dispatch 3145728.0 0.0 0 0 0.5 0.5 0.5 12 12 0.279449462890625 0.16472371419270834 0.09417724609375 12 0 0 0 0.0 0.0 0.0 12 1.0 1.0 1.0 12 0.3949158995862076 0.18817834176075468 0.07608345138535468 12 1.563079833984375 1.291656494140625 1.098876953125 12 1.4608217820834073 0.8516673799234088 0.5323456097372459 12 32.0 0 8.9423828125 5.271158854166667 3.013671875 12 1.08203125 1.0650227864583333 1.046875 96 0 0.3602374118979285 0.21588444472672905 0.08998000444345701 12 1
3
+ echo/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap1p13671875__forcebal/rep1 forced DEEPSEEK_V3_L4 echo 256 8 32 4 1 3 4096 8 static 1.13671875 1 0 token-drop-analysis b099fd84 seq_aux_loss 1e-4 0 0 0.0 0.0 0.0 12 dispatch 3145728.0 0.0 0 0 0.5 0.5 0.5 12 12 0.03521728515625 0.034426371256510414 0.033843994140625 12 0 0 0 0.0 0.0 0.0 12 1.0 1.0 1.0 12 0.009139900015747392 0.006980946768958179 0.0030538170542637444 12 1.014892578125 1.0104904174804688 1.00408935546875 12 0.03876371556245609 0.03570960046443938 0.029816782685728113 12 32.0 0 1.126953125 1.1016438802083333 1.0830078125 12 1.08203125 1.0631510416666667 1.04296875 96 0 0.01467404378157325 0.01036908147130012 0.004072700747674914 12 1
4
+ echo/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__echo__matched__seq4096__gbs8__cap2p00/rep1 natural DEEPSEEK_V3_L4 echo 256 8 32 4 1 3 4096 8 static 2.00 0 0 token-drop-analysis b099fd84 seq_aux_loss 1e-4 0 0 0.0 0.0 0.0 12 dispatch 3145728.0 0.0 0 0 0.5 0.5 0.5 12 12 0.279449462890625 0.16472371419270834 0.09417724609375 12 0 0 0 0.0 0.0 0.0 12 1.0 1.0 1.0 12 0.3949158995862076 0.18817834176075468 0.07608345138535468 12 1.563079833984375 1.291656494140625 1.098876953125 12 1.4608217820834073 0.8516673799234088 0.5323456097372459 12 32.0 0 8.9423828125 5.271158854166667 3.013671875 12 1.08203125 1.0650227864583333 1.046875 96 0 0.3602374118979285 0.21588444472672905 0.08998000444345701 12 1
5
+ moonep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__async/rep1 natural DEEPSEEK_V3_L4 moonep 256 8 32 4 1 3 4096 8 none none 0 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 seq_aux_loss 1e-4 0 0.0 0.0 0.0 10 10 0.277313232421875 0.1758575439453125 0.092315673828125 10 0 0 10 0 0 3822120960.0 3501221068.8 3346653184.0 10 3822120960.0 3501221068.8 3346653184.0 10 2466250752.0 1867304140.8 1409286144.0 10 2466250752.0 1867304140.8 1409286144.0 10 7646072992.0 7004270806.4 6695137352.0 10 12578574496.0 10738879088.0 9546567752.0 10 4932501504.0 3734608281.6 2818572288.0 10 0.6762953708910178 0.5305300760379464 0.4096318958168878 10 4.0 1.325 0.0 80 0.125 0.04140625 0.0 80 64.0 61.35 56.0 80 0.054662301330415754 0.03173931531099595 0.019783416729893442 10 1.0904317633276406 1.051604154450151 1.0214255000518957 10 0.39536184425362714 0.19281063246441604 0.08594336137506545 10 1.55706787109375 1.3085418701171876 1.113128662109375 10 1.4500281317834722 0.9091278053879644 0.5606721744656151 10 32.0 0 8.8740234375 5.62744140625 2.9541015625 10
6
+ moonep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1 forced DEEPSEEK_V3_L4 moonep 256 8 32 4 1 3 4096 8 none none 1 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 seq_aux_loss 1e-4 0 0.0 0.0 0.0 10 10 0.03497314453125 0.034405517578125 0.03399658203125 10 0 0 10 0 0 3276406784.0 3268117708.8 3260407808.0 10 3276406784.0 3268117708.8 3260407808.0 10 1233125376.0 1233125376.0 1233125376.0 10 1233125376.0 1233125376.0 1233125376.0 10 6554650816.0 6538068851.2 6522648928.0 10 9020901568.0 9004319603.2 8988899680.0 10 2466250752.0 2466250752.0 2466250752.0 10 0.37810570202744476 0.37721470246811023 0.3762596698484449 10 1.0 0.875 0.0 80 0.03125 0.02734375 0.0 80 64.0 62.25 62.0 80 0.00471813998380015 0.003239235393271 0.0017429035970738013 10 1.0078812597749087 1.0051071175236657 1.002130562069864 10 0.009139900015747392 0.007327007135739279 0.0030538170542637444 10 1.014892578125 1.0111572265625 1.00408935546875 10 0.03876371556245609 0.0363651698149183 0.0339868829612938 10 32.0 0 1.119140625 1.1009765625 1.087890625 10
7
+ ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p0625/rep1 natural DEEPSEEK_V3_L4 ultraep 256 8 32 4 1 3 4096 8 static 1.0625 0 0 ultraep-token-drop-analysis eaf829e7 seq_aux_loss 1e-4 0 0 3145728.0 0.0 0.0 0.0 96 0.0 0.0 0 12 0 0 0.0 33097.0 32768.0 31546.0 96 0.0 0.0 0.0 96 0.017713692879015048 0.014273621042617956 0.010671796788645344 12 1.010040283203125 1.010022481282552 1.010009765625 12 0.39595445620995434 0.18964812495168137 0.08292629783502173 12 1.5654296875 1.2918370564778645 1.105194091796875 12 1.01004 1.0000000312500001 0.962708 96 0.3548034934497817 0.20816004531911061 0.08612453403285932 12 1
8
+ ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p25/rep1 natural DEEPSEEK_V3_L4 ultraep 256 8 32 4 1 3 4096 8 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 seq_aux_loss 1e-4 0 0 3145728.0 0.0 0.0 0.0 96 0.0 0.0 0 12 0 0 0.0 33097.0 32768.0 31280.0 96 0.0 0.0 0.0 96 0.018971492434756838 0.014175223070035277 0.010671796788645344 12 1.010040283203125 1.0100301106770833 1.010009765625 12 0.3967451453024771 0.18882615919733423 0.08482749505976411 12 1.568817138671875 1.2910486857096355 1.114654541015625 12 1.01004 1.0000000312500001 0.95459 96 0.35619662691851306 0.2084693322073339 0.09388090349075975 12 1
9
+ ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1 forced DEEPSEEK_V3_L4 ultraep 256 8 32 4 1 3 4096 8 static 1.25 1 0 ultraep-token-drop-analysis eaf829e7 seq_aux_loss 1e-4 0 0 3145728.0 0.0 0.0 0.0 96 0.0 0.0 0 12 0 0 0.0 33097.0 32768.0 32330.0 96 0.0 0.0 0.0 96 0.007872884560844515 0.006243076586697642 0.0030538170542637444 12 1.010040283203125 1.0086746215820312 1.00408935546875 12 0.009139900015747392 0.006980946768958179 0.0030538170542637444 12 1.014892578125 1.0104904174804688 1.00408935546875 12 1.01004 0.9999999791666667 0.986633 96 0.004841231657445273 0.0017907127529931067 0.0 12 1
granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/steps.tsv CHANGED
@@ -4,3 +4,4 @@ OK ultraep tierbprobe 8 2026-09-21T06:46:22+00:00 2026-09-21T06:49:53+00:00 /wor
4
  OK moonep tierbprobe 8 2026-09-21T06:49:55+00:00 2026-09-21T06:52:36+00:00 /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/moonep/tierbprobe
5
  OK echo tierb 8 2026-09-21T06:52:38+00:00 2026-09-21T06:56:36+00:00 /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/echo/tierb
6
  OK ultraep tierb 8 2026-09-21T06:56:38+00:00 2026-09-21T07:00:08+00:00 /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/ultraep/tierb
 
 
4
  OK moonep tierbprobe 8 2026-09-21T06:49:55+00:00 2026-09-21T06:52:36+00:00 /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/moonep/tierbprobe
5
  OK echo tierb 8 2026-09-21T06:52:38+00:00 2026-09-21T06:56:36+00:00 /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/echo/tierb
6
  OK ultraep tierb 8 2026-09-21T06:56:38+00:00 2026-09-21T07:00:08+00:00 /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/ultraep/tierb
7
+ OK moonep tierb 8 2026-09-21T07:00:11+00:00 2026-09-21T07:04:03+00:00 /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/moonep/tierb
granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/study.log CHANGED
@@ -371,3 +371,65 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
371
  [guard] iter 3/12 (60s)
372
  [guard] iter 12/12 (70s)
373
  [guard] tearing down process group (outcome=DONE)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
371
  [guard] iter 3/12 (60s)
372
  [guard] iter 12/12 (70s)
373
  [guard] tearing down process group (outcome=DONE)
374
+ [guard] iter 3/12 (60s)
375
+ [guard] iter 12/12 (70s)
376
+ [guard] tearing down process group (outcome=DONE)
377
+ [guard] iter 1/12 (60s)
378
+ [guard] iter 12/12 (70s)
379
+ [guard] tearing down process group (outcome=DONE)
380
+ --------------------------------------------------------------------------
381
+
382
+ ==============================================================
383
+ SWEEP COMPLETE ok=3 failed=0
384
+ ==============================================================
385
+ config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
386
+ DEEPSEEK_V3_L4 moonep matched 32 4096 8 none none 1 OK 192.67 192.30 97849 /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/moonep/tierb/tierb_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__async/rep1
387
+ DEEPSEEK_V3_L4 moonep matched 32 4096 8 none none 2 OK 192.97 192.10 97849 /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/moonep/tierb/tierb_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__async/rep2
388
+ DEEPSEEK_V3_L4 moonep matched 32 4096 8 none none 3 OK 192.90 192.10 97849 /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/moonep/tierb/tierb_gbs8/DEEPSEEK_V3_L4__moonep__matched__seq4096__gbs8__capnone__async/rep3
389
+
390
+ Results: /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/moonep/tierb/tierb_gbs8/results.tsv
391
+ Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/moonep/tierb/tierb_gbs8
392
+ ==============================================================
393
+
394
+ Campaign results: /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/moonep/tierb
395
+ Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/moonep/tierb --compare
396
+ Already logged in to Hugging Face as: AbhishekGhosh1998
397
+ uploaded: /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/moonep/tierb -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/moonep/tierb
398
+ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
399
+
400
+ === ROLL-UP
401
+ 17 runs ok=17 oom=0 -> /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/collected_all.tsv
402
+
403
+ WARNING: 3 run(s) mention capacity saturation. A run that drops assignments does less expert work and must not be quoted as a win:
404
+ ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p0625/rep1: WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=1 total_expected_recv=32094 total_dropped_token
405
+ ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p25/rep1: WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=1 total_expected_recv=33097 total_dropped_token
406
+ ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1: WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=1 total_expected_recv=33019 total_dropped_token
407
+
408
+ phase config seq gbs policy rc cg asy backend iter_ms vs_best
409
+ -------------------------------------------------------------------------------------
410
+ tierb DEEPSEEK_V3_L4 4096 8 matched none 1 0 ultraep 195.66 1.00x
411
+ echo 197.67 1.01x
412
+
413
+ tierb DEEPSEEK_V3_L4 4096 8 matched none 1 1 moonep 192.90 1.00x
414
+
415
+ tierbprobeDEEPSEEK_V3_L4 4096 8 matched none 0 0 ultraep 210.03 1.00x
416
+ echo 223.55 1.06x
417
+
418
+ tierbprobeDEEPSEEK_V3_L4 4096 8 matched none 0 1 moonep 208.62 1.00x
419
+
420
+ 8 probe runs -> /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4/probe_summary_all.tsv
421
+ run arm cap rawExpI preRankI postRankI drop% reqCap
422
+ --------------------------------------------------------------------------------------------------------------------
423
+ echo/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__echo natural 1.13671875 5.2712 1.2917 1.0000 1.0820
424
+ echo/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__echo forced 1.13671875 1.1016 1.0105 1.0000 1.0820
425
+ echo/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__echo natural 2.00 5.2712 1.2917 1.0000 1.0820
426
+ moonep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__mo natural none 5.6274 1.3085 1.0516
427
+ moonep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__mo forced none 1.1010 1.0112 1.0051
428
+ ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__u natural 1.0625 1.2918 1.0100 0.0000 1.0100
429
+ ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__u natural 1.25 1.2910 1.0100 0.0000 1.0100
430
+ ultraep/tierbprobe/tierbprobe_gbs8/DEEPSEEK_V3_L4__u forced 1.25 1.0105 1.0087 0.0000 1.0100
431
+
432
+ ECHO note: echo_logged_pre_rank_metric_valid=0 and echo_logged_load_reduction_valid=0 are intentional.
433
+ The branch's built-in fields mix source-rank and destination-rank axes; this tool reconstructs both globally.
434
+ no memory-dump runs found under /workspace/granularity_next_b200/tierb_models/attempts/DEEPSEEK_V3_L4
435
+ Already logged in to Hugging Face as: AbhishekGhosh1998