AbhishekGhosh1998 commited on
Commit
ddcd788
·
verified ·
1 Parent(s): 667bb86

granularity results granularity_next_b200/tierb_models/attempts/KIMI_K2_L2 @ 2026-09-21T07:54:42Z from sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0

Browse files
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/.completed/moonep__tierb__gbs8 ADDED
@@ -0,0 +1 @@
 
 
1
+ 2026-09-21T07:54:39+00:00
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/STATUS.txt CHANGED
@@ -1,11 +1,12 @@
1
  study_root: /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2
2
  gbs: 8
3
- updated: 2026-09-21T07:50:45+00:00
4
- current: moonep__tierb__gbs8
5
 
6
  completed steps:
7
  echo__tierb__gbs8
8
  echo__tierbprobe__gbs8
 
9
  moonep__tierbprobe__gbs8
10
  ultraep__tierb__gbs8
11
  ultraep__tierbprobe__gbs8
 
1
  study_root: /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2
2
  gbs: 8
3
+ updated: 2026-09-21T07:54:41+00:00
4
+ current: idle
5
 
6
  completed steps:
7
  echo__tierb__gbs8
8
  echo__tierbprobe__gbs8
9
+ moonep__tierb__gbs8
10
  moonep__tierbprobe__gbs8
11
  ultraep__tierb__gbs8
12
  ultraep__tierbprobe__gbs8
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/collected_all.tsv ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ run_dir rep phase is_trace async config backend slot_policy slots_per_rank num_experts topk moe_ffn_hidden experts_per_rank expert_params num_layers seq_length global_batch_size num_microbatches assignments_per_rank mean_tokens_per_expert capacity capacity_mode recompute force_balance cuda_graph memory_dump_mode memory_snapshot_path ultraep_quota_min_tokens shared_expert_overlap branch commit dense_first_layers moe_layers router_balancing aux_loss_coeff router_pre_softmax router_fusion rotary_base qkv_bias source_model standard_num_layers base_rung q_lora_rank kv_lora_rank qk_head_dim qk_pos_emb_head_dim v_head_dim rotary_percent rotary_scaling mscale mscale_all_dim router_groups router_group_topk router_score_function router_expert_bias router_bias_update_rate router_topk_scaling_factor shared_expert_gate apply_layernorm_1p attention_output_gate linear_attention_type linear_attention_freq linear_conv_kernel_dim linear_key_head_dim linear_value_head_dim linear_num_key_heads linear_num_value_heads mtp_num_layers mtp_loss_scaling_factor no_weight_decay_cond_type max_position_embeddings init_method_std status iters_seen iter_ms_mean iter_ms_min iter_ms_stdev tflops peak_mib_sampled megatron_max_reserved_mib drop_warning
2
+ echo/tierb/tierb_gbs8/KIMI_K2_L2__echo__matched__seq4096__gbs8__cap1p17578125/rep1 rep1 tierb 0 0 KIMI_K2_L2 echo matched 48 384 8 2048 48 44040192 2 4096 8 1 32768 682 1.17578125 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 1 1 seq_aux_loss 0 0 0 50000 0 Kimi-K2 61 1536 512 128 64 128 1.0 32 1.0 1.0 1 1 sigmoid 1 1e-3 2.827 0 0 0 0 0.1 4096 0.02 OK 12 119.40 118.30 0.88 296.3 132219 74666
3
+ echo/tierb/tierb_gbs8/KIMI_K2_L2__echo__matched__seq4096__gbs8__cap1p17578125/rep2 rep2 tierb 0 0 KIMI_K2_L2 echo matched 48 384 8 2048 48 44040192 2 4096 8 1 32768 682 1.17578125 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 1 1 seq_aux_loss 0 0 0 50000 0 Kimi-K2 61 1536 512 128 64 128 1.0 32 1.0 1.0 1 1 sigmoid 1 1e-3 2.827 0 0 0 0 0.1 4096 0.02 OK 12 119.74 118.30 1.33 295.4 132219 74666
4
+ echo/tierb/tierb_gbs8/KIMI_K2_L2__echo__matched__seq4096__gbs8__cap1p17578125/rep3 rep3 tierb 0 0 KIMI_K2_L2 echo matched 48 384 8 2048 48 44040192 2 4096 8 1 32768 682 1.17578125 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 1 1 seq_aux_loss 0 0 0 50000 0 Kimi-K2 61 1536 512 128 64 128 1.0 32 1.0 1.0 1 1 sigmoid 1 1e-3 2.827 0 0 0 0 0.1 4096 0.02 OK 12 119.67 119.00 0.56 295.6 132219 74666
5
+ echo/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__echo__matched__seq4096__gbs8__cap1p17578125/rep1 rep1 tierbprobe 0 0 KIMI_K2_L2 echo matched 48 384 8 2048 48 44040192 2 4096 8 1 32768 682 1.17578125 static none 0 0 none 0 token-drop-analysis b099fd84 1 1 seq_aux_loss 0 0 0 50000 0 Kimi-K2 61 1536 512 128 64 128 1.0 32 1.0 1.0 1 1 sigmoid 1 1e-3 2.827 0 0 0 0 0.1 4096 0.02 OK 12 144.69 144.10 0.27 244.5 124715 74552
6
+ echo/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__echo__matched__seq4096__gbs8__cap1p17578125__forcebal/rep1 rep1 tierbprobe 0 0 KIMI_K2_L2 echo matched 48 384 8 2048 48 44040192 2 4096 8 1 32768 682 1.17578125 static none 1 0 none 0 token-drop-analysis b099fd84 1 1 seq_aux_loss 0 0 0 50000 0 Kimi-K2 61 1536 512 128 64 128 1.0 32 1.0 1.0 1 1 sigmoid 1 1e-3 2.827 0 0 0 0 0.1 4096 0.02 OK 12 142.60 142.10 0.42 248.1 124703 74552
7
+ echo/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__echo__matched__seq4096__gbs8__cap2p00/rep1 rep1 tierbprobe 0 0 KIMI_K2_L2 echo matched 48 384 8 2048 48 44040192 2 4096 8 1 32768 682 2.00 static none 0 0 none 0 token-drop-analysis b099fd84 1 1 seq_aux_loss 0 0 0 50000 0 Kimi-K2 61 1536 512 128 64 128 1.0 32 1.0 1.0 1 1 sigmoid 1 1e-3 2.827 0 0 0 0 0.1 4096 0.02 OK 12 145.50 144.20 1.21 243.2 126049 75400
8
+ moonep/tierb/tierb_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__async/rep1 rep1 tierb 0 1 KIMI_K2_L2 moonep matched 48 384 8 2048 48 44040192 2 4096 8 1 32768 682 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 1 1 seq_aux_loss 0 0 0 50000 0 Kimi-K2 61 1536 512 128 64 128 1.0 32 1.0 1.0 1 1 sigmoid 1 1e-3 2.827 0 0 0 0 0.1 4096 0.02 OK 12 108.60 107.70 0.59 325.7 86491 45790
9
+ moonep/tierb/tierb_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__async/rep2 rep2 tierb 0 1 KIMI_K2_L2 moonep matched 48 384 8 2048 48 44040192 2 4096 8 1 32768 682 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 1 1 seq_aux_loss 0 0 0 50000 0 Kimi-K2 61 1536 512 128 64 128 1.0 32 1.0 1.0 1 1 sigmoid 1 1e-3 2.827 0 0 0 0 0.1 4096 0.02 OK 12 108.87 108.10 0.57 324.9 86491 45790
10
+ moonep/tierb/tierb_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__async/rep3 rep3 tierb 0 1 KIMI_K2_L2 moonep matched 48 384 8 2048 48 44040192 2 4096 8 1 32768 682 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 1 1 seq_aux_loss 0 0 0 50000 0 Kimi-K2 61 1536 512 128 64 128 1.0 32 1.0 1.0 1 1 sigmoid 1 1e-3 2.827 0 0 0 0 0.1 4096 0.02 OK 12 109.00 108.40 0.47 324.6 86491 45790
11
+ moonep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__async/rep1 rep1 tierbprobe 0 1 KIMI_K2_L2 moonep matched 48 384 8 2048 48 44040192 2 4096 8 1 32768 682 none none none 0 0 none 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 1 1 seq_aux_loss 0 0 0 50000 0 Kimi-K2 61 1536 512 128 64 128 1.0 32 1.0 1.0 1 1 sigmoid 1 1e-3 2.827 0 0 0 0 0.1 4096 0.02 OK 12 121.60 120.60 0.79 290.9 86157 45788
12
+ moonep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1 rep1 tierbprobe 0 1 KIMI_K2_L2 moonep matched 48 384 8 2048 48 44040192 2 4096 8 1 32768 682 none none none 1 0 none 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 1 1 seq_aux_loss 0 0 0 50000 0 Kimi-K2 61 1536 512 128 64 128 1.0 32 1.0 1.0 1 1 sigmoid 1 1e-3 2.827 0 0 0 0 0.1 4096 0.02 OK 12 120.39 119.90 0.41 293.9 86157 45788
13
+ ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep1 rep1 tierb 0 0 KIMI_K2_L2 ultraep matched 48 384 8 2048 48 44040192 2 4096 8 1 32768 682 1.17578125 static none 0 1 none 85 0 ultraep-bracing-regions-through-kernels 03d57ef7 1 1 seq_aux_loss 0 0 0 50000 0 Kimi-K2 61 1536 512 128 64 128 1.0 32 1.0 1.0 1 1 sigmoid 1 1e-3 2.827 0 0 0 0 0.1 4096 0.02 OK 12 110.06 108.20 2.46 321.6 90375 54394
14
+ ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep2 rep2 tierb 0 0 KIMI_K2_L2 ultraep matched 48 384 8 2048 48 44040192 2 4096 8 1 32768 682 1.17578125 static none 0 1 none 85 0 ultraep-bracing-regions-through-kernels 03d57ef7 1 1 seq_aux_loss 0 0 0 50000 0 Kimi-K2 61 1536 512 128 64 128 1.0 32 1.0 1.0 1 1 sigmoid 1 1e-3 2.827 0 0 0 0 0.1 4096 0.02 OK 12 110.10 108.60 2.30 321.4 90375 54394
15
+ ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep3 rep3 tierb 0 0 KIMI_K2_L2 ultraep matched 48 384 8 2048 48 44040192 2 4096 8 1 32768 682 1.17578125 static none 0 1 none 85 0 ultraep-bracing-regions-through-kernels 03d57ef7 1 1 seq_aux_loss 0 0 0 50000 0 Kimi-K2 61 1536 512 128 64 128 1.0 32 1.0 1.0 1 1 sigmoid 1 1e-3 2.827 0 0 0 0 0.1 4096 0.02 OK 12 110.09 108.50 2.25 321.5 90375 54394
16
+ ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p0625/rep1 rep1 tierbprobe 0 0 KIMI_K2_L2 ultraep matched 48 384 8 2048 48 44040192 2 4096 8 1 32768 682 1.0625 static none 0 0 none 85 0 ultraep-token-drop-analysis eaf829e7 1 1 seq_aux_loss 0 0 0 50000 0 Kimi-K2 61 1536 512 128 64 128 1.0 32 1.0 1.0 1 1 sigmoid 1 1e-3 2.827 0 0 0 0 0.1 4096 0.02 OK 12 125.07 122.00 2.42 282.9 89513 54172 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=1 total_expected_recv=33098 total_dropped_token
17
+ ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25/rep1 rep1 tierbprobe 0 0 KIMI_K2_L2 ultraep matched 48 384 8 2048 48 44040192 2 4096 8 1 32768 682 1.25 static none 0 0 none 85 0 ultraep-token-drop-analysis eaf829e7 1 1 seq_aux_loss 0 0 0 50000 0 Kimi-K2 61 1536 512 128 64 128 1.0 32 1.0 1.0 1 1 sigmoid 1 1e-3 2.827 0 0 0 0 0.1 4096 0.02 OK 12 124.24 122.40 2.44 284.9 89825 54522 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=1 total_expected_recv=33098 total_dropped_token
18
+ ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1 rep1 tierbprobe 0 0 KIMI_K2_L2 ultraep matched 48 384 8 2048 48 44040192 2 4096 8 1 32768 682 1.25 static none 1 0 none 85 0 ultraep-token-drop-analysis eaf829e7 1 1 seq_aux_loss 0 0 0 50000 0 Kimi-K2 61 1536 512 128 64 128 1.0 32 1.0 1.0 1 1 sigmoid 1 1e-3 2.827 0 0 0 0 0.1 4096 0.02 OK 12 116.93 115.90 1.42 302.5 89825 54508 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=1 total_expected_recv=32672 total_dropped_token
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/hf_watcher.log CHANGED
@@ -7,3 +7,5 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
7
  uploaded: /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2
8
  https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
9
  [2026-09-21T07:52:54+00:00] backup pass
 
 
 
7
  uploaded: /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2
8
  https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
9
  [2026-09-21T07:52:54+00:00] backup pass
10
+ uploaded: /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2
11
+ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/probe_summary_all.json ADDED
@@ -0,0 +1,723 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "arm": "natural",
4
+ "assignment_sum_mismatches": 0,
5
+ "aux_loss_coeff": "0",
6
+ "backend": "echo",
7
+ "branch": "token-drop-analysis",
8
+ "capacity": "1.17578125",
9
+ "capacity_mode": "static",
10
+ "cold_expert_fraction_max": 0.0,
11
+ "cold_expert_fraction_mean": 0.0,
12
+ "cold_expert_fraction_min": 0.0,
13
+ "cold_expert_fraction_n": 12,
14
+ "commit": "b099fd84",
15
+ "config": "KIMI_K2_L2",
16
+ "cuda_graph": "0",
17
+ "dense_first_layers": "1",
18
+ "drop_accounting_source": "dispatch",
19
+ "drop_expected_assignments": 3145728.0,
20
+ "drop_rate_pct_overall": 0.0,
21
+ "echo_logged_load_reduction_valid": 0,
22
+ "echo_logged_pre_rank_metric_valid": 0,
23
+ "echo_moved_assignment_fraction_max": 0.5,
24
+ "echo_moved_assignment_fraction_mean": 0.5,
25
+ "echo_moved_assignment_fraction_min": 0.5,
26
+ "echo_moved_assignment_fraction_n": 12,
27
+ "experts_per_rank": "48",
28
+ "force_balance": "0",
29
+ "global_batch_size": "8",
30
+ "global_snapshots": 12,
31
+ "hottest_expert_token_fraction_max": 0.30047607421875,
32
+ "hottest_expert_token_fraction_mean": 0.16150410970052084,
33
+ "hottest_expert_token_fraction_min": 0.10980224609375,
34
+ "hottest_expert_token_fraction_n": 12,
35
+ "incomplete_snapshots": 0,
36
+ "malformed_records": 0,
37
+ "moe_layers": "1",
38
+ "num_experts": "384",
39
+ "num_layers": "2",
40
+ "observed_dropped_tokens": 0,
41
+ "post_physical_rank_cv_max": 0.0,
42
+ "post_physical_rank_cv_mean": 0.0,
43
+ "post_physical_rank_cv_min": 0.0,
44
+ "post_physical_rank_cv_n": 12,
45
+ "post_physical_rank_imax_max": 1.0,
46
+ "post_physical_rank_imax_mean": 1.0,
47
+ "post_physical_rank_imax_min": 1.0,
48
+ "post_physical_rank_imax_n": 12,
49
+ "pre_home_rank_cv_max": 0.23490023124048143,
50
+ "pre_home_rank_cv_mean": 0.1314172003975923,
51
+ "pre_home_rank_cv_min": 0.09022193849823706,
52
+ "pre_home_rank_cv_n": 12,
53
+ "pre_home_rank_imax_max": 1.393341064453125,
54
+ "pre_home_rank_imax_mean": 1.2213694254557292,
55
+ "pre_home_rank_imax_min": 1.133331298828125,
56
+ "pre_home_rank_imax_n": 12,
57
+ "raw_expert_cv_max": 1.7754821522938804,
58
+ "raw_expert_cv_mean": 1.0537211199459462,
59
+ "raw_expert_cv_min": 0.7435680098626484,
60
+ "raw_expert_cv_n": 12,
61
+ "raw_expert_imax_ceiling": 48.0,
62
+ "raw_expert_imax_hit_ceiling": 0,
63
+ "raw_expert_imax_max": 14.4228515625,
64
+ "raw_expert_imax_mean": 7.752197265625,
65
+ "raw_expert_imax_min": 5.2705078125,
66
+ "raw_expert_imax_n": 12,
67
+ "records": {
68
+ "drop_dispatch": 96,
69
+ "drop_experts": 96,
70
+ "drop_setup": 96,
71
+ "echo_post": 96,
72
+ "echo_pre": 96
73
+ },
74
+ "required_capacity_max": 1.1171875,
75
+ "required_capacity_mean": 1.0969645182291667,
76
+ "required_capacity_min": 1.0703125,
77
+ "required_capacity_n": 96,
78
+ "router_balancing": "seq_aux_loss",
79
+ "run_dir": "echo/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__echo__matched__seq4096__gbs8__cap1p17578125/rep1",
80
+ "saturated_dispatch_rows": 0,
81
+ "seq_length": "4096",
82
+ "shared_expert_overlap": "0",
83
+ "topk": "8",
84
+ "valid_max_rank_reduction_max": 0.28230063298070396,
85
+ "valid_max_rank_reduction_mean": 0.17806762817557978,
86
+ "valid_max_rank_reduction_min": 0.11764547486334384,
87
+ "valid_max_rank_reduction_n": 12,
88
+ "zero_drop": 1
89
+ },
90
+ {
91
+ "arm": "forced",
92
+ "assignment_sum_mismatches": 0,
93
+ "aux_loss_coeff": "0",
94
+ "backend": "echo",
95
+ "branch": "token-drop-analysis",
96
+ "capacity": "1.17578125",
97
+ "capacity_mode": "static",
98
+ "cold_expert_fraction_max": 0.0,
99
+ "cold_expert_fraction_mean": 0.0,
100
+ "cold_expert_fraction_min": 0.0,
101
+ "cold_expert_fraction_n": 12,
102
+ "commit": "b099fd84",
103
+ "config": "KIMI_K2_L2",
104
+ "cuda_graph": "0",
105
+ "dense_first_layers": "1",
106
+ "drop_accounting_source": "dispatch",
107
+ "drop_expected_assignments": 3145728.0,
108
+ "drop_rate_pct_overall": 0.0,
109
+ "echo_logged_load_reduction_valid": 0,
110
+ "echo_logged_pre_rank_metric_valid": 0,
111
+ "echo_moved_assignment_fraction_max": 0.5,
112
+ "echo_moved_assignment_fraction_mean": 0.5,
113
+ "echo_moved_assignment_fraction_min": 0.5,
114
+ "echo_moved_assignment_fraction_n": 12,
115
+ "experts_per_rank": "48",
116
+ "force_balance": "1",
117
+ "global_batch_size": "8",
118
+ "global_snapshots": 12,
119
+ "hottest_expert_token_fraction_max": 0.0240478515625,
120
+ "hottest_expert_token_fraction_mean": 0.0235595703125,
121
+ "hottest_expert_token_fraction_min": 0.02313232421875,
122
+ "hottest_expert_token_fraction_n": 12,
123
+ "incomplete_snapshots": 0,
124
+ "malformed_records": 0,
125
+ "moe_layers": "1",
126
+ "num_experts": "384",
127
+ "num_layers": "2",
128
+ "observed_dropped_tokens": 0,
129
+ "post_physical_rank_cv_max": 0.0,
130
+ "post_physical_rank_cv_mean": 0.0,
131
+ "post_physical_rank_cv_min": 0.0,
132
+ "post_physical_rank_cv_n": 12,
133
+ "post_physical_rank_imax_max": 1.0,
134
+ "post_physical_rank_imax_mean": 1.0,
135
+ "post_physical_rank_imax_min": 1.0,
136
+ "post_physical_rank_imax_n": 12,
137
+ "pre_home_rank_cv_max": 0.009327227347999425,
138
+ "pre_home_rank_cv_mean": 0.006896456793407019,
139
+ "pre_home_rank_cv_min": 0.0033077098989994554,
140
+ "pre_home_rank_cv_n": 12,
141
+ "pre_home_rank_imax_max": 1.02001953125,
142
+ "pre_home_rank_imax_mean": 1.0104878743489583,
143
+ "pre_home_rank_imax_min": 1.005340576171875,
144
+ "pre_home_rank_imax_n": 12,
145
+ "raw_expert_cv_max": 0.04836014600641425,
146
+ "raw_expert_cv_mean": 0.04494131821754616,
147
+ "raw_expert_cv_min": 0.038893381234906065,
148
+ "raw_expert_cv_n": 12,
149
+ "raw_expert_imax_ceiling": 48.0,
150
+ "raw_expert_imax_hit_ceiling": 0,
151
+ "raw_expert_imax_max": 1.154296875,
152
+ "raw_expert_imax_mean": 1.130859375,
153
+ "raw_expert_imax_min": 1.1103515625,
154
+ "raw_expert_imax_n": 12,
155
+ "records": {
156
+ "drop_dispatch": 96,
157
+ "drop_experts": 96,
158
+ "drop_setup": 96,
159
+ "echo_post": 96,
160
+ "echo_pre": 96
161
+ },
162
+ "required_capacity_max": 1.1328125,
163
+ "required_capacity_mean": 1.1121826171875,
164
+ "required_capacity_min": 1.09375,
165
+ "required_capacity_n": 96,
166
+ "router_balancing": "seq_aux_loss",
167
+ "run_dir": "echo/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__echo__matched__seq4096__gbs8__cap1p17578125__forcebal/rep1",
168
+ "saturated_dispatch_rows": 0,
169
+ "seq_length": "4096",
170
+ "shared_expert_overlap": "0",
171
+ "topk": "8",
172
+ "valid_max_rank_reduction_max": 0.019626615605552896,
173
+ "valid_max_rank_reduction_mean": 0.010357864282149248,
174
+ "valid_max_rank_reduction_min": 0.005312205931457366,
175
+ "valid_max_rank_reduction_n": 12,
176
+ "zero_drop": 1
177
+ },
178
+ {
179
+ "arm": "natural",
180
+ "assignment_sum_mismatches": 0,
181
+ "aux_loss_coeff": "0",
182
+ "backend": "echo",
183
+ "branch": "token-drop-analysis",
184
+ "capacity": "2.00",
185
+ "capacity_mode": "static",
186
+ "cold_expert_fraction_max": 0.0,
187
+ "cold_expert_fraction_mean": 0.0,
188
+ "cold_expert_fraction_min": 0.0,
189
+ "cold_expert_fraction_n": 12,
190
+ "commit": "b099fd84",
191
+ "config": "KIMI_K2_L2",
192
+ "cuda_graph": "0",
193
+ "dense_first_layers": "1",
194
+ "drop_accounting_source": "dispatch",
195
+ "drop_expected_assignments": 3145728.0,
196
+ "drop_rate_pct_overall": 0.0,
197
+ "echo_logged_load_reduction_valid": 0,
198
+ "echo_logged_pre_rank_metric_valid": 0,
199
+ "echo_moved_assignment_fraction_max": 0.5,
200
+ "echo_moved_assignment_fraction_mean": 0.5,
201
+ "echo_moved_assignment_fraction_min": 0.5,
202
+ "echo_moved_assignment_fraction_n": 12,
203
+ "experts_per_rank": "48",
204
+ "force_balance": "0",
205
+ "global_batch_size": "8",
206
+ "global_snapshots": 12,
207
+ "hottest_expert_token_fraction_max": 0.30047607421875,
208
+ "hottest_expert_token_fraction_mean": 0.16150410970052084,
209
+ "hottest_expert_token_fraction_min": 0.10980224609375,
210
+ "hottest_expert_token_fraction_n": 12,
211
+ "incomplete_snapshots": 0,
212
+ "malformed_records": 0,
213
+ "moe_layers": "1",
214
+ "num_experts": "384",
215
+ "num_layers": "2",
216
+ "observed_dropped_tokens": 0,
217
+ "post_physical_rank_cv_max": 0.0,
218
+ "post_physical_rank_cv_mean": 0.0,
219
+ "post_physical_rank_cv_min": 0.0,
220
+ "post_physical_rank_cv_n": 12,
221
+ "post_physical_rank_imax_max": 1.0,
222
+ "post_physical_rank_imax_mean": 1.0,
223
+ "post_physical_rank_imax_min": 1.0,
224
+ "post_physical_rank_imax_n": 12,
225
+ "pre_home_rank_cv_max": 0.23490023124048143,
226
+ "pre_home_rank_cv_mean": 0.1314172003975923,
227
+ "pre_home_rank_cv_min": 0.09022193849823706,
228
+ "pre_home_rank_cv_n": 12,
229
+ "pre_home_rank_imax_max": 1.393341064453125,
230
+ "pre_home_rank_imax_mean": 1.2213694254557292,
231
+ "pre_home_rank_imax_min": 1.133331298828125,
232
+ "pre_home_rank_imax_n": 12,
233
+ "raw_expert_cv_max": 1.7754821522938804,
234
+ "raw_expert_cv_mean": 1.0537211199459462,
235
+ "raw_expert_cv_min": 0.7435680098626484,
236
+ "raw_expert_cv_n": 12,
237
+ "raw_expert_imax_ceiling": 48.0,
238
+ "raw_expert_imax_hit_ceiling": 0,
239
+ "raw_expert_imax_max": 14.4228515625,
240
+ "raw_expert_imax_mean": 7.752197265625,
241
+ "raw_expert_imax_min": 5.2705078125,
242
+ "raw_expert_imax_n": 12,
243
+ "records": {
244
+ "drop_dispatch": 96,
245
+ "drop_experts": 96,
246
+ "drop_setup": 96,
247
+ "echo_post": 96,
248
+ "echo_pre": 96
249
+ },
250
+ "required_capacity_max": 1.1171875,
251
+ "required_capacity_mean": 1.0969645182291667,
252
+ "required_capacity_min": 1.0703125,
253
+ "required_capacity_n": 96,
254
+ "router_balancing": "seq_aux_loss",
255
+ "run_dir": "echo/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__echo__matched__seq4096__gbs8__cap2p00/rep1",
256
+ "saturated_dispatch_rows": 0,
257
+ "seq_length": "4096",
258
+ "shared_expert_overlap": "0",
259
+ "topk": "8",
260
+ "valid_max_rank_reduction_max": 0.28230063298070396,
261
+ "valid_max_rank_reduction_mean": 0.17806762817557978,
262
+ "valid_max_rank_reduction_min": 0.11764547486334384,
263
+ "valid_max_rank_reduction_n": 12,
264
+ "zero_drop": 1
265
+ },
266
+ {
267
+ "arm": "natural",
268
+ "aux_loss_coeff": "0",
269
+ "backend": "moonep",
270
+ "branch": "moonep_backend_async_grad_reduce_token_routing_analysis",
271
+ "capacity": "none",
272
+ "capacity_mode": "none",
273
+ "cold_expert_fraction_max": 0.0,
274
+ "cold_expert_fraction_mean": 0.0,
275
+ "cold_expert_fraction_min": 0.0,
276
+ "cold_expert_fraction_n": 10,
277
+ "commit": "a6a1a221",
278
+ "config": "KIMI_K2_L2",
279
+ "cuda_graph": "0",
280
+ "dense_first_layers": "1",
281
+ "experts_per_rank": "48",
282
+ "force_balance": "0",
283
+ "global_batch_size": "8",
284
+ "global_snapshots": 10,
285
+ "hottest_expert_token_fraction_max": 0.2982177734375,
286
+ "hottest_expert_token_fraction_mean": 0.170477294921875,
287
+ "hottest_expert_token_fraction_min": 0.110565185546875,
288
+ "hottest_expert_token_fraction_n": 10,
289
+ "incomplete_snapshots": 0,
290
+ "malformed_records": 0,
291
+ "moe_layers": "1",
292
+ "moonep_bytes_global_snapshots": 10,
293
+ "moonep_bytes_incomplete_snapshots": 0,
294
+ "moonep_bytes_malformed_records": 0,
295
+ "moonep_global_combine_payload_bytes_max": 4395991040.0,
296
+ "moonep_global_combine_payload_bytes_mean": 4330650419.2,
297
+ "moonep_global_combine_payload_bytes_min": 4301602816.0,
298
+ "moonep_global_combine_payload_bytes_n": 10,
299
+ "moonep_global_dispatch_payload_bytes_max": 4395991040.0,
300
+ "moonep_global_dispatch_payload_bytes_mean": 4330650419.2,
301
+ "moonep_global_dispatch_payload_bytes_min": 4301602816.0,
302
+ "moonep_global_dispatch_payload_bytes_n": 10,
303
+ "moonep_global_prefetch_bytes_max": 1937768448.0,
304
+ "moonep_global_prefetch_bytes_mean": 1603062988.8,
305
+ "moonep_global_prefetch_bytes_min": 1233125376.0,
306
+ "moonep_global_prefetch_bytes_n": 10,
307
+ "moonep_global_reduce_bytes_max": 1937768448.0,
308
+ "moonep_global_reduce_bytes_mean": 1603062988.8,
309
+ "moonep_global_reduce_bytes_min": 1233125376.0,
310
+ "moonep_global_reduce_bytes_n": 10,
311
+ "moonep_global_token_bytes_max": 8793816424.0,
312
+ "moonep_global_token_bytes_mean": 8663126381.6,
313
+ "moonep_global_token_bytes_min": 8605026376.0,
314
+ "moonep_global_token_bytes_n": 10,
315
+ "moonep_global_total_nvlink_bytes_max": 12480563272.0,
316
+ "moonep_global_total_nvlink_bytes_mean": 11869252359.2,
317
+ "moonep_global_total_nvlink_bytes_min": 11211418776.0,
318
+ "moonep_global_total_nvlink_bytes_n": 10,
319
+ "moonep_global_weight_movement_bytes_max": 3875536896.0,
320
+ "moonep_global_weight_movement_bytes_mean": 3206125977.6,
321
+ "moonep_global_weight_movement_bytes_min": 2466250752.0,
322
+ "moonep_global_weight_movement_bytes_n": 10,
323
+ "moonep_global_weight_to_token_ratio_max": 0.4503805946265469,
324
+ "moonep_global_weight_to_token_ratio_mean": 0.37043439150680124,
325
+ "moonep_global_weight_to_token_ratio_min": 0.280452835616301,
326
+ "moonep_global_weight_to_token_ratio_n": 10,
327
+ "moonep_remote_demand_max": 2.0,
328
+ "moonep_remote_demand_mean": 1.1375,
329
+ "moonep_remote_demand_min": 0.0,
330
+ "moonep_remote_demand_n": 80,
331
+ "moonep_slot_utilization_max": 0.041667,
332
+ "moonep_slot_utilization_mean": 0.0236978,
333
+ "moonep_slot_utilization_min": 0.0,
334
+ "moonep_slot_utilization_n": 80,
335
+ "moonep_unused_prefetch_slots_max": 96.0,
336
+ "moonep_unused_prefetch_slots_mean": 93.725,
337
+ "moonep_unused_prefetch_slots_min": 92.0,
338
+ "moonep_unused_prefetch_slots_n": 80,
339
+ "num_experts": "384",
340
+ "num_layers": "2",
341
+ "post_physical_rank_cv_max": 0.022939437595188024,
342
+ "post_physical_rank_cv_mean": 0.016598105480502157,
343
+ "post_physical_rank_cv_min": 0.0017300051649072432,
344
+ "post_physical_rank_cv_n": 10,
345
+ "post_physical_rank_imax_max": 1.0368468027524256,
346
+ "post_physical_rank_imax_mean": 1.0260113076815665,
347
+ "post_physical_rank_imax_min": 1.003556609365738,
348
+ "post_physical_rank_imax_n": 10,
349
+ "pre_home_rank_cv_max": 0.23147253286871253,
350
+ "pre_home_rank_cv_mean": 0.1336617698353712,
351
+ "pre_home_rank_cv_min": 0.08928583511808427,
352
+ "pre_home_rank_cv_n": 10,
353
+ "pre_home_rank_imax_max": 1.389129638671875,
354
+ "pre_home_rank_imax_mean": 1.230499267578125,
355
+ "pre_home_rank_imax_min": 1.129058837890625,
356
+ "pre_home_rank_imax_n": 10,
357
+ "raw_expert_cv_max": 1.7760509001770588,
358
+ "raw_expert_cv_mean": 1.0733812106366694,
359
+ "raw_expert_cv_min": 0.7388867791341617,
360
+ "raw_expert_cv_n": 10,
361
+ "raw_expert_imax_ceiling": 48.0,
362
+ "raw_expert_imax_hit_ceiling": 0,
363
+ "raw_expert_imax_max": 14.314453125,
364
+ "raw_expert_imax_mean": 8.18291015625,
365
+ "raw_expert_imax_min": 5.30712890625,
366
+ "raw_expert_imax_n": 10,
367
+ "records": {
368
+ "moonep_bytes": 80,
369
+ "moonep_routing": 80,
370
+ "moonep_summary": 80,
371
+ "moonep_weights": 80
372
+ },
373
+ "router_balancing": "seq_aux_loss",
374
+ "run_dir": "moonep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__async/rep1",
375
+ "seq_length": "4096",
376
+ "shared_expert_overlap": "0",
377
+ "topk": "8"
378
+ },
379
+ {
380
+ "arm": "forced",
381
+ "aux_loss_coeff": "0",
382
+ "backend": "moonep",
383
+ "branch": "moonep_backend_async_grad_reduce_token_routing_analysis",
384
+ "capacity": "none",
385
+ "capacity_mode": "none",
386
+ "cold_expert_fraction_max": 0.0,
387
+ "cold_expert_fraction_mean": 0.0,
388
+ "cold_expert_fraction_min": 0.0,
389
+ "cold_expert_fraction_n": 10,
390
+ "commit": "a6a1a221",
391
+ "config": "KIMI_K2_L2",
392
+ "cuda_graph": "0",
393
+ "dense_first_layers": "1",
394
+ "experts_per_rank": "48",
395
+ "force_balance": "1",
396
+ "global_batch_size": "8",
397
+ "global_snapshots": 10,
398
+ "hottest_expert_token_fraction_max": 0.0240478515625,
399
+ "hottest_expert_token_fraction_mean": 0.0235992431640625,
400
+ "hottest_expert_token_fraction_min": 0.02313232421875,
401
+ "hottest_expert_token_fraction_n": 10,
402
+ "incomplete_snapshots": 0,
403
+ "malformed_records": 0,
404
+ "moe_layers": "1",
405
+ "moonep_bytes_global_snapshots": 10,
406
+ "moonep_bytes_incomplete_snapshots": 0,
407
+ "moonep_bytes_malformed_records": 0,
408
+ "moonep_global_combine_payload_bytes_max": 4344438784.0,
409
+ "moonep_global_combine_payload_bytes_mean": 4338601164.8,
410
+ "moonep_global_combine_payload_bytes_min": 4333916160.0,
411
+ "moonep_global_combine_payload_bytes_n": 10,
412
+ "moonep_global_dispatch_payload_bytes_max": 4344438784.0,
413
+ "moonep_global_dispatch_payload_bytes_mean": 4338601164.8,
414
+ "moonep_global_dispatch_payload_bytes_min": 4333916160.0,
415
+ "moonep_global_dispatch_payload_bytes_n": 10,
416
+ "moonep_global_prefetch_bytes_max": 1233125376.0,
417
+ "moonep_global_prefetch_bytes_mean": 1233125376.0,
418
+ "moonep_global_prefetch_bytes_min": 1233125376.0,
419
+ "moonep_global_prefetch_bytes_n": 10,
420
+ "moonep_global_reduce_bytes_max": 1233125376.0,
421
+ "moonep_global_reduce_bytes_mean": 1233125376.0,
422
+ "moonep_global_reduce_bytes_min": 1233125376.0,
423
+ "moonep_global_reduce_bytes_n": 10,
424
+ "moonep_global_token_bytes_max": 8690713000.0,
425
+ "moonep_global_token_bytes_mean": 8679036183.2,
426
+ "moonep_global_token_bytes_min": 8669663568.0,
427
+ "moonep_global_token_bytes_n": 10,
428
+ "moonep_global_total_nvlink_bytes_max": 11156963752.0,
429
+ "moonep_global_total_nvlink_bytes_mean": 11145286935.2,
430
+ "moonep_global_total_nvlink_bytes_min": 11135914320.0,
431
+ "moonep_global_total_nvlink_bytes_n": 10,
432
+ "moonep_global_weight_movement_bytes_max": 2466250752.0,
433
+ "moonep_global_weight_movement_bytes_mean": 2466250752.0,
434
+ "moonep_global_weight_movement_bytes_min": 2466250752.0,
435
+ "moonep_global_weight_movement_bytes_n": 10,
436
+ "moonep_global_weight_to_token_ratio_max": 0.2844690261226524,
437
+ "moonep_global_weight_to_token_ratio_mean": 0.28416198294879025,
438
+ "moonep_global_weight_to_token_ratio_min": 0.2837800249530735,
439
+ "moonep_global_weight_to_token_ratio_n": 10,
440
+ "moonep_remote_demand_max": 1.0,
441
+ "moonep_remote_demand_mean": 0.875,
442
+ "moonep_remote_demand_min": 0.0,
443
+ "moonep_remote_demand_n": 80,
444
+ "moonep_slot_utilization_max": 0.020833,
445
+ "moonep_slot_utilization_mean": 0.018228875,
446
+ "moonep_slot_utilization_min": 0.0,
447
+ "moonep_slot_utilization_n": 80,
448
+ "moonep_unused_prefetch_slots_max": 96.0,
449
+ "moonep_unused_prefetch_slots_mean": 94.25,
450
+ "moonep_unused_prefetch_slots_min": 94.0,
451
+ "moonep_unused_prefetch_slots_n": 80,
452
+ "num_experts": "384",
453
+ "num_layers": "2",
454
+ "post_physical_rank_cv_max": 0.0031320054168418494,
455
+ "post_physical_rank_cv_mean": 0.0022532777674585456,
456
+ "post_physical_rank_cv_min": 0.0014149963886086814,
457
+ "post_physical_rank_cv_n": 10,
458
+ "post_physical_rank_imax_max": 1.004496538012507,
459
+ "post_physical_rank_imax_mean": 1.0033084889047725,
460
+ "post_physical_rank_imax_min": 1.0019296319833613,
461
+ "post_physical_rank_imax_n": 10,
462
+ "pre_home_rank_cv_max": 0.009327227347999425,
463
+ "pre_home_rank_cv_mean": 0.007192100098074482,
464
+ "pre_home_rank_cv_min": 0.003541485721042321,
465
+ "pre_home_rank_cv_n": 10,
466
+ "pre_home_rank_imax_max": 1.02001953125,
467
+ "pre_home_rank_imax_mean": 1.01103515625,
468
+ "pre_home_rank_imax_min": 1.005340576171875,
469
+ "pre_home_rank_imax_n": 10,
470
+ "raw_expert_cv_max": 0.04836014600641425,
471
+ "raw_expert_cv_mean": 0.045649240379339776,
472
+ "raw_expert_cv_min": 0.04141931327746496,
473
+ "raw_expert_cv_n": 10,
474
+ "raw_expert_imax_ceiling": 48.0,
475
+ "raw_expert_imax_hit_ceiling": 0,
476
+ "raw_expert_imax_max": 1.154296875,
477
+ "raw_expert_imax_mean": 1.132763671875,
478
+ "raw_expert_imax_min": 1.1103515625,
479
+ "raw_expert_imax_n": 10,
480
+ "records": {
481
+ "moonep_bytes": 80,
482
+ "moonep_routing": 80,
483
+ "moonep_summary": 80,
484
+ "moonep_weights": 80
485
+ },
486
+ "router_balancing": "seq_aux_loss",
487
+ "run_dir": "moonep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1",
488
+ "seq_length": "4096",
489
+ "shared_expert_overlap": "0",
490
+ "topk": "8"
491
+ },
492
+ {
493
+ "arm": "natural",
494
+ "assignment_sum_mismatches": 0,
495
+ "aux_loss_coeff": "0",
496
+ "backend": "ultraep",
497
+ "branch": "ultraep-token-drop-analysis",
498
+ "capacity": "1.0625",
499
+ "capacity_mode": "static",
500
+ "commit": "eaf829e7",
501
+ "config": "KIMI_K2_L2",
502
+ "cuda_graph": "0",
503
+ "dense_first_layers": "1",
504
+ "drop_expected_assignments": 3145728.0,
505
+ "drop_rate_pct_max": 0.0,
506
+ "drop_rate_pct_mean": 0.0,
507
+ "drop_rate_pct_min": 0.0,
508
+ "drop_rate_pct_n": 96,
509
+ "drop_rate_pct_overall": 0.0,
510
+ "dropped_tokens_sum_all_records": 0.0,
511
+ "duplicate_vector_mismatches": 0,
512
+ "experts_per_rank": "48",
513
+ "force_balance": "0",
514
+ "global_batch_size": "8",
515
+ "global_snapshots": 12,
516
+ "incomplete_snapshots": 0,
517
+ "malformed_records": 0,
518
+ "moe_layers": "1",
519
+ "num_experts": "384",
520
+ "num_layers": "2",
521
+ "observed_dropped_tokens": 0.0,
522
+ "peak_demand_max": 33098.0,
523
+ "peak_demand_mean": 32768.0,
524
+ "peak_demand_min": 31213.0,
525
+ "peak_demand_n": 96,
526
+ "peak_rank_overflow_max": 0.0,
527
+ "peak_rank_overflow_mean": 0.0,
528
+ "peak_rank_overflow_min": 0.0,
529
+ "peak_rank_overflow_n": 96,
530
+ "post_physical_rank_cv_max": 0.018551700753199882,
531
+ "post_physical_rank_cv_mean": 0.014736188098268745,
532
+ "post_physical_rank_cv_min": 0.01142687710780522,
533
+ "post_physical_rank_cv_n": 12,
534
+ "post_physical_rank_imax_max": 1.01007080078125,
535
+ "post_physical_rank_imax_mean": 1.0100326538085938,
536
+ "post_physical_rank_imax_min": 1.010009765625,
537
+ "post_physical_rank_imax_n": 12,
538
+ "pre_home_rank_cv_max": 0.2350733530310411,
539
+ "pre_home_rank_cv_mean": 0.13086798523811746,
540
+ "pre_home_rank_cv_min": 0.0905917891947787,
541
+ "pre_home_rank_cv_n": 12,
542
+ "pre_home_rank_imax_max": 1.406707763671875,
543
+ "pre_home_rank_imax_mean": 1.2260996500651042,
544
+ "pre_home_rank_imax_min": 1.131988525390625,
545
+ "pre_home_rank_imax_n": 12,
546
+ "records": {
547
+ "drop_dispatch": 96,
548
+ "drop_experts": 96,
549
+ "drop_setup": 96,
550
+ "ultraep_drop": 96,
551
+ "ultraep_post": 96,
552
+ "ultraep_pre": 96
553
+ },
554
+ "required_capacity_max": 1.010071,
555
+ "required_capacity_mean": 1.0000000833333333,
556
+ "required_capacity_min": 0.952545,
557
+ "required_capacity_n": 96,
558
+ "router_balancing": "seq_aux_loss",
559
+ "run_dir": "ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p0625/rep1",
560
+ "seq_length": "4096",
561
+ "shared_expert_overlap": "0",
562
+ "topk": "8",
563
+ "valid_max_rank_reduction_max": 0.282004555808656,
564
+ "valid_max_rank_reduction_mean": 0.17248486518149106,
565
+ "valid_max_rank_reduction_min": 0.10772922114684712,
566
+ "valid_max_rank_reduction_n": 12,
567
+ "zero_drop": 1
568
+ },
569
+ {
570
+ "arm": "natural",
571
+ "assignment_sum_mismatches": 0,
572
+ "aux_loss_coeff": "0",
573
+ "backend": "ultraep",
574
+ "branch": "ultraep-token-drop-analysis",
575
+ "capacity": "1.25",
576
+ "capacity_mode": "static",
577
+ "commit": "eaf829e7",
578
+ "config": "KIMI_K2_L2",
579
+ "cuda_graph": "0",
580
+ "dense_first_layers": "1",
581
+ "drop_expected_assignments": 3145728.0,
582
+ "drop_rate_pct_max": 0.0,
583
+ "drop_rate_pct_mean": 0.0,
584
+ "drop_rate_pct_min": 0.0,
585
+ "drop_rate_pct_n": 96,
586
+ "drop_rate_pct_overall": 0.0,
587
+ "dropped_tokens_sum_all_records": 0.0,
588
+ "duplicate_vector_mismatches": 0,
589
+ "experts_per_rank": "48",
590
+ "force_balance": "0",
591
+ "global_batch_size": "8",
592
+ "global_snapshots": 12,
593
+ "incomplete_snapshots": 0,
594
+ "malformed_records": 0,
595
+ "moe_layers": "1",
596
+ "num_experts": "384",
597
+ "num_layers": "2",
598
+ "observed_dropped_tokens": 0.0,
599
+ "peak_demand_max": 33098.0,
600
+ "peak_demand_mean": 32768.0,
601
+ "peak_demand_min": 31238.0,
602
+ "peak_demand_n": 96,
603
+ "peak_rank_overflow_max": 0.0,
604
+ "peak_rank_overflow_mean": 0.0,
605
+ "peak_rank_overflow_min": 0.0,
606
+ "peak_rank_overflow_n": 96,
607
+ "post_physical_rank_cv_max": 0.018343139886457832,
608
+ "post_physical_rank_cv_mean": 0.014707084226669935,
609
+ "post_physical_rank_cv_min": 0.011387320530400494,
610
+ "post_physical_rank_cv_n": 12,
611
+ "post_physical_rank_imax_max": 1.01007080078125,
612
+ "post_physical_rank_imax_mean": 1.0100351969401042,
613
+ "post_physical_rank_imax_min": 1.010009765625,
614
+ "post_physical_rank_imax_n": 12,
615
+ "pre_home_rank_cv_max": 0.23003629950316515,
616
+ "pre_home_rank_cv_mean": 0.12688407124108866,
617
+ "pre_home_rank_cv_min": 0.08877670743785306,
618
+ "pre_home_rank_cv_n": 12,
619
+ "pre_home_rank_imax_max": 1.385650634765625,
620
+ "pre_home_rank_imax_mean": 1.2170664469401042,
621
+ "pre_home_rank_imax_min": 1.12664794921875,
622
+ "pre_home_rank_imax_n": 12,
623
+ "records": {
624
+ "drop_dispatch": 96,
625
+ "drop_experts": 96,
626
+ "drop_setup": 96,
627
+ "ultraep_drop": 96,
628
+ "ultraep_post": 96,
629
+ "ultraep_pre": 96
630
+ },
631
+ "required_capacity_max": 1.010071,
632
+ "required_capacity_mean": 1.0000000208333335,
633
+ "required_capacity_min": 0.953308,
634
+ "required_capacity_n": 96,
635
+ "router_balancing": "seq_aux_loss",
636
+ "run_dir": "ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25/rep1",
637
+ "seq_length": "4096",
638
+ "shared_expert_overlap": "0",
639
+ "topk": "8",
640
+ "valid_max_rank_reduction_max": 0.271071467900011,
641
+ "valid_max_rank_reduction_mean": 0.16650987487084534,
642
+ "valid_max_rank_reduction_min": 0.10352673492605233,
643
+ "valid_max_rank_reduction_n": 12,
644
+ "zero_drop": 1
645
+ },
646
+ {
647
+ "arm": "forced",
648
+ "assignment_sum_mismatches": 0,
649
+ "aux_loss_coeff": "0",
650
+ "backend": "ultraep",
651
+ "branch": "ultraep-token-drop-analysis",
652
+ "capacity": "1.25",
653
+ "capacity_mode": "static",
654
+ "commit": "eaf829e7",
655
+ "config": "KIMI_K2_L2",
656
+ "cuda_graph": "0",
657
+ "dense_first_layers": "1",
658
+ "drop_expected_assignments": 3145728.0,
659
+ "drop_rate_pct_max": 0.0,
660
+ "drop_rate_pct_mean": 0.0,
661
+ "drop_rate_pct_min": 0.0,
662
+ "drop_rate_pct_n": 96,
663
+ "drop_rate_pct_overall": 0.0,
664
+ "dropped_tokens_sum_all_records": 0.0,
665
+ "duplicate_vector_mismatches": 0,
666
+ "experts_per_rank": "48",
667
+ "force_balance": "1",
668
+ "global_batch_size": "8",
669
+ "global_snapshots": 12,
670
+ "incomplete_snapshots": 0,
671
+ "malformed_records": 0,
672
+ "moe_layers": "1",
673
+ "num_experts": "384",
674
+ "num_layers": "2",
675
+ "observed_dropped_tokens": 0.0,
676
+ "peak_demand_max": 33097.0,
677
+ "peak_demand_mean": 32768.0,
678
+ "peak_demand_min": 32318.0,
679
+ "peak_demand_n": 96,
680
+ "peak_rank_overflow_max": 0.0,
681
+ "peak_rank_overflow_mean": 0.0,
682
+ "peak_rank_overflow_min": 0.0,
683
+ "peak_rank_overflow_n": 96,
684
+ "post_physical_rank_cv_max": 0.008620891733745346,
685
+ "post_physical_rank_cv_mean": 0.00609096140081787,
686
+ "post_physical_rank_cv_min": 0.0033077098989994554,
687
+ "post_physical_rank_cv_n": 12,
688
+ "post_physical_rank_imax_max": 1.010040283203125,
689
+ "post_physical_rank_imax_mean": 1.0083948771158855,
690
+ "post_physical_rank_imax_min": 1.005340576171875,
691
+ "post_physical_rank_imax_n": 12,
692
+ "pre_home_rank_cv_max": 0.009327227347999425,
693
+ "pre_home_rank_cv_mean": 0.006896456793407019,
694
+ "pre_home_rank_cv_min": 0.0033077098989994554,
695
+ "pre_home_rank_cv_n": 12,
696
+ "pre_home_rank_imax_max": 1.02001953125,
697
+ "pre_home_rank_imax_mean": 1.0104878743489583,
698
+ "pre_home_rank_imax_min": 1.005340576171875,
699
+ "pre_home_rank_imax_n": 12,
700
+ "records": {
701
+ "drop_dispatch": 96,
702
+ "drop_experts": 96,
703
+ "drop_setup": 96,
704
+ "ultraep_drop": 96,
705
+ "ultraep_post": 96,
706
+ "ultraep_pre": 96
707
+ },
708
+ "required_capacity_max": 1.01004,
709
+ "required_capacity_mean": 0.9999999895833334,
710
+ "required_capacity_min": 0.986267,
711
+ "required_capacity_n": 96,
712
+ "router_balancing": "seq_aux_loss",
713
+ "run_dir": "ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1",
714
+ "seq_length": "4096",
715
+ "shared_expert_overlap": "0",
716
+ "topk": "8",
717
+ "valid_max_rank_reduction_max": 0.009843226424126376,
718
+ "valid_max_rank_reduction_mean": 0.002057005642129182,
719
+ "valid_max_rank_reduction_min": 0.0,
720
+ "valid_max_rank_reduction_n": 12,
721
+ "zero_drop": 1
722
+ }
723
+ ]
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/probe_summary_all.tsv ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ run_dir arm config backend num_experts topk experts_per_rank num_layers moe_layers dense_first_layers seq_length global_batch_size capacity_mode capacity force_balance cuda_graph branch commit router_balancing aux_loss_coeff shared_expert_overlap assignment_sum_mismatches cold_expert_fraction_max cold_expert_fraction_mean cold_expert_fraction_min cold_expert_fraction_n drop_accounting_source drop_expected_assignments drop_rate_pct_max drop_rate_pct_mean drop_rate_pct_min drop_rate_pct_n drop_rate_pct_overall dropped_tokens_sum_all_records duplicate_vector_mismatches echo_logged_load_reduction_valid echo_logged_pre_rank_metric_valid echo_moved_assignment_fraction_max echo_moved_assignment_fraction_mean echo_moved_assignment_fraction_min echo_moved_assignment_fraction_n global_snapshots hottest_expert_token_fraction_max hottest_expert_token_fraction_mean hottest_expert_token_fraction_min hottest_expert_token_fraction_n incomplete_snapshots malformed_records moonep_bytes_global_snapshots moonep_bytes_incomplete_snapshots moonep_bytes_malformed_records moonep_global_combine_payload_bytes_max moonep_global_combine_payload_bytes_mean moonep_global_combine_payload_bytes_min moonep_global_combine_payload_bytes_n moonep_global_dispatch_payload_bytes_max moonep_global_dispatch_payload_bytes_mean moonep_global_dispatch_payload_bytes_min moonep_global_dispatch_payload_bytes_n moonep_global_prefetch_bytes_max moonep_global_prefetch_bytes_mean moonep_global_prefetch_bytes_min moonep_global_prefetch_bytes_n moonep_global_reduce_bytes_max moonep_global_reduce_bytes_mean moonep_global_reduce_bytes_min moonep_global_reduce_bytes_n moonep_global_token_bytes_max moonep_global_token_bytes_mean moonep_global_token_bytes_min moonep_global_token_bytes_n moonep_global_total_nvlink_bytes_max moonep_global_total_nvlink_bytes_mean moonep_global_total_nvlink_bytes_min moonep_global_total_nvlink_bytes_n moonep_global_weight_movement_bytes_max moonep_global_weight_movement_bytes_mean moonep_global_weight_movement_bytes_min moonep_global_weight_movement_bytes_n moonep_global_weight_to_token_ratio_max moonep_global_weight_to_token_ratio_mean moonep_global_weight_to_token_ratio_min moonep_global_weight_to_token_ratio_n moonep_remote_demand_max moonep_remote_demand_mean moonep_remote_demand_min moonep_remote_demand_n moonep_slot_utilization_max moonep_slot_utilization_mean moonep_slot_utilization_min moonep_slot_utilization_n moonep_unused_prefetch_slots_max moonep_unused_prefetch_slots_mean moonep_unused_prefetch_slots_min moonep_unused_prefetch_slots_n observed_dropped_tokens peak_demand_max peak_demand_mean peak_demand_min peak_demand_n peak_rank_overflow_max peak_rank_overflow_mean peak_rank_overflow_min peak_rank_overflow_n post_physical_rank_cv_max post_physical_rank_cv_mean post_physical_rank_cv_min post_physical_rank_cv_n post_physical_rank_imax_max post_physical_rank_imax_mean post_physical_rank_imax_min post_physical_rank_imax_n pre_home_rank_cv_max pre_home_rank_cv_mean pre_home_rank_cv_min pre_home_rank_cv_n pre_home_rank_imax_max pre_home_rank_imax_mean pre_home_rank_imax_min pre_home_rank_imax_n raw_expert_cv_max raw_expert_cv_mean raw_expert_cv_min raw_expert_cv_n raw_expert_imax_ceiling raw_expert_imax_hit_ceiling raw_expert_imax_max raw_expert_imax_mean raw_expert_imax_min raw_expert_imax_n required_capacity_max required_capacity_mean required_capacity_min required_capacity_n saturated_dispatch_rows valid_max_rank_reduction_max valid_max_rank_reduction_mean valid_max_rank_reduction_min valid_max_rank_reduction_n zero_drop
2
+ echo/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__echo__matched__seq4096__gbs8__cap1p17578125/rep1 natural KIMI_K2_L2 echo 384 8 48 2 1 1 4096 8 static 1.17578125 0 0 token-drop-analysis b099fd84 seq_aux_loss 0 0 0 0.0 0.0 0.0 12 dispatch 3145728.0 0.0 0 0 0.5 0.5 0.5 12 12 0.30047607421875 0.16150410970052084 0.10980224609375 12 0 0 0 0.0 0.0 0.0 12 1.0 1.0 1.0 12 0.23490023124048143 0.1314172003975923 0.09022193849823706 12 1.393341064453125 1.2213694254557292 1.133331298828125 12 1.7754821522938804 1.0537211199459462 0.7435680098626484 12 48.0 0 14.4228515625 7.752197265625 5.2705078125 12 1.1171875 1.0969645182291667 1.0703125 96 0 0.28230063298070396 0.17806762817557978 0.11764547486334384 12 1
3
+ echo/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__echo__matched__seq4096__gbs8__cap1p17578125__forcebal/rep1 forced KIMI_K2_L2 echo 384 8 48 2 1 1 4096 8 static 1.17578125 1 0 token-drop-analysis b099fd84 seq_aux_loss 0 0 0 0.0 0.0 0.0 12 dispatch 3145728.0 0.0 0 0 0.5 0.5 0.5 12 12 0.0240478515625 0.0235595703125 0.02313232421875 12 0 0 0 0.0 0.0 0.0 12 1.0 1.0 1.0 12 0.009327227347999425 0.006896456793407019 0.0033077098989994554 12 1.02001953125 1.0104878743489583 1.005340576171875 12 0.04836014600641425 0.04494131821754616 0.038893381234906065 12 48.0 0 1.154296875 1.130859375 1.1103515625 12 1.1328125 1.1121826171875 1.09375 96 0 0.019626615605552896 0.010357864282149248 0.005312205931457366 12 1
4
+ echo/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__echo__matched__seq4096__gbs8__cap2p00/rep1 natural KIMI_K2_L2 echo 384 8 48 2 1 1 4096 8 static 2.00 0 0 token-drop-analysis b099fd84 seq_aux_loss 0 0 0 0.0 0.0 0.0 12 dispatch 3145728.0 0.0 0 0 0.5 0.5 0.5 12 12 0.30047607421875 0.16150410970052084 0.10980224609375 12 0 0 0 0.0 0.0 0.0 12 1.0 1.0 1.0 12 0.23490023124048143 0.1314172003975923 0.09022193849823706 12 1.393341064453125 1.2213694254557292 1.133331298828125 12 1.7754821522938804 1.0537211199459462 0.7435680098626484 12 48.0 0 14.4228515625 7.752197265625 5.2705078125 12 1.1171875 1.0969645182291667 1.0703125 96 0 0.28230063298070396 0.17806762817557978 0.11764547486334384 12 1
5
+ moonep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__async/rep1 natural KIMI_K2_L2 moonep 384 8 48 2 1 1 4096 8 none none 0 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 seq_aux_loss 0 0 0.0 0.0 0.0 10 10 0.2982177734375 0.170477294921875 0.110565185546875 10 0 0 10 0 0 4395991040.0 4330650419.2 4301602816.0 10 4395991040.0 4330650419.2 4301602816.0 10 1937768448.0 1603062988.8 1233125376.0 10 1937768448.0 1603062988.8 1233125376.0 10 8793816424.0 8663126381.6 8605026376.0 10 12480563272.0 11869252359.2 11211418776.0 10 3875536896.0 3206125977.6 2466250752.0 10 0.4503805946265469 0.37043439150680124 0.280452835616301 10 2.0 1.1375 0.0 80 0.041667 0.0236978 0.0 80 96.0 93.725 92.0 80 0.022939437595188024 0.016598105480502157 0.0017300051649072432 10 1.0368468027524256 1.0260113076815665 1.003556609365738 10 0.23147253286871253 0.1336617698353712 0.08928583511808427 10 1.389129638671875 1.230499267578125 1.129058837890625 10 1.7760509001770588 1.0733812106366694 0.7388867791341617 10 48.0 0 14.314453125 8.18291015625 5.30712890625 10
6
+ moonep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1 forced KIMI_K2_L2 moonep 384 8 48 2 1 1 4096 8 none none 1 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 seq_aux_loss 0 0 0.0 0.0 0.0 10 10 0.0240478515625 0.0235992431640625 0.02313232421875 10 0 0 10 0 0 4344438784.0 4338601164.8 4333916160.0 10 4344438784.0 4338601164.8 4333916160.0 10 1233125376.0 1233125376.0 1233125376.0 10 1233125376.0 1233125376.0 1233125376.0 10 8690713000.0 8679036183.2 8669663568.0 10 11156963752.0 11145286935.2 11135914320.0 10 2466250752.0 2466250752.0 2466250752.0 10 0.2844690261226524 0.28416198294879025 0.2837800249530735 10 1.0 0.875 0.0 80 0.020833 0.018228875 0.0 80 96.0 94.25 94.0 80 0.0031320054168418494 0.0022532777674585456 0.0014149963886086814 10 1.004496538012507 1.0033084889047725 1.0019296319833613 10 0.009327227347999425 0.007192100098074482 0.003541485721042321 10 1.02001953125 1.01103515625 1.005340576171875 10 0.04836014600641425 0.045649240379339776 0.04141931327746496 10 48.0 0 1.154296875 1.132763671875 1.1103515625 10
7
+ ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p0625/rep1 natural KIMI_K2_L2 ultraep 384 8 48 2 1 1 4096 8 static 1.0625 0 0 ultraep-token-drop-analysis eaf829e7 seq_aux_loss 0 0 0 3145728.0 0.0 0.0 0.0 96 0.0 0.0 0 12 0 0 0.0 33098.0 32768.0 31213.0 96 0.0 0.0 0.0 96 0.018551700753199882 0.014736188098268745 0.01142687710780522 12 1.01007080078125 1.0100326538085938 1.010009765625 12 0.2350733530310411 0.13086798523811746 0.0905917891947787 12 1.406707763671875 1.2260996500651042 1.131988525390625 12 1.010071 1.0000000833333333 0.952545 96 0.282004555808656 0.17248486518149106 0.10772922114684712 12 1
8
+ ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25/rep1 natural KIMI_K2_L2 ultraep 384 8 48 2 1 1 4096 8 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 seq_aux_loss 0 0 0 3145728.0 0.0 0.0 0.0 96 0.0 0.0 0 12 0 0 0.0 33098.0 32768.0 31238.0 96 0.0 0.0 0.0 96 0.018343139886457832 0.014707084226669935 0.011387320530400494 12 1.01007080078125 1.0100351969401042 1.010009765625 12 0.23003629950316515 0.12688407124108866 0.08877670743785306 12 1.385650634765625 1.2170664469401042 1.12664794921875 12 1.010071 1.0000000208333335 0.953308 96 0.271071467900011 0.16650987487084534 0.10352673492605233 12 1
9
+ ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1 forced KIMI_K2_L2 ultraep 384 8 48 2 1 1 4096 8 static 1.25 1 0 ultraep-token-drop-analysis eaf829e7 seq_aux_loss 0 0 0 3145728.0 0.0 0.0 0.0 96 0.0 0.0 0 12 0 0 0.0 33097.0 32768.0 32318.0 96 0.0 0.0 0.0 96 0.008620891733745346 0.00609096140081787 0.0033077098989994554 12 1.010040283203125 1.0083948771158855 1.005340576171875 12 0.009327227347999425 0.006896456793407019 0.0033077098989994554 12 1.02001953125 1.0104878743489583 1.005340576171875 12 1.01004 0.9999999895833334 0.986267 96 0.009843226424126376 0.002057005642129182 0.0 12 1
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/steps.tsv CHANGED
@@ -4,3 +4,4 @@ OK ultraep tierbprobe 8 2026-09-21T07:36:53+00:00 2026-09-21T07:40:24+00:00 /wor
4
  OK moonep tierbprobe 8 2026-09-21T07:40:26+00:00 2026-09-21T07:43:07+00:00 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe
5
  OK echo tierb 8 2026-09-21T07:43:09+00:00 2026-09-21T07:47:09+00:00 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/echo/tierb
6
  OK ultraep tierb 8 2026-09-21T07:47:12+00:00 2026-09-21T07:50:42+00:00 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb
 
 
4
  OK moonep tierbprobe 8 2026-09-21T07:40:26+00:00 2026-09-21T07:43:07+00:00 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierbprobe
5
  OK echo tierb 8 2026-09-21T07:43:09+00:00 2026-09-21T07:47:09+00:00 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/echo/tierb
6
  OK ultraep tierb 8 2026-09-21T07:47:12+00:00 2026-09-21T07:50:42+00:00 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb
7
+ OK moonep tierb 8 2026-09-21T07:50:45+00:00 2026-09-21T07:54:39+00:00 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierb
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/study.log CHANGED
@@ -369,3 +369,63 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
369
  [07:51:00] CELL KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__async (slots=48)
370
  [guard] iter 12/12 (70s)
371
  [guard] tearing down process group (outcome=DONE)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
369
  [07:51:00] CELL KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__async (slots=48)
370
  [guard] iter 12/12 (70s)
371
  [guard] tearing down process group (outcome=DONE)
372
+ [guard] iter 12/12 (70s)
373
+ [guard] tearing down process group (outcome=DONE)
374
+ [guard] iter 12/12 (70s)
375
+ [guard] tearing down process group (outcome=DONE)
376
+ --------------------------------------------------------------------------
377
+
378
+ ==============================================================
379
+ SWEEP COMPLETE ok=3 failed=0
380
+ ==============================================================
381
+ config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
382
+ KIMI_K2_L2 moonep matched 48 4096 8 none none 1 OK 108.60 107.70 86491 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierb/tierb_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__async/rep1
383
+ KIMI_K2_L2 moonep matched 48 4096 8 none none 2 OK 108.74 107.70 86491 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierb/tierb_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__async/rep2
384
+ KIMI_K2_L2 moonep matched 48 4096 8 none none 3 OK 108.82 107.70 86491 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierb/tierb_gbs8/KIMI_K2_L2__moonep__matched__seq4096__gbs8__capnone__async/rep3
385
+
386
+ Results: /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierb/tierb_gbs8/results.tsv
387
+ Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierb/tierb_gbs8
388
+ ==============================================================
389
+
390
+ Campaign results: /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierb
391
+ Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierb --compare
392
+ Already logged in to Hugging Face as: AbhishekGhosh1998
393
+ uploaded: /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierb -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/moonep/tierb
394
+ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
395
+
396
+ === ROLL-UP
397
+ 17 runs ok=17 oom=0 -> /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/collected_all.tsv
398
+
399
+ WARNING: 3 run(s) mention capacity saturation. A run that drops assignments does less expert work and must not be quoted as a win:
400
+ ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p0625/rep1: WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=1 total_expected_recv=33098 total_dropped_token
401
+ ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25/rep1: WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=1 total_expected_recv=33098 total_dropped_token
402
+ ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1: WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=1 total_expected_recv=32672 total_dropped_token
403
+
404
+ phase config seq gbs policy rc cg asy backend iter_ms vs_best
405
+ -------------------------------------------------------------------------------------
406
+ tierb KIMI_K2_L2 4096 8 matched none 1 0 ultraep 110.08 1.00x
407
+ echo 119.60 1.09x
408
+
409
+ tierb KIMI_K2_L2 4096 8 matched none 1 1 moonep 108.82 1.00x
410
+
411
+ tierbprobeKIMI_K2_L2 4096 8 matched none 0 0 ultraep 122.08 1.00x
412
+ echo 144.26 1.18x
413
+
414
+ tierbprobeKIMI_K2_L2 4096 8 matched none 0 1 moonep 121.00 1.00x
415
+
416
+ 8 probe runs -> /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/probe_summary_all.tsv
417
+ run arm cap rawExpI preRankI postRankI drop% reqCap
418
+ --------------------------------------------------------------------------------------------------------------------
419
+ echo/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__echo__ma natural 1.17578125 7.7522 1.2214 1.0000 1.1172
420
+ echo/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__echo__ma forced 1.17578125 1.1309 1.0105 1.0000 1.1328
421
+ echo/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__echo__ma natural 2.00 7.7522 1.2214 1.0000 1.1172
422
+ moonep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__moonep natural none 8.1829 1.2305 1.0260
423
+ moonep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__moonep forced none 1.1328 1.0110 1.0033
424
+ ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultra natural 1.0625 1.2261 1.0100 0.0000 1.0101
425
+ ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultra natural 1.25 1.2171 1.0100 0.0000 1.0101
426
+ ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultra forced 1.25 1.0105 1.0084 0.0000 1.0100
427
+
428
+ ECHO note: echo_logged_pre_rank_metric_valid=0 and echo_logged_load_reduction_valid=0 are intentional.
429
+ The branch's built-in fields mix source-rank and destination-rank axes; this tool reconstructs both globally.
430
+ no memory-dump runs found under /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2
431
+ Already logged in to Hugging Face as: AbhishekGhosh1998