AbhishekGhosh1998 commited on
Commit
846693e
·
verified ·
1 Parent(s): 33b3d07

granularity results tiera-rerun-fixed-moonep @ 2026-08-30T08:03:45Z from sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0

Browse files
tiera-rerun-fixed-moonep/.completed/moonep__a2__gbs16 ADDED
@@ -0,0 +1 @@
 
 
1
+ 2026-08-30T08:03:34+00:00
tiera-rerun-fixed-moonep/STATUS.txt CHANGED
@@ -1,13 +1,14 @@
1
  study_root: /workspace/tiera-rerun-fixed-moonep
2
  gbs: 16
3
- updated: 2026-08-30T07:48:22+00:00
4
- current: moonep__a2__gbs16
5
 
6
  completed steps:
7
  echo__a1__gbs16
8
  echo__a2__gbs16
9
  echo__ponly__gbs16
10
  moonep__a1__gbs16
 
11
  moonep__ponly__gbs16
12
  ultraep__a1__gbs16
13
  ultraep__a2__gbs16
 
1
  study_root: /workspace/tiera-rerun-fixed-moonep
2
  gbs: 16
3
+ updated: 2026-08-30T08:03:36+00:00
4
+ current: idle
5
 
6
  completed steps:
7
  echo__a1__gbs16
8
  echo__a2__gbs16
9
  echo__ponly__gbs16
10
  moonep__a1__gbs16
11
+ moonep__a2__gbs16
12
  moonep__ponly__gbs16
13
  ultraep__a1__gbs16
14
  ultraep__a2__gbs16
tiera-rerun-fixed-moonep/collected_all.tsv CHANGED
@@ -1,4 +1,36 @@
1
  run_dir rep phase is_trace async config backend slot_policy slots_per_rank num_experts topk moe_ffn_hidden experts_per_rank expert_params num_layers seq_length global_batch_size num_microbatches assignments_per_rank mean_tokens_per_expert capacity capacity_mode recompute force_balance cuda_graph memory_dump_mode memory_snapshot_path ultraep_quota_min_tokens shared_expert_overlap branch commit dense_first_layers moe_layers router_balancing aux_loss_coeff router_pre_softmax router_fusion rotary_base qkv_bias status iters_seen iter_ms_mean iter_ms_min iter_ms_stdev tflops peak_mib_sampled megatron_max_reserved_mib drop_warning
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
  echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1 rep1 tierbprobe 0 0 G1 echo matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.00 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 671.53 666.90 4.51 496.5 125477 94730
3
  echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1 rep1 tierbprobe 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 660.43 642.00 14.73 505.0 119267 88146
4
  echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1 rep1 tierbprobe 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 1 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 628.07 616.10 9.43 530.9 119267 88146
@@ -9,6 +41,57 @@ echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap2p00/rep1
9
  echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375/rep1 rep1 tierbprobe 0 0 G4 echo matched 16 128 32 896 16 11010048 16 4096 16 2 131072 8192 1.068359375 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 775.73 738.90 77.17 432.9 129621 96956
10
  echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375__forcebal/rep1 rep1 tierbprobe 0 0 G4 echo matched 16 128 32 896 16 11010048 16 4096 16 2 131072 8192 1.068359375 static none 1 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 768.04 733.40 78.16 437.3 129579 96956
11
  echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap2p00/rep1 rep1 tierbprobe 0 0 G4 echo matched 16 128 32 896 16 11010048 16 4096 16 2 131072 8192 2.00 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 829.14 790.80 78.61 404.7 146037 114976
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
12
  ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1 rep1 tierbprobe 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 1.25 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 893.49 886.90 5.95 373.1 108731 80984 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=283999 total_dropped_tok
13
  ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1 rep1 tierbprobe 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 1.8125 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 914.84 903.60 9.13 364.4 113767 84920 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=284540 total_dropped_tok
14
  ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 rep1 tierbprobe 0 0 G2 ultraep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.0625 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 551.94 527.00 15.90 604.4 107987 75842 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=1048800 total_dropped_to
 
1
  run_dir rep phase is_trace async config backend slot_policy slots_per_rank num_experts topk moe_ffn_hidden experts_per_rank expert_params num_layers seq_length global_batch_size num_microbatches assignments_per_rank mean_tokens_per_expert capacity capacity_mode recompute force_balance cuda_graph memory_dump_mode memory_snapshot_path ultraep_quota_min_tokens shared_expert_overlap branch commit dense_first_layers moe_layers router_balancing aux_loss_coeff router_pre_softmax router_fusion rotary_base qkv_bias status iters_seen iter_ms_mean iter_ms_min iter_ms_stdev tflops peak_mib_sampled megatron_max_reserved_mib drop_warning
2
+ echo/a1/a1_oneslot_gbs16/G1__echo__one__seq4096__gbs16__cap2p40625/nsys a1 1 0 G1 echo one 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 8 1396.60 467.10 1600.68 504.7 99120
3
+ echo/a1/a1_oneslot_gbs16/G1__echo__one__seq4096__gbs16__cap2p40625/rep1 rep1 a1 0 0 G1 echo one 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 468.13 465.80 2.27 712.2 161723 99122
4
+ echo/a1/a1_oneslot_gbs16/G1__echo__one__seq4096__gbs16__cap2p40625/rep2 rep2 a1 0 0 G1 echo one 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 466.30 463.40 2.08 715.0 161723 99122
5
+ echo/a1/a1_oneslot_gbs16/G1__echo__one__seq4096__gbs16__cap2p40625/rep3 rep3 a1 0 0 G1 echo one 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 465.61 463.00 1.36 716.0 161723 99122
6
+ echo/a1/a1_oneslot_gbs16/G2__echo__one__seq4096__gbs16__cap1p08203125/nsys a1 1 0 G2 echo one 1 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 8 1305.10 396.30 1565.87 590.0 72832
7
+ echo/a1/a1_oneslot_gbs16/G2__echo__one__seq4096__gbs16__cap1p08203125/rep1 rep1 a1 0 0 G2 echo one 1 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 394.21 392.20 1.81 845.7 120227 73168
8
+ echo/a1/a1_oneslot_gbs16/G2__echo__one__seq4096__gbs16__cap1p08203125/rep2 rep2 a1 0 0 G2 echo one 1 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 394.19 393.00 0.88 845.7 120225 73280
9
+ echo/a1/a1_oneslot_gbs16/G2__echo__one__seq4096__gbs16__cap1p08203125/rep3 rep3 a1 0 0 G2 echo one 1 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 394.44 393.20 1.11 845.2 120225 73392
10
+ echo/a1/a1_oneslot_gbs16/G3__echo__one__seq4096__gbs16__cap1p072265625/rep1 rep1 a1 0 0 G3 echo one 1 64 16 1792 8 22020096 16 4096 16 2 65536 8192 1.072265625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 412.23 410.50 0.97 808.7 116311 73028
11
+ echo/a1/a1_oneslot_gbs16/G3__echo__one__seq4096__gbs16__cap1p072265625/rep2 rep2 a1 0 0 G3 echo one 1 64 16 1792 8 22020096 16 4096 16 2 65536 8192 1.072265625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 413.76 412.10 1.42 805.8 116423 73028
12
+ echo/a1/a1_oneslot_gbs16/G3__echo__one__seq4096__gbs16__cap1p072265625/rep3 rep3 a1 0 0 G3 echo one 1 64 16 1792 8 22020096 16 4096 16 2 65536 8192 1.072265625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 412.51 410.80 0.82 808.2 116531 72692
13
+ echo/a1/a1_oneslot_gbs16/G4__echo__one__seq4096__gbs16__cap1p095703125/rep1 rep1 a1 0 0 G4 echo one 1 128 32 896 16 11010048 16 4096 16 2 131072 8192 1.095703125 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 459.80 458.70 0.67 725.1 127009 77116
14
+ echo/a1/a1_oneslot_gbs16/G4__echo__one__seq4096__gbs16__cap1p095703125/rep2 rep2 a1 0 0 G4 echo one 1 128 32 896 16 11010048 16 4096 16 2 131072 8192 1.095703125 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 462.49 459.90 3.04 720.9 127043 77004
15
+ echo/a1/a1_oneslot_gbs16/G4__echo__one__seq4096__gbs16__cap1p095703125/rep3 rep3 a1 0 0 G4 echo one 1 128 32 896 16 11010048 16 4096 16 2 131072 8192 1.095703125 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 460.79 458.90 1.61 723.5 127009 77116
16
+ echo/a2/a2_matched_gbs16/G1__echo__matched__seq4096__gbs16__cap2p40625/rep1 rep1 a2 0 0 G1 echo matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 466.86 462.10 2.38 714.1 161723 99122
17
+ echo/a2/a2_matched_gbs16/G1__echo__matched__seq4096__gbs16__cap2p40625/rep2 rep2 a2 0 0 G1 echo matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 466.49 462.60 2.84 714.7 161723 99122
18
+ echo/a2/a2_matched_gbs16/G1__echo__matched__seq4096__gbs16__cap2p40625/rep3 rep3 a2 0 0 G1 echo matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 467.39 461.80 3.50 713.3 161723 99122
19
+ echo/a2/a2_matched_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1 rep1 a2 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 415.60 413.70 1.10 802.2 145689 88178
20
+ echo/a2/a2_matched_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep2 rep2 a2 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 414.94 414.20 0.58 803.5 145691 88178
21
+ echo/a2/a2_matched_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep3 rep3 a2 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 415.09 412.60 1.56 803.2 145691 88178
22
+ echo/a2/a2_matched_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625/rep1 rep1 a2 0 0 G3 echo matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 1.072265625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 436.97 435.10 1.53 762.9 148157 91570
23
+ echo/a2/a2_matched_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625/rep2 rep2 a2 0 0 G3 echo matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 1.072265625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 437.63 436.00 1.95 761.8 148273 91570
24
+ echo/a2/a2_matched_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625/rep3 rep3 a2 0 0 G3 echo matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 1.072265625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 437.44 436.10 1.35 762.1 147933 91570
25
+ echo/a2/a2_matched_gbs16/G4__echo__matched__seq4096__gbs16__cap1p095703125/rep1 rep1 a2 0 0 G4 echo matched 16 128 32 896 16 11010048 16 4096 16 2 131072 8192 1.095703125 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 486.56 484.60 2.13 685.2 168485 97406
26
+ echo/a2/a2_matched_gbs16/G4__echo__matched__seq4096__gbs16__cap1p095703125/rep2 rep2 a2 0 0 G4 echo matched 16 128 32 896 16 11010048 16 4096 16 2 131072 8192 1.095703125 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 487.47 485.00 1.80 683.9 168709 97406
27
+ echo/a2/a2_matched_gbs16/G4__echo__matched__seq4096__gbs16__cap1p095703125/rep3 rep3 a2 0 0 G4 echo matched 16 128 32 896 16 11010048 16 4096 16 2 131072 8192 1.095703125 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 485.59 482.40 1.60 686.6 168485 97406
28
+ echo/ponly/ponly_gbs16/PONLY__echo__one__seq4096__gbs16__cap2p40625/rep1 rep1 ponly 0 0 PONLY echo one 1 8 2 1536 1 18874368 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 175.19 174.10 1.79 490.9 40157 24124
29
+ echo/ponly/ponly_gbs16/PONLY__echo__one__seq4096__gbs16__cap2p40625/rep2 rep2 ponly 0 0 PONLY echo one 1 8 2 1536 1 18874368 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 174.66 173.90 1.14 492.3 40241 24132
30
+ echo/ponly/ponly_gbs16/PONLY__echo__one__seq4096__gbs16__cap2p40625/rep3 rep3 ponly 0 0 PONLY echo one 1 8 2 1536 1 18874368 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 174.59 173.80 0.63 492.6 40243 24132
31
+ echo/ponly/ponly_gbs16/PONLY__echo__one__seq8192__gbs16__cap2p40625/rep1 rep1 ponly 0 0 PONLY echo one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 316.36 315.50 0.77 627.1 53279 32528
32
+ echo/ponly/ponly_gbs16/PONLY__echo__one__seq8192__gbs16__cap2p40625/rep2 rep2 ponly 0 0 PONLY echo one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 316.59 315.80 0.88 626.6 53283 32496
33
+ echo/ponly/ponly_gbs16/PONLY__echo__one__seq8192__gbs16__cap2p40625/rep3 rep3 ponly 0 0 PONLY echo one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 317.16 315.40 1.68 625.5 53283 32496
34
  echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1 rep1 tierbprobe 0 0 G1 echo matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.00 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 671.53 666.90 4.51 496.5 125477 94730
35
  echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1 rep1 tierbprobe 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 660.43 642.00 14.73 505.0 119267 88146
36
  echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1 rep1 tierbprobe 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 1 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 628.07 616.10 9.43 530.9 119267 88146
 
41
  echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375/rep1 rep1 tierbprobe 0 0 G4 echo matched 16 128 32 896 16 11010048 16 4096 16 2 131072 8192 1.068359375 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 775.73 738.90 77.17 432.9 129621 96956
42
  echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375__forcebal/rep1 rep1 tierbprobe 0 0 G4 echo matched 16 128 32 896 16 11010048 16 4096 16 2 131072 8192 1.068359375 static none 1 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 768.04 733.40 78.16 437.3 129579 96956
43
  echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap2p00/rep1 rep1 tierbprobe 0 0 G4 echo matched 16 128 32 896 16 11010048 16 4096 16 2 131072 8192 2.00 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 829.14 790.80 78.61 404.7 146037 114976
44
+ moonep/a1/a1_oneslot_gbs16/G1__moonep__one__seq4096__gbs16__capnone__async/rep1 rep1 a1 0 1 G1 moonep one 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 477.56 460.30 13.57 698.6 96849 58996
45
+ moonep/a1/a1_oneslot_gbs16/G1__moonep__one__seq4096__gbs16__capnone__async/rep2 rep2 a1 0 1 G1 moonep one 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 481.51 464.00 11.89 692.7 96849 58996
46
+ moonep/a1/a1_oneslot_gbs16/G1__moonep__one__seq4096__gbs16__capnone__async/rep3 rep3 a1 0 1 G1 moonep one 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 481.83 462.20 13.14 692.4 96849 58996
47
+ moonep/a2/a2_matched_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/rep1 rep1 a2 0 1 G1 moonep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 479.69 459.20 12.89 695.4 96849 58996
48
+ moonep/a2/a2_matched_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/rep2 rep2 a2 0 1 G1 moonep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 482.00 473.40 7.38 691.8 96849 58996
49
+ moonep/a2/a2_matched_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/rep3 rep3 a2 0 1 G1 moonep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 482.41 464.00 10.58 691.4 96849 58996
50
+ moonep/a2/a2_matched_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/rep1 rep1 a2 0 1 G2 moonep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 432.44 427.10 3.39 771.0 97663 56294
51
+ moonep/a2/a2_matched_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/rep2 rep2 a2 0 1 G2 moonep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 431.89 426.20 3.57 772.0 97663 56294
52
+ moonep/a2/a2_matched_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/rep3 rep3 a2 0 1 G2 moonep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 431.74 426.20 3.67 772.2 97665 56294
53
+ moonep/a2/a2_matched_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/rep1 rep1 a2 0 1 G3 moonep matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 559.74 558.50 1.17 595.6 99427 55728
54
+ moonep/a2/a2_matched_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/rep2 rep2 a2 0 1 G3 moonep matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 560.20 558.70 1.25 595.1 99407 55708
55
+ moonep/a2/a2_matched_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/rep3 rep3 a2 0 1 G3 moonep matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 559.71 557.60 1.63 595.6 99409 55708
56
+ moonep/a2/a2_matched_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__async/rep1 rep1 a2 0 1 G4 moonep matched 16 128 32 896 16 11010048 16 4096 16 2 131072 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 782.76 779.80 1.39 425.9 106031 57154
57
+ moonep/a2/a2_matched_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__async/rep2 rep2 a2 0 1 G4 moonep matched 16 128 32 896 16 11010048 16 4096 16 2 131072 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 784.30 780.70 3.69 425.1 106031 57154
58
+ moonep/a2/a2_matched_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__async/rep3 rep3 a2 0 1 G4 moonep matched 16 128 32 896 16 11010048 16 4096 16 2 131072 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 782.96 781.00 1.35 425.8 106029 57154
59
+ moonep/ponly/ponly_gbs16/PONLY__moonep__one__seq4096__gbs16__capnone__async/rep1 rep1 ponly 0 1 PONLY moonep one 1 8 2 1536 1 18874368 16 4096 16 2 8192 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 165.09 164.60 0.41 520.9 31983 17966
60
+ moonep/ponly/ponly_gbs16/PONLY__moonep__one__seq4096__gbs16__capnone__async/rep2 rep2 ponly 0 1 PONLY moonep one 1 8 2 1536 1 18874368 16 4096 16 2 8192 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 165.21 164.40 0.82 520.5 31983 17966
61
+ moonep/ponly/ponly_gbs16/PONLY__moonep__one__seq4096__gbs16__capnone__async/rep3 rep3 ponly 0 1 PONLY moonep one 1 8 2 1536 1 18874368 16 4096 16 2 8192 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 165.36 164.20 0.70 520.1 31983 17966
62
+ moonep/ponly/ponly_gbs16/PONLY__moonep__one__seq8192__gbs16__capnone__async/rep1 rep1 ponly 0 1 PONLY moonep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 300.76 299.30 0.82 659.6 41295 25036
63
+ moonep/ponly/ponly_gbs16/PONLY__moonep__one__seq8192__gbs16__capnone__async/rep2 rep2 ponly 0 1 PONLY moonep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 300.27 297.30 2.48 660.7 41297 25038
64
+ moonep/ponly/ponly_gbs16/PONLY__moonep__one__seq8192__gbs16__capnone__async/rep3 rep3 ponly 0 1 PONLY moonep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 300.81 297.90 1.79 659.5 41295 25038
65
+ ultraep/a1/a1_oneslot_gbs16/G1__ultraep__one__seq4096__gbs16__cap2p40625/rep1 rep1 a1 0 0 G1 ultraep one 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 888.60 869.80 12.39 375.2 129475 87692
66
+ ultraep/a1/a1_oneslot_gbs16/G1__ultraep__one__seq4096__gbs16__cap2p40625/rep2 rep2 a1 0 0 G1 ultraep one 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 894.61 880.90 11.04 372.7 129475 87692
67
+ ultraep/a1/a1_oneslot_gbs16/G1__ultraep__one__seq4096__gbs16__cap2p40625/rep3 rep3 a1 0 0 G1 ultraep one 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 889.80 872.80 10.69 374.7 129475 87692
68
+ ultraep/a1/a1_oneslot_gbs16/G2__ultraep__one__seq4096__gbs16__cap1p08203125/rep1 rep1 a1 0 0 G2 ultraep one 1 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 466.57 441.70 13.62 715.1 102733 68232
69
+ ultraep/a1/a1_oneslot_gbs16/G2__ultraep__one__seq4096__gbs16__cap1p08203125/rep2 rep2 a1 0 0 G2 ultraep one 1 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 466.79 444.20 13.66 714.7 102733 68232
70
+ ultraep/a1/a1_oneslot_gbs16/G2__ultraep__one__seq4096__gbs16__cap1p08203125/rep3 rep3 a1 0 0 G2 ultraep one 1 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 466.59 444.70 13.01 715.0 102733 68232
71
+ ultraep/a1/a1_oneslot_gbs16/G3__ultraep__one__seq4096__gbs16__cap1p072265625/rep1 rep1 a1 0 0 G3 ultraep one 1 64 16 1792 8 22020096 16 4096 16 2 65536 8192 1.072265625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 448.07 438.50 6.56 744.2 102043 68904
72
+ ultraep/a1/a1_oneslot_gbs16/G3__ultraep__one__seq4096__gbs16__cap1p072265625/rep2 rep2 a1 0 0 G3 ultraep one 1 64 16 1792 8 22020096 16 4096 16 2 65536 8192 1.072265625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 448.20 435.80 7.95 744.0 102043 68904
73
+ ultraep/a1/a1_oneslot_gbs16/G3__ultraep__one__seq4096__gbs16__cap1p072265625/rep3 rep3 a1 0 0 G3 ultraep one 1 64 16 1792 8 22020096 16 4096 16 2 65536 8192 1.072265625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 447.26 432.10 8.50 745.7 102043 68904
74
+ ultraep/a1/a1_oneslot_gbs16/G4__ultraep__one__seq4096__gbs16__cap1p095703125/rep1 rep1 a1 0 0 G4 ultraep one 1 128 32 896 16 11010048 16 4096 16 2 131072 8192 1.095703125 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 467.36 461.70 3.68 713.4 114075 73446
75
+ ultraep/a1/a1_oneslot_gbs16/G4__ultraep__one__seq4096__gbs16__cap1p095703125/rep2 rep2 a1 0 0 G4 ultraep one 1 128 32 896 16 11010048 16 4096 16 2 131072 8192 1.095703125 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 468.20 462.70 4.17 712.1 114077 73446
76
+ ultraep/a1/a1_oneslot_gbs16/G4__ultraep__one__seq4096__gbs16__cap1p095703125/rep3 rep3 a1 0 0 G4 ultraep one 1 128 32 896 16 11010048 16 4096 16 2 131072 8192 1.095703125 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 467.16 462.70 4.50 713.7 114077 73446
77
+ ultraep/a2/a2_matched_gbs16/G1__ultraep__matched__seq4096__gbs16__cap2p40625/rep1 rep1 a2 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 893.80 876.90 10.94 373.0 129475 87692
78
+ ultraep/a2/a2_matched_gbs16/G1__ultraep__matched__seq4096__gbs16__cap2p40625/rep2 rep2 a2 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 890.86 875.20 12.19 374.3 129475 87692
79
+ ultraep/a2/a2_matched_gbs16/G1__ultraep__matched__seq4096__gbs16__cap2p40625/rep3 rep3 a2 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 891.10 874.90 11.80 374.2 129475 87692
80
+ ultraep/a2/a2_matched_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p08203125/rep1 rep1 a2 0 0 G2 ultraep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 492.27 465.20 19.15 678.1 112315 76502
81
+ ultraep/a2/a2_matched_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p08203125/rep2 rep2 a2 0 0 G2 ultraep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 492.16 462.50 18.61 678.3 112315 76502
82
+ ultraep/a2/a2_matched_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p08203125/rep3 rep3 a2 0 0 G2 ultraep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 493.20 468.10 19.41 676.9 112315 76502
83
+ ultraep/a2/a2_matched_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p072265625/rep1 rep1 a2 0 0 G3 ultraep matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 1.072265625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 460.01 449.60 7.38 724.9 113951 79598
84
+ ultraep/a2/a2_matched_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p072265625/rep2 rep2 a2 0 0 G3 ultraep matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 1.072265625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 458.99 451.10 5.71 726.4 113951 79598
85
+ ultraep/a2/a2_matched_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p072265625/rep3 rep3 a2 0 0 G3 ultraep matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 1.072265625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 459.80 455.50 3.60 725.1 113951 79598
86
+ ultraep/a2/a2_matched_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1 rep1 a2 0 0 G4 ultraep matched 16 128 32 896 16 11010048 16 4096 16 2 131072 8192 1.095703125 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 483.19 475.20 5.58 690.1 132437 85038
87
+ ultraep/a2/a2_matched_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep2 rep2 a2 0 0 G4 ultraep matched 16 128 32 896 16 11010048 16 4096 16 2 131072 8192 1.095703125 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 482.57 474.50 5.23 690.9 132437 85038
88
+ ultraep/a2/a2_matched_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep3 rep3 a2 0 0 G4 ultraep matched 16 128 32 896 16 11010048 16 4096 16 2 131072 8192 1.095703125 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 482.17 475.60 4.19 691.5 132437 85038
89
+ ultraep/ponly/ponly_gbs16/PONLY__ultraep__one__seq4096__gbs16__cap2p40625/rep1 rep1 ponly 0 0 PONLY ultraep one 1 8 2 1536 1 18874368 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 176.80 175.80 0.52 486.5 38329 22904
90
+ ultraep/ponly/ponly_gbs16/PONLY__ultraep__one__seq4096__gbs16__cap2p40625/rep2 rep2 ponly 0 0 PONLY ultraep one 1 8 2 1536 1 18874368 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 176.80 176.40 0.33 486.4 38329 22904
91
+ ultraep/ponly/ponly_gbs16/PONLY__ultraep__one__seq4096__gbs16__cap2p40625/rep3 rep3 ponly 0 0 PONLY ultraep one 1 8 2 1536 1 18874368 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 176.67 175.90 0.47 486.8 38331 22904
92
+ ultraep/ponly/ponly_gbs16/PONLY__ultraep__one__seq8192__gbs16__cap2p40625/rep1 rep1 ponly 0 0 PONLY ultraep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 2048 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 292.76 290.70 1.82 677.6 51017 31264
93
+ ultraep/ponly/ponly_gbs16/PONLY__ultraep__one__seq8192__gbs16__cap2p40625/rep2 rep2 ponly 0 0 PONLY ultraep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 2048 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 292.47 291.30 1.15 678.3 51015 31264
94
+ ultraep/ponly/ponly_gbs16/PONLY__ultraep__one__seq8192__gbs16__cap2p40625/rep3 rep3 ponly 0 0 PONLY ultraep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 2048 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 292.64 290.20 1.85 677.9 51013 31264
95
  ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1 rep1 tierbprobe 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 1.25 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 893.49 886.90 5.95 373.1 108731 80984 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=283999 total_dropped_tok
96
  ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1 rep1 tierbprobe 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 1.8125 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 914.84 903.60 9.13 364.4 113767 84920 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=284540 total_dropped_tok
97
  ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 rep1 tierbprobe 0 0 G2 ultraep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.0625 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 551.94 527.00 15.90 604.4 107987 75842 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=1048800 total_dropped_to
tiera-rerun-fixed-moonep/hf_watcher.log CHANGED
@@ -53,3 +53,5 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
53
  uploaded: /workspace/tiera-rerun-fixed-moonep -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tiera-rerun-fixed-moonep
54
  https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
55
  [2026-08-30T08:00:57+00:00] backup pass
 
 
 
53
  uploaded: /workspace/tiera-rerun-fixed-moonep -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tiera-rerun-fixed-moonep
54
  https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
55
  [2026-08-30T08:00:57+00:00] backup pass
56
+ uploaded: /workspace/tiera-rerun-fixed-moonep -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tiera-rerun-fixed-moonep
57
+ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
tiera-rerun-fixed-moonep/steps.tsv CHANGED
@@ -9,3 +9,4 @@ OK ultraep ponly 16 2026-08-30T07:15:19+00:00 2026-08-30T07:22:01+00:00 /workspa
9
  OK ultraep a2 16 2026-08-30T07:22:03+00:00 2026-08-30T07:37:07+00:00 /workspace/tiera-rerun-fixed-moonep/ultraep/a2
10
  OK moonep a1 16 2026-08-30T07:37:09+00:00 2026-08-30T07:41:05+00:00 /workspace/tiera-rerun-fixed-moonep/moonep/a1
11
  OK moonep ponly 16 2026-08-30T07:41:07+00:00 2026-08-30T07:48:20+00:00 /workspace/tiera-rerun-fixed-moonep/moonep/ponly
 
 
9
  OK ultraep a2 16 2026-08-30T07:22:03+00:00 2026-08-30T07:37:07+00:00 /workspace/tiera-rerun-fixed-moonep/ultraep/a2
10
  OK moonep a1 16 2026-08-30T07:37:09+00:00 2026-08-30T07:41:05+00:00 /workspace/tiera-rerun-fixed-moonep/moonep/a1
11
  OK moonep ponly 16 2026-08-30T07:41:07+00:00 2026-08-30T07:48:20+00:00 /workspace/tiera-rerun-fixed-moonep/moonep/ponly
12
+ OK moonep a2 16 2026-08-30T07:48:22+00:00 2026-08-30T08:03:34+00:00 /workspace/tiera-rerun-fixed-moonep/moonep/a2
tiera-rerun-fixed-moonep/study.log CHANGED
@@ -962,3 +962,142 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
962
  --------------------------------------------------------------------------
963
  [07:59:42] CELL G4__moonep__matched__seq4096__gbs16__capnone__async (slots=16)
964
  [guard] iter 11/12 (70s)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
962
  --------------------------------------------------------------------------
963
  [07:59:42] CELL G4__moonep__matched__seq4096__gbs16__capnone__async (slots=16)
964
  [guard] iter 11/12 (70s)
965
+ [guard] iter 12/12 (80s)
966
+ [guard] tearing down process group (outcome=DONE)
967
+ [guard] iter 1/12 (60s)
968
+ [guard] iter 12/12 (70s)
969
+ [guard] tearing down process group (outcome=DONE)
970
+ [guard] iter 1/12 (60s)
971
+ [guard] iter 12/12 (70s)
972
+ [guard] tearing down process group (outcome=DONE)
973
+ --------------------------------------------------------------------------
974
+
975
+ ==============================================================
976
+ SWEEP COMPLETE ok=12 failed=0
977
+ ==============================================================
978
+ config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
979
+ G1 moonep matched 1 4096 16 none none 1 OK 479.69 459.20 96849 /workspace/tiera-rerun-fixed-moonep/moonep/a2/a2_matched_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/rep1
980
+ G1 moonep matched 1 4096 16 none none 2 OK 480.84 459.20 96849 /workspace/tiera-rerun-fixed-moonep/moonep/a2/a2_matched_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/rep2
981
+ G1 moonep matched 1 4096 16 none none 3 OK 481.37 459.20 96849 /workspace/tiera-rerun-fixed-moonep/moonep/a2/a2_matched_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/rep3
982
+ G2 moonep matched 4 4096 16 none none 1 OK 432.44 427.10 97663 /workspace/tiera-rerun-fixed-moonep/moonep/a2/a2_matched_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/rep1
983
+ G2 moonep matched 4 4096 16 none none 2 OK 432.16 426.20 97663 /workspace/tiera-rerun-fixed-moonep/moonep/a2/a2_matched_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/rep2
984
+ G2 moonep matched 4 4096 16 none none 3 OK 432.02 426.20 97665 /workspace/tiera-rerun-fixed-moonep/moonep/a2/a2_matched_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/rep3
985
+ G3 moonep matched 8 4096 16 none none 1 OK 559.74 558.50 99427 /workspace/tiera-rerun-fixed-moonep/moonep/a2/a2_matched_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/rep1
986
+ G3 moonep matched 8 4096 16 none none 2 OK 559.97 558.50 99407 /workspace/tiera-rerun-fixed-moonep/moonep/a2/a2_matched_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/rep2
987
+ G3 moonep matched 8 4096 16 none none 3 OK 559.89 557.60 99409 /workspace/tiera-rerun-fixed-moonep/moonep/a2/a2_matched_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/rep3
988
+ G4 moonep matched 16 4096 16 none none 1 OK 782.76 779.80 106031 /workspace/tiera-rerun-fixed-moonep/moonep/a2/a2_matched_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__async/rep1
989
+ G4 moonep matched 16 4096 16 none none 2 OK 783.53 779.80 106031 /workspace/tiera-rerun-fixed-moonep/moonep/a2/a2_matched_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__async/rep2
990
+ G4 moonep matched 16 4096 16 none none 3 OK 783.34 779.80 106029 /workspace/tiera-rerun-fixed-moonep/moonep/a2/a2_matched_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__async/rep3
991
+
992
+ Results: /workspace/tiera-rerun-fixed-moonep/moonep/a2/a2_matched_gbs16/results.tsv
993
+ Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/tiera-rerun-fixed-moonep/moonep/a2/a2_matched_gbs16
994
+ ==============================================================
995
+
996
+ Campaign results: /workspace/tiera-rerun-fixed-moonep/moonep/a2
997
+ Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/tiera-rerun-fixed-moonep/moonep/a2 --compare
998
+ Already logged in to Hugging Face as: AbhishekGhosh1998
999
+ uploaded: /workspace/tiera-rerun-fixed-moonep/moonep/a2 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tiera-rerun-fixed-moonep/moonep/a2
1000
+ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
1001
+
1002
+ === ROLL-UP
1003
+ 104 runs ok=104 oom=0 -> /workspace/tiera-rerun-fixed-moonep/collected_all.tsv
1004
+
1005
+ WARNING: 11 run(s) mention capacity saturation. A run that drops assignments does less expert work and must not be quoted as a win:
1006
+ ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1: WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=283999 total_dropped_tok
1007
+ ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1: WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=284540 total_dropped_tok
1008
+ ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1: WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=1048800 total_dropped_to
1009
+ ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1: WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=1048800 total_dropped_to
1010
+ ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1: WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=32 total_expected_recv=1047984 total_dropped_to
1011
+
1012
+ (excluded 2 nsys-traced run(s) from the timing comparison)
1013
+
1014
+ phase config seq gbs policy rc cg asy backend iter_ms vs_best
1015
+ -------------------------------------------------------------------------------------
1016
+ a1 G1 4096 16 one none 1 0 echo 466.68 1.00x
1017
+ ultraep 891.00 1.91x
1018
+
1019
+ a1 G1 4096 16 one none 1 1 moonep 480.30 1.00x
1020
+
1021
+ a1 G2 4096 16 one none 1 0 echo 394.28 1.00x
1022
+ ultraep 466.65 1.18x
1023
+
1024
+ a1 G3 4096 16 one none 1 0 echo 412.83 1.00x
1025
+ ultraep 447.84 1.08x
1026
+
1027
+ a1 G4 4096 16 one none 1 0 echo 461.03 1.00x
1028
+ ultraep 467.57 1.01x
1029
+
1030
+ a2 G1 4096 16 matched none 1 0 echo 466.91 1.00x
1031
+ ultraep 891.92 1.91x
1032
+
1033
+ a2 G1 4096 16 matched none 1 1 moonep 481.37 1.00x
1034
+
1035
+ a2 G2 4096 16 matched none 1 0 echo 415.21 1.00x
1036
+ ultraep 492.54 1.19x
1037
+
1038
+ a2 G2 4096 16 matched none 1 1 moonep 432.02 1.00x
1039
+
1040
+ a2 G3 4096 16 matched none 1 0 echo 437.35 1.00x
1041
+ ultraep 459.60 1.05x
1042
+
1043
+ a2 G3 4096 16 matched none 1 1 moonep 559.88 1.00x
1044
+
1045
+ a2 G4 4096 16 matched none 1 0 ultraep 482.64 1.00x
1046
+ echo 486.54 1.01x
1047
+
1048
+ a2 G4 4096 16 matched none 1 1 moonep 783.34 1.00x
1049
+
1050
+ ponly PONLY 4096 16 one none 1 0 echo 174.81 1.00x
1051
+ ultraep 176.76 1.01x
1052
+
1053
+ ponly PONLY 4096 16 one none 1 1 moonep 165.22 1.00x
1054
+
1055
+ ponly PONLY 8192 16 one none 1 0 ultraep 292.62 1.00x
1056
+ echo 316.70 1.08x
1057
+
1058
+ ponly PONLY 8192 16 one none 1 1 moonep 300.61 1.00x
1059
+
1060
+ tierbprobeG1 4096 16 matched none 0 0 echo 671.53 1.00x
1061
+ ultraep 904.16 1.35x
1062
+
1063
+ tierbprobeG2 4096 16 matched none 0 0 ultraep 512.37 1.00x
1064
+ echo 654.76 1.28x
1065
+
1066
+ tierbprobeG3 4096 16 matched none 0 0 ultraep 512.31 1.00x
1067
+ echo 689.13 1.35x
1068
+
1069
+ tierbprobeG4 4096 16 matched none 0 0 ultraep 600.19 1.00x
1070
+ echo 790.97 1.32x
1071
+
1072
+ 21 probe runs -> /workspace/tiera-rerun-fixed-moonep/probe_summary_all.tsv
1073
+ run arm cap rawExpI preRankI postRankI drop% reqCap
1074
+ --------------------------------------------------------------------------------------------------------------------
1075
+ echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__ natural 2.00 3.6198* 3.6198 1.0827 1.9062
1076
+ echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__ natural 1.08203125 3.8720* 2.1703 1.0000 1.0273
1077
+ echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__ forced 1.08203125 1.0201 1.0069 1.0000 1.0234
1078
+ echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__ natural 2.00 3.8724* 2.1875 1.0000 1.0273
1079
+ echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__ natural 1.072265625 3.9096* 1.7838 1.0000 1.0195
1080
+ echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__ forced 1.072265625 1.0231 1.0048 1.0000 1.0176
1081
+ echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__ natural 2.00 3.9096* 1.7916 1.0000 1.0195
1082
+ echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__ natural 1.068359375 3.9303* 1.5507 1.0000 1.0166
1083
+ echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__ forced 1.068359375 1.0249 1.0034 1.0000 1.0137
1084
+ echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__ natural 2.00 3.9301* 1.5543 1.0000 1.0166
1085
+ ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__mat natural 1.25 3.6085 1.0640 2.8918 1.7246
1086
+ ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__mat natural 1.8125 3.6152 1.0747 1.5460 2.3091
1087
+ ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__mat natural 1.0625 2.2032 1.0100 0.0000 1.0101
1088
+ ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__mat natural 1.25 2.2136 1.0100 0.0000 1.0101
1089
+ ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__mat forced 1.25 1.0069 1.0067 0.0000 1.0100
1090
+ ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__mat natural 1.0625 1.7842 1.0100 0.0000 1.0101
1091
+ ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__mat natural 1.25 1.7752 1.0100 0.0000 1.0101
1092
+ ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__mat forced 1.25 1.0047 1.0047 0.0000 1.0100
1093
+ ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__mat natural 1.095703125 1.5612 1.0100 0.0000 1.0429
1094
+ ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__mat natural 1.25 1.5504 1.0100 0.0000 1.0429
1095
+ ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__mat forced 1.25 1.0033 1.0033 0.0000 1.0071
1096
+
1097
+ * At least one globally aggregated snapshot reached the E/K expert-Imax ceiling.
1098
+ Inspect raw_expert_imax_mean and hottest_expert_token_fraction_mean before generalising.
1099
+
1100
+ ECHO note: echo_logged_pre_rank_metric_valid=0 and echo_logged_load_reduction_valid=0 are intentional.
1101
+ The branch's built-in fields mix source-rank and destination-rank axes; this tool reconstructs both globally.
1102
+ no memory-dump runs found under /workspace/tiera-rerun-fixed-moonep
1103
+ Already logged in to Hugging Face as: AbhishekGhosh1998