AbhishekGhosh1998 commited on
Commit
334ff7f
·
verified ·
1 Parent(s): 774a4d5

granularity results tiera-rerun-fixed-moonep @ 2026-08-30T08:12:09Z from sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0

Browse files
tiera-rerun-fixed-moonep/.completed/echo__tierbmem__gbs16 ADDED
@@ -0,0 +1 @@
 
 
1
+ 2026-08-30T08:06:37+00:00
tiera-rerun-fixed-moonep/.completed/moonep__tierbmem__gbs16 ADDED
@@ -0,0 +1 @@
 
 
1
+ 2026-08-30T08:11:55+00:00
tiera-rerun-fixed-moonep/.completed/ultraep__tierbmem__gbs16 ADDED
@@ -0,0 +1 @@
 
 
1
+ 2026-08-30T08:09:23+00:00
tiera-rerun-fixed-moonep/STATUS.txt CHANGED
@@ -1,18 +1,21 @@
1
  study_root: /workspace/tiera-rerun-fixed-moonep
2
  gbs: 16
3
- updated: 2026-08-30T08:03:47+00:00
4
- current: echo__tierbmem__gbs16
5
 
6
  completed steps:
7
  echo__a1__gbs16
8
  echo__a2__gbs16
9
  echo__ponly__gbs16
 
10
  moonep__a1__gbs16
11
  moonep__a2__gbs16
12
  moonep__ponly__gbs16
 
13
  ultraep__a1__gbs16
14
  ultraep__a2__gbs16
15
  ultraep__ponly__gbs16
 
16
 
17
  failed steps:
18
  FAIL echo tierbprobe 16 2026-08-30T04:52:38+00:00 2026-08-30T05:06:38+00:00 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe
 
1
  study_root: /workspace/tiera-rerun-fixed-moonep
2
  gbs: 16
3
+ updated: 2026-08-30T08:12:00+00:00
4
+ current: idle
5
 
6
  completed steps:
7
  echo__a1__gbs16
8
  echo__a2__gbs16
9
  echo__ponly__gbs16
10
+ echo__tierbmem__gbs16
11
  moonep__a1__gbs16
12
  moonep__a2__gbs16
13
  moonep__ponly__gbs16
14
+ moonep__tierbmem__gbs16
15
  ultraep__a1__gbs16
16
  ultraep__a2__gbs16
17
  ultraep__ponly__gbs16
18
+ ultraep__tierbmem__gbs16
19
 
20
  failed steps:
21
  FAIL echo tierbprobe 16 2026-08-30T04:52:38+00:00 2026-08-30T05:06:38+00:00 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe
tiera-rerun-fixed-moonep/collected_all.tsv CHANGED
@@ -31,6 +31,8 @@ echo/ponly/ponly_gbs16/PONLY__echo__one__seq4096__gbs16__cap2p40625/rep3 rep3 po
31
  echo/ponly/ponly_gbs16/PONLY__echo__one__seq8192__gbs16__cap2p40625/rep1 rep1 ponly 0 0 PONLY echo one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 316.36 315.50 0.77 627.1 53279 32528
32
  echo/ponly/ponly_gbs16/PONLY__echo__one__seq8192__gbs16__cap2p40625/rep2 rep2 ponly 0 0 PONLY echo one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 316.59 315.80 0.88 626.6 53283 32496
33
  echo/ponly/ponly_gbs16/PONLY__echo__one__seq8192__gbs16__cap2p40625/rep3 rep3 ponly 0 0 PONLY echo one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 317.16 315.40 1.68 625.5 53283 32496
 
 
34
  echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1 rep1 tierbprobe 0 0 G1 echo matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.00 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 671.53 666.90 4.51 496.5 125477 94730
35
  echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1 rep1 tierbprobe 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 660.43 642.00 14.73 505.0 119267 88146
36
  echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1 rep1 tierbprobe 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 1 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 628.07 616.10 9.43 530.9 119267 88146
@@ -62,6 +64,8 @@ moonep/ponly/ponly_gbs16/PONLY__moonep__one__seq4096__gbs16__capnone__async/rep3
62
  moonep/ponly/ponly_gbs16/PONLY__moonep__one__seq8192__gbs16__capnone__async/rep1 rep1 ponly 0 1 PONLY moonep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 300.76 299.30 0.82 659.6 41295 25036
63
  moonep/ponly/ponly_gbs16/PONLY__moonep__one__seq8192__gbs16__capnone__async/rep2 rep2 ponly 0 1 PONLY moonep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 300.27 297.30 2.48 660.7 41297 25038
64
  moonep/ponly/ponly_gbs16/PONLY__moonep__one__seq8192__gbs16__capnone__async/rep3 rep3 ponly 0 1 PONLY moonep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 300.81 297.90 1.79 659.5 41295 25038
 
 
65
  ultraep/a1/a1_oneslot_gbs16/G1__ultraep__one__seq4096__gbs16__cap2p40625/rep1 rep1 a1 0 0 G1 ultraep one 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 888.60 869.80 12.39 375.2 129475 87692
66
  ultraep/a1/a1_oneslot_gbs16/G1__ultraep__one__seq4096__gbs16__cap2p40625/rep2 rep2 a1 0 0 G1 ultraep one 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 894.61 880.90 11.04 372.7 129475 87692
67
  ultraep/a1/a1_oneslot_gbs16/G1__ultraep__one__seq4096__gbs16__cap2p40625/rep3 rep3 a1 0 0 G1 ultraep one 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 889.80 872.80 10.69 374.7 129475 87692
@@ -92,6 +96,8 @@ ultraep/ponly/ponly_gbs16/PONLY__ultraep__one__seq4096__gbs16__cap2p40625/rep3 r
92
  ultraep/ponly/ponly_gbs16/PONLY__ultraep__one__seq8192__gbs16__cap2p40625/rep1 rep1 ponly 0 0 PONLY ultraep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 2048 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 292.76 290.70 1.82 677.6 51017 31264
93
  ultraep/ponly/ponly_gbs16/PONLY__ultraep__one__seq8192__gbs16__cap2p40625/rep2 rep2 ponly 0 0 PONLY ultraep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 2048 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 292.47 291.30 1.15 678.3 51015 31264
94
  ultraep/ponly/ponly_gbs16/PONLY__ultraep__one__seq8192__gbs16__cap2p40625/rep3 rep3 ponly 0 0 PONLY ultraep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 2048 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 292.64 290.20 1.85 677.9 51013 31264
 
 
95
  ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1 rep1 tierbprobe 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 1.25 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 893.49 886.90 5.95 373.1 108731 80984 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=283999 total_dropped_tok
96
  ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1 rep1 tierbprobe 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 1.8125 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 914.84 903.60 9.13 364.4 113767 84920 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=284540 total_dropped_tok
97
  ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 rep1 tierbprobe 0 0 G2 ultraep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.0625 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 551.94 527.00 15.90 604.4 107987 75842 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=1048800 total_dropped_to
 
31
  echo/ponly/ponly_gbs16/PONLY__echo__one__seq8192__gbs16__cap2p40625/rep1 rep1 ponly 0 0 PONLY echo one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 316.36 315.50 0.77 627.1 53279 32528
32
  echo/ponly/ponly_gbs16/PONLY__echo__one__seq8192__gbs16__cap2p40625/rep2 rep2 ponly 0 0 PONLY echo one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 316.59 315.80 0.88 626.6 53283 32496
33
  echo/ponly/ponly_gbs16/PONLY__echo__one__seq8192__gbs16__cap2p40625/rep3 rep3 ponly 0 0 PONLY echo one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 317.16 315.40 1.68 625.5 53283 32496
34
+ echo/tierbmem/tierbmem_gbs16/G1__echo__matched__seq4096__gbs16__cap2p40625__memiteration__it6/rep1 rep1 tierbmem 0 0 G1 echo matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 iteration /workspace/tiera-rerun-fixed-moonep/echo/tierbmem/tierbmem_gbs16/G1__echo__matched__seq4096__gbs16__cap2p40625__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 0 better-memory-dump-support b7c52442 0 16 aux_loss 1e-2 0 0 1000000 0 OK 6 474.40 474.40 702.7 161723 99122
35
+ echo/tierbmem/tierbmem_gbs16/G1__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 G1 echo matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 dynamic dynamic none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/echo/tierbmem/tierbmem_gbs16/G1__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 better-memory-dump-support b7c52442 0 16 aux_loss 1e-2 0 0 1000000 0 INCOMPLETE 5 115283 88418
36
  echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1 rep1 tierbprobe 0 0 G1 echo matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.00 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 671.53 666.90 4.51 496.5 125477 94730
37
  echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1 rep1 tierbprobe 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 660.43 642.00 14.73 505.0 119267 88146
38
  echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1 rep1 tierbprobe 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 1 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 628.07 616.10 9.43 530.9 119267 88146
 
64
  moonep/ponly/ponly_gbs16/PONLY__moonep__one__seq8192__gbs16__capnone__async/rep1 rep1 ponly 0 1 PONLY moonep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 300.76 299.30 0.82 659.6 41295 25036
65
  moonep/ponly/ponly_gbs16/PONLY__moonep__one__seq8192__gbs16__capnone__async/rep2 rep2 ponly 0 1 PONLY moonep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 300.27 297.30 2.48 660.7 41297 25038
66
  moonep/ponly/ponly_gbs16/PONLY__moonep__one__seq8192__gbs16__capnone__async/rep3 rep3 ponly 0 1 PONLY moonep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 300.81 297.90 1.79 659.5 41295 25038
67
+ moonep/tierbmem/tierbmem_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1 rep1 tierbmem 0 1 G1 moonep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 none none none 0 1 iteration /workspace/tiera-rerun-fixed-moonep/moonep/tierbmem/tierbmem_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 0 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384 0 16 aux_loss 1e-2 0 0 1000000 0 OK 6 497.40 497.40 670.2 96849 58996
68
+ moonep/tierbmem/tierbmem_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1 rep1 tierbmem 0 1 G1 moonep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 none none none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/moonep/tierbmem/tierbmem_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384 0 16 aux_loss 1e-2 0 0 1000000 0 FAIL 5 92579 59020
69
  ultraep/a1/a1_oneslot_gbs16/G1__ultraep__one__seq4096__gbs16__cap2p40625/rep1 rep1 a1 0 0 G1 ultraep one 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 888.60 869.80 12.39 375.2 129475 87692
70
  ultraep/a1/a1_oneslot_gbs16/G1__ultraep__one__seq4096__gbs16__cap2p40625/rep2 rep2 a1 0 0 G1 ultraep one 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 894.61 880.90 11.04 372.7 129475 87692
71
  ultraep/a1/a1_oneslot_gbs16/G1__ultraep__one__seq4096__gbs16__cap2p40625/rep3 rep3 a1 0 0 G1 ultraep one 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 889.80 872.80 10.69 374.7 129475 87692
 
96
  ultraep/ponly/ponly_gbs16/PONLY__ultraep__one__seq8192__gbs16__cap2p40625/rep1 rep1 ponly 0 0 PONLY ultraep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 2048 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 292.76 290.70 1.82 677.6 51017 31264
97
  ultraep/ponly/ponly_gbs16/PONLY__ultraep__one__seq8192__gbs16__cap2p40625/rep2 rep2 ponly 0 0 PONLY ultraep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 2048 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 292.47 291.30 1.15 678.3 51015 31264
98
  ultraep/ponly/ponly_gbs16/PONLY__ultraep__one__seq8192__gbs16__cap2p40625/rep3 rep3 ponly 0 0 PONLY ultraep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 2048 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 292.64 290.20 1.85 677.9 51013 31264
99
+ ultraep/tierbmem/tierbmem_gbs16/G1__ultraep__matched__seq4096__gbs16__cap2p40625__memiteration__it6/rep1 rep1 tierbmem 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 iteration /workspace/tiera-rerun-fixed-moonep/ultraep/tierbmem/tierbmem_gbs16/G1__ultraep__matched__seq4096__gbs16__cap2p40625__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 1024 0 ultraep-better-memory-dump-support c3a8a8a7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 6 909.90 909.90 366.4 129473 87692
100
+ ultraep/tierbmem/tierbmem_gbs16/G1__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 dynamic dynamic none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/ultraep/tierbmem/tierbmem_gbs16/G1__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 1024 0 ultraep-better-memory-dump-support c3a8a8a7 0 16 aux_loss 1e-2 0 0 1000000 0 FAIL 5 109261 80286
101
  ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1 rep1 tierbprobe 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 1.25 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 893.49 886.90 5.95 373.1 108731 80984 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=283999 total_dropped_tok
102
  ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1 rep1 tierbprobe 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 1.8125 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 914.84 903.60 9.13 364.4 113767 84920 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=284540 total_dropped_tok
103
  ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 rep1 tierbprobe 0 0 G2 ultraep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.0625 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 551.94 527.00 15.90 604.4 107987 75842 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=1048800 total_dropped_to