AbhishekGhosh1998 commited on
Commit
069ab73
·
verified ·
1 Parent(s): 3e80e18

granularity results tiera-rerun-fixed-moonep @ 2026-08-30T10:17:03Z from sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0

Browse files
tiera-rerun-fixed-moonep/.completed/echo__tierbmem__gbs16__G3 ADDED
@@ -0,0 +1 @@
 
 
1
+ 2026-08-30T10:11:03+00:00
tiera-rerun-fixed-moonep/.completed/moonep__tierbmem__gbs16__G3 ADDED
@@ -0,0 +1 @@
 
 
1
+ 2026-08-30T10:16:46+00:00
tiera-rerun-fixed-moonep/.completed/ultraep__tierbmem__gbs16__G3 ADDED
@@ -0,0 +1 @@
 
 
1
+ 2026-08-30T10:13:56+00:00
tiera-rerun-fixed-moonep/STATUS.txt CHANGED
@@ -1,7 +1,7 @@
1
  study_root: /workspace/tiera-rerun-fixed-moonep
2
  gbs: 16
3
- updated: 2026-08-30T10:08:06+00:00
4
- current: echo__tierbmem__gbs16__G3
5
 
6
  completed steps:
7
  echo__a1__gbs16
@@ -9,18 +9,21 @@ completed steps:
9
  echo__ponly__gbs16
10
  echo__tierbmem__gbs16__G1
11
  echo__tierbmem__gbs16__G2
 
12
  echo__tierbtrace__gbs16
13
  moonep__a1__gbs16
14
  moonep__a2__gbs16
15
  moonep__ponly__gbs16
16
  moonep__tierbmem__gbs16__G1
17
  moonep__tierbmem__gbs16__G2
 
18
  moonep__tierbtrace__gbs16
19
  ultraep__a1__gbs16
20
  ultraep__a2__gbs16
21
  ultraep__ponly__gbs16
22
  ultraep__tierbmem__gbs16__G1
23
  ultraep__tierbmem__gbs16__G2
 
24
  ultraep__tierbtrace__gbs16
25
 
26
  failed steps:
 
1
  study_root: /workspace/tiera-rerun-fixed-moonep
2
  gbs: 16
3
+ updated: 2026-08-30T10:16:54+00:00
4
+ current: idle
5
 
6
  completed steps:
7
  echo__a1__gbs16
 
9
  echo__ponly__gbs16
10
  echo__tierbmem__gbs16__G1
11
  echo__tierbmem__gbs16__G2
12
+ echo__tierbmem__gbs16__G3
13
  echo__tierbtrace__gbs16
14
  moonep__a1__gbs16
15
  moonep__a2__gbs16
16
  moonep__ponly__gbs16
17
  moonep__tierbmem__gbs16__G1
18
  moonep__tierbmem__gbs16__G2
19
+ moonep__tierbmem__gbs16__G3
20
  moonep__tierbtrace__gbs16
21
  ultraep__a1__gbs16
22
  ultraep__a2__gbs16
23
  ultraep__ponly__gbs16
24
  ultraep__tierbmem__gbs16__G1
25
  ultraep__tierbmem__gbs16__G2
26
+ ultraep__tierbmem__gbs16__G3
27
  ultraep__tierbtrace__gbs16
28
 
29
  failed steps:
tiera-rerun-fixed-moonep/collected_all.tsv CHANGED
@@ -35,6 +35,8 @@ echo/tierbmem/tierbmem_gbs16/G1__echo__matched__seq4096__gbs16__cap2p40625__memi
35
  echo/tierbmem/tierbmem_gbs16/G1__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 G1 echo matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 dynamic dynamic none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/echo/tierbmem/tierbmem_gbs16/G1__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 better-memory-dump-support b7c52442 0 16 aux_loss 1e-2 0 0 1000000 0 INCOMPLETE 5 115283 88418
36
  echo/tierbmem/tierbmem_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__memiteration__it6/rep1 rep1 tierbmem 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 1 iteration /workspace/tiera-rerun-fixed-moonep/echo/tierbmem/tierbmem_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 0 better-memory-dump-support b7c52442 0 16 aux_loss 1e-2 0 0 1000000 0 OK 6 430.10 430.10 775.2 145895 88178
37
  echo/tierbmem/tierbmem_gbs16/G2__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 dynamic dynamic none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/echo/tierbmem/tierbmem_gbs16/G2__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 better-memory-dump-support b7c52442 0 16 aux_loss 1e-2 0 0 1000000 0 INCOMPLETE 5 117877 83562
 
 
38
  echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1 rep1 tierbprobe 0 0 G1 echo matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.00 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 671.53 666.90 4.51 496.5 125477 94730
39
  echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1 rep1 tierbprobe 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 660.43 642.00 14.73 505.0 119267 88146
40
  echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1 rep1 tierbprobe 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 1 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 628.07 616.10 9.43 530.9 119267 88146
@@ -74,6 +76,8 @@ moonep/tierbmem/tierbmem_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__asy
74
  moonep/tierbmem/tierbmem_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1 rep1 tierbmem 0 1 G1 moonep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 none none none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/moonep/tierbmem/tierbmem_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384 0 16 aux_loss 1e-2 0 0 1000000 0 FAIL 5 92579 59020
75
  moonep/tierbmem/tierbmem_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1 rep1 tierbmem 0 1 G2 moonep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 none none none 0 1 iteration /workspace/tiera-rerun-fixed-moonep/moonep/tierbmem/tierbmem_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 0 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384 0 16 aux_loss 1e-2 0 0 1000000 0 OK 6 439.70 439.70 758.2 97663 56294
76
  moonep/tierbmem/tierbmem_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1 rep1 tierbmem 0 1 G2 moonep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 none none none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/moonep/tierbmem/tierbmem_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384 0 16 aux_loss 1e-2 0 0 1000000 0 FAIL 5 95485 56290
 
 
77
  moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys tierbtrace 1 1 G1 moonep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 10 9437.68 487.60 18511.57 423.5 58996
78
  moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys tierbtrace 1 1 G2 moonep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 8 1344.80 429.70 1575.40 545.9 56294
79
  moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys tierbtrace 1 1 G3 moonep matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 8 1483.70 563.20 1577.58 421.7 55708
@@ -112,6 +116,8 @@ ultraep/tierbmem/tierbmem_gbs16/G1__ultraep__matched__seq4096__gbs16__cap2p40625
112
  ultraep/tierbmem/tierbmem_gbs16/G1__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 dynamic dynamic none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/ultraep/tierbmem/tierbmem_gbs16/G1__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 1024 0 ultraep-better-memory-dump-support c3a8a8a7 0 16 aux_loss 1e-2 0 0 1000000 0 FAIL 5 109261 80286
113
  ultraep/tierbmem/tierbmem_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p08203125__memiteration__it6/rep1 rep1 tierbmem 0 0 G2 ultraep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 1 iteration /workspace/tiera-rerun-fixed-moonep/ultraep/tierbmem/tierbmem_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p08203125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 1024 0 ultraep-better-memory-dump-support c3a8a8a7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 6 475.40 475.40 701.2 112317 76502
114
  ultraep/tierbmem/tierbmem_gbs16/G2__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 G2 ultraep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 dynamic dynamic none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/ultraep/tierbmem/tierbmem_gbs16/G2__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 1024 0 ultraep-better-memory-dump-support c3a8a8a7 0 16 aux_loss 1e-2 0 0 1000000 0 FAIL 5 108213 75714
 
 
115
  ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1 rep1 tierbprobe 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 1.25 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 893.49 886.90 5.95 373.1 108731 80984 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=283999 total_dropped_tok
116
  ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1 rep1 tierbprobe 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 1.8125 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 914.84 903.60 9.13 364.4 113767 84920 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=284540 total_dropped_tok
117
  ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 rep1 tierbprobe 0 0 G2 ultraep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.0625 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 551.94 527.00 15.90 604.4 107987 75842 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=1048800 total_dropped_to
 
35
  echo/tierbmem/tierbmem_gbs16/G1__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 G1 echo matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 dynamic dynamic none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/echo/tierbmem/tierbmem_gbs16/G1__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 better-memory-dump-support b7c52442 0 16 aux_loss 1e-2 0 0 1000000 0 INCOMPLETE 5 115283 88418
36
  echo/tierbmem/tierbmem_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__memiteration__it6/rep1 rep1 tierbmem 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 1 iteration /workspace/tiera-rerun-fixed-moonep/echo/tierbmem/tierbmem_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 0 better-memory-dump-support b7c52442 0 16 aux_loss 1e-2 0 0 1000000 0 OK 6 430.10 430.10 775.2 145895 88178
37
  echo/tierbmem/tierbmem_gbs16/G2__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 dynamic dynamic none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/echo/tierbmem/tierbmem_gbs16/G2__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 better-memory-dump-support b7c52442 0 16 aux_loss 1e-2 0 0 1000000 0 INCOMPLETE 5 117877 83562
38
+ echo/tierbmem/tierbmem_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625__memiteration__it6/rep1 rep1 tierbmem 0 0 G3 echo matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 1.072265625 static none 0 1 iteration /workspace/tiera-rerun-fixed-moonep/echo/tierbmem/tierbmem_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 0 better-memory-dump-support b7c52442 0 16 aux_loss 1e-2 0 0 1000000 0 OK 6 452.40 452.40 736.9 148009 91570
39
+ echo/tierbmem/tierbmem_gbs16/G3__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 G3 echo matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 dynamic dynamic none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/echo/tierbmem/tierbmem_gbs16/G3__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 better-memory-dump-support b7c52442 0 16 aux_loss 1e-2 0 0 1000000 0 INCOMPLETE 5 118901 86906
40
  echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1 rep1 tierbprobe 0 0 G1 echo matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.00 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 671.53 666.90 4.51 496.5 125477 94730
41
  echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1 rep1 tierbprobe 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 660.43 642.00 14.73 505.0 119267 88146
42
  echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1 rep1 tierbprobe 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 1 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 628.07 616.10 9.43 530.9 119267 88146
 
76
  moonep/tierbmem/tierbmem_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1 rep1 tierbmem 0 1 G1 moonep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 none none none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/moonep/tierbmem/tierbmem_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384 0 16 aux_loss 1e-2 0 0 1000000 0 FAIL 5 92579 59020
77
  moonep/tierbmem/tierbmem_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1 rep1 tierbmem 0 1 G2 moonep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 none none none 0 1 iteration /workspace/tiera-rerun-fixed-moonep/moonep/tierbmem/tierbmem_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 0 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384 0 16 aux_loss 1e-2 0 0 1000000 0 OK 6 439.70 439.70 758.2 97663 56294
78
  moonep/tierbmem/tierbmem_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1 rep1 tierbmem 0 1 G2 moonep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 none none none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/moonep/tierbmem/tierbmem_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384 0 16 aux_loss 1e-2 0 0 1000000 0 FAIL 5 95485 56290
79
+ moonep/tierbmem/tierbmem_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1 rep1 tierbmem 0 1 G3 moonep matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 none none none 0 1 iteration /workspace/tiera-rerun-fixed-moonep/moonep/tierbmem/tierbmem_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 0 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384 0 16 aux_loss 1e-2 0 0 1000000 0 OK 6 578.30 578.30 576.5 99411 55708
80
+ moonep/tierbmem/tierbmem_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1 rep1 tierbmem 0 1 G3 moonep matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 none none none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/moonep/tierbmem/tierbmem_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384 0 16 aux_loss 1e-2 0 0 1000000 0 FAIL 5 98611 55624
81
  moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys tierbtrace 1 1 G1 moonep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 10 9437.68 487.60 18511.57 423.5 58996
82
  moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys tierbtrace 1 1 G2 moonep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 8 1344.80 429.70 1575.40 545.9 56294
83
  moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys tierbtrace 1 1 G3 moonep matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 8 1483.70 563.20 1577.58 421.7 55708
 
116
  ultraep/tierbmem/tierbmem_gbs16/G1__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 dynamic dynamic none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/ultraep/tierbmem/tierbmem_gbs16/G1__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 1024 0 ultraep-better-memory-dump-support c3a8a8a7 0 16 aux_loss 1e-2 0 0 1000000 0 FAIL 5 109261 80286
117
  ultraep/tierbmem/tierbmem_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p08203125__memiteration__it6/rep1 rep1 tierbmem 0 0 G2 ultraep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 1 iteration /workspace/tiera-rerun-fixed-moonep/ultraep/tierbmem/tierbmem_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p08203125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 1024 0 ultraep-better-memory-dump-support c3a8a8a7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 6 475.40 475.40 701.2 112317 76502
118
  ultraep/tierbmem/tierbmem_gbs16/G2__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 G2 ultraep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 dynamic dynamic none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/ultraep/tierbmem/tierbmem_gbs16/G2__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 1024 0 ultraep-better-memory-dump-support c3a8a8a7 0 16 aux_loss 1e-2 0 0 1000000 0 FAIL 5 108213 75714
119
+ ultraep/tierbmem/tierbmem_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p072265625__memiteration__it6/rep1 rep1 tierbmem 0 0 G3 ultraep matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 1.072265625 static none 0 1 iteration /workspace/tiera-rerun-fixed-moonep/ultraep/tierbmem/tierbmem_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p072265625__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 1024 0 ultraep-better-memory-dump-support c3a8a8a7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 6 477.80 477.80 697.8 113951 79598
120
+ ultraep/tierbmem/tierbmem_gbs16/G3__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 G3 ultraep matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 dynamic dynamic none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/ultraep/tierbmem/tierbmem_gbs16/G3__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 1024 0 ultraep-better-memory-dump-support c3a8a8a7 0 16 aux_loss 1e-2 0 0 1000000 0 FAIL 5 111707 78996
121
  ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1 rep1 tierbprobe 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 1.25 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 893.49 886.90 5.95 373.1 108731 80984 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=283999 total_dropped_tok
122
  ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1 rep1 tierbprobe 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 1.8125 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 914.84 903.60 9.13 364.4 113767 84920 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=284540 total_dropped_tok
123
  ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 rep1 tierbprobe 0 0 G2 ultraep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.0625 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 551.94 527.00 15.90 604.4 107987 75842 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=1048800 total_dropped_to