granularity results tiera-rerun-fixed-moonep @ 2026-08-30T10:17:03Z from sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0
Browse files
tiera-rerun-fixed-moonep/.completed/echo__tierbmem__gbs16__G3
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
2026-08-30T10:11:03+00:00
|
tiera-rerun-fixed-moonep/.completed/moonep__tierbmem__gbs16__G3
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
2026-08-30T10:16:46+00:00
|
tiera-rerun-fixed-moonep/.completed/ultraep__tierbmem__gbs16__G3
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
2026-08-30T10:13:56+00:00
|
tiera-rerun-fixed-moonep/STATUS.txt
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
study_root: /workspace/tiera-rerun-fixed-moonep
|
| 2 |
gbs: 16
|
| 3 |
-
updated: 2026-08-30T10:
|
| 4 |
-
current:
|
| 5 |
|
| 6 |
completed steps:
|
| 7 |
echo__a1__gbs16
|
|
@@ -9,18 +9,21 @@ completed steps:
|
|
| 9 |
echo__ponly__gbs16
|
| 10 |
echo__tierbmem__gbs16__G1
|
| 11 |
echo__tierbmem__gbs16__G2
|
|
|
|
| 12 |
echo__tierbtrace__gbs16
|
| 13 |
moonep__a1__gbs16
|
| 14 |
moonep__a2__gbs16
|
| 15 |
moonep__ponly__gbs16
|
| 16 |
moonep__tierbmem__gbs16__G1
|
| 17 |
moonep__tierbmem__gbs16__G2
|
|
|
|
| 18 |
moonep__tierbtrace__gbs16
|
| 19 |
ultraep__a1__gbs16
|
| 20 |
ultraep__a2__gbs16
|
| 21 |
ultraep__ponly__gbs16
|
| 22 |
ultraep__tierbmem__gbs16__G1
|
| 23 |
ultraep__tierbmem__gbs16__G2
|
|
|
|
| 24 |
ultraep__tierbtrace__gbs16
|
| 25 |
|
| 26 |
failed steps:
|
|
|
|
| 1 |
study_root: /workspace/tiera-rerun-fixed-moonep
|
| 2 |
gbs: 16
|
| 3 |
+
updated: 2026-08-30T10:16:54+00:00
|
| 4 |
+
current: idle
|
| 5 |
|
| 6 |
completed steps:
|
| 7 |
echo__a1__gbs16
|
|
|
|
| 9 |
echo__ponly__gbs16
|
| 10 |
echo__tierbmem__gbs16__G1
|
| 11 |
echo__tierbmem__gbs16__G2
|
| 12 |
+
echo__tierbmem__gbs16__G3
|
| 13 |
echo__tierbtrace__gbs16
|
| 14 |
moonep__a1__gbs16
|
| 15 |
moonep__a2__gbs16
|
| 16 |
moonep__ponly__gbs16
|
| 17 |
moonep__tierbmem__gbs16__G1
|
| 18 |
moonep__tierbmem__gbs16__G2
|
| 19 |
+
moonep__tierbmem__gbs16__G3
|
| 20 |
moonep__tierbtrace__gbs16
|
| 21 |
ultraep__a1__gbs16
|
| 22 |
ultraep__a2__gbs16
|
| 23 |
ultraep__ponly__gbs16
|
| 24 |
ultraep__tierbmem__gbs16__G1
|
| 25 |
ultraep__tierbmem__gbs16__G2
|
| 26 |
+
ultraep__tierbmem__gbs16__G3
|
| 27 |
ultraep__tierbtrace__gbs16
|
| 28 |
|
| 29 |
failed steps:
|
tiera-rerun-fixed-moonep/collected_all.tsv
CHANGED
|
@@ -35,6 +35,8 @@ echo/tierbmem/tierbmem_gbs16/G1__echo__matched__seq4096__gbs16__cap2p40625__memi
|
|
| 35 |
echo/tierbmem/tierbmem_gbs16/G1__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 G1 echo matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 dynamic dynamic none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/echo/tierbmem/tierbmem_gbs16/G1__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 better-memory-dump-support b7c52442 0 16 aux_loss 1e-2 0 0 1000000 0 INCOMPLETE 5 115283 88418
|
| 36 |
echo/tierbmem/tierbmem_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__memiteration__it6/rep1 rep1 tierbmem 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 1 iteration /workspace/tiera-rerun-fixed-moonep/echo/tierbmem/tierbmem_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 0 better-memory-dump-support b7c52442 0 16 aux_loss 1e-2 0 0 1000000 0 OK 6 430.10 430.10 775.2 145895 88178
|
| 37 |
echo/tierbmem/tierbmem_gbs16/G2__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 dynamic dynamic none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/echo/tierbmem/tierbmem_gbs16/G2__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 better-memory-dump-support b7c52442 0 16 aux_loss 1e-2 0 0 1000000 0 INCOMPLETE 5 117877 83562
|
|
|
|
|
|
|
| 38 |
echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1 rep1 tierbprobe 0 0 G1 echo matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.00 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 671.53 666.90 4.51 496.5 125477 94730
|
| 39 |
echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1 rep1 tierbprobe 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 660.43 642.00 14.73 505.0 119267 88146
|
| 40 |
echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1 rep1 tierbprobe 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 1 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 628.07 616.10 9.43 530.9 119267 88146
|
|
@@ -74,6 +76,8 @@ moonep/tierbmem/tierbmem_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__asy
|
|
| 74 |
moonep/tierbmem/tierbmem_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1 rep1 tierbmem 0 1 G1 moonep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 none none none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/moonep/tierbmem/tierbmem_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384 0 16 aux_loss 1e-2 0 0 1000000 0 FAIL 5 92579 59020
|
| 75 |
moonep/tierbmem/tierbmem_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1 rep1 tierbmem 0 1 G2 moonep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 none none none 0 1 iteration /workspace/tiera-rerun-fixed-moonep/moonep/tierbmem/tierbmem_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 0 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384 0 16 aux_loss 1e-2 0 0 1000000 0 OK 6 439.70 439.70 758.2 97663 56294
|
| 76 |
moonep/tierbmem/tierbmem_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1 rep1 tierbmem 0 1 G2 moonep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 none none none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/moonep/tierbmem/tierbmem_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384 0 16 aux_loss 1e-2 0 0 1000000 0 FAIL 5 95485 56290
|
|
|
|
|
|
|
| 77 |
moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys tierbtrace 1 1 G1 moonep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 10 9437.68 487.60 18511.57 423.5 58996
|
| 78 |
moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys tierbtrace 1 1 G2 moonep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 8 1344.80 429.70 1575.40 545.9 56294
|
| 79 |
moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys tierbtrace 1 1 G3 moonep matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 8 1483.70 563.20 1577.58 421.7 55708
|
|
@@ -112,6 +116,8 @@ ultraep/tierbmem/tierbmem_gbs16/G1__ultraep__matched__seq4096__gbs16__cap2p40625
|
|
| 112 |
ultraep/tierbmem/tierbmem_gbs16/G1__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 dynamic dynamic none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/ultraep/tierbmem/tierbmem_gbs16/G1__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 1024 0 ultraep-better-memory-dump-support c3a8a8a7 0 16 aux_loss 1e-2 0 0 1000000 0 FAIL 5 109261 80286
|
| 113 |
ultraep/tierbmem/tierbmem_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p08203125__memiteration__it6/rep1 rep1 tierbmem 0 0 G2 ultraep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 1 iteration /workspace/tiera-rerun-fixed-moonep/ultraep/tierbmem/tierbmem_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p08203125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 1024 0 ultraep-better-memory-dump-support c3a8a8a7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 6 475.40 475.40 701.2 112317 76502
|
| 114 |
ultraep/tierbmem/tierbmem_gbs16/G2__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 G2 ultraep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 dynamic dynamic none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/ultraep/tierbmem/tierbmem_gbs16/G2__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 1024 0 ultraep-better-memory-dump-support c3a8a8a7 0 16 aux_loss 1e-2 0 0 1000000 0 FAIL 5 108213 75714
|
|
|
|
|
|
|
| 115 |
ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1 rep1 tierbprobe 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 1.25 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 893.49 886.90 5.95 373.1 108731 80984 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=283999 total_dropped_tok
|
| 116 |
ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1 rep1 tierbprobe 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 1.8125 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 914.84 903.60 9.13 364.4 113767 84920 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=284540 total_dropped_tok
|
| 117 |
ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 rep1 tierbprobe 0 0 G2 ultraep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.0625 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 551.94 527.00 15.90 604.4 107987 75842 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=1048800 total_dropped_to
|
|
|
|
| 35 |
echo/tierbmem/tierbmem_gbs16/G1__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 G1 echo matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 dynamic dynamic none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/echo/tierbmem/tierbmem_gbs16/G1__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 better-memory-dump-support b7c52442 0 16 aux_loss 1e-2 0 0 1000000 0 INCOMPLETE 5 115283 88418
|
| 36 |
echo/tierbmem/tierbmem_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__memiteration__it6/rep1 rep1 tierbmem 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 1 iteration /workspace/tiera-rerun-fixed-moonep/echo/tierbmem/tierbmem_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 0 better-memory-dump-support b7c52442 0 16 aux_loss 1e-2 0 0 1000000 0 OK 6 430.10 430.10 775.2 145895 88178
|
| 37 |
echo/tierbmem/tierbmem_gbs16/G2__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 dynamic dynamic none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/echo/tierbmem/tierbmem_gbs16/G2__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 better-memory-dump-support b7c52442 0 16 aux_loss 1e-2 0 0 1000000 0 INCOMPLETE 5 117877 83562
|
| 38 |
+
echo/tierbmem/tierbmem_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625__memiteration__it6/rep1 rep1 tierbmem 0 0 G3 echo matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 1.072265625 static none 0 1 iteration /workspace/tiera-rerun-fixed-moonep/echo/tierbmem/tierbmem_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 0 better-memory-dump-support b7c52442 0 16 aux_loss 1e-2 0 0 1000000 0 OK 6 452.40 452.40 736.9 148009 91570
|
| 39 |
+
echo/tierbmem/tierbmem_gbs16/G3__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 G3 echo matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 dynamic dynamic none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/echo/tierbmem/tierbmem_gbs16/G3__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 better-memory-dump-support b7c52442 0 16 aux_loss 1e-2 0 0 1000000 0 INCOMPLETE 5 118901 86906
|
| 40 |
echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1 rep1 tierbprobe 0 0 G1 echo matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.00 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 671.53 666.90 4.51 496.5 125477 94730
|
| 41 |
echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1 rep1 tierbprobe 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 660.43 642.00 14.73 505.0 119267 88146
|
| 42 |
echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1 rep1 tierbprobe 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 1 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 628.07 616.10 9.43 530.9 119267 88146
|
|
|
|
| 76 |
moonep/tierbmem/tierbmem_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1 rep1 tierbmem 0 1 G1 moonep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 none none none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/moonep/tierbmem/tierbmem_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384 0 16 aux_loss 1e-2 0 0 1000000 0 FAIL 5 92579 59020
|
| 77 |
moonep/tierbmem/tierbmem_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1 rep1 tierbmem 0 1 G2 moonep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 none none none 0 1 iteration /workspace/tiera-rerun-fixed-moonep/moonep/tierbmem/tierbmem_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 0 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384 0 16 aux_loss 1e-2 0 0 1000000 0 OK 6 439.70 439.70 758.2 97663 56294
|
| 78 |
moonep/tierbmem/tierbmem_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1 rep1 tierbmem 0 1 G2 moonep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 none none none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/moonep/tierbmem/tierbmem_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384 0 16 aux_loss 1e-2 0 0 1000000 0 FAIL 5 95485 56290
|
| 79 |
+
moonep/tierbmem/tierbmem_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1 rep1 tierbmem 0 1 G3 moonep matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 none none none 0 1 iteration /workspace/tiera-rerun-fixed-moonep/moonep/tierbmem/tierbmem_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 0 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384 0 16 aux_loss 1e-2 0 0 1000000 0 OK 6 578.30 578.30 576.5 99411 55708
|
| 80 |
+
moonep/tierbmem/tierbmem_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1 rep1 tierbmem 0 1 G3 moonep matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 none none none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/moonep/tierbmem/tierbmem_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384 0 16 aux_loss 1e-2 0 0 1000000 0 FAIL 5 98611 55624
|
| 81 |
moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys tierbtrace 1 1 G1 moonep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 10 9437.68 487.60 18511.57 423.5 58996
|
| 82 |
moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys tierbtrace 1 1 G2 moonep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 8 1344.80 429.70 1575.40 545.9 56294
|
| 83 |
moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys tierbtrace 1 1 G3 moonep matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 8 1483.70 563.20 1577.58 421.7 55708
|
|
|
|
| 116 |
ultraep/tierbmem/tierbmem_gbs16/G1__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 dynamic dynamic none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/ultraep/tierbmem/tierbmem_gbs16/G1__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 1024 0 ultraep-better-memory-dump-support c3a8a8a7 0 16 aux_loss 1e-2 0 0 1000000 0 FAIL 5 109261 80286
|
| 117 |
ultraep/tierbmem/tierbmem_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p08203125__memiteration__it6/rep1 rep1 tierbmem 0 0 G2 ultraep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 1 iteration /workspace/tiera-rerun-fixed-moonep/ultraep/tierbmem/tierbmem_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p08203125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 1024 0 ultraep-better-memory-dump-support c3a8a8a7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 6 475.40 475.40 701.2 112317 76502
|
| 118 |
ultraep/tierbmem/tierbmem_gbs16/G2__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 G2 ultraep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 dynamic dynamic none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/ultraep/tierbmem/tierbmem_gbs16/G2__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 1024 0 ultraep-better-memory-dump-support c3a8a8a7 0 16 aux_loss 1e-2 0 0 1000000 0 FAIL 5 108213 75714
|
| 119 |
+
ultraep/tierbmem/tierbmem_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p072265625__memiteration__it6/rep1 rep1 tierbmem 0 0 G3 ultraep matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 1.072265625 static none 0 1 iteration /workspace/tiera-rerun-fixed-moonep/ultraep/tierbmem/tierbmem_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p072265625__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 1024 0 ultraep-better-memory-dump-support c3a8a8a7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 6 477.80 477.80 697.8 113951 79598
|
| 120 |
+
ultraep/tierbmem/tierbmem_gbs16/G3__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 G3 ultraep matched 8 64 16 1792 8 22020096 16 4096 16 2 65536 8192 dynamic dynamic none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/ultraep/tierbmem/tierbmem_gbs16/G3__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 1024 0 ultraep-better-memory-dump-support c3a8a8a7 0 16 aux_loss 1e-2 0 0 1000000 0 FAIL 5 111707 78996
|
| 121 |
ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1 rep1 tierbprobe 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 1.25 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 893.49 886.90 5.95 373.1 108731 80984 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=283999 total_dropped_tok
|
| 122 |
ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1 rep1 tierbprobe 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 1.8125 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 914.84 903.60 9.13 364.4 113767 84920 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=284540 total_dropped_tok
|
| 123 |
ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 rep1 tierbprobe 0 0 G2 ultraep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.0625 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 551.94 527.00 15.90 604.4 107987 75842 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=1048800 total_dropped_to
|