granularity results tiera-rerun-fixed-moonep @ 2026-08-30T08:12:09Z from sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0
Browse files
tiera-rerun-fixed-moonep/.completed/echo__tierbmem__gbs16
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
2026-08-30T08:06:37+00:00
|
tiera-rerun-fixed-moonep/.completed/moonep__tierbmem__gbs16
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
2026-08-30T08:11:55+00:00
|
tiera-rerun-fixed-moonep/.completed/ultraep__tierbmem__gbs16
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
2026-08-30T08:09:23+00:00
|
tiera-rerun-fixed-moonep/STATUS.txt
CHANGED
|
@@ -1,18 +1,21 @@
|
|
| 1 |
study_root: /workspace/tiera-rerun-fixed-moonep
|
| 2 |
gbs: 16
|
| 3 |
-
updated: 2026-08-30T08:
|
| 4 |
-
current:
|
| 5 |
|
| 6 |
completed steps:
|
| 7 |
echo__a1__gbs16
|
| 8 |
echo__a2__gbs16
|
| 9 |
echo__ponly__gbs16
|
|
|
|
| 10 |
moonep__a1__gbs16
|
| 11 |
moonep__a2__gbs16
|
| 12 |
moonep__ponly__gbs16
|
|
|
|
| 13 |
ultraep__a1__gbs16
|
| 14 |
ultraep__a2__gbs16
|
| 15 |
ultraep__ponly__gbs16
|
|
|
|
| 16 |
|
| 17 |
failed steps:
|
| 18 |
FAIL echo tierbprobe 16 2026-08-30T04:52:38+00:00 2026-08-30T05:06:38+00:00 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe
|
|
|
|
| 1 |
study_root: /workspace/tiera-rerun-fixed-moonep
|
| 2 |
gbs: 16
|
| 3 |
+
updated: 2026-08-30T08:12:00+00:00
|
| 4 |
+
current: idle
|
| 5 |
|
| 6 |
completed steps:
|
| 7 |
echo__a1__gbs16
|
| 8 |
echo__a2__gbs16
|
| 9 |
echo__ponly__gbs16
|
| 10 |
+
echo__tierbmem__gbs16
|
| 11 |
moonep__a1__gbs16
|
| 12 |
moonep__a2__gbs16
|
| 13 |
moonep__ponly__gbs16
|
| 14 |
+
moonep__tierbmem__gbs16
|
| 15 |
ultraep__a1__gbs16
|
| 16 |
ultraep__a2__gbs16
|
| 17 |
ultraep__ponly__gbs16
|
| 18 |
+
ultraep__tierbmem__gbs16
|
| 19 |
|
| 20 |
failed steps:
|
| 21 |
FAIL echo tierbprobe 16 2026-08-30T04:52:38+00:00 2026-08-30T05:06:38+00:00 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe
|
tiera-rerun-fixed-moonep/collected_all.tsv
CHANGED
|
@@ -31,6 +31,8 @@ echo/ponly/ponly_gbs16/PONLY__echo__one__seq4096__gbs16__cap2p40625/rep3 rep3 po
|
|
| 31 |
echo/ponly/ponly_gbs16/PONLY__echo__one__seq8192__gbs16__cap2p40625/rep1 rep1 ponly 0 0 PONLY echo one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 316.36 315.50 0.77 627.1 53279 32528
|
| 32 |
echo/ponly/ponly_gbs16/PONLY__echo__one__seq8192__gbs16__cap2p40625/rep2 rep2 ponly 0 0 PONLY echo one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 316.59 315.80 0.88 626.6 53283 32496
|
| 33 |
echo/ponly/ponly_gbs16/PONLY__echo__one__seq8192__gbs16__cap2p40625/rep3 rep3 ponly 0 0 PONLY echo one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 317.16 315.40 1.68 625.5 53283 32496
|
|
|
|
|
|
|
| 34 |
echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1 rep1 tierbprobe 0 0 G1 echo matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.00 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 671.53 666.90 4.51 496.5 125477 94730
|
| 35 |
echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1 rep1 tierbprobe 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 660.43 642.00 14.73 505.0 119267 88146
|
| 36 |
echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1 rep1 tierbprobe 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 1 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 628.07 616.10 9.43 530.9 119267 88146
|
|
@@ -62,6 +64,8 @@ moonep/ponly/ponly_gbs16/PONLY__moonep__one__seq4096__gbs16__capnone__async/rep3
|
|
| 62 |
moonep/ponly/ponly_gbs16/PONLY__moonep__one__seq8192__gbs16__capnone__async/rep1 rep1 ponly 0 1 PONLY moonep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 300.76 299.30 0.82 659.6 41295 25036
|
| 63 |
moonep/ponly/ponly_gbs16/PONLY__moonep__one__seq8192__gbs16__capnone__async/rep2 rep2 ponly 0 1 PONLY moonep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 300.27 297.30 2.48 660.7 41297 25038
|
| 64 |
moonep/ponly/ponly_gbs16/PONLY__moonep__one__seq8192__gbs16__capnone__async/rep3 rep3 ponly 0 1 PONLY moonep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 300.81 297.90 1.79 659.5 41295 25038
|
|
|
|
|
|
|
| 65 |
ultraep/a1/a1_oneslot_gbs16/G1__ultraep__one__seq4096__gbs16__cap2p40625/rep1 rep1 a1 0 0 G1 ultraep one 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 888.60 869.80 12.39 375.2 129475 87692
|
| 66 |
ultraep/a1/a1_oneslot_gbs16/G1__ultraep__one__seq4096__gbs16__cap2p40625/rep2 rep2 a1 0 0 G1 ultraep one 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 894.61 880.90 11.04 372.7 129475 87692
|
| 67 |
ultraep/a1/a1_oneslot_gbs16/G1__ultraep__one__seq4096__gbs16__cap2p40625/rep3 rep3 a1 0 0 G1 ultraep one 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 889.80 872.80 10.69 374.7 129475 87692
|
|
@@ -92,6 +96,8 @@ ultraep/ponly/ponly_gbs16/PONLY__ultraep__one__seq4096__gbs16__cap2p40625/rep3 r
|
|
| 92 |
ultraep/ponly/ponly_gbs16/PONLY__ultraep__one__seq8192__gbs16__cap2p40625/rep1 rep1 ponly 0 0 PONLY ultraep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 2048 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 292.76 290.70 1.82 677.6 51017 31264
|
| 93 |
ultraep/ponly/ponly_gbs16/PONLY__ultraep__one__seq8192__gbs16__cap2p40625/rep2 rep2 ponly 0 0 PONLY ultraep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 2048 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 292.47 291.30 1.15 678.3 51015 31264
|
| 94 |
ultraep/ponly/ponly_gbs16/PONLY__ultraep__one__seq8192__gbs16__cap2p40625/rep3 rep3 ponly 0 0 PONLY ultraep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 2048 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 292.64 290.20 1.85 677.9 51013 31264
|
|
|
|
|
|
|
| 95 |
ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1 rep1 tierbprobe 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 1.25 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 893.49 886.90 5.95 373.1 108731 80984 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=283999 total_dropped_tok
|
| 96 |
ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1 rep1 tierbprobe 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 1.8125 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 914.84 903.60 9.13 364.4 113767 84920 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=284540 total_dropped_tok
|
| 97 |
ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 rep1 tierbprobe 0 0 G2 ultraep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.0625 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 551.94 527.00 15.90 604.4 107987 75842 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=1048800 total_dropped_to
|
|
|
|
| 31 |
echo/ponly/ponly_gbs16/PONLY__echo__one__seq8192__gbs16__cap2p40625/rep1 rep1 ponly 0 0 PONLY echo one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 316.36 315.50 0.77 627.1 53279 32528
|
| 32 |
echo/ponly/ponly_gbs16/PONLY__echo__one__seq8192__gbs16__cap2p40625/rep2 rep2 ponly 0 0 PONLY echo one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 316.59 315.80 0.88 626.6 53283 32496
|
| 33 |
echo/ponly/ponly_gbs16/PONLY__echo__one__seq8192__gbs16__cap2p40625/rep3 rep3 ponly 0 0 PONLY echo one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 0 bracing-regions-through-kernels 9a236793 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 317.16 315.40 1.68 625.5 53283 32496
|
| 34 |
+
echo/tierbmem/tierbmem_gbs16/G1__echo__matched__seq4096__gbs16__cap2p40625__memiteration__it6/rep1 rep1 tierbmem 0 0 G1 echo matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 iteration /workspace/tiera-rerun-fixed-moonep/echo/tierbmem/tierbmem_gbs16/G1__echo__matched__seq4096__gbs16__cap2p40625__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 0 better-memory-dump-support b7c52442 0 16 aux_loss 1e-2 0 0 1000000 0 OK 6 474.40 474.40 702.7 161723 99122
|
| 35 |
+
echo/tierbmem/tierbmem_gbs16/G1__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 G1 echo matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 dynamic dynamic none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/echo/tierbmem/tierbmem_gbs16/G1__echo__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 better-memory-dump-support b7c52442 0 16 aux_loss 1e-2 0 0 1000000 0 INCOMPLETE 5 115283 88418
|
| 36 |
echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1 rep1 tierbprobe 0 0 G1 echo matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.00 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 671.53 666.90 4.51 496.5 125477 94730
|
| 37 |
echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1 rep1 tierbprobe 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 0 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 660.43 642.00 14.73 505.0 119267 88146
|
| 38 |
echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1 rep1 tierbprobe 0 0 G2 echo matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.08203125 static none 1 0 none 0 token-drop-analysis b099fd84 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 628.07 616.10 9.43 530.9 119267 88146
|
|
|
|
| 64 |
moonep/ponly/ponly_gbs16/PONLY__moonep__one__seq8192__gbs16__capnone__async/rep1 rep1 ponly 0 1 PONLY moonep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 300.76 299.30 0.82 659.6 41295 25036
|
| 65 |
moonep/ponly/ponly_gbs16/PONLY__moonep__one__seq8192__gbs16__capnone__async/rep2 rep2 ponly 0 1 PONLY moonep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 300.27 297.30 2.48 660.7 41297 25038
|
| 66 |
moonep/ponly/ponly_gbs16/PONLY__moonep__one__seq8192__gbs16__capnone__async/rep3 rep3 ponly 0 1 PONLY moonep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 none none none 0 1 none 0 moonep_backend_async_grad_reduce 2e68c8ad 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 300.81 297.90 1.79 659.5 41295 25038
|
| 67 |
+
moonep/tierbmem/tierbmem_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1 rep1 tierbmem 0 1 G1 moonep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 none none none 0 1 iteration /workspace/tiera-rerun-fixed-moonep/moonep/tierbmem/tierbmem_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 0 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384 0 16 aux_loss 1e-2 0 0 1000000 0 OK 6 497.40 497.40 670.2 96849 58996
|
| 68 |
+
moonep/tierbmem/tierbmem_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1 rep1 tierbmem 0 1 G1 moonep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 none none none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/moonep/tierbmem/tierbmem_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 0 moonep_backend_async_grad_reduce_better_memory_dump_support 7949f384 0 16 aux_loss 1e-2 0 0 1000000 0 FAIL 5 92579 59020
|
| 69 |
ultraep/a1/a1_oneslot_gbs16/G1__ultraep__one__seq4096__gbs16__cap2p40625/rep1 rep1 a1 0 0 G1 ultraep one 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 888.60 869.80 12.39 375.2 129475 87692
|
| 70 |
ultraep/a1/a1_oneslot_gbs16/G1__ultraep__one__seq4096__gbs16__cap2p40625/rep2 rep2 a1 0 0 G1 ultraep one 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 894.61 880.90 11.04 372.7 129475 87692
|
| 71 |
ultraep/a1/a1_oneslot_gbs16/G1__ultraep__one__seq4096__gbs16__cap2p40625/rep3 rep3 a1 0 0 G1 ultraep one 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 none 1024 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 889.80 872.80 10.69 374.7 129475 87692
|
|
|
|
| 96 |
ultraep/ponly/ponly_gbs16/PONLY__ultraep__one__seq8192__gbs16__cap2p40625/rep1 rep1 ponly 0 0 PONLY ultraep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 2048 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 292.76 290.70 1.82 677.6 51017 31264
|
| 97 |
ultraep/ponly/ponly_gbs16/PONLY__ultraep__one__seq8192__gbs16__cap2p40625/rep2 rep2 ponly 0 0 PONLY ultraep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 2048 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 292.47 291.30 1.15 678.3 51015 31264
|
| 98 |
ultraep/ponly/ponly_gbs16/PONLY__ultraep__one__seq8192__gbs16__cap2p40625/rep3 rep3 ponly 0 0 PONLY ultraep one 1 8 2 1536 1 18874368 16 8192 16 2 16384 16384 2.40625 static none 0 1 none 2048 0 ultraep-bracing-regions-through-kernels 03d57ef7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 292.64 290.20 1.85 677.9 51013 31264
|
| 99 |
+
ultraep/tierbmem/tierbmem_gbs16/G1__ultraep__matched__seq4096__gbs16__cap2p40625__memiteration__it6/rep1 rep1 tierbmem 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 2.40625 static none 0 1 iteration /workspace/tiera-rerun-fixed-moonep/ultraep/tierbmem/tierbmem_gbs16/G1__ultraep__matched__seq4096__gbs16__cap2p40625__memiteration__it6/rep1/memory_snapshots/snapshot.pickle 1024 0 ultraep-better-memory-dump-support c3a8a8a7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 6 909.90 909.90 366.4 129473 87692
|
| 100 |
+
ultraep/tierbmem/tierbmem_gbs16/G1__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1 rep1 tierbmem 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 dynamic dynamic none 0 0 iteration /workspace/tiera-rerun-fixed-moonep/ultraep/tierbmem/tierbmem_gbs16/G1__ultraep__matched__seq4096__gbs16__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle 1024 0 ultraep-better-memory-dump-support c3a8a8a7 0 16 aux_loss 1e-2 0 0 1000000 0 FAIL 5 109261 80286
|
| 101 |
ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1 rep1 tierbprobe 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 1.25 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 893.49 886.90 5.95 373.1 108731 80984 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=283999 total_dropped_tok
|
| 102 |
ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1 rep1 tierbprobe 0 0 G1 ultraep matched 1 8 2 14336 1 176160768 16 4096 16 2 8192 8192 1.8125 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 914.84 903.60 9.13 364.4 113767 84920 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=284540 total_dropped_tok
|
| 103 |
ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 rep1 tierbprobe 0 0 G2 ultraep matched 4 32 8 3584 4 44040192 16 4096 16 2 32768 8192 1.0625 static none 0 0 none 1024 0 ultraep-token-drop-analysis eaf829e7 0 16 aux_loss 1e-2 0 0 1000000 0 OK 12 551.94 527.00 15.90 604.4 107987 75842 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=1048800 total_dropped_to
|