AbhishekGhosh1998 commited on
Commit
026f6fd
·
verified ·
1 Parent(s): 22628da

granularity results tiera-rerun-fixed-moonep @ 2026-08-30T07:00:41Z from sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0

Browse files
tiera-rerun-fixed-moonep/.completed/echo__a2__gbs16 ADDED
@@ -0,0 +1 @@
 
 
1
+ 2026-08-30T07:00:25+00:00
tiera-rerun-fixed-moonep/STATUS.txt CHANGED
@@ -1,10 +1,11 @@
1
  study_root: /workspace/tiera-rerun-fixed-moonep
2
  gbs: 16
3
- updated: 2026-08-30T06:44:13+00:00
4
- current: echo__a2__gbs16
5
 
6
  completed steps:
7
  echo__a1__gbs16
 
8
  echo__ponly__gbs16
9
 
10
  failed steps:
 
1
  study_root: /workspace/tiera-rerun-fixed-moonep
2
  gbs: 16
3
+ updated: 2026-08-30T07:00:27+00:00
4
+ current: ultraep__a1__gbs16
5
 
6
  completed steps:
7
  echo__a1__gbs16
8
+ echo__a2__gbs16
9
  echo__ponly__gbs16
10
 
11
  failed steps:
tiera-rerun-fixed-moonep/hf_watcher.log CHANGED
@@ -32,3 +32,6 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
32
  uploaded: /workspace/tiera-rerun-fixed-moonep -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tiera-rerun-fixed-moonep
33
  https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
34
  [2026-08-30T06:50:38+00:00] backup pass
 
 
 
 
32
  uploaded: /workspace/tiera-rerun-fixed-moonep -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tiera-rerun-fixed-moonep
33
  https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
34
  [2026-08-30T06:50:38+00:00] backup pass
35
+ uploaded: /workspace/tiera-rerun-fixed-moonep -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tiera-rerun-fixed-moonep
36
+ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
37
+ [2026-08-30T07:00:41+00:00] backup pass
tiera-rerun-fixed-moonep/steps.tsv CHANGED
@@ -3,3 +3,4 @@ FAIL echo tierbprobe 16 2026-08-30T04:52:38+00:00 2026-08-30T05:06:38+00:00 /wor
3
  FAIL ultraep tierbprobe 16 2026-08-30T05:06:44+00:00 2026-08-30T05:20:40+00:00 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe
4
  OK echo a1 16 2026-08-30T06:20:31+00:00 2026-08-30T06:36:28+00:00 /workspace/tiera-rerun-fixed-moonep/echo/a1
5
  OK echo ponly 16 2026-08-30T06:36:31+00:00 2026-08-30T06:44:11+00:00 /workspace/tiera-rerun-fixed-moonep/echo/ponly
 
 
3
  FAIL ultraep tierbprobe 16 2026-08-30T05:06:44+00:00 2026-08-30T05:20:40+00:00 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe
4
  OK echo a1 16 2026-08-30T06:20:31+00:00 2026-08-30T06:36:28+00:00 /workspace/tiera-rerun-fixed-moonep/echo/a1
5
  OK echo ponly 16 2026-08-30T06:36:31+00:00 2026-08-30T06:44:11+00:00 /workspace/tiera-rerun-fixed-moonep/echo/ponly
6
+ OK echo a2 16 2026-08-30T06:44:13+00:00 2026-08-30T07:00:25+00:00 /workspace/tiera-rerun-fixed-moonep/echo/a2
tiera-rerun-fixed-moonep/study.log CHANGED
@@ -569,3 +569,57 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
569
  [06:48:19] CELL G2__echo__matched__seq4096__gbs16__cap1p08203125 (slots=4)
570
  [guard] iter 8/12 (70s)
571
  [guard] iter 12/12 (80s)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
569
  [06:48:19] CELL G2__echo__matched__seq4096__gbs16__cap1p08203125 (slots=4)
570
  [guard] iter 8/12 (70s)
571
  [guard] iter 12/12 (80s)
572
+ [guard] iter 8/12 (70s)
573
+ [guard] iter 12/12 (80s)
574
+ [guard] iter 7/12 (70s)
575
+ [guard] iter 12/12 (80s)
576
+ --------------------------------------------------------------------------
577
+ [06:52:19] CELL G3__echo__matched__seq4096__gbs16__cap1p072265625 (slots=8)
578
+ [guard] iter 4/12 (70s)
579
+ [guard] iter 12/12 (80s)
580
+ [guard] tearing down process group (outcome=DONE)
581
+ [guard] iter 3/12 (70s)
582
+ [guard] iter 12/12 (80s)
583
+ [guard] tearing down process group (outcome=DONE)
584
+ [guard] iter 4/12 (70s)
585
+ [guard] iter 12/12 (80s)
586
+ [guard] tearing down process group (outcome=DONE)
587
+ --------------------------------------------------------------------------
588
+ [06:56:23] CELL G4__echo__matched__seq4096__gbs16__cap1p095703125 (slots=16)
589
+ [guard] iter 7/12 (70s)
590
+ [guard] iter 12/12 (80s)
591
+ [guard] iter 4/12 (70s)
592
+ [guard] iter 12/12 (80s)
593
+ [guard] tearing down process group (outcome=DONE)
594
+ [guard] iter 6/12 (70s)
595
+ [guard] iter 12/12 (80s)
596
+ --------------------------------------------------------------------------
597
+
598
+ ==============================================================
599
+ SWEEP COMPLETE ok=12 failed=0
600
+ ==============================================================
601
+ config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
602
+ G1 echo matched 1 4096 16 2.40625 none 1 OK 466.86 462.10 161723 /workspace/tiera-rerun-fixed-moonep/echo/a2/a2_matched_gbs16/G1__echo__matched__seq4096__gbs16__cap2p40625/rep1
603
+ G1 echo matched 1 4096 16 2.40625 none 2 OK 466.67 462.10 161723 /workspace/tiera-rerun-fixed-moonep/echo/a2/a2_matched_gbs16/G1__echo__matched__seq4096__gbs16__cap2p40625/rep2
604
+ G1 echo matched 1 4096 16 2.40625 none 3 OK 466.91 461.80 161723 /workspace/tiera-rerun-fixed-moonep/echo/a2/a2_matched_gbs16/G1__echo__matched__seq4096__gbs16__cap2p40625/rep3
605
+ G2 echo matched 4 4096 16 1.08203125 none 1 OK 415.60 413.70 145689 /workspace/tiera-rerun-fixed-moonep/echo/a2/a2_matched_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1
606
+ G2 echo matched 4 4096 16 1.08203125 none 2 OK 415.27 413.70 145691 /workspace/tiera-rerun-fixed-moonep/echo/a2/a2_matched_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep2
607
+ G2 echo matched 4 4096 16 1.08203125 none 3 OK 415.21 412.60 145691 /workspace/tiera-rerun-fixed-moonep/echo/a2/a2_matched_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep3
608
+ G3 echo matched 8 4096 16 1.072265625 none 1 OK 436.97 435.10 148157 /workspace/tiera-rerun-fixed-moonep/echo/a2/a2_matched_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625/rep1
609
+ G3 echo matched 8 4096 16 1.072265625 none 2 OK 437.30 435.10 148273 /workspace/tiera-rerun-fixed-moonep/echo/a2/a2_matched_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625/rep2
610
+ G3 echo matched 8 4096 16 1.072265625 none 3 OK 437.35 435.10 147933 /workspace/tiera-rerun-fixed-moonep/echo/a2/a2_matched_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625/rep3
611
+ G4 echo matched 16 4096 16 1.095703125 none 1 OK 486.56 484.60 168485 /workspace/tiera-rerun-fixed-moonep/echo/a2/a2_matched_gbs16/G4__echo__matched__seq4096__gbs16__cap1p095703125/rep1
612
+ G4 echo matched 16 4096 16 1.095703125 none 2 OK 487.01 484.60 168709 /workspace/tiera-rerun-fixed-moonep/echo/a2/a2_matched_gbs16/G4__echo__matched__seq4096__gbs16__cap1p095703125/rep2
613
+ G4 echo matched 16 4096 16 1.095703125 none 3 OK 486.54 482.40 168485 /workspace/tiera-rerun-fixed-moonep/echo/a2/a2_matched_gbs16/G4__echo__matched__seq4096__gbs16__cap1p095703125/rep3
614
+
615
+ Results: /workspace/tiera-rerun-fixed-moonep/echo/a2/a2_matched_gbs16/results.tsv
616
+ Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/tiera-rerun-fixed-moonep/echo/a2/a2_matched_gbs16
617
+ ==============================================================
618
+
619
+ Campaign results: /workspace/tiera-rerun-fixed-moonep/echo/a2
620
+ Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/tiera-rerun-fixed-moonep/echo/a2 --compare
621
+ Already logged in to Hugging Face as: AbhishekGhosh1998
622
+ uploaded: /workspace/tiera-rerun-fixed-moonep/echo/a2 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tiera-rerun-fixed-moonep/echo/a2
623
+ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
624
+
625
+ === STEP ultraep__a1__gbs16
tiera-rerun-fixed-moonep/ultraep/a1/provenance.txt ADDED
@@ -0,0 +1,90 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ==============================================================
2
+ PROVENANCE: ultraep__a1__gbs16
3
+ ==============================================================
4
+ captured_at : 2026-08-30T07:00:27+00:00
5
+ hostname : sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0
6
+ kernel : 6.8.0-1043-nvidia
7
+ cwd : /workspace/blackwell_run_scripts
8
+ command : run_ladder.sh --phase a1 --gbs 16 --out /workspace/tiera-rerun-fixed-moonep/ultraep/a1
9
+
10
+ ---- GPU / driver --------------------------------------------
11
+ 0, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
12
+ 1, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
13
+ 2, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
14
+ 3, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
15
+ 4, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
16
+ 5, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
17
+ 6, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
18
+ 7, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
19
+ persistence/clocks:
20
+ 0, 135 MHz, 1965 MHz, 1000.00 W, 30
21
+ 1, 120 MHz, 1965 MHz, 1000.00 W, 35
22
+ 2, 135 MHz, 1965 MHz, 1000.00 W, 35
23
+ 3, 337 MHz, 1965 MHz, 1000.00 W, 30
24
+ 4, 120 MHz, 1965 MHz, 1000.00 W, 31
25
+ 5, 225 MHz, 1965 MHz, 1000.00 W, 35
26
+ 6, 120 MHz, 1965 MHz, 1000.00 W, 35
27
+ 7, 180 MHz, 1965 MHz, 1000.00 W, 30
28
+
29
+ NVLink topology:
30
+ GPU0 GPU1 GPU2 GPU3 GPU4 GPU5 GPU6 GPU7 NIC0 NIC1 NIC2 NIC3 NIC4 NIC5 NIC6 NIC7 NIC8 CPU Affinity NUMA Affinity GPU NUMA ID
31
+ GPU0 X NV18 NV18 NV18 NV18 NV18 NV18 NV18 PIX NODE SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A
32
+ GPU1 NV18 X NV18 NV18 NV18 NV18 NV18 NV18 NODE PIX SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A
33
+ GPU2 NV18 NV18 X NV18 NV18 NV18 NV18 NV18 SYS SYS PIX NODE SYS SYS SYS SYS SYS 28-55 1 N/A
34
+ GPU3 NV18 NV18 NV18 X NV18 NV18 NV18 NV18 SYS SYS NODE PIX SYS SYS SYS SYS SYS 28-55 1 N/A
35
+ GPU4 NV18 NV18 NV18 NV18 X NV18 NV18 NV18 SYS SYS SYS SYS PIX NODE SYS SYS SYS 56-83 2 N/A
36
+ GPU5 NV18 NV18 NV18 NV18 NV18 X NV18 NV18 SYS SYS SYS SYS NODE PIX SYS SYS SYS 56-83 2 N/A
37
+ GPU6 NV18 NV18 NV18 NV18 NV18 NV18 X NV18 SYS SYS SYS SYS SYS SYS PIX NODE SYS 84-111 3 N/A
38
+ GPU7 NV18 NV18 NV18 NV18 NV18 NV18 NV18 X SYS SYS SYS SYS SYS SYS NODE PIX SYS 84-111 3 N/A
39
+ NIC0 PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS SYS NODE
40
+ NIC1 NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS SYS NODE
41
+ NIC2 SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS
42
+ NIC3 SYS SYS NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS
43
+ NIC4 SYS SYS SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS
44
+ unavailable
45
+
46
+ ---- toolchain -----------------------------------------------
47
+ nvcc : release 12.9, V12.9.86
48
+ gcc : gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0
49
+ python : 3.12.3
50
+ torch : 2.8.0a0+5228986c39.nv25.06 cuda=12.9 nccl=2.27.3
51
+ torch arch : ['sm_75', 'sm_80', 'sm_86', 'sm_90', 'sm_100', 'sm_120', 'compute_120']
52
+ transformer_engine: 2.10.0.dev0+10f6128d [/usr/local/lib/python3.12/dist-packages/transformer_engine/__init__.py]
53
+ deep_ep : 1.2.1+34152ae [/workspace/DeepEP/deep_ep/__init__.py]
54
+ ultra_ep : 1.0.0 [/workspace/UltraEP/ultra_ep/__init__.py]
55
+ moonep : 0.0.1 [/workspace/MoonEP/moonep/__init__.py]
56
+ megatron : 0.14.0rc5 [None]
57
+ flash_attn : 2.7.4.post1 [/usr/local/lib/python3.12/dist-packages/flash_attn/__init__.py]
58
+ numpy : 1.26.4 [/usr/local/lib/python3.12/dist-packages/numpy/__init__.py]
59
+ huggingface_hub: 1.29.0 [/usr/local/lib/python3.12/dist-packages/huggingface_hub/__init__.py]
60
+
61
+ ---- repositories --------------------------------------------
62
+ Megatron-MoE-Echo bracing-regions-through-kernels 9a23679390d7
63
+ Megatron-UltraEP ultraep-bracing-regions-through-kernels 03d57ef7d476
64
+ UltraEP main 94cab099b44f
65
+ DeepEP HEAD 34152ae28f80 (+3 untracked: build output)
66
+ MoonEP megatron-ddp-symmetric-params d8bc6667fad7
67
+ blackwell_run_scripts granularity-ultraep 5e3993b15bed
68
+
69
+ ---- environment ---------------------------------------------
70
+ CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0
71
+ CUDA_DRIVER_VERSION=575.57.08
72
+ CUDA_HOME=/usr/local/cuda
73
+ CUDA_MODULE_LOADING=LAZY
74
+ CUDA_VERSION=12.9.1.010
75
+ HF_REMOTE_ROOT=tiera-rerun-fixed-moonep
76
+ NCCL_VERSION=2.27.3
77
+ NVSHMEM_VERSION=3.2.5
78
+ PYTORCH_BUILD_NUMBER=0
79
+ PYTORCH_BUILD_VERSION=2.8.0a0+5228986
80
+ PYTORCH_HOME=/opt/pytorch/pytorch
81
+ PYTORCH_VERSION=2.8.0a0+5228986
82
+ TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1
83
+ TORCH_CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0+PTX
84
+ TORCH_NCCL_USE_COMM_NONBLOCKING=0
85
+
86
+ ---- shared library resolution -------------------------------
87
+ libnvshmem_host.so: not in ldconfig
88
+ libnccl.so: /lib/x86_64-linux-gnu/libnccl.so.2
89
+ lrwxrwxrwx 1 root root 20 Aug 29 16:41 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so -> libnvshmem_host.so.3
90
+ -rw-r--r-- 1 root root 179045744 Aug 29 16:37 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so.3