AbhishekGhosh1998 commited on
Commit
3a7ba41
·
verified ·
1 Parent(s): 823502f

granularity results tierb_study_20260826_144644 @ 2026-08-26T16:35:54Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0

Browse files
Files changed (29) hide show
  1. .gitattributes +2 -0
  2. tierb_study_20260826_144644/echo/tierbprobe/manifest.txt +2 -2
  3. tierb_study_20260826_144644/echo/tierbprobe/provenance.txt +61 -0
  4. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv +48 -0
  5. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25/rep1/run.env +61 -0
  6. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25/rep1/train.log +620 -0
  7. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv +81 -81
  8. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log +0 -0
  9. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv +248 -0
  10. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25/rep1/run.env +61 -0
  11. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25/rep1/train.log +0 -0
  12. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__forcebal/rep1/mem_samples.csv +240 -0
  13. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__forcebal/rep1/run.env +61 -0
  14. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__forcebal/rep1/train.log +0 -0
  15. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv +88 -88
  16. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log +0 -0
  17. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1/mem_samples.csv +272 -0
  18. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1/run.env +61 -0
  19. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1/train.log +3 -0
  20. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal/rep1/mem_samples.csv +272 -0
  21. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal/rep1/run.env +61 -0
  22. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log +3 -0
  23. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/mem_samples.csv +134 -134
  24. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/train.log +2 -2
  25. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv +7 -1
  26. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv +2 -0
  27. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/results.tsv +7 -1
  28. tierb_study_20260826_144644/hf_watcher.log +3 -0
  29. tierb_study_20260826_144644/study.log +43 -0
.gitattributes CHANGED
@@ -35,3 +35,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
  tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/train.log filter=lfs diff=lfs merge=lfs -text
37
  tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log filter=lfs diff=lfs merge=lfs -text
 
 
 
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
  tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/train.log filter=lfs diff=lfs merge=lfs -text
37
  tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log filter=lfs diff=lfs merge=lfs -text
38
+ tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1/train.log filter=lfs diff=lfs merge=lfs -text
39
+ tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log filter=lfs diff=lfs merge=lfs -text
tierb_study_20260826_144644/echo/tierbprobe/manifest.txt CHANGED
@@ -1,9 +1,9 @@
1
- timestamp: 2026-08-26T16:16:58+00:00
2
  host: sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0
3
  gpu: NVIDIA B200 x8
4
  driver: 580.95.05
5
  study_scripts_branch: granularity-echo
6
- study_scripts_commit: b99bce1
7
  campaign: echo
8
  backend: echo
9
  phase: tierbprobe
 
1
+ timestamp: 2026-08-26T16:26:06+00:00
2
  host: sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0
3
  gpu: NVIDIA B200 x8
4
  driver: 580.95.05
5
  study_scripts_branch: granularity-echo
6
+ study_scripts_commit: 6e4b1c8
7
  campaign: echo
8
  backend: echo
9
  phase: tierbprobe
tierb_study_20260826_144644/echo/tierbprobe/provenance.txt CHANGED
@@ -27,3 +27,64 @@ command : run_ladder.sh --phase tierbprobe --gbs 8 --out /workspace/tierb_
27
  7, 120 MHz, 1965 MHz, 1000.00 W, 28
28
 
29
  NVLink topology:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
  7, 120 MHz, 1965 MHz, 1000.00 W, 28
28
 
29
  NVLink topology:
30
+ GPU0 GPU1 GPU2 GPU3 GPU4 GPU5 GPU6 GPU7 NIC0 NIC1 NIC2 NIC3 NIC4 NIC5 NIC6 NIC7 NIC8 CPU Affinity NUMA Affinity GPU NUMA ID
31
+ GPU0 X NV18 NV18 NV18 NV18 NV18 NV18 NV18 PIX NODE SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A
32
+ GPU1 NV18 X NV18 NV18 NV18 NV18 NV18 NV18 NODE PIX SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A
33
+ GPU2 NV18 NV18 X NV18 NV18 NV18 NV18 NV18 SYS SYS PIX NODE SYS SYS SYS SYS SYS 28-55 1 N/A
34
+ GPU3 NV18 NV18 NV18 X NV18 NV18 NV18 NV18 SYS SYS NODE PIX SYS SYS SYS SYS SYS 28-55 1 N/A
35
+ GPU4 NV18 NV18 NV18 NV18 X NV18 NV18 NV18 SYS SYS SYS SYS PIX NODE SYS SYS SYS 56-83 2 N/A
36
+ GPU5 NV18 NV18 NV18 NV18 NV18 X NV18 NV18 SYS SYS SYS SYS NODE PIX SYS SYS SYS 56-83 2 N/A
37
+ GPU6 NV18 NV18 NV18 NV18 NV18 NV18 X NV18 SYS SYS SYS SYS SYS SYS PIX NODE SYS 84-111 3 N/A
38
+ GPU7 NV18 NV18 NV18 NV18 NV18 NV18 NV18 X SYS SYS SYS SYS SYS SYS NODE PIX SYS 84-111 3 N/A
39
+ NIC0 PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS SYS NODE
40
+ NIC1 NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS SYS NODE
41
+ NIC2 SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS
42
+ NIC3 SYS SYS NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS
43
+ NIC4 SYS SYS SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS
44
+ unavailable
45
+
46
+ ---- toolchain -----------------------------------------------
47
+ nvcc : release 12.9, V12.9.86
48
+ gcc : gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0
49
+ python : 3.12.3
50
+ torch : 2.8.0a0+5228986c39.nv25.06 cuda=12.9 nccl=2.27.3
51
+ torch arch : ['sm_75', 'sm_80', 'sm_86', 'sm_90', 'sm_100', 'sm_120', 'compute_120']
52
+ transformer_engine: 2.10.0.dev0+10f6128d [/usr/local/lib/python3.12/dist-packages/transformer_engine/__init__.py]
53
+ deep_ep : 1.2.1+34152ae [/workspace/DeepEP/deep_ep/__init__.py]
54
+ ultra_ep : 1.0.0 [/workspace/UltraEP/ultra_ep/__init__.py]
55
+ moonep : 0.0.1 [/workspace/MoonEP/moonep/__init__.py]
56
+ megatron : 0.14.0rc5 [None]
57
+ flash_attn : 2.7.4.post1 [/usr/local/lib/python3.12/dist-packages/flash_attn/__init__.py]
58
+ numpy : 1.26.4 [/usr/local/lib/python3.12/dist-packages/numpy/__init__.py]
59
+ huggingface_hub: 1.28.0 [/usr/local/lib/python3.12/dist-packages/huggingface_hub/__init__.py]
60
+
61
+ ---- repositories --------------------------------------------
62
+ Megatron-MoE-Echo token-drop-analysis b099fd8494b0
63
+ Megatron-UltraEP ultraep-token-drop-analysis eaf829e7ad21
64
+ UltraEP main 94cab099b44f
65
+ DeepEP HEAD 34152ae28f80 (+3 untracked: build output)
66
+ MoonEP megatron-integration c27d35da15f2
67
+ blackwell_run_scripts granularity-echo 6e4b1c8e7fae
68
+
69
+ ---- environment ---------------------------------------------
70
+ CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0
71
+ CUDA_DRIVER_VERSION=575.57.08
72
+ CUDA_HOME=/usr/local/cuda
73
+ CUDA_MODULE_LOADING=LAZY
74
+ CUDA_VERSION=12.9.1.010
75
+ HF_REMOTE_ROOT=tierb_study_20260826_144644
76
+ NCCL_VERSION=2.27.3
77
+ NVSHMEM_VERSION=3.2.5
78
+ PYTORCH_BUILD_NUMBER=0
79
+ PYTORCH_BUILD_VERSION=2.8.0a0+5228986
80
+ PYTORCH_HOME=/opt/pytorch/pytorch
81
+ PYTORCH_VERSION=2.8.0a0+5228986
82
+ TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1
83
+ TORCH_CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0+PTX
84
+ TORCH_NCCL_USE_COMM_NONBLOCKING=0
85
+
86
+ ---- shared library resolution -------------------------------
87
+ libnvshmem_host.so: not in ldconfig
88
+ libnccl.so: /lib/x86_64-linux-gnu/libnccl.so.2
89
+ lrwxrwxrwx 1 root root 20 Aug 26 14:37 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so -> libnvshmem_host.so.3
90
+ -rw-r--r-- 1 root root 179045744 Aug 26 14:09 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so.3
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 27
41
+ 0, 819
42
+ 1, 817
43
+ 2, 799
44
+ 3, 799
45
+ 4, 801
46
+ 5, 801
47
+ 6, 803
48
+ 7, 805
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=echo
2
+ config=DSV2_LITE_L24
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=token-drop-analysis
6
+ commit=b099fd84
7
+ num_layers=24
8
+ hidden=2048
9
+ ffn_hidden=10944
10
+ moe_ffn_hidden=1408
11
+ num_experts=64
12
+ topk=6
13
+ experts_per_rank=8
14
+ expert_params=8650752
15
+ heads=16
16
+ kv_groups=0
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=2816
20
+ shared_expert_overlap=0
21
+ mla=1
22
+ dense_first_layers=1
23
+ moe_layers=23
24
+ rotary_base=10000
25
+ rotary_scaling=40
26
+ qkv_bias=0
27
+ router_balancing=seq_aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=1
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=8192
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=49152
39
+ mean_tokens_per_expert=6144
40
+ slot_policy=matched
41
+ slots_per_rank=8
42
+ capacity_mode=static
43
+ capacity=1.25
44
+ cuda_graph=0
45
+ probe_mode=1
46
+ async_grad_reduce=0
47
+ branch_role=probe
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25/rep1/train.log ADDED
@@ -0,0 +1,620 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [16:35:35] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe)
2
+ ==============================================================
3
+ DSV2_LITE_L24 / echo (slots/rank=8 policy=matched)
4
+ E=64 K=6 H'=1408 m=8
5
+ L=24 seq=8192 gbs=8 (1 microbatches)
6
+ cap_mode=static cap=1.25 cg=0 recompute=none
7
+ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25/rep1/memory_snapshots/snapshot.pickle
8
+ repo=/workspace/Megatron-MoE-Echo @ b099fd84
9
+ ==============================================================
10
+ W0826 16:35:36.740000 237450 torch/distributed/run.py:766]
11
+ W0826 16:35:36.740000 237450 torch/distributed/run.py:766] *****************************************
12
+ W0826 16:35:36.740000 237450 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
13
+ W0826 16:35:36.740000 237450 torch/distributed/run.py:766] *****************************************
14
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
15
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
16
+ WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it
17
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
18
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
19
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
20
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
21
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
22
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
23
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
24
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
25
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
26
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
27
+ using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1
28
+ WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer
29
+ Number of virtual stages per pipeline stage: None
30
+ accumulate and all-reduce gradients in fp32 for bfloat16 data type.
31
+ using torch.bfloat16 for parameters ...
32
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
33
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
34
+ ------------------------ arguments ------------------------
35
+ account_for_embedding_in_pipeline_split ......... False
36
+ account_for_loss_in_pipeline_split .............. False
37
+ accumulate_allreduce_grads_in_fp32 .............. True
38
+ adam_beta1 ...................................... 0.9
39
+ adam_beta2 ...................................... 0.999
40
+ adam_eps ........................................ 1e-08
41
+ add_bias_linear ................................. False
42
+ add_position_embedding .......................... False
43
+ add_qkv_bias .................................... False
44
+ adlr_autoresume ................................. False
45
+ adlr_autoresume_interval ........................ 1000
46
+ align_grad_reduce ............................... True
47
+ align_param_gather .............................. False
48
+ app_tag_run_name ................................ None
49
+ app_tag_run_version ............................. 0.0.0
50
+ apply_layernorm_1p .............................. False
51
+ apply_query_key_layer_scaling ................... False
52
+ apply_residual_connection_post_layernorm ........ False
53
+ apply_rope_fusion ............................... False
54
+ async_save ...................................... None
55
+ async_tensor_model_parallel_allreduce ........... True
56
+ attention_backend ............................... AttnBackend.auto
57
+ attention_dropout ............................... 0.0
58
+ attention_softmax_in_fp32 ....................... False
59
+ auto_detect_ckpt_format ......................... False
60
+ barrier_with_L1_time ............................ True
61
+ bert_binary_head ................................ True
62
+ bert_embedder_type .............................. megatron
63
+ bert_load ....................................... None
64
+ bf16 ............................................ True
65
+ bias_dropout_fusion ............................. True
66
+ bias_gelu_fusion ................................ False
67
+ bias_swiglu_fusion .............................. True
68
+ biencoder_projection_dim ........................ 0
69
+ biencoder_shared_query_context_model ............ False
70
+ block_data_path ................................. None
71
+ calc_ft_timeouts ................................ False
72
+ calculate_per_token_loss ........................ False
73
+ check_for_large_grads ........................... False
74
+ check_for_nan_in_loss_and_grad .................. True
75
+ check_for_spiky_loss ............................ False
76
+ check_weight_hash_across_dp_replicas_interval ... None
77
+ ckpt_assume_constant_structure .................. False
78
+ ckpt_convert_format ............................. None
79
+ ckpt_convert_save ............................... None
80
+ ckpt_convert_update_legacy_dist_opt_format ...... False
81
+ ckpt_format ..................................... torch_dist
82
+ ckpt_fully_parallel_load ........................ False
83
+ ckpt_fully_parallel_save ........................ True
84
+ ckpt_fully_parallel_save_deprecated ............. False
85
+ ckpt_step ....................................... None
86
+ classes_fraction ................................ 1.0
87
+ clip_grad ....................................... 1.0
88
+ clone_scatter_output_in_embedding ............... True
89
+ config_logger_dir ...............................
90
+ consumed_train_samples .......................... 0
91
+ consumed_valid_samples .......................... 0
92
+ context_parallel_size ........................... 1
93
+ cp_comm_type .................................... ['p2p']
94
+ create_attention_mask_in_dataloader ............. True
95
+ cross_entropy_fusion_impl ....................... native
96
+ cross_entropy_loss_fusion ....................... False
97
+ cuda_graph_scope ................................ full
98
+ cuda_graph_warmup_steps ......................... 3
99
+ data_args_path .................................. None
100
+ data_cache_path ................................. None
101
+ data_parallel_random_init ....................... False
102
+ data_parallel_sharding_strategy ................. no_shard
103
+ data_parallel_size .............................. 8
104
+ data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document']
105
+ data_per_class_fraction ......................... 1.0
106
+ data_sharding ................................... True
107
+ dataloader_type ................................. single
108
+ ddp_average_in_collective ....................... False
109
+ ddp_bucket_size ................................. None
110
+ ddp_num_buckets ................................. None
111
+ ddp_pad_buckets_for_high_nccl_busbw ............. False
112
+ decoder_first_pipeline_num_layers ............... None
113
+ decoder_last_pipeline_num_layers ................ None
114
+ decoder_num_layers .............................. None
115
+ decoder_seq_length .............................. None
116
+ decoupled_lr .................................... None
117
+ decoupled_min_lr ................................ None
118
+ decrease_batch_size_if_needed ................... False
119
+ defer_embedding_wgrad_compute ................... False
120
+ delay_wgrad_compute ............................. False
121
+ deprecated_use_mcore_models ..................... True
122
+ deterministic_mode .............................. False
123
+ dino_bottleneck_size ............................ 256
124
+ dino_freeze_last_layer .......................... 1
125
+ dino_head_hidden_size ........................... 2048
126
+ dino_local_crops_number ......................... 10
127
+ dino_local_img_size ............................. 96
128
+ dino_norm_last_layer ............................ False
129
+ dino_teacher_temp ............................... 0.07
130
+ dino_warmup_teacher_temp ........................ 0.04
131
+ dino_warmup_teacher_temp_epochs ................. 30
132
+ disable_bf16_reduced_precision_matmul ........... False
133
+ disable_mamba_mem_eff_path ...................... False
134
+ disable_straggler_on_startup .................... False
135
+ dist_ckpt_format_deprecated ..................... None
136
+ dist_ckpt_strictness ............................ assume_ok_unexpected
137
+ distribute_saved_activations .................... False
138
+ distributed_backend ............................. nccl
139
+ distributed_timeout_minutes ..................... 10
140
+ embedding_init_method_std ....................... None
141
+ embedding_path .................................. None
142
+ empty_unused_memory_level ....................... 0
143
+ enable_cuda_graph ............................... False
144
+ enable_experimental ............................. False
145
+ enable_ft_package ............................... False
146
+ enable_gloo_process_groups ...................... True
147
+ enable_msc ...................................... True
148
+ enable_one_logger ............................... True
149
+ encoder_num_layers .............................. 24
150
+ encoder_seq_length .............................. 8192
151
+ end_weight_decay ................................ 0.1
152
+ eod_mask_loss ................................... False
153
+ error_injection_rate ............................ 0
154
+ error_injection_type ............................ transient_error
155
+ eval_interval ................................... 1000000
156
+ eval_iters ...................................... 0
157
+ evidence_data_path .............................. None
158
+ exit_duration_in_mins ........................... None
159
+ exit_interval ................................... None
160
+ exit_on_missing_checkpoint ...................... False
161
+ exit_signal_handler ............................. False
162
+ exp_avg_dtype ................................... torch.float32
163
+ exp_avg_sq_dtype ................................ torch.float32
164
+ expert_model_parallel_size ...................... 8
165
+ expert_tensor_parallel_size ..................... 1
166
+ export_force_local_attention .................... False
167
+ export_kd_cfg ................................... None
168
+ export_kd_teacher_ckpt_format ................... None
169
+ export_kd_teacher_load .......................... None
170
+ export_kv_cache_quant ........................... False
171
+ export_legacy_megatron .......................... False
172
+ export_model_type ............................... GPTModel
173
+ export_moe_apply_probs_on_input ................. False
174
+ export_qk_l2_norm ............................... False
175
+ export_quant_cfg ................................ None
176
+ export_real_quant_cfg ........................... None
177
+ export_te_mcore_model ........................... False
178
+ external_cuda_graph ............................. False
179
+ ffn_hidden_size ................................. 10944
180
+ finetune ........................................ False
181
+ finetune_data_split ............................. train
182
+ finetune_hf_dataset ............................. None
183
+ first_last_layers_bf16 .......................... False
184
+ flash_decode .................................... False
185
+ fp16 ............................................ False
186
+ fp16_lm_cross_entropy ........................... False
187
+ fp32_residual_connection ........................ False
188
+ fp8 ............................................. e4m3
189
+ fp8_amax_compute_algo ........................... most_recent
190
+ fp8_amax_history_len ............................ 1
191
+ fp8_interval .................................... 1
192
+ fp8_margin ...................................... 0
193
+ fp8_param_gather ................................ False
194
+ fp8_recipe ...................................... mxfp8
195
+ fp8_wgrad ....................................... True
196
+ fsdp_double_buffer .............................. False
197
+ global_batch_size ............................... 8
198
+ grad_reduce_in_bf16 ............................. False
199
+ gradient_accumulation_fusion .................... True
200
+ gradient_reduce_div_fusion ...................... True
201
+ group_query_attention ........................... False
202
+ head_lr_mult .................................... 1.0
203
+ heterogeneous_layers_config_encoded_json ........ None
204
+ heterogeneous_layers_config_path ................ None
205
+ hidden_dropout .................................. 0.0
206
+ hidden_size ..................................... 2048
207
+ hierarchical_context_parallel_sizes ............. None
208
+ high_priority_stream_groups ..................... []
209
+ hybrid_attention_ratio .......................... 0.0
210
+ hybrid_mlp_ratio ................................ 0.0
211
+ hybrid_override_pattern ......................... None
212
+ hysteresis ...................................... 2
213
+ ict_head_size ................................... None
214
+ ict_load ........................................ None
215
+ img_h ........................................... 224
216
+ img_w ........................................... 224
217
+ indexer_batch_size .............................. 128
218
+ indexer_log_interval ............................ 1000
219
+ inference_batch_times_seqlen_threshold .......... -1
220
+ inference_dynamic_batching ...................... False
221
+ inference_dynamic_batching_buffer_guaranteed_fraction 0.2
222
+ inference_dynamic_batching_buffer_overflow_factor None
223
+ inference_dynamic_batching_buffer_size_gb ....... 40.0
224
+ inference_dynamic_batching_chunk_size ........... 256
225
+ inference_dynamic_batching_max_requests_override None
226
+ inference_dynamic_batching_max_tokens_override .. None
227
+ inference_dynamic_batching_num_cuda_graphs ...... 16
228
+ inference_max_batch_size ........................ 8
229
+ inference_max_seq_length ........................ 2560
230
+ inference_rng_tracker ........................... False
231
+ init_method_std ................................. 0.01
232
+ init_method_xavier_uniform ...................... False
233
+ init_model_with_meta_device ..................... False
234
+ initial_loss_scale .............................. 4294967296
235
+ inprocess_active_world_size ..................... 8
236
+ inprocess_barrier_timeout ....................... 120
237
+ inprocess_completion_timeout .................... 120
238
+ inprocess_empty_cuda_cache ...................... False
239
+ inprocess_granularity ........................... node
240
+ inprocess_hard_timeout .......................... 90
241
+ inprocess_heartbeat_interval .................... 30
242
+ inprocess_heartbeat_timeout ..................... 60
243
+ inprocess_last_call_wait ........................ 1
244
+ inprocess_max_iterations ........................ None
245
+ inprocess_monitor_process_interval .............. 1.0
246
+ inprocess_monitor_thread_interval ............... 1.0
247
+ inprocess_progress_watchdog_interval ............ 1.0
248
+ inprocess_restart ............................... False
249
+ inprocess_soft_timeout .......................... 60
250
+ inprocess_termination_grace_time ................ 1
251
+ is_hybrid_model ................................. False
252
+ iter_per_epoch .................................. 1250
253
+ iterations_to_skip .............................. []
254
+ keep_fp8_transpose_cache_when_using_custom_fsdp . False
255
+ kitchen_config_file ............................. None
256
+ kitchen_recipe_number ........................... None
257
+ kv_channels ..................................... 128
258
+ kv_lora_rank .................................... 512
259
+ lazy_mpu_init ................................... None
260
+ load ............................................ None
261
+ load_main_params_from_ckpt ...................... None
262
+ load_model_opt_format ........................... False
263
+ local_rank ...................................... 0
264
+ log_energy ...................................... False
265
+ log_interval .................................... 1
266
+ log_loss_scale_to_tensorboard ................... True
267
+ log_memory_to_tensorboard ....................... False
268
+ log_num_zeros_in_grad ........................... False
269
+ log_params_norm ................................. False
270
+ log_progress .................................... False
271
+ log_straggler ................................... False
272
+ log_throughput .................................. True
273
+ log_timers_to_tensorboard ....................... False
274
+ log_validation_ppl_to_tensorboard ............... False
275
+ log_world_size_to_tensorboard ................... False
276
+ logging_level ................................... None
277
+ loss_scale ...................................... None
278
+ loss_scale_window ............................... 1000
279
+ lr .............................................. 0.0001
280
+ lr_decay_iters .................................. 8
281
+ lr_decay_samples ................................ None
282
+ lr_decay_style .................................. cosine
283
+ lr_warmup_fraction .............................. None
284
+ lr_warmup_init .................................. 0.0
285
+ lr_warmup_iters ................................. 5
286
+ lr_warmup_samples ............................... 0
287
+ lr_wsd_decay_iters .............................. None
288
+ lr_wsd_decay_samples ............................ None
289
+ lr_wsd_decay_style .............................. exponential
290
+ main_grads_dtype ................................ torch.float32
291
+ main_params_dtype ............................... torch.float32
292
+ make_vocab_size_divisible_by .................... 128
293
+ mamba_head_dim .................................. 64
294
+ mamba_num_groups ................................ 8
295
+ mamba_num_heads ................................. None
296
+ mamba_state_dim ................................. 128
297
+ manual_gc ....................................... False
298
+ manual_gc_eval .................................. True
299
+ manual_gc_interval .............................. 0
300
+ mask_factor ..................................... 1.0
301
+ mask_prob ....................................... 0.15
302
+ mask_type ....................................... random
303
+ masked_softmax_fusion ........................... False
304
+ max_position_embeddings ......................... 32768
305
+ max_tokens_to_oom ............................... 12000
306
+ memory_snapshot_path ............................ snapshot.pickle
307
+ merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt
308
+ micro_batch_size ................................ 1
309
+ microbatch_group_size_per_vp_stage .............. None
310
+ mid_level_dataset_surplus ....................... 0.005
311
+ min_loss_scale .................................. 1.0
312
+ min_lr .......................................... 1e-05
313
+ mlp_chunks_for_prefill .......................... 1
314
+ mmap_bin_files .................................. True
315
+ mock_data ....................................... False
316
+ moe_apply_probs_on_input ........................ False
317
+ moe_aux_loss_coeff .............................. 0.001
318
+ moe_debug_drop_logging .......................... True
319
+ moe_debug_drop_logging_interval ................. 1
320
+ moe_deepep_num_sms .............................. 20
321
+ moe_echo_enable_random_offloading ............... False
322
+ moe_echo_expert_dispatch_overlap ................ True
323
+ moe_echo_expert_dispatcher_type ................. hybridep
324
+ moe_echo_recompute_expert_dispatch .............. False
325
+ moe_enable_deepep ............................... False
326
+ moe_enable_echo ................................. True
327
+ moe_enable_hybridep ............................. True
328
+ moe_expert_capacity_factor ...................... None
329
+ moe_extended_tp ................................. False
330
+ moe_ffn_hidden_size ............................. 1408
331
+ moe_grouped_gemm ................................ True
332
+ moe_hybridep_num_sms ............................ 16
333
+ moe_input_jitter_eps ............................ None
334
+ moe_layer_freq .................................. [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
335
+ moe_layer_recompute ............................. False
336
+ moe_num_echo_experts ............................ 64
337
+ moe_pad_expert_input_to_capacity ................ False
338
+ moe_per_layer_logging ........................... False
339
+ moe_permute_fusion .............................. False
340
+ moe_received_token_capacity ..................... 1.25
341
+ moe_router_bias_update_rate ..................... 0.001
342
+ moe_router_dtype ................................ fp32
343
+ moe_router_enable_expert_bias ................... False
344
+ moe_router_force_load_balancing ................. False
345
+ moe_router_group_topk ........................... None
346
+ moe_router_load_balancing_type .................. seq_aux_loss
347
+ moe_router_num_groups ........................... None
348
+ moe_router_padding_for_fp8 ...................... False
349
+ moe_router_pre_softmax .......................... True
350
+ moe_router_score_function ....................... softmax
351
+ moe_router_topk ................................. 6
352
+ moe_router_topk_scaling_factor .................. None
353
+ moe_shared_expert_intermediate_size ............. 2816
354
+ moe_shared_expert_overlap ....................... False
355
+ moe_token_dispatcher_type ....................... flex
356
+ moe_token_drop_policy ........................... probs
357
+ moe_upcycling_granularity ....................... 1
358
+ moe_use_device_initiated_grouped_gemm ........... True
359
+ moe_use_legacy_grouped_gemm ..................... False
360
+ moe_use_upcycling ............................... False
361
+ moe_z_loss_coeff ................................ None
362
+ mrope_section ................................... None
363
+ mscale .......................................... 0.707
364
+ mscale_all_dim .................................. 0.707
365
+ mtp_loss_scaling_factor ......................... 0.1
366
+ mtp_num_layers .................................. None
367
+ multi_latent_attention .......................... True
368
+ nccl_all_reduce_for_prefill ..................... False
369
+ nccl_communicator_config_path ................... None
370
+ nccl_ub ......................................... False
371
+ no_load_optim ................................... None
372
+ no_load_rng ..................................... None
373
+ no_persist_layer_norm ........................... False
374
+ no_rope_freq .................................... None
375
+ no_save_optim ................................... None
376
+ no_save_rng ..................................... None
377
+ non_persistent_ckpt_type ........................ None
378
+ non_persistent_global_ckpt_dir .................. None
379
+ non_persistent_local_ckpt_algo .................. fully_parallel
380
+ non_persistent_local_ckpt_dir ................... None
381
+ non_persistent_save_interval .................... None
382
+ norm_epsilon .................................... 1e-06
383
+ normalization ................................... RMSNorm
384
+ num_attention_heads ............................. 16
385
+ num_channels .................................... 3
386
+ num_classes ..................................... 1000
387
+ num_dataset_builder_threads ..................... 1
388
+ num_distributed_optimizer_instances ............. 1
389
+ num_experts ..................................... 64
390
+ num_layers ...................................... 24
391
+ num_layers_at_end_in_bf16 ....................... 1
392
+ num_layers_at_start_in_bf16 ..................... 1
393
+ num_layers_per_virtual_pipeline_stage ........... None
394
+ num_query_groups ................................ 1
395
+ num_virtual_stages_per_pipeline_rank ............ None
396
+ num_workers ..................................... 2
397
+ object_storage_cache_path ....................... None
398
+ one_logger_async ................................ False
399
+ one_logger_project .............................. megatron-lm
400
+ one_logger_run_name ............................. None
401
+ onnx_safe ....................................... None
402
+ openai_gelu ..................................... False
403
+ optimizer ....................................... adam
404
+ optimizer_cpu_offload ........................... False
405
+ optimizer_offload_fraction ...................... 1.0
406
+ output_bert_embeddings .......................... False
407
+ overlap_cpu_optimizer_d2h_h2d ................... False
408
+ overlap_grad_reduce ............................. True
409
+ overlap_moe_expert_parallel_comm ................ False
410
+ overlap_p2p_comm ................................ False
411
+ overlap_p2p_comm_warmup_flush ................... False
412
+ overlap_param_gather ............................ True
413
+ overlap_param_gather_with_optimizer_step ........ False
414
+ override_opt_param_scheduler .................... False
415
+ params_dtype .................................... torch.bfloat16
416
+ patch_dim ....................................... 16
417
+ per_split_data_args_path ........................ None
418
+ perform_initialization .......................... True
419
+ pin_cpu_grads ................................... True
420
+ pin_cpu_params .................................. True
421
+ pipeline_model_parallel_comm_backend ............ None
422
+ pipeline_model_parallel_layout .................. None
423
+ pipeline_model_parallel_size .................... 1
424
+ position_embedding_type ......................... rope
425
+ pretrained_checkpoint ........................... None
426
+ profile ......................................... False
427
+ profile_ranks ................................... [0]
428
+ profile_step_end ................................ 12
429
+ profile_step_start .............................. 10
430
+ q_lora_rank ..................................... None
431
+ qk_head_dim ..................................... 128
432
+ qk_l2_norm ...................................... False
433
+ qk_layernorm .................................... True
434
+ qk_pos_emb_head_dim ............................. 64
435
+ query_in_block_prob ............................. 0.1
436
+ rampup_batch_size ............................... None
437
+ rank ............................................ 0
438
+ recompute_granularity ........................... None
439
+ recompute_method ................................ None
440
+ recompute_modules ............................... None
441
+ recompute_num_layers ............................ None
442
+ record_memory_history ........................... False
443
+ relative_attention_max_distance ................. 128
444
+ relative_attention_num_buckets .................. 32
445
+ replication ..................................... False
446
+ replication_factor .............................. 2
447
+ replication_jump ................................ None
448
+ rerun_mode ...................................... validate_results
449
+ reset_attention_mask ............................ False
450
+ reset_position_ids .............................. False
451
+ result_rejected_tracker_filename ................ None
452
+ retriever_report_topk_accuracies ................ []
453
+ retriever_score_scaling ......................... False
454
+ retriever_seq_length ............................ 256
455
+ retro_add_retriever ............................. False
456
+ retro_attention_gate ............................ 1
457
+ retro_cyclic_train_iters ........................ None
458
+ retro_encoder_attention_dropout ................. 0.1
459
+ retro_encoder_hidden_dropout .................... 0.1
460
+ retro_encoder_layers ............................ 2
461
+ retro_num_neighbors ............................. 2
462
+ retro_num_retrieved_chunks ...................... 2
463
+ retro_project_dir ............................... None
464
+ retro_verify_neighbor_count ..................... True
465
+ reuse_grad_buf_for_mxfp8_param_ag ............... False
466
+ rope_scaling_factor ............................. 8.0
467
+ rope_type ....................................... None
468
+ rotary_base ..................................... 10000
469
+ rotary_interleaved .............................. False
470
+ rotary_percent .................................. 1.0
471
+ rotary_scaling_factor ........................... 40.0
472
+ rotary_seq_len_interpolation_factor ............. None
473
+ run_workload_inspector_server ................... False
474
+ sample_rate ..................................... 1.0
475
+ save ............................................ None
476
+ save_interval ................................... None
477
+ save_retain_interval ............................ None
478
+ scatter_gather_tensors_in_pipeline .............. True
479
+ seed ............................................ 1234
480
+ seq_length ...................................... 8192
481
+ sequence_parallel ............................... False
482
+ sft ............................................. False
483
+ sft_tokenizer_prompt_format ..................... nemotron-h-aligned
484
+ sgd_momentum .................................... 0.9
485
+ short_seq_prob .................................. 0.1
486
+ skip_train ...................................... False
487
+ skipped_train_samples ........................... 0
488
+ spec ............................................ None
489
+ split ........................................... 100,0,0
490
+ squared_relu .................................... False
491
+ start_weight_decay .............................. 0.1
492
+ straggler_ctrlr_port ............................ 65535
493
+ straggler_minmax_count .......................... 1
494
+ suggested_communication_unit_size ............... None
495
+ swiglu .......................................... True
496
+ swin_backbone_type .............................. tiny
497
+ symmetric_ar_type ............................... None
498
+ te_rng_tracker .................................. False
499
+ tensor_model_parallel_size ...................... 1
500
+ tensorboard_dir ................................. None
501
+ tensorboard_log_interval ........................ 1
502
+ tensorboard_queue_size .......................... 1000
503
+ test_data_path .................................. None
504
+ test_mode ....................................... False
505
+ tiktoken_num_special_tokens ..................... 1000
506
+ tiktoken_pattern ................................ None
507
+ tiktoken_special_tokens ......................... None
508
+ timing_log_level ................................ 0
509
+ timing_log_option ............................... minmax
510
+ titles_data_path ................................ None
511
+ tokenizer_model ................................. None
512
+ tokenizer_type .................................. GPT2BPETokenizer
513
+ torch_fsdp2_reshard_after_forward ............... True
514
+ tp_comm_bootstrap_backend ....................... nccl
515
+ tp_comm_bulk_dgrad .............................. True
516
+ tp_comm_bulk_wgrad .............................. True
517
+ tp_comm_overlap ................................. False
518
+ tp_comm_overlap_ag .............................. True
519
+ tp_comm_overlap_cfg ............................. None
520
+ tp_comm_overlap_rs .............................. True
521
+ tp_comm_overlap_rs_dgrad ........................ False
522
+ tp_comm_split_ag ................................ True
523
+ tp_comm_split_rs ................................ True
524
+ train_data_path ................................. None
525
+ train_iters ..................................... 12
526
+ train_samples ................................... None
527
+ train_sync_interval ............................. None
528
+ transformer_impl ................................ transformer_engine
529
+ transformer_pipeline_model_parallel_size ........ 1
530
+ untie_embeddings_and_output_weights ............. True
531
+ use_checkpoint_args ............................. False
532
+ use_checkpoint_opt_param_scheduler .............. False
533
+ use_cpu_initialization .......................... None
534
+ use_custom_fsdp ................................. False
535
+ use_dist_ckpt ................................... True
536
+ use_dist_ckpt_deprecated ........................ False
537
+ use_distributed_optimizer ....................... True
538
+ use_flash_attn .................................. False
539
+ use_legacy_models ............................... False
540
+ use_mp_args_from_checkpoint_args ................ False
541
+ use_one_sent_docs ............................... False
542
+ use_persistent_ckpt_worker ...................... False
543
+ use_precision_aware_optimizer ................... False
544
+ use_pytorch_profiler ............................ False
545
+ use_ring_exchange_p2p ........................... False
546
+ use_rope_scaling ................................ False
547
+ use_rotary_position_embeddings .................. False
548
+ use_sharp ....................................... False
549
+ use_tokenizer_model_from_checkpoint_args ........ True
550
+ use_torch_fsdp2 ................................. False
551
+ use_torch_optimizer_for_cpu_offload ............. False
552
+ use_tp_pp_dp_mapping ............................ False
553
+ v_head_dim ...................................... 128
554
+ valid_data_path ................................. None
555
+ variable_seq_lengths ............................ False
556
+ virtual_pipeline_model_parallel_size ............ None
557
+ vision_backbone_type ............................ vit
558
+ vision_pretraining .............................. False
559
+ vision_pretraining_type ......................... classify
560
+ vocab_extra_ids ................................. 0
561
+ vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json
562
+ vocab_size ...................................... None
563
+ wandb_exp_name ..................................
564
+ wandb_project ...................................
565
+ wandb_save_dir ..................................
566
+ weight_decay .................................... 0.1
567
+ weight_decay_incr_style ......................... constant
568
+ wgrad_deferral_limit ............................ 0
569
+ world_size ...................................... 8
570
+ yaml_cfg ........................................ None
571
+ -------------------- end of arguments ---------------------
572
+ INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1
573
+ > building GPT2BPETokenizer tokenizer ...
574
+ > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304)
575
+ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS
576
+ > initializing torch distributed ...
577
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
578
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
579
+ [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
580
+ [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
581
+ [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
582
+ [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
583
+ [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
584
+ [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
585
+ [Gloo] Rank [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
586
+ 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
587
+ [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
588
+ [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
589
+ [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
590
+ [Gloo] Rank [Gloo] Rank 5 is connected to 47 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is :
591
+ 7
592
+ [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
593
+ [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
594
+ [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
595
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
596
+ > initialized tensor model parallel with size 1
597
+ > initialized pipeline model parallel with size 1
598
+ > setting random seeds to 1234 ...
599
+ [Gloo] Rank 0 is connected to 0 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : [Gloo] Rank 000
600
+ is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00
601
+
602
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
603
+ [Gloo] Rank 0[Gloo] Rank is connected to 00 is connected to peer ranks. 0Expected number of connected peer ranks is : peer ranks. 0Expected number of connected peer ranks is :
604
+ 0
605
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
606
+ [rank2]:[W826 16:35:54.040729199 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
607
+ [rank1]:[W826 16:35:54.040744910 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
608
+ [rank7]:[W826 16:35:54.042537371 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
609
+ [rank5]:[W826 16:35:55.045097105 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
610
+ [rank6]:[W826 16:35:55.045219285 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
611
+ [rank3]:[W826 16:35:55.045217490 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
612
+ [rank4]:[W826 16:35:55.045741026 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
613
+ > compiling dataset index builder ...
614
+ make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets'
615
+ make: Nothing to be done for 'default'.
616
+ make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets'
617
+ >>> done with dataset index builder. Compilation time: 0.035 seconds
618
+ WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations.
619
+ > compiling and loading fused kernels ...
620
+ [rank0]:[W826 16:35:55.205831838 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv CHANGED
@@ -14,7 +14,7 @@
14
  5, 0
15
  6, 0
16
  7, 0
17
- 0, 2
18
  1, 1
19
  2, 1
20
  3, 1
@@ -36,24 +36,24 @@
36
  3, 4
37
  4, 4
38
  5, 4
39
- 6, 1
40
- 7, 1
41
- 0, 4
42
- 1, 45
43
- 2, 4
44
- 3, 4
45
- 4, 4
46
- 5, 4
47
- 6, 27
48
  7, 4
49
- 0, 821
50
- 1, 821
51
- 2, 821
52
- 3, 821
53
- 4, 821
54
- 5, 821
55
- 6, 821
56
- 7, 821
 
 
 
 
 
 
 
 
57
  0, 1955
58
  1, 1955
59
  2, 1955
@@ -70,22 +70,22 @@
70
  5, 26223
71
  6, 26355
72
  7, 26377
73
- 0, 26887
74
- 1, 27039
75
- 2, 26773
76
- 3, 26885
77
- 4, 26885
78
- 5, 26885
79
- 6, 26953
80
- 7, 26927
81
- 0, 36511
82
- 1, 36595
83
- 2, 36445
84
- 3, 36443
85
- 4, 36443
86
- 5, 36441
87
- 6, 36577
88
- 7, 36595
89
  0, 37783
90
  1, 37867
91
  2, 37717
@@ -102,14 +102,14 @@
102
  5, 37801
103
  6, 37937
104
  7, 37955
105
- 0, 38003
106
- 1, 38123
107
- 2, 37971
108
- 3, 37933
109
- 4, 37931
110
- 5, 37955
111
- 6, 38077
112
- 7, 38117
113
  0, 38819
114
  1, 38903
115
  2, 38753
@@ -126,14 +126,6 @@
126
  5, 38751
127
  6, 38885
128
  7, 38905
129
- 0, 38825
130
- 1, 38909
131
- 2, 38759
132
- 3, 38759
133
- 4, 38759
134
- 5, 38755
135
- 6, 38891
136
- 7, 38909
137
  0, 38827
138
  1, 38913
139
  2, 38763
@@ -142,14 +134,22 @@
142
  5, 38759
143
  6, 38893
144
  7, 38913
145
- 0, 121349
146
- 1, 121649
147
- 2, 121471
148
- 3, 121465
149
- 4, 121453
150
- 5, 121559
151
- 6, 121627
152
- 7, 121653
 
 
 
 
 
 
 
 
153
  0, 121857
154
  1, 122153
155
  2, 121983
@@ -184,12 +184,12 @@
184
  7, 122153
185
  0, 122541
186
  1, 122837
187
- 2, 122627
188
- 3, 122621
189
  4, 122667
190
- 5, 122745
191
- 6, 122801
192
- 7, 122769
193
  0, 123719
194
  1, 126063
195
  2, 125893
@@ -198,14 +198,14 @@
198
  5, 125973
199
  6, 126043
200
  7, 126063
201
- 0, 124389
202
- 1, 126747
203
- 2, 126543
204
- 3, 126587
205
- 4, 126547
206
- 5, 126593
207
- 6, 126697
208
- 7, 126687
209
  0, 139007
210
  1, 138657
211
  2, 138497
@@ -238,19 +238,19 @@
238
  5, 138567
239
  6, 138707
240
  7, 140961
241
- 0, 138381
242
- 1, 137775
243
  2, 138197
244
- 3, 137867
245
- 4, 137547
246
- 5, 137621
247
- 6, 137825
248
  7, 140661
249
- 0, 2055
250
  1, 778
251
  2, 778
252
  3, 778
253
  4, 778
254
- 5, 778
255
- 6, 1034
256
- 7, 140013
 
14
  5, 0
15
  6, 0
16
  7, 0
17
+ 0, 1
18
  1, 1
19
  2, 1
20
  3, 1
 
36
  3, 4
37
  4, 4
38
  5, 4
39
+ 6, 4
 
 
 
 
 
 
 
 
40
  7, 4
41
+ 0, 4
42
+ 1, 629
43
+ 2, 629
44
+ 3, 629
45
+ 4, 629
46
+ 5, 629
47
+ 6, 47
48
+ 7, 47
49
+ 0, 975
50
+ 1, 959
51
+ 2, 949
52
+ 3, 979
53
+ 4, 955
54
+ 5, 951
55
+ 6, 953
56
+ 7, 957
57
  0, 1955
58
  1, 1955
59
  2, 1955
 
70
  5, 26223
71
  6, 26355
72
  7, 26377
73
+ 0, 32143
74
+ 1, 32229
75
+ 2, 32079
76
+ 3, 32077
77
+ 4, 32077
78
+ 5, 32075
79
+ 6, 32209
80
+ 7, 32229
81
+ 0, 36899
82
+ 1, 36983
83
+ 2, 36833
84
+ 3, 36831
85
+ 4, 36831
86
+ 5, 36829
87
+ 6, 36965
88
+ 7, 36983
89
  0, 37783
90
  1, 37867
91
  2, 37717
 
102
  5, 37801
103
  6, 37937
104
  7, 37955
105
+ 0, 38131
106
+ 1, 38211
107
+ 2, 38063
108
+ 3, 38063
109
+ 4, 38037
110
+ 5, 38047
111
+ 6, 38197
112
+ 7, 38203
113
  0, 38819
114
  1, 38903
115
  2, 38753
 
126
  5, 38751
127
  6, 38885
128
  7, 38905
 
 
 
 
 
 
 
 
129
  0, 38827
130
  1, 38913
131
  2, 38763
 
134
  5, 38759
135
  6, 38893
136
  7, 38913
137
+ 0, 38827
138
+ 1, 38913
139
+ 2, 38763
140
+ 3, 38761
141
+ 4, 38761
142
+ 5, 38759
143
+ 6, 38893
144
+ 7, 38913
145
+ 0, 121341
146
+ 1, 121633
147
+ 2, 121465
148
+ 3, 121469
149
+ 4, 121457
150
+ 5, 121539
151
+ 6, 121593
152
+ 7, 121615
153
  0, 121857
154
  1, 122153
155
  2, 121983
 
184
  7, 122153
185
  0, 122541
186
  1, 122837
187
+ 2, 122667
188
+ 3, 122667
189
  4, 122667
190
+ 5, 122747
191
+ 6, 122817
192
+ 7, 122837
193
  0, 123719
194
  1, 126063
195
  2, 125893
 
198
  5, 125973
199
  6, 126043
200
  7, 126063
201
+ 0, 124305
202
+ 1, 126599
203
+ 2, 126513
204
+ 3, 126471
205
+ 4, 126481
206
+ 5, 126525
207
+ 6, 126613
208
+ 7, 126617
209
  0, 139007
210
  1, 138657
211
  2, 138497
 
238
  5, 138567
239
  6, 138707
240
  7, 140961
241
+ 0, 138707
242
+ 1, 138357
243
  2, 138197
244
+ 3, 138129
245
+ 4, 137867
246
+ 5, 137685
247
+ 6, 138087
248
  7, 140661
249
+ 0, 775
250
  1, 778
251
  2, 778
252
  3, 778
253
  4, 778
254
+ 5, 1034
255
+ 6, 137759
256
+ 7, 2058
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log CHANGED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25/rep1/mem_samples.csv ADDED
@@ -0,0 +1,248 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 4
43
+ 2, 4
44
+ 3, 4
45
+ 4, 27
46
+ 5, 4
47
+ 6, 4
48
+ 7, 4
49
+ 0, 821
50
+ 1, 821
51
+ 2, 821
52
+ 3, 821
53
+ 4, 821
54
+ 5, 821
55
+ 6, 821
56
+ 7, 821
57
+ 0, 1975
58
+ 1, 1975
59
+ 2, 1975
60
+ 3, 1975
61
+ 4, 1975
62
+ 5, 1975
63
+ 6, 1975
64
+ 7, 1975
65
+ 0, 47803
66
+ 1, 47803
67
+ 2, 47803
68
+ 3, 47803
69
+ 4, 47515
70
+ 5, 47803
71
+ 6, 47803
72
+ 7, 47803
73
+ 0, 48483
74
+ 1, 48483
75
+ 2, 48483
76
+ 3, 48483
77
+ 4, 48373
78
+ 5, 48661
79
+ 6, 48661
80
+ 7, 48661
81
+ 0, 49539
82
+ 1, 49539
83
+ 2, 49539
84
+ 3, 49539
85
+ 4, 49251
86
+ 5, 49539
87
+ 6, 49539
88
+ 7, 49539
89
+ 0, 53697
90
+ 1, 53697
91
+ 2, 53697
92
+ 3, 53697
93
+ 4, 53409
94
+ 5, 53697
95
+ 6, 53697
96
+ 7, 53697
97
+ 0, 54105
98
+ 1, 54105
99
+ 2, 54105
100
+ 3, 54105
101
+ 4, 53817
102
+ 5, 54105
103
+ 6, 54105
104
+ 7, 54105
105
+ 0, 54421
106
+ 1, 54409
107
+ 2, 54421
108
+ 3, 54459
109
+ 4, 54185
110
+ 5, 54431
111
+ 6, 54413
112
+ 7, 54443
113
+ 0, 55543
114
+ 1, 55543
115
+ 2, 55543
116
+ 3, 55543
117
+ 4, 55255
118
+ 5, 55543
119
+ 6, 55543
120
+ 7, 55543
121
+ 0, 56183
122
+ 1, 56183
123
+ 2, 56183
124
+ 3, 56183
125
+ 4, 55895
126
+ 5, 56183
127
+ 6, 56183
128
+ 7, 56183
129
+ 0, 57321
130
+ 1, 57321
131
+ 2, 57321
132
+ 3, 57321
133
+ 4, 57033
134
+ 5, 57321
135
+ 6, 57321
136
+ 7, 57321
137
+ 0, 58095
138
+ 1, 58095
139
+ 2, 58095
140
+ 3, 58095
141
+ 4, 57807
142
+ 5, 58095
143
+ 6, 58095
144
+ 7, 58095
145
+ 0, 98719
146
+ 1, 98761
147
+ 2, 98761
148
+ 3, 98761
149
+ 4, 98731
150
+ 5, 98761
151
+ 6, 98761
152
+ 7, 98761
153
+ 0, 99365
154
+ 1, 99407
155
+ 2, 99407
156
+ 3, 99407
157
+ 4, 99377
158
+ 5, 99407
159
+ 6, 99407
160
+ 7, 99407
161
+ 0, 99367
162
+ 1, 99409
163
+ 2, 99409
164
+ 3, 99409
165
+ 4, 99379
166
+ 5, 99409
167
+ 6, 99409
168
+ 7, 99409
169
+ 0, 99367
170
+ 1, 99409
171
+ 2, 99409
172
+ 3, 99409
173
+ 4, 99379
174
+ 5, 99409
175
+ 6, 99409
176
+ 7, 99409
177
+ 0, 99367
178
+ 1, 99409
179
+ 2, 99409
180
+ 3, 99409
181
+ 4, 99379
182
+ 5, 99409
183
+ 6, 99409
184
+ 7, 99409
185
+ 0, 100731
186
+ 1, 101157
187
+ 2, 101157
188
+ 3, 101157
189
+ 4, 101127
190
+ 5, 101157
191
+ 6, 101157
192
+ 7, 101157
193
+ 0, 106371
194
+ 1, 105621
195
+ 2, 106653
196
+ 3, 106125
197
+ 4, 106335
198
+ 5, 106557
199
+ 6, 106749
200
+ 7, 107181
201
+ 0, 108169
202
+ 1, 107891
203
+ 2, 108169
204
+ 3, 107837
205
+ 4, 108269
206
+ 5, 108295
207
+ 6, 107829
208
+ 7, 108181
209
+ 0, 134563
210
+ 1, 134329
211
+ 2, 134713
212
+ 3, 134329
213
+ 4, 134627
214
+ 5, 134713
215
+ 6, 134329
216
+ 7, 134713
217
+ 0, 134563
218
+ 1, 134329
219
+ 2, 134713
220
+ 3, 134329
221
+ 4, 134627
222
+ 5, 134713
223
+ 6, 134329
224
+ 7, 134713
225
+ 0, 5886
226
+ 1, 134327
227
+ 2, 134711
228
+ 3, 134327
229
+ 4, 134625
230
+ 5, 134711
231
+ 6, 134327
232
+ 7, 134711
233
+ 0, 3963
234
+ 1, 4285
235
+ 2, 134711
236
+ 3, 5566
237
+ 4, 3966
238
+ 5, 3966
239
+ 6, 3966
240
+ 7, 3966
241
+ 0, 0
242
+ 1, 0
243
+ 2, 0
244
+ 3, 0
245
+ 4, 0
246
+ 5, 0
247
+ 6, 0
248
+ 7, 0
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=echo
2
+ config=QWEN3_235B
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=token-drop-analysis
6
+ commit=b099fd84
7
+ num_layers=12
8
+ hidden=4096
9
+ ffn_hidden=12288
10
+ moe_ffn_hidden=1536
11
+ num_experts=128
12
+ topk=8
13
+ experts_per_rank=16
14
+ expert_params=18874368
15
+ heads=64
16
+ kv_groups=4
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=0
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=12
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=0
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=0
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=8192
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=65536
39
+ mean_tokens_per_expert=4096
40
+ slot_policy=matched
41
+ slots_per_rank=16
42
+ capacity_mode=static
43
+ capacity=1.25
44
+ cuda_graph=0
45
+ probe_mode=1
46
+ async_grad_reduce=0
47
+ branch_role=probe
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__forcebal/rep1/mem_samples.csv ADDED
@@ -0,0 +1,240 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 4
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 2
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 47
43
+ 2, 4
44
+ 3, 629
45
+ 4, 629
46
+ 5, 629
47
+ 6, 629
48
+ 7, 4
49
+ 0, 909
50
+ 1, 889
51
+ 2, 897
52
+ 3, 899
53
+ 4, 897
54
+ 5, 893
55
+ 6, 905
56
+ 7, 907
57
+ 0, 1975
58
+ 1, 1975
59
+ 2, 1975
60
+ 3, 1975
61
+ 4, 1975
62
+ 5, 1975
63
+ 6, 1975
64
+ 7, 1975
65
+ 0, 47803
66
+ 1, 47803
67
+ 2, 47803
68
+ 3, 47803
69
+ 4, 47515
70
+ 5, 47803
71
+ 6, 47803
72
+ 7, 47803
73
+ 0, 49055
74
+ 1, 49055
75
+ 2, 49055
76
+ 3, 49055
77
+ 4, 48767
78
+ 5, 49055
79
+ 6, 49055
80
+ 7, 49055
81
+ 0, 49927
82
+ 1, 49927
83
+ 2, 49927
84
+ 3, 49927
85
+ 4, 49639
86
+ 5, 49927
87
+ 6, 49927
88
+ 7, 49927
89
+ 0, 53697
90
+ 1, 53697
91
+ 2, 53697
92
+ 3, 53697
93
+ 4, 53409
94
+ 5, 53697
95
+ 6, 53697
96
+ 7, 53697
97
+ 0, 54105
98
+ 1, 54105
99
+ 2, 54105
100
+ 3, 54105
101
+ 4, 53817
102
+ 5, 54105
103
+ 6, 54105
104
+ 7, 54105
105
+ 0, 54451
106
+ 1, 54443
107
+ 2, 54489
108
+ 3, 54479
109
+ 4, 54179
110
+ 5, 54459
111
+ 6, 54527
112
+ 7, 54471
113
+ 0, 55543
114
+ 1, 55543
115
+ 2, 55543
116
+ 3, 55543
117
+ 4, 55255
118
+ 5, 55543
119
+ 6, 55543
120
+ 7, 55543
121
+ 0, 56183
122
+ 1, 56183
123
+ 2, 56183
124
+ 3, 56183
125
+ 4, 55895
126
+ 5, 56183
127
+ 6, 56183
128
+ 7, 56183
129
+ 0, 58095
130
+ 1, 58095
131
+ 2, 58095
132
+ 3, 58095
133
+ 4, 57807
134
+ 5, 58095
135
+ 6, 58095
136
+ 7, 58095
137
+ 0, 58095
138
+ 1, 58095
139
+ 2, 58095
140
+ 3, 58095
141
+ 4, 57807
142
+ 5, 58095
143
+ 6, 58095
144
+ 7, 58095
145
+ 0, 98817
146
+ 1, 98913
147
+ 2, 98913
148
+ 3, 98915
149
+ 4, 98871
150
+ 5, 98905
151
+ 6, 98911
152
+ 7, 98897
153
+ 0, 99355
154
+ 1, 99407
155
+ 2, 99407
156
+ 3, 99407
157
+ 4, 99377
158
+ 5, 99407
159
+ 6, 99407
160
+ 7, 99407
161
+ 0, 99357
162
+ 1, 99409
163
+ 2, 99409
164
+ 3, 99409
165
+ 4, 99379
166
+ 5, 99409
167
+ 6, 99409
168
+ 7, 99409
169
+ 0, 99357
170
+ 1, 99409
171
+ 2, 99409
172
+ 3, 99409
173
+ 4, 99379
174
+ 5, 99409
175
+ 6, 99409
176
+ 7, 99409
177
+ 0, 99357
178
+ 1, 99409
179
+ 2, 99409
180
+ 3, 99409
181
+ 4, 99379
182
+ 5, 99409
183
+ 6, 99409
184
+ 7, 99409
185
+ 0, 103409
186
+ 1, 103461
187
+ 2, 103461
188
+ 3, 103461
189
+ 4, 103431
190
+ 5, 103461
191
+ 6, 103461
192
+ 7, 103461
193
+ 0, 104057
194
+ 1, 104109
195
+ 2, 104109
196
+ 3, 104109
197
+ 4, 104079
198
+ 5, 104109
199
+ 6, 104109
200
+ 7, 104109
201
+ 0, 108145
202
+ 1, 108271
203
+ 2, 108171
204
+ 3, 108343
205
+ 4, 108331
206
+ 5, 108227
207
+ 6, 108239
208
+ 7, 108207
209
+ 0, 134577
210
+ 1, 134699
211
+ 2, 134699
212
+ 3, 134699
213
+ 4, 134627
214
+ 5, 134699
215
+ 6, 134699
216
+ 7, 134699
217
+ 0, 134577
218
+ 1, 134699
219
+ 2, 134699
220
+ 3, 134699
221
+ 4, 134627
222
+ 5, 134699
223
+ 6, 134699
224
+ 7, 134699
225
+ 0, 134577
226
+ 1, 134699
227
+ 2, 134699
228
+ 3, 134699
229
+ 4, 134627
230
+ 5, 134699
231
+ 6, 134699
232
+ 7, 134699
233
+ 0, 3963
234
+ 1, 3966
235
+ 2, 3966
236
+ 3, 3966
237
+ 4, 3966
238
+ 5, 4286
239
+ 6, 134697
240
+ 7, 5566
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__forcebal/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=echo
2
+ config=QWEN3_235B
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=token-drop-analysis
6
+ commit=b099fd84
7
+ num_layers=12
8
+ hidden=4096
9
+ ffn_hidden=12288
10
+ moe_ffn_hidden=1536
11
+ num_experts=128
12
+ topk=8
13
+ experts_per_rank=16
14
+ expert_params=18874368
15
+ heads=64
16
+ kv_groups=4
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=0
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=12
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=0
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=0
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=8192
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=65536
39
+ mean_tokens_per_expert=4096
40
+ slot_policy=matched
41
+ slots_per_rank=16
42
+ capacity_mode=static
43
+ capacity=1.25
44
+ cuda_graph=0
45
+ probe_mode=1
46
+ async_grad_reduce=0
47
+ branch_role=probe
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=1
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__forcebal/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv CHANGED
@@ -14,7 +14,7 @@
14
  5, 0
15
  6, 0
16
  7, 0
17
- 0, 1
18
  1, 1
19
  2, 1
20
  3, 1
@@ -38,22 +38,22 @@
38
  5, 4
39
  6, 4
40
  7, 4
41
- 0, 4
42
- 1, 4
43
- 2, 4
44
- 3, 4
45
- 4, 4
46
- 5, 4
47
- 6, 4
48
- 7, 4
49
- 0, 821
50
- 1, 821
51
- 2, 821
52
- 3, 821
53
- 4, 821
54
- 5, 821
55
- 6, 821
56
- 7, 821
57
  0, 1975
58
  1, 1975
59
  2, 1975
@@ -62,30 +62,30 @@
62
  5, 1975
63
  6, 1975
64
  7, 1975
65
- 0, 46263
66
- 1, 45927
67
- 2, 46359
68
- 3, 45975
69
- 4, 45879
70
- 5, 46071
71
- 6, 47079
72
- 7, 46263
73
- 0, 48205
74
- 1, 48219
75
- 2, 48231
76
- 3, 48209
77
- 4, 47925
78
- 5, 48153
79
- 6, 48155
80
- 7, 48221
81
- 0, 49543
82
- 1, 49543
83
- 2, 49543
84
- 3, 49543
85
- 4, 49255
86
- 5, 49543
87
- 6, 49543
88
- 7, 49543
89
  0, 53697
90
  1, 53697
91
  2, 53697
@@ -102,14 +102,14 @@
102
  5, 54105
103
  6, 54105
104
  7, 54105
105
- 0, 54409
106
- 1, 54419
107
- 2, 54411
108
- 3, 54443
109
- 4, 54111
110
- 5, 54419
111
- 6, 54413
112
- 7, 54425
113
  0, 55543
114
  1, 55543
115
  2, 55543
@@ -142,14 +142,14 @@
142
  5, 59799
143
  6, 59799
144
  7, 59799
145
- 0, 105823
146
- 1, 105869
147
- 2, 105847
148
- 3, 105885
149
- 4, 105795
150
- 5, 105829
151
- 6, 105841
152
- 7, 105823
153
  0, 106307
154
  1, 106347
155
  2, 106347
@@ -182,54 +182,54 @@
182
  5, 106349
183
  6, 106349
184
  7, 106349
185
- 0, 110361
186
- 1, 108481
187
- 2, 110401
188
- 3, 110401
189
- 4, 110381
190
  5, 107713
191
- 6, 110401
192
- 7, 108097
193
  0, 111393
194
- 1, 111049
195
  2, 111049
196
  3, 111433
197
  4, 111029
198
  5, 111433
199
- 6, 111433
200
- 7, 111049
201
- 0, 112059
202
- 1, 111733
203
- 2, 111737
204
- 3, 112119
205
- 4, 111717
206
- 5, 112109
207
- 6, 112135
208
- 7, 111769
209
  0, 141823
210
- 1, 141559
211
  2, 141559
212
  3, 141943
213
  4, 141539
214
  5, 141943
215
- 6, 141943
216
- 7, 141559
217
  0, 141823
218
- 1, 141559
219
  2, 141559
220
  3, 141943
221
  4, 141539
222
  5, 141943
223
- 6, 141943
224
- 7, 141559
225
- 0, 141823
226
- 1, 141559
227
- 2, 141559
228
- 3, 141943
229
  4, 141539
230
- 5, 141943
231
- 6, 141943
232
- 7, 141559
233
  0, 3963
234
  1, 3966
235
  2, 3966
 
14
  5, 0
15
  6, 0
16
  7, 0
17
+ 0, 2
18
  1, 1
19
  2, 1
20
  3, 1
 
38
  5, 4
39
  6, 4
40
  7, 4
41
+ 0, 725
42
+ 1, 725
43
+ 2, 725
44
+ 3, 725
45
+ 4, 725
46
+ 5, 725
47
+ 6, 725
48
+ 7, 725
49
+ 0, 935
50
+ 1, 949
51
+ 2, 949
52
+ 3, 961
53
+ 4, 973
54
+ 5, 933
55
+ 6, 971
56
+ 7, 951
57
  0, 1975
58
  1, 1975
59
  2, 1975
 
62
  5, 1975
63
  6, 1975
64
  7, 1975
65
+ 0, 47803
66
+ 1, 47803
67
+ 2, 47803
68
+ 3, 47803
69
+ 4, 47515
70
+ 5, 47803
71
+ 6, 47803
72
+ 7, 47803
73
+ 0, 49279
74
+ 1, 49279
75
+ 2, 49055
76
+ 3, 49279
77
+ 4, 48767
78
+ 5, 49055
79
+ 6, 49119
80
+ 7, 49279
81
+ 0, 49927
82
+ 1, 49927
83
+ 2, 49927
84
+ 3, 49927
85
+ 4, 49639
86
+ 5, 49927
87
+ 6, 49927
88
+ 7, 49927
89
  0, 53697
90
  1, 53697
91
  2, 53697
 
102
  5, 54105
103
  6, 54105
104
  7, 54105
105
+ 0, 54429
106
+ 1, 54455
107
+ 2, 54439
108
+ 3, 54421
109
+ 4, 54163
110
+ 5, 54407
111
+ 6, 54431
112
+ 7, 54437
113
  0, 55543
114
  1, 55543
115
  2, 55543
 
142
  5, 59799
143
  6, 59799
144
  7, 59799
145
+ 0, 105919
146
+ 1, 105959
147
+ 2, 105959
148
+ 3, 105959
149
+ 4, 105915
150
+ 5, 105957
151
+ 6, 105939
152
+ 7, 105957
153
  0, 106307
154
  1, 106347
155
  2, 106347
 
182
  5, 106349
183
  6, 106349
184
  7, 106349
185
+ 0, 109593
186
+ 1, 107713
187
+ 2, 107713
188
+ 3, 107713
189
+ 4, 107693
190
  5, 107713
191
+ 6, 107713
192
+ 7, 107713
193
  0, 111393
194
+ 1, 111433
195
  2, 111049
196
  3, 111433
197
  4, 111029
198
  5, 111433
199
+ 6, 111049
200
+ 7, 111433
201
+ 0, 112087
202
+ 1, 112135
203
+ 2, 111769
204
+ 3, 112131
205
+ 4, 111733
206
+ 5, 112175
207
+ 6, 111787
208
+ 7, 112075
209
  0, 141823
210
+ 1, 141943
211
  2, 141559
212
  3, 141943
213
  4, 141539
214
  5, 141943
215
+ 6, 141559
216
+ 7, 141943
217
  0, 141823
218
+ 1, 141943
219
  2, 141559
220
  3, 141943
221
  4, 141539
222
  5, 141943
223
+ 6, 141559
224
+ 7, 141943
225
+ 0, 141821
226
+ 1, 141941
227
+ 2, 141557
228
+ 3, 141941
229
  4, 141539
230
+ 5, 141941
231
+ 6, 141559
232
+ 7, 141943
233
  0, 3963
234
  1, 3966
235
  2, 3966
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log CHANGED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1/mem_samples.csv ADDED
@@ -0,0 +1,272 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 0
25
+ 0, 4
26
+ 1, 1
27
+ 2, 1
28
+ 3, 1
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 1
38
+ 5, 1
39
+ 6, 1
40
+ 7, 1
41
+ 0, 4
42
+ 1, 4
43
+ 2, 4
44
+ 3, 4
45
+ 4, 4
46
+ 5, 4
47
+ 6, 4
48
+ 7, 4
49
+ 0, 4
50
+ 1, 629
51
+ 2, 629
52
+ 3, 629
53
+ 4, 629
54
+ 5, 629
55
+ 6, 629
56
+ 7, 629
57
+ 0, 877
58
+ 1, 899
59
+ 2, 895
60
+ 3, 899
61
+ 4, 885
62
+ 5, 897
63
+ 6, 897
64
+ 7, 905
65
+ 0, 1831
66
+ 1, 1831
67
+ 2, 1831
68
+ 3, 1831
69
+ 4, 1831
70
+ 5, 1831
71
+ 6, 1831
72
+ 7, 1831
73
+ 0, 39169
74
+ 1, 38995
75
+ 2, 39197
76
+ 3, 38995
77
+ 4, 38995
78
+ 5, 39197
79
+ 6, 38995
80
+ 7, 39197
81
+ 0, 44989
82
+ 1, 44991
83
+ 2, 45001
84
+ 3, 44963
85
+ 4, 44987
86
+ 5, 45001
87
+ 6, 44991
88
+ 7, 45001
89
+ 0, 45895
90
+ 1, 45897
91
+ 2, 45907
92
+ 3, 45869
93
+ 4, 45893
94
+ 5, 45907
95
+ 6, 45897
96
+ 7, 45907
97
+ 0, 46917
98
+ 1, 46919
99
+ 2, 46929
100
+ 3, 46891
101
+ 4, 46915
102
+ 5, 46929
103
+ 6, 46919
104
+ 7, 46929
105
+ 0, 47013
106
+ 1, 47015
107
+ 2, 47025
108
+ 3, 46987
109
+ 4, 47011
110
+ 5, 47025
111
+ 6, 47015
112
+ 7, 47025
113
+ 0, 47193
114
+ 1, 47203
115
+ 2, 47233
116
+ 3, 47171
117
+ 4, 47181
118
+ 5, 47217
119
+ 6, 47209
120
+ 7, 47221
121
+ 0, 47877
122
+ 1, 47879
123
+ 2, 47889
124
+ 3, 47851
125
+ 4, 47875
126
+ 5, 47889
127
+ 6, 47879
128
+ 7, 47889
129
+ 0, 48037
130
+ 1, 48039
131
+ 2, 48049
132
+ 3, 48011
133
+ 4, 48035
134
+ 5, 48049
135
+ 6, 48039
136
+ 7, 48049
137
+ 0, 48737
138
+ 1, 48739
139
+ 2, 48749
140
+ 3, 48711
141
+ 4, 48735
142
+ 5, 48749
143
+ 6, 48699
144
+ 7, 48749
145
+ 0, 48737
146
+ 1, 48739
147
+ 2, 48749
148
+ 3, 48711
149
+ 4, 48735
150
+ 5, 48749
151
+ 6, 48699
152
+ 7, 48749
153
+ 0, 92911
154
+ 1, 93679
155
+ 2, 93683
156
+ 3, 92905
157
+ 4, 93695
158
+ 5, 93683
159
+ 6, 93679
160
+ 7, 92897
161
+ 0, 94411
162
+ 1, 94393
163
+ 2, 94397
164
+ 3, 94405
165
+ 4, 94409
166
+ 5, 94397
167
+ 6, 94393
168
+ 7, 94397
169
+ 0, 94411
170
+ 1, 94393
171
+ 2, 94397
172
+ 3, 94405
173
+ 4, 94409
174
+ 5, 94397
175
+ 6, 94393
176
+ 7, 94397
177
+ 0, 94413
178
+ 1, 94395
179
+ 2, 94399
180
+ 3, 94407
181
+ 4, 94411
182
+ 5, 94399
183
+ 6, 94395
184
+ 7, 94399
185
+ 0, 94413
186
+ 1, 94395
187
+ 2, 94399
188
+ 3, 94407
189
+ 4, 94411
190
+ 5, 94399
191
+ 6, 94395
192
+ 7, 94399
193
+ 0, 95007
194
+ 1, 95015
195
+ 2, 94969
196
+ 3, 94999
197
+ 4, 95003
198
+ 5, 94975
199
+ 6, 94985
200
+ 7, 94977
201
+ 0, 95965
202
+ 1, 95947
203
+ 2, 95759
204
+ 3, 95959
205
+ 4, 95867
206
+ 5, 95951
207
+ 6, 95755
208
+ 7, 95951
209
+ 0, 96663
210
+ 1, 96621
211
+ 2, 96465
212
+ 3, 96637
213
+ 4, 96539
214
+ 5, 96625
215
+ 6, 96425
216
+ 7, 96633
217
+ 0, 126025
218
+ 1, 126025
219
+ 2, 125839
220
+ 3, 126019
221
+ 4, 125965
222
+ 5, 126031
223
+ 6, 125795
224
+ 7, 126031
225
+ 0, 126025
226
+ 1, 126025
227
+ 2, 125839
228
+ 3, 126019
229
+ 4, 125965
230
+ 5, 126031
231
+ 6, 125795
232
+ 7, 126031
233
+ 0, 126025
234
+ 1, 126025
235
+ 2, 125839
236
+ 3, 126019
237
+ 4, 125965
238
+ 5, 126031
239
+ 6, 125795
240
+ 7, 126031
241
+ 0, 126025
242
+ 1, 126025
243
+ 2, 125839
244
+ 3, 126019
245
+ 4, 125965
246
+ 5, 126031
247
+ 6, 125795
248
+ 7, 126031
249
+ 0, 126025
250
+ 1, 126025
251
+ 2, 125839
252
+ 3, 126019
253
+ 4, 125965
254
+ 5, 126031
255
+ 6, 125795
256
+ 7, 126031
257
+ 0, 126025
258
+ 1, 126025
259
+ 2, 125839
260
+ 3, 126019
261
+ 4, 125965
262
+ 5, 126031
263
+ 6, 125795
264
+ 7, 126031
265
+ 0, 906
266
+ 1, 1162
267
+ 2, 125023
268
+ 3, 2442
269
+ 4, 125149
270
+ 5, 2442
271
+ 6, 124979
272
+ 7, 2186
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=echo
2
+ config=QWEN3_30B
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=token-drop-analysis
6
+ commit=b099fd84
7
+ num_layers=48
8
+ hidden=2048
9
+ ffn_hidden=6144
10
+ moe_ffn_hidden=768
11
+ num_experts=128
12
+ topk=8
13
+ experts_per_rank=16
14
+ expert_params=4718592
15
+ heads=32
16
+ kv_groups=4
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=0
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=48
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=0
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=0
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=4096
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=32768
39
+ mean_tokens_per_expert=2048
40
+ slot_policy=matched
41
+ slots_per_rank=16
42
+ capacity_mode=static
43
+ capacity=1.25
44
+ cuda_graph=0
45
+ probe_mode=1
46
+ async_grad_reduce=0
47
+ branch_role=probe
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1/train.log ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f59d8f2658c1bf9973fd1c6a8738fea92ed503364de4076a4c3ab080dda98768
3
+ size 13020943
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal/rep1/mem_samples.csv ADDED
@@ -0,0 +1,272 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 4
43
+ 2, 4
44
+ 3, 4
45
+ 4, 4
46
+ 5, 4
47
+ 6, 4
48
+ 7, 4
49
+ 0, 821
50
+ 1, 821
51
+ 2, 821
52
+ 3, 821
53
+ 4, 821
54
+ 5, 821
55
+ 6, 821
56
+ 7, 821
57
+ 0, 1831
58
+ 1, 1831
59
+ 2, 1831
60
+ 3, 1831
61
+ 4, 1831
62
+ 5, 1831
63
+ 6, 1831
64
+ 7, 1831
65
+ 0, 11795
66
+ 1, 11815
67
+ 2, 11819
68
+ 3, 11805
69
+ 4, 11803
70
+ 5, 11819
71
+ 6, 11813
72
+ 7, 18257
73
+ 0, 44891
74
+ 1, 44855
75
+ 2, 44895
76
+ 3, 44813
77
+ 4, 44837
78
+ 5, 44819
79
+ 6, 44849
80
+ 7, 44859
81
+ 0, 45895
82
+ 1, 45897
83
+ 2, 45907
84
+ 3, 45869
85
+ 4, 45893
86
+ 5, 45907
87
+ 6, 45897
88
+ 7, 45907
89
+ 0, 46917
90
+ 1, 46919
91
+ 2, 46929
92
+ 3, 46891
93
+ 4, 46915
94
+ 5, 46929
95
+ 6, 46919
96
+ 7, 46929
97
+ 0, 47013
98
+ 1, 47015
99
+ 2, 47025
100
+ 3, 46987
101
+ 4, 47011
102
+ 5, 47025
103
+ 6, 47015
104
+ 7, 47025
105
+ 0, 47153
106
+ 1, 47151
107
+ 2, 47155
108
+ 3, 47119
109
+ 4, 47171
110
+ 5, 47155
111
+ 6, 47163
112
+ 7, 47155
113
+ 0, 47877
114
+ 1, 47879
115
+ 2, 47889
116
+ 3, 47851
117
+ 4, 47875
118
+ 5, 47889
119
+ 6, 47879
120
+ 7, 47889
121
+ 0, 48037
122
+ 1, 48039
123
+ 2, 48049
124
+ 3, 48011
125
+ 4, 48035
126
+ 5, 48049
127
+ 6, 48039
128
+ 7, 48049
129
+ 0, 48513
130
+ 1, 48515
131
+ 2, 48529
132
+ 3, 48487
133
+ 4, 48511
134
+ 5, 48529
135
+ 6, 48475
136
+ 7, 48529
137
+ 0, 48737
138
+ 1, 48739
139
+ 2, 48749
140
+ 3, 48711
141
+ 4, 48735
142
+ 5, 48749
143
+ 6, 48699
144
+ 7, 48749
145
+ 0, 92891
146
+ 1, 92893
147
+ 2, 92897
148
+ 3, 92905
149
+ 4, 92889
150
+ 5, 92897
151
+ 6, 92893
152
+ 7, 92897
153
+ 0, 94391
154
+ 1, 94393
155
+ 2, 94397
156
+ 3, 94405
157
+ 4, 94389
158
+ 5, 94397
159
+ 6, 94393
160
+ 7, 94397
161
+ 0, 94391
162
+ 1, 94393
163
+ 2, 94397
164
+ 3, 94405
165
+ 4, 94389
166
+ 5, 94397
167
+ 6, 94393
168
+ 7, 94397
169
+ 0, 94393
170
+ 1, 94395
171
+ 2, 94399
172
+ 3, 94407
173
+ 4, 94391
174
+ 5, 94399
175
+ 6, 94395
176
+ 7, 94399
177
+ 0, 94393
178
+ 1, 94395
179
+ 2, 94399
180
+ 3, 94407
181
+ 4, 94391
182
+ 5, 94399
183
+ 6, 94395
184
+ 7, 94399
185
+ 0, 95031
186
+ 1, 95031
187
+ 2, 94977
188
+ 3, 95001
189
+ 4, 94983
190
+ 5, 94985
191
+ 6, 94977
192
+ 7, 94997
193
+ 0, 96041
194
+ 1, 95947
195
+ 2, 95759
196
+ 3, 96055
197
+ 4, 95943
198
+ 5, 95759
199
+ 6, 95755
200
+ 7, 96047
201
+ 0, 96685
202
+ 1, 96653
203
+ 2, 96427
204
+ 3, 96719
205
+ 4, 96597
206
+ 5, 96417
207
+ 6, 96419
208
+ 7, 96723
209
+ 0, 126121
210
+ 1, 126025
211
+ 2, 125859
212
+ 3, 126115
213
+ 4, 126061
214
+ 5, 125859
215
+ 6, 125815
216
+ 7, 126147
217
+ 0, 126121
218
+ 1, 126025
219
+ 2, 125859
220
+ 3, 126115
221
+ 4, 126061
222
+ 5, 125859
223
+ 6, 125815
224
+ 7, 126147
225
+ 0, 126121
226
+ 1, 126025
227
+ 2, 125859
228
+ 3, 126115
229
+ 4, 126061
230
+ 5, 125859
231
+ 6, 125815
232
+ 7, 126147
233
+ 0, 126121
234
+ 1, 126025
235
+ 2, 125859
236
+ 3, 126115
237
+ 4, 126061
238
+ 5, 125859
239
+ 6, 125815
240
+ 7, 126147
241
+ 0, 126121
242
+ 1, 126025
243
+ 2, 125859
244
+ 3, 126115
245
+ 4, 126061
246
+ 5, 125859
247
+ 6, 125815
248
+ 7, 126147
249
+ 0, 126121
250
+ 1, 126025
251
+ 2, 125859
252
+ 3, 126115
253
+ 4, 126061
254
+ 5, 125859
255
+ 6, 125815
256
+ 7, 126147
257
+ 0, 125305
258
+ 1, 125209
259
+ 2, 125043
260
+ 3, 2186
261
+ 4, 1162
262
+ 5, 125043
263
+ 6, 124999
264
+ 7, 125331
265
+ 0, 125302
266
+ 1, 2183
267
+ 2, 903
268
+ 3, 903
269
+ 4, 903
270
+ 5, 903
271
+ 6, 905
272
+ 7, 1162
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=echo
2
+ config=QWEN3_30B
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=token-drop-analysis
6
+ commit=b099fd84
7
+ num_layers=48
8
+ hidden=2048
9
+ ffn_hidden=6144
10
+ moe_ffn_hidden=768
11
+ num_experts=128
12
+ topk=8
13
+ experts_per_rank=16
14
+ expert_params=4718592
15
+ heads=32
16
+ kv_groups=4
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=0
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=48
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=0
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=0
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=4096
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=32768
39
+ mean_tokens_per_expert=2048
40
+ slot_policy=matched
41
+ slots_per_rank=16
42
+ capacity_mode=static
43
+ capacity=1.25
44
+ cuda_graph=0
45
+ probe_mode=1
46
+ async_grad_reduce=0
47
+ branch_role=probe
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=1
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2dd6f58e7eee63ad96bb2cf53ad35b2b7bae74f4e30d7d440a04ee800647c5f1
3
+ size 15159073
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/mem_samples.csv CHANGED
@@ -14,7 +14,7 @@
14
  5, 0
15
  6, 0
16
  7, 0
17
- 0, 1
18
  1, 1
19
  2, 1
20
  3, 1
@@ -39,21 +39,21 @@
39
  6, 4
40
  7, 4
41
  0, 4
42
- 1, 629
43
- 2, 4
44
  3, 4
45
- 4, 27
46
  5, 4
47
- 6, 4
48
- 7, 4
49
- 0, 821
50
- 1, 821
51
- 2, 821
52
- 3, 821
53
- 4, 821
54
- 5, 821
55
- 6, 821
56
- 7, 821
57
  0, 1831
58
  1, 1831
59
  2, 1831
@@ -70,22 +70,22 @@
70
  5, 39197
71
  6, 38995
72
  7, 39197
73
- 0, 44953
74
- 1, 44991
75
- 2, 44965
76
- 3, 44963
77
- 4, 44951
78
- 5, 44965
79
- 6, 44991
80
- 7, 45001
81
- 0, 45895
82
- 1, 45897
83
- 2, 45907
84
- 3, 45869
85
- 4, 45893
86
- 5, 45907
87
- 6, 45897
88
- 7, 45907
89
  0, 46917
90
  1, 46919
91
  2, 46929
@@ -102,14 +102,14 @@
102
  5, 47025
103
  6, 47015
104
  7, 47025
105
- 0, 47275
106
- 1, 47265
107
- 2, 47281
108
- 3, 47245
109
- 4, 47255
110
- 5, 47251
111
- 6, 47245
112
- 7, 47281
113
  0, 47877
114
  1, 47879
115
  2, 47889
@@ -142,14 +142,22 @@
142
  5, 49145
143
  6, 49117
144
  7, 49145
145
- 0, 100829
146
- 1, 100835
147
- 2, 100849
148
- 3, 100875
149
- 4, 100853
150
- 5, 100861
151
- 6, 100833
152
- 7, 100861
 
 
 
 
 
 
 
 
153
  0, 101347
154
  1, 101369
155
  2, 101353
@@ -174,99 +182,91 @@
174
  5, 101355
175
  6, 101355
176
  7, 101355
177
- 0, 101349
178
- 1, 101371
179
- 2, 101355
180
- 3, 101363
181
- 4, 101367
182
- 5, 101355
183
- 6, 101355
184
- 7, 101355
185
- 0, 102061
186
- 1, 102851
187
- 2, 102835
188
- 3, 102075
189
- 4, 102847
190
- 5, 102835
191
- 6, 102355
192
- 7, 102067
193
- 0, 102805
194
- 1, 103499
195
- 2, 103483
196
- 3, 102915
197
- 4, 103495
198
- 5, 103483
199
- 6, 103195
200
  7, 102907
201
- 0, 103503
202
- 1, 104195
203
- 2, 104173
204
- 3, 103585
205
- 4, 104153
206
- 5, 104183
207
- 6, 103863
208
- 7, 103593
209
- 0, 132853
210
- 1, 133533
211
- 2, 133539
212
- 3, 132951
213
- 4, 133569
214
- 5, 133539
215
- 6, 133231
216
  7, 132963
217
- 0, 132853
218
- 1, 133533
219
- 2, 133539
220
- 3, 132951
221
- 4, 133569
222
- 5, 133539
223
- 6, 133231
224
  7, 132963
225
- 0, 132853
226
- 1, 133533
227
- 2, 133539
228
- 3, 132951
229
- 4, 133569
230
- 5, 133539
231
- 6, 133231
232
  7, 132963
233
- 0, 132853
234
- 1, 133533
235
- 2, 133539
236
- 3, 132951
237
- 4, 133569
238
- 5, 133539
239
- 6, 133231
240
  7, 132963
241
- 0, 132853
242
- 1, 133533
243
- 2, 133539
244
- 3, 132951
245
- 4, 133569
246
- 5, 133539
247
- 6, 133231
248
  7, 132963
249
- 0, 132853
250
- 1, 133533
251
- 2, 133539
252
- 3, 132951
253
- 4, 133569
254
- 5, 133539
255
- 6, 133231
256
  7, 132963
257
- 0, 1162
258
- 1, 132717
259
- 2, 132723
260
- 3, 132135
261
  4, 2442
262
- 5, 132723
263
- 6, 132415
264
- 7, 2186
265
- 0, 0
266
- 1, 0
267
- 2, 0
268
- 3, 0
269
- 4, 0
270
- 5, 0
271
- 6, 0
272
- 7, 0
 
14
  5, 0
15
  6, 0
16
  7, 0
17
+ 0, 2
18
  1, 1
19
  2, 1
20
  3, 1
 
39
  6, 4
40
  7, 4
41
  0, 4
42
+ 1, 4
43
+ 2, 27
44
  3, 4
45
+ 4, 4
46
  5, 4
47
+ 6, 41
48
+ 7, 629
49
+ 0, 895
50
+ 1, 903
51
+ 2, 917
52
+ 3, 929
53
+ 4, 925
54
+ 5, 899
55
+ 6, 915
56
+ 7, 913
57
  0, 1831
58
  1, 1831
59
  2, 1831
 
70
  5, 39197
71
  6, 38995
72
  7, 39197
73
+ 0, 44847
74
+ 1, 44875
75
+ 2, 44877
76
+ 3, 44859
77
+ 4, 44821
78
+ 5, 44837
79
+ 6, 44841
80
+ 7, 44875
81
+ 0, 45791
82
+ 1, 45839
83
+ 2, 45845
84
+ 3, 45775
85
+ 4, 45789
86
+ 5, 45793
87
+ 6, 45835
88
+ 7, 45855
89
  0, 46917
90
  1, 46919
91
  2, 46929
 
102
  5, 47025
103
  6, 47015
104
  7, 47025
105
+ 0, 47191
106
+ 1, 47219
107
+ 2, 47209
108
+ 3, 47163
109
+ 4, 47213
110
+ 5, 47213
111
+ 6, 47191
112
+ 7, 47211
113
  0, 47877
114
  1, 47879
115
  2, 47889
 
142
  5, 49145
143
  6, 49117
144
  7, 49145
145
+ 0, 100699
146
+ 1, 100655
147
+ 2, 100703
148
+ 3, 100647
149
+ 4, 100715
150
+ 5, 100639
151
+ 6, 100639
152
+ 7, 100703
153
+ 0, 101347
154
+ 1, 101369
155
+ 2, 101353
156
+ 3, 101361
157
+ 4, 101365
158
+ 5, 101353
159
+ 6, 101353
160
+ 7, 101353
161
  0, 101347
162
  1, 101369
163
  2, 101353
 
182
  5, 101355
183
  6, 101355
184
  7, 101355
185
+ 0, 101957
186
+ 1, 101953
187
+ 2, 101945
188
+ 3, 101963
189
+ 4, 101903
190
+ 5, 101965
191
+ 6, 101923
192
+ 7, 101951
193
+ 0, 102901
194
+ 1, 102731
195
+ 2, 102907
196
+ 3, 102723
197
+ 4, 102727
198
+ 5, 102907
199
+ 6, 102811
 
 
 
 
 
 
 
 
200
  7, 102907
201
+ 0, 103543
202
+ 1, 103383
203
+ 2, 103557
204
+ 3, 103389
205
+ 4, 103383
206
+ 5, 103559
207
+ 6, 103459
208
+ 7, 103559
209
+ 0, 132949
210
+ 1, 132765
211
+ 2, 132963
212
+ 3, 132759
213
+ 4, 132801
214
+ 5, 132963
215
+ 6, 132847
216
  7, 132963
217
+ 0, 132949
218
+ 1, 132765
219
+ 2, 132963
220
+ 3, 132759
221
+ 4, 132801
222
+ 5, 132963
223
+ 6, 132847
224
  7, 132963
225
+ 0, 132949
226
+ 1, 132765
227
+ 2, 132963
228
+ 3, 132759
229
+ 4, 132801
230
+ 5, 132963
231
+ 6, 132847
232
  7, 132963
233
+ 0, 132949
234
+ 1, 132765
235
+ 2, 132963
236
+ 3, 132759
237
+ 4, 132801
238
+ 5, 132963
239
+ 6, 132847
240
  7, 132963
241
+ 0, 132949
242
+ 1, 132765
243
+ 2, 132963
244
+ 3, 132759
245
+ 4, 132801
246
+ 5, 132963
247
+ 6, 132847
248
  7, 132963
249
+ 0, 132949
250
+ 1, 132765
251
+ 2, 132963
252
+ 3, 132759
253
+ 4, 132801
254
+ 5, 132963
255
+ 6, 132847
256
  7, 132963
257
+ 0, 132133
258
+ 1, 131949
259
+ 2, 2442
260
+ 3, 131943
261
  4, 2442
262
+ 5, 132147
263
+ 6, 132031
264
+ 7, 132147
265
+ 0, 132130
266
+ 1, 2183
267
+ 2, 903
268
+ 3, 903
269
+ 4, 903
270
+ 5, 903
271
+ 6, 905
272
+ 7, 1162
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/train.log CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:df595e47799be5bbf50d80509f6ea6b2cba57e19228e6f7973711658afe74dc1
3
- size 13026229
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:01c7900ca48884e0bc2e1036053aa7a996f875bd01f2f8370ebc63610dc4f90b
3
+ size 13023349
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv CHANGED
@@ -1,2 +1,8 @@
1
  model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir
2
- QWEN3_235B echo natural 2.00 OK 141943 1 0.0 1.02734375 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1
 
 
 
 
 
 
 
1
  model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir
2
+ QWEN3_235B echo measure 2.00 OK 141943 1 0.0 1.029296875 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1
3
+ QWEN3_235B echo validate 1.25 OK 134713 1 0.0 1.03125 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25/rep1
4
+ QWEN3_235B echo forced 1.25 OK 134699 1 0.0 1.025390625 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__forcebal/rep1
5
+ QWEN3_30B echo measure 2.00 OK 132963 1 0.0 1.05859375 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1
6
+ QWEN3_30B echo validate 1.25 OK 126031 1 0.0 1.05859375 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1
7
+ QWEN3_30B echo forced 1.25 OK 126147 1 0.0 1.0546875 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal/rep1
8
+ DSV2_LITE_L24 echo measure 2.00 OK 140961 1 0.0 1.0234375 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv CHANGED
@@ -1 +1,3 @@
1
  model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir
 
 
 
1
  model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir
2
+ QWEN3_235B echo SELECTED 1.25 1.03125 0.0 134713 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25/rep1
3
+ QWEN3_30B echo SELECTED 1.25 1.05859375 0.0 126031 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/results.tsv CHANGED
@@ -1,2 +1,8 @@
1
  config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
2
- QWEN3_235B echo matched 16 8192 8 2.00 none 1 OK 447.80 439.80 141943 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1
 
 
 
 
 
 
 
1
  config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
2
+ QWEN3_235B echo matched 16 8192 8 2.00 none 1 OK 439.09 433.90 141943 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1
3
+ QWEN3_235B echo matched 16 8192 8 1.25 none 1 OK 443.50 437.00 134713 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25/rep1
4
+ QWEN3_235B echo matched 16 8192 8 1.25 none+forcebal 1 OK 415.86 413.80 134699 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__forcebal/rep1
5
+ QWEN3_30B echo matched 16 4096 8 2.00 none 1 OK 1002.86 951.60 132963 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1
6
+ QWEN3_30B echo matched 16 4096 8 1.25 none 1 OK 1007.40 966.70 126031 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1
7
+ QWEN3_30B echo matched 16 4096 8 1.25 none+forcebal 1 OK 1020.94 952.60 126147 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal/rep1
8
+ DSV2_LITE_L24 echo matched 8 8192 8 2.00 none 1 OK 817.63 810.40 140961 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1
tierb_study_20260826_144644/hf_watcher.log CHANGED
@@ -36,3 +36,6 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
36
  Already logged in to Hugging Face as: AbhishekGhosh1998
37
  Watching /workspace/tierb_study_20260826_144644 -> HF every 600s (kill to stop)
38
  [2026-08-26T16:25:52+00:00] backup pass
 
 
 
 
36
  Already logged in to Hugging Face as: AbhishekGhosh1998
37
  Watching /workspace/tierb_study_20260826_144644 -> HF every 600s (kill to stop)
38
  [2026-08-26T16:25:52+00:00] backup pass
39
+ uploaded: /workspace/tierb_study_20260826_144644 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tierb_study_20260826_144644
40
+ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
41
+ [2026-08-26T16:35:54+00:00] backup pass
tierb_study_20260826_144644/study.log CHANGED
@@ -910,3 +910,46 @@ driver_repo: /workspace/blackwell_run_scripts @ 6e4b1c8
910
  === STAGE 7 -- Tier B probe and zero-drop capacity selection
911
 
912
  === STEP echo__tierbprobe__gbs8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
910
  === STAGE 7 -- Tier B probe and zero-drop capacity selection
911
 
912
  === STEP echo__tierbprobe__gbs8
913
+ ==============================================================
914
+ CAMPAIGN: echo backend=echo phase=tierbprobe
915
+ Megatron: /workspace/Megatron-MoE-Echo @ token-drop-analysis
916
+ GBS=8 seq=4096 repeats=3 nsys=1
917
+ results -> /workspace/tierb_study_20260826_144644/echo/tierbprobe
918
+ ==============================================================
919
+ --------------------------------------------------------------------------
920
+ [16:26:07] CELL QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00 (slots=16)
921
+ [guard] iter 12/12 (70s)
922
+ [guard] tearing down process group (outcome=DONE)
923
+ [16:27:23] QWEN3_235B echo: measured required=1.029296875 at C=2.00 -> validating C=1.25
924
+ --------------------------------------------------------------------------
925
+ [16:27:23] CELL QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25 (slots=16)
926
+ [guard] iter 12/12 (70s)
927
+ [guard] tearing down process group (outcome=DONE)
928
+ --------------------------------------------------------------------------
929
+ [16:28:40] CELL QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__forcebal (slots=16)
930
+ [guard] iter 12/12 (70s)
931
+ [guard] tearing down process group (outcome=DONE)
932
+ --------------------------------------------------------------------------
933
+ [16:29:56] CELL QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00 (slots=16)
934
+ [guard] iter 5/12 (70s)
935
+ [guard] iter 12/12 (80s)
936
+ [guard] tearing down process group (outcome=DONE)
937
+ [16:31:22] QWEN3_30B echo: measured required=1.05859375 at C=2.00 -> validating C=1.25
938
+ --------------------------------------------------------------------------
939
+ [16:31:22] CELL QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25 (slots=16)
940
+ [guard] iter 5/12 (70s)
941
+ [guard] iter 12/12 (80s)
942
+ [guard] tearing down process group (outcome=DONE)
943
+ --------------------------------------------------------------------------
944
+ [16:32:47] CELL QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal (slots=16)
945
+ [guard] iter 4/12 (70s)
946
+ [guard] iter 12/12 (80s)
947
+ [guard] tearing down process group (outcome=DONE)
948
+ --------------------------------------------------------------------------
949
+ [16:34:13] CELL DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00 (slots=8)
950
+ [guard] iter 6/12 (70s)
951
+ [guard] iter 12/12 (80s)
952
+ [guard] tearing down process group (outcome=DONE)
953
+ [16:35:35] DSV2_LITE_L24 echo: measured required=1.0234375 at C=2.00 -> validating C=1.25
954
+ --------------------------------------------------------------------------
955
+ [16:35:35] CELL DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25 (slots=8)