AbhishekGhosh1998 commited on
Commit
f23b68c
·
verified ·
1 Parent(s): 5beb25c

granularity results tierb_study_20260826_144644 @ 2026-08-26T17:19:17Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0

Browse files
Files changed (28) hide show
  1. .gitattributes +1 -0
  2. tierb_study_20260826_144644/echo/tierbprobe/manifest.txt +2 -2
  3. tierb_study_20260826_144644/echo/tierbprobe/provenance.txt +42 -0
  4. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0234375/rep1/mem_samples.csv +40 -0
  5. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0234375/rep1/run.env +61 -0
  6. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0234375/rep1/train.log +13 -0
  7. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv +67 -67
  8. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log +0 -0
  9. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125/rep1/mem_samples.csv +256 -0
  10. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125/rep1/run.env +61 -0
  11. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125/rep1/train.log +0 -0
  12. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125__forcebal/rep1/mem_samples.csv +232 -0
  13. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125__forcebal/rep1/run.env +61 -0
  14. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125__forcebal/rep1/train.log +0 -0
  15. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv +62 -62
  16. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log +0 -0
  17. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p05859375/rep1/mem_samples.csv +272 -0
  18. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p05859375/rep1/run.env +61 -0
  19. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p05859375/rep1/train.log +3 -0
  20. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/mem_samples.csv +137 -145
  21. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/train.log +2 -2
  22. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/mem_samples.csv +109 -109
  23. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log +2 -2
  24. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv +7 -2
  25. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv +2 -0
  26. tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/results.tsv +7 -2
  27. tierb_study_20260826_144644/hf_watcher.log +3 -0
  28. tierb_study_20260826_144644/study.log +44 -0
.gitattributes CHANGED
@@ -37,3 +37,4 @@ tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__mat
37
  tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log filter=lfs diff=lfs merge=lfs -text
38
  tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1/train.log filter=lfs diff=lfs merge=lfs -text
39
  tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log filter=lfs diff=lfs merge=lfs -text
 
 
37
  tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log filter=lfs diff=lfs merge=lfs -text
38
  tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25/rep1/train.log filter=lfs diff=lfs merge=lfs -text
39
  tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log filter=lfs diff=lfs merge=lfs -text
40
+ tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p05859375/rep1/train.log filter=lfs diff=lfs merge=lfs -text
tierb_study_20260826_144644/echo/tierbprobe/manifest.txt CHANGED
@@ -1,9 +1,9 @@
1
- timestamp: 2026-08-26T17:05:10+00:00
2
  host: sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0
3
  gpu: NVIDIA B200 x8
4
  driver: 580.95.05
5
  study_scripts_branch: granularity-echo
6
- study_scripts_commit: 9340d0b
7
  campaign: echo
8
  backend: echo
9
  phase: tierbprobe
 
1
+ timestamp: 2026-08-26T17:09:26+00:00
2
  host: sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0
3
  gpu: NVIDIA B200 x8
4
  driver: 580.95.05
5
  study_scripts_branch: granularity-echo
6
+ study_scripts_commit: 6553011
7
  campaign: echo
8
  backend: echo
9
  phase: tierbprobe
tierb_study_20260826_144644/echo/tierbprobe/provenance.txt CHANGED
@@ -46,3 +46,45 @@ command : run_ladder.sh --phase tierbprobe --gbs 8 --out /workspace/tierb_
46
  ---- toolchain -----------------------------------------------
47
  nvcc : release 12.9, V12.9.86
48
  gcc : gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
46
  ---- toolchain -----------------------------------------------
47
  nvcc : release 12.9, V12.9.86
48
  gcc : gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0
49
+ python : 3.12.3
50
+ torch : 2.8.0a0+5228986c39.nv25.06 cuda=12.9 nccl=2.27.3
51
+ torch arch : ['sm_75', 'sm_80', 'sm_86', 'sm_90', 'sm_100', 'sm_120', 'compute_120']
52
+ transformer_engine: 2.10.0.dev0+10f6128d [/usr/local/lib/python3.12/dist-packages/transformer_engine/__init__.py]
53
+ deep_ep : 1.2.1+34152ae [/workspace/DeepEP/deep_ep/__init__.py]
54
+ ultra_ep : 1.0.0 [/workspace/UltraEP/ultra_ep/__init__.py]
55
+ moonep : 0.0.1 [/workspace/MoonEP/moonep/__init__.py]
56
+ megatron : 0.14.0rc5 [None]
57
+ flash_attn : 2.7.4.post1 [/usr/local/lib/python3.12/dist-packages/flash_attn/__init__.py]
58
+ numpy : 1.26.4 [/usr/local/lib/python3.12/dist-packages/numpy/__init__.py]
59
+ huggingface_hub: 1.28.0 [/usr/local/lib/python3.12/dist-packages/huggingface_hub/__init__.py]
60
+
61
+ ---- repositories --------------------------------------------
62
+ Megatron-MoE-Echo token-drop-analysis b099fd8494b0
63
+ Megatron-UltraEP ultraep-token-drop-analysis eaf829e7ad21
64
+ UltraEP main 94cab099b44f
65
+ DeepEP HEAD 34152ae28f80 (+3 untracked: build output)
66
+ MoonEP megatron-integration c27d35da15f2
67
+ blackwell_run_scripts granularity-echo 655301106287
68
+
69
+ ---- environment ---------------------------------------------
70
+ CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0
71
+ CUDA_DRIVER_VERSION=575.57.08
72
+ CUDA_HOME=/usr/local/cuda
73
+ CUDA_MODULE_LOADING=LAZY
74
+ CUDA_VERSION=12.9.1.010
75
+ HF_REMOTE_ROOT=tierb_study_20260826_144644
76
+ NCCL_VERSION=2.27.3
77
+ NVSHMEM_VERSION=3.2.5
78
+ PYTORCH_BUILD_NUMBER=0
79
+ PYTORCH_BUILD_VERSION=2.8.0a0+5228986
80
+ PYTORCH_HOME=/opt/pytorch/pytorch
81
+ PYTORCH_VERSION=2.8.0a0+5228986
82
+ TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1
83
+ TORCH_CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0+PTX
84
+ TORCH_NCCL_USE_COMM_NONBLOCKING=0
85
+
86
+ ---- shared library resolution -------------------------------
87
+ libnvshmem_host.so: not in ldconfig
88
+ libnccl.so: /lib/x86_64-linux-gnu/libnccl.so.2
89
+ lrwxrwxrwx 1 root root 20 Aug 26 14:37 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so -> libnvshmem_host.so.3
90
+ -rw-r--r-- 1 root root 179045744 Aug 26 14:09 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so.3
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0234375/rep1/mem_samples.csv ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 4
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 1
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0234375/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=echo
2
+ config=DSV2_LITE_L24
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=token-drop-analysis
6
+ commit=b099fd84
7
+ num_layers=24
8
+ hidden=2048
9
+ ffn_hidden=10944
10
+ moe_ffn_hidden=1408
11
+ num_experts=64
12
+ topk=6
13
+ experts_per_rank=8
14
+ expert_params=8650752
15
+ heads=16
16
+ kv_groups=0
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=2816
20
+ shared_expert_overlap=0
21
+ mla=1
22
+ dense_first_layers=1
23
+ moe_layers=23
24
+ rotary_base=10000
25
+ rotary_scaling=40
26
+ qkv_bias=0
27
+ router_balancing=seq_aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=1
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=8192
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=49152
39
+ mean_tokens_per_expert=6144
40
+ slot_policy=matched
41
+ slots_per_rank=8
42
+ capacity_mode=static
43
+ capacity=1.0234375
44
+ cuda_graph=0
45
+ probe_mode=1
46
+ async_grad_reduce=0
47
+ branch_role=probe
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0234375/rep1/train.log ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [17:19:04] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe)
2
+ ==============================================================
3
+ DSV2_LITE_L24 / echo (slots/rank=8 policy=matched)
4
+ E=64 K=6 H'=1408 m=8
5
+ L=24 seq=8192 gbs=8 (1 microbatches)
6
+ cap_mode=static cap=1.0234375 cg=0 recompute=none
7
+ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0234375/rep1/memory_snapshots/snapshot.pickle
8
+ repo=/workspace/Megatron-MoE-Echo @ b099fd84
9
+ ==============================================================
10
+ W0826 17:19:06.243000 350624 torch/distributed/run.py:766]
11
+ W0826 17:19:06.243000 350624 torch/distributed/run.py:766] *****************************************
12
+ W0826 17:19:06.243000 350624 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
13
+ W0826 17:19:06.243000 350624 torch/distributed/run.py:766] *****************************************
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv CHANGED
@@ -14,7 +14,7 @@
14
  5, 0
15
  6, 0
16
  7, 0
17
- 0, 1
18
  1, 1
19
  2, 1
20
  3, 1
@@ -43,17 +43,17 @@
43
  2, 629
44
  3, 629
45
  4, 629
46
- 5, 629
47
- 6, 47
48
- 7, 47
49
- 0, 975
50
- 1, 959
51
- 2, 949
52
- 3, 979
53
- 4, 955
54
- 5, 951
55
- 6, 953
56
- 7, 957
57
  0, 1955
58
  1, 1955
59
  2, 1955
@@ -78,14 +78,14 @@
78
  5, 32075
79
  6, 32209
80
  7, 32229
81
- 0, 36899
82
- 1, 36983
83
- 2, 36833
84
- 3, 36831
85
- 4, 36831
86
- 5, 36829
87
- 6, 36965
88
- 7, 36983
89
  0, 37783
90
  1, 37867
91
  2, 37717
@@ -102,14 +102,14 @@
102
  5, 37801
103
  6, 37937
104
  7, 37955
105
- 0, 38131
106
- 1, 38211
107
- 2, 38063
108
- 3, 38063
109
- 4, 38037
110
- 5, 38047
111
- 6, 38197
112
- 7, 38203
113
  0, 38819
114
  1, 38903
115
  2, 38753
@@ -142,13 +142,13 @@
142
  5, 38759
143
  6, 38893
144
  7, 38913
145
- 0, 121341
146
- 1, 121633
147
- 2, 121465
148
- 3, 121469
149
- 4, 121457
150
- 5, 121539
151
- 6, 121593
152
  7, 121615
153
  0, 121857
154
  1, 122153
@@ -182,14 +182,14 @@
182
  5, 122063
183
  6, 122133
184
  7, 122153
185
- 0, 122541
186
- 1, 122837
187
- 2, 122667
188
- 3, 122667
189
- 4, 122667
190
- 5, 122747
191
- 6, 122817
192
- 7, 122837
193
  0, 123719
194
  1, 126063
195
  2, 125893
@@ -198,21 +198,21 @@
198
  5, 125973
199
  6, 126043
200
  7, 126063
201
- 0, 124305
202
- 1, 126599
203
- 2, 126513
204
- 3, 126471
205
- 4, 126481
206
- 5, 126525
207
- 6, 126613
208
- 7, 126617
209
  0, 139007
210
  1, 138657
211
  2, 138497
212
  3, 138493
213
  4, 138493
214
  5, 138567
215
- 6, 138707
216
  7, 140961
217
  0, 139007
218
  1, 138657
@@ -220,7 +220,7 @@
220
  3, 138493
221
  4, 138493
222
  5, 138567
223
- 6, 138707
224
  7, 140961
225
  0, 139007
226
  1, 138657
@@ -228,7 +228,7 @@
228
  3, 138493
229
  4, 138493
230
  5, 138567
231
- 6, 138707
232
  7, 140961
233
  0, 139007
234
  1, 138657
@@ -236,21 +236,21 @@
236
  3, 138493
237
  4, 138493
238
  5, 138567
239
- 6, 138707
240
  7, 140961
241
- 0, 138707
242
- 1, 138357
243
- 2, 138197
244
- 3, 138129
245
- 4, 137867
246
- 5, 137685
247
- 6, 138087
248
- 7, 140661
249
- 0, 775
250
  1, 778
251
  2, 778
252
  3, 778
253
  4, 778
254
- 5, 1034
255
- 6, 137759
256
- 7, 2058
 
14
  5, 0
15
  6, 0
16
  7, 0
17
+ 0, 2
18
  1, 1
19
  2, 1
20
  3, 1
 
43
  2, 629
44
  3, 629
45
  4, 629
46
+ 5, 165
47
+ 6, 629
48
+ 7, 629
49
+ 0, 895
50
+ 1, 903
51
+ 2, 909
52
+ 3, 895
53
+ 4, 893
54
+ 5, 897
55
+ 6, 893
56
+ 7, 911
57
  0, 1955
58
  1, 1955
59
  2, 1955
 
78
  5, 32075
79
  6, 32209
80
  7, 32229
81
+ 0, 36383
82
+ 1, 36469
83
+ 2, 36335
84
+ 3, 36331
85
+ 4, 36321
86
+ 5, 36305
87
+ 6, 36427
88
+ 7, 36471
89
  0, 37783
90
  1, 37867
91
  2, 37717
 
102
  5, 37801
103
  6, 37937
104
  7, 37955
105
+ 0, 38055
106
+ 1, 38143
107
+ 2, 37995
108
+ 3, 37977
109
+ 4, 37977
110
+ 5, 37963
111
+ 6, 38137
112
+ 7, 38137
113
  0, 38819
114
  1, 38903
115
  2, 38753
 
142
  5, 38759
143
  6, 38893
144
  7, 38913
145
+ 0, 121321
146
+ 1, 121611
147
+ 2, 121443
148
+ 3, 121449
149
+ 4, 121437
150
+ 5, 121529
151
+ 6, 121599
152
  7, 121615
153
  0, 121857
154
  1, 122153
 
182
  5, 122063
183
  6, 122133
184
  7, 122153
185
+ 0, 122469
186
+ 1, 122787
187
+ 2, 122655
188
+ 3, 122611
189
+ 4, 122611
190
+ 5, 122655
191
+ 6, 122755
192
+ 7, 122735
193
  0, 123719
194
  1, 126063
195
  2, 125893
 
198
  5, 125973
199
  6, 126043
200
  7, 126063
201
+ 0, 124319
202
+ 1, 126645
203
+ 2, 126519
204
+ 3, 126505
205
+ 4, 126489
206
+ 5, 126613
207
+ 6, 126663
208
+ 7, 126703
209
  0, 139007
210
  1, 138657
211
  2, 138497
212
  3, 138493
213
  4, 138493
214
  5, 138567
215
+ 6, 140755
216
  7, 140961
217
  0, 139007
218
  1, 138657
 
220
  3, 138493
221
  4, 138493
222
  5, 138567
223
+ 6, 140755
224
  7, 140961
225
  0, 139007
226
  1, 138657
 
228
  3, 138493
229
  4, 138493
230
  5, 138567
231
+ 6, 140755
232
  7, 140961
233
  0, 139007
234
  1, 138657
 
236
  3, 138493
237
  4, 138493
238
  5, 138567
239
+ 6, 140755
240
  7, 140961
241
+ 0, 139007
242
+ 1, 138657
243
+ 2, 138497
244
+ 3, 138493
245
+ 4, 138493
246
+ 5, 138567
247
+ 6, 140755
248
+ 7, 140961
249
+ 0, 2057
250
  1, 778
251
  2, 778
252
  3, 778
253
  4, 778
254
+ 5, 778
255
+ 6, 1034
256
+ 7, 140013
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log CHANGED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125/rep1/mem_samples.csv ADDED
@@ -0,0 +1,256 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 4
43
+ 2, 4
44
+ 3, 4
45
+ 4, 4
46
+ 5, 4
47
+ 6, 4
48
+ 7, 4
49
+ 0, 821
50
+ 1, 821
51
+ 2, 821
52
+ 3, 821
53
+ 4, 821
54
+ 5, 821
55
+ 6, 821
56
+ 7, 821
57
+ 0, 1975
58
+ 1, 1975
59
+ 2, 1975
60
+ 3, 1975
61
+ 4, 1975
62
+ 5, 1975
63
+ 6, 1975
64
+ 7, 1975
65
+ 0, 47803
66
+ 1, 47803
67
+ 2, 47803
68
+ 3, 47803
69
+ 4, 47515
70
+ 5, 47803
71
+ 6, 47803
72
+ 7, 47803
73
+ 0, 49279
74
+ 1, 49279
75
+ 2, 49279
76
+ 3, 49279
77
+ 4, 48991
78
+ 5, 49279
79
+ 6, 49279
80
+ 7, 49279
81
+ 0, 49929
82
+ 1, 49929
83
+ 2, 49929
84
+ 3, 49929
85
+ 4, 49641
86
+ 5, 49927
87
+ 6, 49929
88
+ 7, 49929
89
+ 0, 53697
90
+ 1, 53697
91
+ 2, 53697
92
+ 3, 53697
93
+ 4, 53409
94
+ 5, 53697
95
+ 6, 53697
96
+ 7, 53697
97
+ 0, 54105
98
+ 1, 54105
99
+ 2, 54105
100
+ 3, 54105
101
+ 4, 53817
102
+ 5, 54105
103
+ 6, 54105
104
+ 7, 54105
105
+ 0, 54555
106
+ 1, 54553
107
+ 2, 54555
108
+ 3, 54555
109
+ 4, 54267
110
+ 5, 54555
111
+ 6, 54555
112
+ 7, 54527
113
+ 0, 55543
114
+ 1, 55543
115
+ 2, 55543
116
+ 3, 55543
117
+ 4, 55255
118
+ 5, 55543
119
+ 6, 55543
120
+ 7, 55543
121
+ 0, 56071
122
+ 1, 56071
123
+ 2, 56071
124
+ 3, 56071
125
+ 4, 55783
126
+ 5, 56071
127
+ 6, 56071
128
+ 7, 56071
129
+ 0, 57013
130
+ 1, 57013
131
+ 2, 57013
132
+ 3, 57013
133
+ 4, 56725
134
+ 5, 57013
135
+ 6, 57013
136
+ 7, 57013
137
+ 0, 57655
138
+ 1, 57655
139
+ 2, 57655
140
+ 3, 57655
141
+ 4, 57367
142
+ 5, 57655
143
+ 6, 57655
144
+ 7, 57655
145
+ 0, 96515
146
+ 1, 96507
147
+ 2, 96507
148
+ 3, 96507
149
+ 4, 96531
150
+ 5, 96507
151
+ 6, 96507
152
+ 7, 96507
153
+ 0, 97161
154
+ 1, 97153
155
+ 2, 97153
156
+ 3, 97153
157
+ 4, 97177
158
+ 5, 97153
159
+ 6, 97153
160
+ 7, 97153
161
+ 0, 97163
162
+ 1, 97155
163
+ 2, 97155
164
+ 3, 97155
165
+ 4, 97179
166
+ 5, 97155
167
+ 6, 97155
168
+ 7, 97155
169
+ 0, 97163
170
+ 1, 97155
171
+ 2, 97155
172
+ 3, 97155
173
+ 4, 97179
174
+ 5, 97155
175
+ 6, 97155
176
+ 7, 97155
177
+ 0, 97163
178
+ 1, 97155
179
+ 2, 97155
180
+ 3, 97155
181
+ 4, 97179
182
+ 5, 97155
183
+ 6, 97155
184
+ 7, 97155
185
+ 0, 97615
186
+ 1, 97561
187
+ 2, 97597
188
+ 3, 97551
189
+ 4, 97567
190
+ 5, 97529
191
+ 6, 97535
192
+ 7, 97529
193
+ 0, 102247
194
+ 1, 101855
195
+ 2, 101855
196
+ 3, 101855
197
+ 4, 101879
198
+ 5, 102239
199
+ 6, 101855
200
+ 7, 102239
201
+ 0, 108141
202
+ 1, 107647
203
+ 2, 107639
204
+ 3, 107661
205
+ 4, 108051
206
+ 5, 108085
207
+ 6, 107609
208
+ 7, 108057
209
+ 0, 132791
210
+ 1, 132543
211
+ 2, 132543
212
+ 3, 132543
213
+ 4, 132579
214
+ 5, 132927
215
+ 6, 132543
216
+ 7, 132927
217
+ 0, 132791
218
+ 1, 132543
219
+ 2, 132543
220
+ 3, 132543
221
+ 4, 132579
222
+ 5, 132927
223
+ 6, 132543
224
+ 7, 132927
225
+ 0, 132791
226
+ 1, 132543
227
+ 2, 132543
228
+ 3, 132543
229
+ 4, 132559
230
+ 5, 132865
231
+ 6, 132449
232
+ 7, 132893
233
+ 0, 3366
234
+ 1, 3622
235
+ 2, 131295
236
+ 3, 131295
237
+ 4, 4646
238
+ 5, 3366
239
+ 6, 3366
240
+ 7, 3366
241
+ 0, 0
242
+ 1, 0
243
+ 2, 0
244
+ 3, 0
245
+ 4, 0
246
+ 5, 0
247
+ 6, 0
248
+ 7, 0
249
+ 0, 0
250
+ 1, 0
251
+ 2, 0
252
+ 3, 0
253
+ 4, 0
254
+ 5, 0
255
+ 6, 0
256
+ 7, 0
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=echo
2
+ config=QWEN3_235B
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=token-drop-analysis
6
+ commit=b099fd84
7
+ num_layers=12
8
+ hidden=4096
9
+ ffn_hidden=12288
10
+ moe_ffn_hidden=1536
11
+ num_experts=128
12
+ topk=8
13
+ experts_per_rank=16
14
+ expert_params=18874368
15
+ heads=64
16
+ kv_groups=4
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=0
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=12
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=0
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=0
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=8192
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=65536
39
+ mean_tokens_per_expert=4096
40
+ slot_policy=matched
41
+ slots_per_rank=16
42
+ capacity_mode=static
43
+ capacity=1.03125
44
+ cuda_graph=0
45
+ probe_mode=1
46
+ async_grad_reduce=0
47
+ branch_role=probe
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125__forcebal/rep1/mem_samples.csv ADDED
@@ -0,0 +1,232 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 4
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 821
42
+ 1, 819
43
+ 2, 821
44
+ 3, 819
45
+ 4, 819
46
+ 5, 821
47
+ 6, 819
48
+ 7, 819
49
+ 0, 1537
50
+ 1, 1537
51
+ 2, 1537
52
+ 3, 1537
53
+ 4, 1537
54
+ 5, 1537
55
+ 6, 1537
56
+ 7, 1537
57
+ 0, 2551
58
+ 1, 2551
59
+ 2, 2551
60
+ 3, 2551
61
+ 4, 2551
62
+ 5, 2551
63
+ 6, 2551
64
+ 7, 2551
65
+ 0, 48097
66
+ 1, 48085
67
+ 2, 48091
68
+ 3, 48087
69
+ 4, 47783
70
+ 5, 48119
71
+ 6, 48067
72
+ 7, 48125
73
+ 0, 49643
74
+ 1, 49653
75
+ 2, 49703
76
+ 3, 49683
77
+ 4, 49425
78
+ 5, 49713
79
+ 6, 49703
80
+ 7, 49693
81
+ 0, 53697
82
+ 1, 53697
83
+ 2, 53697
84
+ 3, 53697
85
+ 4, 53409
86
+ 5, 53697
87
+ 6, 53697
88
+ 7, 53697
89
+ 0, 54105
90
+ 1, 54105
91
+ 2, 54105
92
+ 3, 54105
93
+ 4, 53817
94
+ 5, 54105
95
+ 6, 54105
96
+ 7, 54105
97
+ 0, 54457
98
+ 1, 54477
99
+ 2, 54459
100
+ 3, 54459
101
+ 4, 54157
102
+ 5, 54431
103
+ 6, 54451
104
+ 7, 54453
105
+ 0, 55543
106
+ 1, 55543
107
+ 2, 55543
108
+ 3, 55543
109
+ 4, 55255
110
+ 5, 55543
111
+ 6, 55543
112
+ 7, 55543
113
+ 0, 56071
114
+ 1, 56071
115
+ 2, 56071
116
+ 3, 56071
117
+ 4, 55783
118
+ 5, 56071
119
+ 6, 56071
120
+ 7, 56071
121
+ 0, 57013
122
+ 1, 57013
123
+ 2, 57013
124
+ 3, 57013
125
+ 4, 56725
126
+ 5, 57013
127
+ 6, 57013
128
+ 7, 57013
129
+ 0, 57655
130
+ 1, 57655
131
+ 2, 57655
132
+ 3, 57655
133
+ 4, 57367
134
+ 5, 57655
135
+ 6, 57655
136
+ 7, 57655
137
+ 0, 96665
138
+ 1, 96677
139
+ 2, 96663
140
+ 3, 96673
141
+ 4, 96683
142
+ 5, 96663
143
+ 6, 96637
144
+ 7, 96643
145
+ 0, 97161
146
+ 1, 97157
147
+ 2, 97157
148
+ 3, 97157
149
+ 4, 97177
150
+ 5, 97157
151
+ 6, 97157
152
+ 7, 97157
153
+ 0, 97163
154
+ 1, 97159
155
+ 2, 97159
156
+ 3, 97159
157
+ 4, 97179
158
+ 5, 97159
159
+ 6, 97159
160
+ 7, 97159
161
+ 0, 97163
162
+ 1, 97159
163
+ 2, 97159
164
+ 3, 97159
165
+ 4, 97179
166
+ 5, 97159
167
+ 6, 97159
168
+ 7, 97159
169
+ 0, 97163
170
+ 1, 97159
171
+ 2, 97159
172
+ 3, 97159
173
+ 4, 97179
174
+ 5, 97159
175
+ 6, 97159
176
+ 7, 97159
177
+ 0, 98527
178
+ 1, 98523
179
+ 2, 98509
180
+ 3, 98523
181
+ 4, 98543
182
+ 5, 98509
183
+ 6, 98509
184
+ 7, 98523
185
+ 0, 102247
186
+ 1, 101859
187
+ 2, 102243
188
+ 3, 101859
189
+ 4, 102263
190
+ 5, 102243
191
+ 6, 101859
192
+ 7, 101859
193
+ 0, 108151
194
+ 1, 107703
195
+ 2, 108077
196
+ 3, 107737
197
+ 4, 108237
198
+ 5, 108049
199
+ 6, 107681
200
+ 7, 107659
201
+ 0, 132791
202
+ 1, 132535
203
+ 2, 132919
204
+ 3, 132535
205
+ 4, 132939
206
+ 5, 132919
207
+ 6, 132535
208
+ 7, 132535
209
+ 0, 132791
210
+ 1, 132535
211
+ 2, 132919
212
+ 3, 132535
213
+ 4, 132939
214
+ 5, 132919
215
+ 6, 132535
216
+ 7, 132535
217
+ 0, 132789
218
+ 1, 5886
219
+ 2, 132917
220
+ 3, 132533
221
+ 4, 132937
222
+ 5, 132917
223
+ 6, 132533
224
+ 7, 132533
225
+ 0, 3963
226
+ 1, 3965
227
+ 2, 3966
228
+ 3, 3966
229
+ 4, 3966
230
+ 5, 4286
231
+ 6, 132533
232
+ 7, 5566
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125__forcebal/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=echo
2
+ config=QWEN3_235B
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=token-drop-analysis
6
+ commit=b099fd84
7
+ num_layers=12
8
+ hidden=4096
9
+ ffn_hidden=12288
10
+ moe_ffn_hidden=1536
11
+ num_experts=128
12
+ topk=8
13
+ experts_per_rank=16
14
+ expert_params=18874368
15
+ heads=64
16
+ kv_groups=4
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=0
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=12
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=0
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=0
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=8192
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=65536
39
+ mean_tokens_per_expert=4096
40
+ slot_policy=matched
41
+ slots_per_rank=16
42
+ capacity_mode=static
43
+ capacity=1.03125
44
+ cuda_graph=0
45
+ probe_mode=1
46
+ async_grad_reduce=0
47
+ branch_role=probe
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=1
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125__forcebal/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv CHANGED
@@ -14,7 +14,7 @@
14
  5, 0
15
  6, 0
16
  7, 0
17
- 0, 2
18
  1, 1
19
  2, 1
20
  3, 1
@@ -42,7 +42,7 @@
42
  1, 4
43
  2, 4
44
  3, 4
45
- 4, 4
46
  5, 4
47
  6, 4
48
  7, 4
@@ -62,14 +62,14 @@
62
  5, 1975
63
  6, 1975
64
  7, 1975
65
- 0, 39733
66
- 1, 39733
67
- 2, 39733
68
- 3, 39733
69
- 4, 39733
70
- 5, 39733
71
- 6, 39733
72
- 7, 39733
73
  0, 48319
74
  1, 48319
75
  2, 48319
@@ -85,7 +85,7 @@
85
  4, 49639
86
  5, 49927
87
  6, 49927
88
- 7, 49927
89
  0, 53697
90
  1, 53697
91
  2, 53697
@@ -102,14 +102,14 @@
102
  5, 54105
103
  6, 54105
104
  7, 54105
105
- 0, 54427
106
- 1, 54421
107
- 2, 54417
108
- 3, 54419
109
- 4, 54127
110
- 5, 54407
111
- 6, 54425
112
- 7, 54411
113
  0, 55543
114
  1, 55543
115
  2, 55543
@@ -142,14 +142,14 @@
142
  5, 59799
143
  6, 59799
144
  7, 59799
145
- 0, 105919
146
- 1, 105959
147
- 2, 105953
148
- 3, 105959
149
- 4, 105901
150
- 5, 105957
151
- 6, 105945
152
- 7, 105925
153
  0, 106307
154
  1, 106347
155
  2, 106347
@@ -182,62 +182,62 @@
182
  5, 106349
183
  6, 106349
184
  7, 106349
185
- 0, 110361
186
- 1, 110785
187
- 2, 110401
188
- 3, 110401
189
  4, 110381
190
  5, 110401
191
  6, 110401
192
- 7, 110785
193
  0, 111009
194
- 1, 111433
195
  2, 111049
196
  3, 111049
197
  4, 111029
198
- 5, 111049
199
- 6, 111049
200
- 7, 111433
201
- 0, 111627
202
- 1, 112083
203
- 2, 111681
204
- 3, 111713
205
- 4, 111739
206
- 5, 111723
207
- 6, 111697
208
- 7, 112065
209
  0, 141439
210
- 1, 141943
211
  2, 141559
212
  3, 141559
213
  4, 141539
214
- 5, 141559
215
- 6, 141559
216
- 7, 141943
217
  0, 141439
218
- 1, 141943
219
  2, 141559
220
  3, 141559
221
  4, 141539
222
- 5, 141559
223
- 6, 141559
224
- 7, 141943
225
  0, 141439
226
- 1, 141943
227
  2, 141559
228
  3, 141559
229
  4, 141539
230
- 5, 141559
231
- 6, 141559
232
- 7, 141943
233
  0, 140191
234
- 1, 140695
235
  2, 140311
236
  3, 140311
237
- 4, 140291
238
- 5, 140311
239
- 6, 4902
240
- 7, 140695
241
  0, 4643
242
  1, 3366
243
  2, 3366
@@ -245,7 +245,7 @@
245
  4, 3366
246
  5, 3366
247
  6, 3622
248
- 7, 140695
249
  0, 0
250
  1, 0
251
  2, 0
 
14
  5, 0
15
  6, 0
16
  7, 0
17
+ 0, 1
18
  1, 1
19
  2, 1
20
  3, 1
 
42
  1, 4
43
  2, 4
44
  3, 4
45
+ 4, 45
46
  5, 4
47
  6, 4
48
  7, 4
 
62
  5, 1975
63
  6, 1975
64
  7, 1975
65
+ 0, 45207
66
+ 1, 45495
67
+ 2, 45111
68
+ 3, 45159
69
+ 4, 44823
70
+ 5, 45063
71
+ 6, 44775
72
+ 7, 45063
73
  0, 48319
74
  1, 48319
75
  2, 48319
 
85
  4, 49639
86
  5, 49927
87
  6, 49927
88
+ 7, 49929
89
  0, 53697
90
  1, 53697
91
  2, 53697
 
102
  5, 54105
103
  6, 54105
104
  7, 54105
105
+ 0, 54439
106
+ 1, 54415
107
+ 2, 54433
108
+ 3, 54427
109
+ 4, 54123
110
+ 5, 54447
111
+ 6, 54409
112
+ 7, 54417
113
  0, 55543
114
  1, 55543
115
  2, 55543
 
142
  5, 59799
143
  6, 59799
144
  7, 59799
145
+ 0, 105853
146
+ 1, 105915
147
+ 2, 105931
148
+ 3, 105909
149
+ 4, 105833
150
+ 5, 105867
151
+ 6, 105931
152
+ 7, 105889
153
  0, 106307
154
  1, 106347
155
  2, 106347
 
182
  5, 106349
183
  6, 106349
184
  7, 106349
185
+ 0, 107673
186
+ 1, 110401
187
+ 2, 108481
188
+ 3, 108481
189
  4, 110381
190
  5, 110401
191
  6, 110401
192
+ 7, 108481
193
  0, 111009
194
+ 1, 111049
195
  2, 111049
196
  3, 111049
197
  4, 111029
198
+ 5, 111433
199
+ 6, 111433
200
+ 7, 111049
201
+ 0, 111665
202
+ 1, 111671
203
+ 2, 111699
204
+ 3, 111667
205
+ 4, 111663
206
+ 5, 112029
207
+ 6, 112079
208
+ 7, 111679
209
  0, 141439
210
+ 1, 141559
211
  2, 141559
212
  3, 141559
213
  4, 141539
214
+ 5, 141943
215
+ 6, 141943
216
+ 7, 141559
217
  0, 141439
218
+ 1, 141559
219
  2, 141559
220
  3, 141559
221
  4, 141539
222
+ 5, 141943
223
+ 6, 141943
224
+ 7, 141559
225
  0, 141439
226
+ 1, 141559
227
  2, 141559
228
  3, 141559
229
  4, 141539
230
+ 5, 141943
231
+ 6, 141943
232
+ 7, 141559
233
  0, 140191
234
+ 1, 140311
235
  2, 140311
236
  3, 140311
237
+ 4, 4902
238
+ 5, 140695
239
+ 6, 140695
240
+ 7, 140311
241
  0, 4643
242
  1, 3366
243
  2, 3366
 
245
  4, 3366
246
  5, 3366
247
  6, 3622
248
+ 7, 140311
249
  0, 0
250
  1, 0
251
  2, 0
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log CHANGED
The diff for this file is too large to render. See raw diff
 
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p05859375/rep1/mem_samples.csv ADDED
@@ -0,0 +1,272 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 41
43
+ 2, 4
44
+ 3, 4
45
+ 4, 4
46
+ 5, 4
47
+ 6, 4
48
+ 7, 4
49
+ 0, 821
50
+ 1, 821
51
+ 2, 821
52
+ 3, 821
53
+ 4, 821
54
+ 5, 821
55
+ 6, 821
56
+ 7, 821
57
+ 0, 1831
58
+ 1, 1831
59
+ 2, 1831
60
+ 3, 1831
61
+ 4, 1831
62
+ 5, 1831
63
+ 6, 1831
64
+ 7, 1831
65
+ 0, 39169
66
+ 1, 38995
67
+ 2, 39197
68
+ 3, 38995
69
+ 4, 38995
70
+ 5, 39197
71
+ 6, 38995
72
+ 7, 39197
73
+ 0, 44847
74
+ 1, 44869
75
+ 2, 44851
76
+ 3, 44811
77
+ 4, 44843
78
+ 5, 44869
79
+ 6, 44841
80
+ 7, 44893
81
+ 0, 45895
82
+ 1, 45897
83
+ 2, 45907
84
+ 3, 45869
85
+ 4, 45893
86
+ 5, 45907
87
+ 6, 45897
88
+ 7, 45907
89
+ 0, 46917
90
+ 1, 46919
91
+ 2, 46929
92
+ 3, 46891
93
+ 4, 46915
94
+ 5, 46929
95
+ 6, 46919
96
+ 7, 46929
97
+ 0, 47013
98
+ 1, 47015
99
+ 2, 47025
100
+ 3, 46987
101
+ 4, 47011
102
+ 5, 47025
103
+ 6, 47015
104
+ 7, 47025
105
+ 0, 47317
106
+ 1, 47311
107
+ 2, 47329
108
+ 3, 47291
109
+ 4, 47315
110
+ 5, 47311
111
+ 6, 47319
112
+ 7, 47329
113
+ 0, 47877
114
+ 1, 47879
115
+ 2, 47889
116
+ 3, 47851
117
+ 4, 47875
118
+ 5, 47889
119
+ 6, 47879
120
+ 7, 47889
121
+ 0, 48013
122
+ 1, 48015
123
+ 2, 48025
124
+ 3, 47987
125
+ 4, 48011
126
+ 5, 48025
127
+ 6, 48015
128
+ 7, 48025
129
+ 0, 48451
130
+ 1, 48449
131
+ 2, 48463
132
+ 3, 48421
133
+ 4, 48445
134
+ 5, 48463
135
+ 6, 48409
136
+ 7, 48463
137
+ 0, 48719
138
+ 1, 48717
139
+ 2, 48727
140
+ 3, 48689
141
+ 4, 48713
142
+ 5, 48727
143
+ 6, 48677
144
+ 7, 48727
145
+ 0, 77033
146
+ 1, 77035
147
+ 2, 77035
148
+ 3, 77055
149
+ 4, 77059
150
+ 5, 77035
151
+ 6, 77031
152
+ 7, 77035
153
+ 0, 92599
154
+ 1, 92601
155
+ 2, 92601
156
+ 3, 92643
157
+ 4, 92647
158
+ 5, 92643
159
+ 6, 92645
160
+ 7, 92611
161
+ 0, 92703
162
+ 1, 92705
163
+ 2, 92705
164
+ 3, 92705
165
+ 4, 92709
166
+ 5, 92705
167
+ 6, 92701
168
+ 7, 92705
169
+ 0, 92705
170
+ 1, 92707
171
+ 2, 92707
172
+ 3, 92707
173
+ 4, 92711
174
+ 5, 92707
175
+ 6, 92703
176
+ 7, 92707
177
+ 0, 92705
178
+ 1, 92707
179
+ 2, 92707
180
+ 3, 92707
181
+ 4, 92711
182
+ 5, 92707
183
+ 6, 92703
184
+ 7, 92707
185
+ 0, 92705
186
+ 1, 92707
187
+ 2, 92707
188
+ 3, 92707
189
+ 4, 92711
190
+ 5, 92707
191
+ 6, 92703
192
+ 7, 92707
193
+ 0, 93353
194
+ 1, 93353
195
+ 2, 93357
196
+ 3, 93395
197
+ 4, 93371
198
+ 5, 93331
199
+ 6, 93337
200
+ 7, 93335
201
+ 0, 94065
202
+ 1, 94067
203
+ 2, 94259
204
+ 3, 94835
205
+ 4, 94263
206
+ 5, 94259
207
+ 6, 94255
208
+ 7, 94163
209
+ 0, 94705
210
+ 1, 94729
211
+ 2, 94883
212
+ 3, 95463
213
+ 4, 94901
214
+ 5, 94881
215
+ 6, 94899
216
+ 7, 94821
217
+ 0, 124273
218
+ 1, 124261
219
+ 2, 124477
220
+ 3, 125063
221
+ 4, 124455
222
+ 5, 124477
223
+ 6, 124443
224
+ 7, 124381
225
+ 0, 124273
226
+ 1, 124261
227
+ 2, 124477
228
+ 3, 125063
229
+ 4, 124455
230
+ 5, 124477
231
+ 6, 124443
232
+ 7, 124381
233
+ 0, 124273
234
+ 1, 124261
235
+ 2, 124477
236
+ 3, 125063
237
+ 4, 124455
238
+ 5, 124477
239
+ 6, 124443
240
+ 7, 124381
241
+ 0, 124273
242
+ 1, 124261
243
+ 2, 124477
244
+ 3, 125063
245
+ 4, 124455
246
+ 5, 124477
247
+ 6, 124443
248
+ 7, 124381
249
+ 0, 124273
250
+ 1, 124261
251
+ 2, 124477
252
+ 3, 125063
253
+ 4, 124455
254
+ 5, 124477
255
+ 6, 124443
256
+ 7, 124381
257
+ 0, 124273
258
+ 1, 124261
259
+ 2, 124477
260
+ 3, 125063
261
+ 4, 124455
262
+ 5, 124477
263
+ 6, 124443
264
+ 7, 124381
265
+ 0, 906
266
+ 1, 906
267
+ 2, 906
268
+ 3, 906
269
+ 4, 1226
270
+ 5, 124307
271
+ 6, 2506
272
+ 7, 906
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p05859375/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=echo
2
+ config=QWEN3_30B
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=token-drop-analysis
6
+ commit=b099fd84
7
+ num_layers=48
8
+ hidden=2048
9
+ ffn_hidden=6144
10
+ moe_ffn_hidden=768
11
+ num_experts=128
12
+ topk=8
13
+ experts_per_rank=16
14
+ expert_params=4718592
15
+ heads=32
16
+ kv_groups=4
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=0
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=48
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=0
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=0
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=4096
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=32768
39
+ mean_tokens_per_expert=2048
40
+ slot_policy=matched
41
+ slots_per_rank=16
42
+ capacity_mode=static
43
+ capacity=1.05859375
44
+ cuda_graph=0
45
+ probe_mode=1
46
+ async_grad_reduce=0
47
+ branch_role=probe
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p05859375/rep1/train.log ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:13812f8ee67386cd1c544bc6e126525ff567e1ac8bb6a759965cd7145a4f0d1a
3
+ size 13017463
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/mem_samples.csv CHANGED
@@ -14,7 +14,7 @@
14
  5, 0
15
  6, 0
16
  7, 0
17
- 0, 2
18
  1, 1
19
  2, 1
20
  3, 1
@@ -40,52 +40,52 @@
40
  7, 4
41
  0, 4
42
  1, 4
43
- 2, 27
44
  3, 4
45
  4, 4
46
  5, 4
47
- 6, 41
48
- 7, 629
49
- 0, 895
50
- 1, 903
51
- 2, 917
52
- 3, 929
53
- 4, 925
54
- 5, 899
55
- 6, 915
56
- 7, 913
57
- 0, 1831
58
- 1, 1831
59
- 2, 1831
60
- 3, 1831
61
- 4, 1831
62
- 5, 1831
63
- 6, 1831
64
- 7, 1831
65
- 0, 39169
66
- 1, 38995
67
- 2, 39197
68
- 3, 38995
69
- 4, 38995
70
- 5, 39197
71
- 6, 38995
72
- 7, 39197
73
- 0, 44847
74
- 1, 44875
75
- 2, 44877
76
- 3, 44859
77
- 4, 44821
78
- 5, 44837
79
- 6, 44841
80
- 7, 44875
81
- 0, 45791
82
- 1, 45839
83
- 2, 45845
84
- 3, 45775
85
- 4, 45789
86
- 5, 45793
87
- 6, 45835
88
- 7, 45855
89
  0, 46917
90
  1, 46919
91
  2, 46929
@@ -102,14 +102,14 @@
102
  5, 47025
103
  6, 47015
104
  7, 47025
105
- 0, 47191
106
- 1, 47219
107
- 2, 47209
108
- 3, 47163
109
- 4, 47213
110
- 5, 47213
111
- 6, 47191
112
- 7, 47211
113
  0, 47877
114
  1, 47879
115
  2, 47889
@@ -142,14 +142,14 @@
142
  5, 49145
143
  6, 49117
144
  7, 49145
145
- 0, 100699
146
- 1, 100655
147
- 2, 100703
148
  3, 100647
149
- 4, 100715
150
  5, 100639
151
  6, 100639
152
- 7, 100703
153
  0, 101347
154
  1, 101369
155
  2, 101353
@@ -182,91 +182,83 @@
182
  5, 101355
183
  6, 101355
184
  7, 101355
185
- 0, 101957
186
- 1, 101953
187
- 2, 101945
188
- 3, 101963
189
- 4, 101903
190
- 5, 101965
191
- 6, 101923
192
- 7, 101951
193
- 0, 102901
194
- 1, 102731
195
- 2, 102907
196
- 3, 102723
197
- 4, 102727
198
- 5, 102907
199
- 6, 102811
200
- 7, 102907
201
- 0, 103543
202
- 1, 103383
203
- 2, 103557
204
- 3, 103389
205
- 4, 103383
206
- 5, 103559
207
- 6, 103459
208
- 7, 103559
209
- 0, 132949
210
- 1, 132765
211
- 2, 132963
212
- 3, 132759
213
- 4, 132801
214
- 5, 132963
215
- 6, 132847
216
- 7, 132963
217
- 0, 132949
218
- 1, 132765
219
- 2, 132963
220
- 3, 132759
221
- 4, 132801
222
- 5, 132963
223
- 6, 132847
224
- 7, 132963
225
- 0, 132949
226
- 1, 132765
227
- 2, 132963
228
- 3, 132759
229
- 4, 132801
230
- 5, 132963
231
- 6, 132847
232
- 7, 132963
233
- 0, 132949
234
- 1, 132765
235
- 2, 132963
236
- 3, 132759
237
- 4, 132801
238
- 5, 132963
239
- 6, 132847
240
- 7, 132963
241
- 0, 132949
242
- 1, 132765
243
- 2, 132963
244
- 3, 132759
245
- 4, 132801
246
- 5, 132963
247
- 6, 132847
248
- 7, 132963
249
- 0, 132949
250
- 1, 132765
251
- 2, 132963
252
- 3, 132759
253
- 4, 132801
254
- 5, 132963
255
- 6, 132847
256
- 7, 132963
257
- 0, 132133
258
- 1, 131949
259
- 2, 2442
260
- 3, 131943
261
- 4, 2442
262
- 5, 132147
263
- 6, 132031
264
- 7, 132147
265
- 0, 132130
266
- 1, 2183
267
- 2, 903
268
- 3, 903
269
- 4, 903
270
- 5, 903
271
- 6, 905
272
- 7, 1162
 
14
  5, 0
15
  6, 0
16
  7, 0
17
+ 0, 4
18
  1, 1
19
  2, 1
20
  3, 1
 
40
  7, 4
41
  0, 4
42
  1, 4
43
+ 2, 4
44
  3, 4
45
  4, 4
46
  5, 4
47
+ 6, 4
48
+ 7, 4
49
+ 0, 821
50
+ 1, 821
51
+ 2, 821
52
+ 3, 821
53
+ 4, 821
54
+ 5, 821
55
+ 6, 821
56
+ 7, 821
57
+ 0, 1537
58
+ 1, 1537
59
+ 2, 1537
60
+ 3, 1537
61
+ 4, 1537
62
+ 5, 1537
63
+ 6, 1537
64
+ 7, 1537
65
+ 0, 1975
66
+ 1, 1975
67
+ 2, 1975
68
+ 3, 1975
69
+ 4, 1975
70
+ 5, 1975
71
+ 6, 1975
72
+ 7, 1975
73
+ 0, 44783
74
+ 1, 44819
75
+ 2, 44805
76
+ 3, 44799
77
+ 4, 44801
78
+ 5, 44813
79
+ 6, 44775
80
+ 7, 44799
81
+ 0, 45561
82
+ 1, 45563
83
+ 2, 45553
84
+ 3, 45535
85
+ 4, 45549
86
+ 5, 45583
87
+ 6, 45573
88
+ 7, 45593
89
  0, 46917
90
  1, 46919
91
  2, 46929
 
102
  5, 47025
103
  6, 47015
104
  7, 47025
105
+ 0, 47217
106
+ 1, 47173
107
+ 2, 47195
108
+ 3, 47167
109
+ 4, 47201
110
+ 5, 47201
111
+ 6, 47185
112
+ 7, 47183
113
  0, 47877
114
  1, 47879
115
  2, 47889
 
142
  5, 49145
143
  6, 49117
144
  7, 49145
145
+ 0, 100697
146
+ 1, 100719
147
+ 2, 100639
148
  3, 100647
149
+ 4, 100651
150
  5, 100639
151
  6, 100639
152
+ 7, 100639
153
  0, 101347
154
  1, 101369
155
  2, 101353
 
182
  5, 101355
183
  6, 101355
184
  7, 101355
185
+ 0, 101891
186
+ 1, 101973
187
+ 2, 101989
188
+ 3, 101947
189
+ 4, 101965
190
+ 5, 101925
191
+ 6, 101951
192
+ 7, 101955
193
+ 0, 102709
194
+ 1, 102923
195
+ 2, 103483
196
+ 3, 103491
197
+ 4, 102919
198
+ 5, 102811
199
+ 6, 102715
200
+ 7, 102715
201
+ 0, 103339
202
+ 1, 103489
203
+ 2, 104065
204
+ 3, 104051
205
+ 4, 103483
206
+ 5, 103405
207
+ 6, 103283
208
+ 7, 103271
209
+ 0, 132757
210
+ 1, 132957
211
+ 2, 133539
212
+ 3, 133527
213
+ 4, 132993
214
+ 5, 132867
215
+ 6, 132751
216
+ 7, 132771
217
+ 0, 132757
218
+ 1, 132957
219
+ 2, 133539
220
+ 3, 133527
221
+ 4, 132993
222
+ 5, 132867
223
+ 6, 132751
224
+ 7, 132771
225
+ 0, 132757
226
+ 1, 132957
227
+ 2, 133539
228
+ 3, 133527
229
+ 4, 132993
230
+ 5, 132867
231
+ 6, 132751
232
+ 7, 132771
233
+ 0, 132757
234
+ 1, 132957
235
+ 2, 133539
236
+ 3, 133527
237
+ 4, 132993
238
+ 5, 132867
239
+ 6, 132751
240
+ 7, 132771
241
+ 0, 132757
242
+ 1, 132957
243
+ 2, 133539
244
+ 3, 133527
245
+ 4, 132993
246
+ 5, 132867
247
+ 6, 132751
248
+ 7, 132771
249
+ 0, 132587
250
+ 1, 132787
251
+ 2, 133369
252
+ 3, 133357
253
+ 4, 132823
254
+ 5, 132697
255
+ 6, 132581
256
+ 7, 132601
257
+ 0, 1162
258
+ 1, 132141
259
+ 2, 2186
260
+ 3, 906
261
+ 4, 906
262
+ 5, 906
263
+ 6, 906
264
+ 7, 906
 
 
 
 
 
 
 
 
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/train.log CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:01c7900ca48884e0bc2e1036053aa7a996f875bd01f2f8370ebc63610dc4f90b
3
- size 13023349
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b3cc59379cd8890580a7c6d403bc7e87dbf0a1ff3a78a68cd8dc0e79d3aae086
3
+ size 13020062
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/mem_samples.csv CHANGED
@@ -25,7 +25,7 @@
25
  0, 4
26
  1, 4
27
  2, 4
28
- 3, 2
29
  4, 1
30
  5, 1
31
  6, 1
@@ -38,38 +38,38 @@
38
  5, 4
39
  6, 4
40
  7, 4
41
- 0, 819
42
- 1, 819
43
- 2, 807
44
- 3, 801
45
- 4, 803
46
- 5, 819
47
- 6, 799
48
- 7, 799
49
- 0, 1537
50
- 1, 1537
51
- 2, 1537
52
- 3, 1537
53
- 4, 1537
54
- 5, 1537
55
- 6, 1537
56
- 7, 1537
57
- 0, 1975
58
- 1, 1975
59
- 2, 1975
60
- 3, 1975
61
- 4, 1975
62
- 5, 1975
63
- 6, 1975
64
- 7, 1975
65
- 0, 44533
66
- 1, 44535
67
- 2, 44545
68
- 3, 44507
69
- 4, 44531
70
- 5, 44545
71
- 6, 44535
72
- 7, 44545
73
  0, 45247
74
  1, 45249
75
  2, 45259
@@ -79,10 +79,10 @@
79
  6, 45249
80
  7, 45259
81
  0, 45895
82
- 1, 45899
83
  2, 45907
84
  3, 45869
85
- 4, 45895
86
  5, 45907
87
  6, 45897
88
  7, 45907
@@ -102,14 +102,14 @@
102
  5, 47025
103
  6, 47015
104
  7, 47025
105
- 0, 47271
106
- 1, 47265
107
- 2, 47315
108
- 3, 47275
109
- 4, 47269
110
- 5, 47293
111
- 6, 47275
112
- 7, 47291
113
  0, 47877
114
  1, 47879
115
  2, 47889
@@ -182,86 +182,86 @@
182
  5, 101355
183
  6, 101355
184
  7, 101355
185
- 0, 102637
186
- 1, 102659
187
- 2, 102067
188
- 3, 102651
189
- 4, 102079
190
- 5, 102067
191
- 6, 102067
192
- 7, 102067
193
  0, 103477
194
- 1, 103499
195
- 2, 102715
196
  3, 103491
197
- 4, 102727
198
- 5, 102715
199
- 6, 102907
200
- 7, 102715
201
- 0, 104099
202
- 1, 104145
203
- 2, 103361
204
  3, 104141
205
- 4, 103387
206
- 5, 103329
207
- 6, 103539
208
- 7, 103349
 
 
 
 
 
 
 
 
209
  0, 133525
210
- 1, 133533
211
- 2, 132771
212
  3, 133527
213
- 4, 132801
214
- 5, 132771
215
- 6, 132943
216
- 7, 132771
217
  0, 133525
218
- 1, 133533
219
- 2, 132771
220
  3, 133527
221
- 4, 132801
222
- 5, 132771
223
- 6, 132943
224
- 7, 132771
225
  0, 133525
226
- 1, 133533
227
- 2, 132771
228
  3, 133527
229
- 4, 132801
230
- 5, 132771
231
- 6, 132943
232
- 7, 132771
233
  0, 133525
234
- 1, 133533
235
- 2, 132771
236
  3, 133527
237
- 4, 132801
238
- 5, 132771
239
- 6, 132943
240
- 7, 132771
241
  0, 133525
242
- 1, 133533
243
- 2, 132771
244
  3, 133527
245
- 4, 132801
246
- 5, 132771
247
- 6, 132943
248
- 7, 132771
249
- 0, 133291
250
- 1, 133101
251
- 2, 132601
252
- 3, 133095
253
- 4, 132631
254
- 5, 132601
255
- 6, 132773
256
- 7, 132601
257
  0, 132709
258
- 1, 2442
259
- 2, 131955
260
- 3, 2442
261
- 4, 131985
262
- 5, 131955
263
- 6, 132127
264
- 7, 131955
265
  0, 0
266
  1, 0
267
  2, 0
 
25
  0, 4
26
  1, 4
27
  2, 4
28
+ 3, 4
29
  4, 1
30
  5, 1
31
  6, 1
 
38
  5, 4
39
  6, 4
40
  7, 4
41
+ 0, 4
42
+ 1, 4
43
+ 2, 4
44
+ 3, 629
45
+ 4, 4
46
+ 5, 4
47
+ 6, 4
48
+ 7, 4
49
+ 0, 821
50
+ 1, 821
51
+ 2, 821
52
+ 3, 821
53
+ 4, 821
54
+ 5, 821
55
+ 6, 821
56
+ 7, 821
57
+ 0, 1831
58
+ 1, 1831
59
+ 2, 1831
60
+ 3, 1831
61
+ 4, 1831
62
+ 5, 1831
63
+ 6, 1831
64
+ 7, 1831
65
+ 0, 39169
66
+ 1, 38995
67
+ 2, 39197
68
+ 3, 38995
69
+ 4, 38995
70
+ 5, 39197
71
+ 6, 38995
72
+ 7, 39197
73
  0, 45247
74
  1, 45249
75
  2, 45259
 
79
  6, 45249
80
  7, 45259
81
  0, 45895
82
+ 1, 45897
83
  2, 45907
84
  3, 45869
85
+ 4, 45893
86
  5, 45907
87
  6, 45897
88
  7, 45907
 
102
  5, 47025
103
  6, 47015
104
  7, 47025
105
+ 0, 47243
106
+ 1, 47263
107
+ 2, 47227
108
+ 3, 47201
109
+ 4, 47221
110
+ 5, 47243
111
+ 6, 47241
112
+ 7, 47235
113
  0, 47877
114
  1, 47879
115
  2, 47889
 
182
  5, 101355
183
  6, 101355
184
  7, 101355
185
+ 0, 101965
186
+ 1, 101987
187
+ 2, 101995
188
+ 3, 102033
189
+ 4, 102013
190
+ 5, 101961
191
+ 6, 101937
192
+ 7, 101977
193
  0, 103477
194
+ 1, 102731
195
+ 2, 103483
196
  3, 103491
197
+ 4, 103015
198
+ 5, 103483
199
+ 6, 103291
200
+ 7, 103291
201
+ 0, 104139
202
+ 1, 103385
203
+ 2, 104175
204
  3, 104141
205
+ 4, 103665
206
+ 5, 104111
207
+ 6, 103931
208
+ 7, 103909
209
+ 0, 133525
210
+ 1, 132765
211
+ 2, 133539
212
+ 3, 133527
213
+ 4, 133089
214
+ 5, 133539
215
+ 6, 133327
216
+ 7, 133347
217
  0, 133525
218
+ 1, 132765
219
+ 2, 133539
220
  3, 133527
221
+ 4, 133089
222
+ 5, 133539
223
+ 6, 133327
224
+ 7, 133347
225
  0, 133525
226
+ 1, 132765
227
+ 2, 133539
228
  3, 133527
229
+ 4, 133089
230
+ 5, 133539
231
+ 6, 133327
232
+ 7, 133347
233
  0, 133525
234
+ 1, 132765
235
+ 2, 133539
236
  3, 133527
237
+ 4, 133089
238
+ 5, 133539
239
+ 6, 133327
240
+ 7, 133347
241
  0, 133525
242
+ 1, 132765
243
+ 2, 133539
244
  3, 133527
245
+ 4, 133089
246
+ 5, 133539
247
+ 6, 133327
248
+ 7, 133347
249
  0, 133525
250
+ 1, 132765
251
+ 2, 133539
252
  3, 133527
253
+ 4, 133089
254
+ 5, 133539
255
+ 6, 133327
256
+ 7, 133347
 
 
 
 
 
 
 
 
257
  0, 132709
258
+ 1, 131949
259
+ 2, 2442
260
+ 3, 132711
261
+ 4, 2442
262
+ 5, 132723
263
+ 6, 132511
264
+ 7, 132531
265
  0, 0
266
  1, 0
267
  2, 0
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0f3db7afc14e8cc4e9ef89fe36136bd5e3f294d0051eea03a82ef014ab6943a9
3
- size 15163423
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:79645046d96acc4d96e5897dd6ab188f7aa4c5a066c78e9ad5c0517f53376cdb
3
+ size 15163552
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv CHANGED
@@ -1,3 +1,8 @@
1
  model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir
2
- QWEN3_235B echo measure 2.00 OK 141943 1 0.0 1.029296875 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1
3
- QWEN3_235B echo validate 1.25 OK 135097 1 0.0 1.029296875 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25/rep1
 
 
 
 
 
 
1
  model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir
2
+ QWEN3_235B echo measure 2.00 OK 141943 1 0.0 1.03125 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1
3
+ QWEN3_235B echo validate 1.03125 OK 132927 1 0.0 1.029296875 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125/rep1
4
+ QWEN3_235B echo forced 1.03125 OK 132939 1 0.0 1.025390625 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125__forcebal/rep1
5
+ QWEN3_30B echo measure 2.00 OK 133539 1 0.0 1.05859375 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1
6
+ QWEN3_30B echo validate 1.05859375 OK 125063 0 0.0 1.05859375 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p05859375/rep1
7
+ QWEN3_30B echo forced 2.00 OK 133539 1 0.0 1.0546875 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1
8
+ DSV2_LITE_L24 echo measure 2.00 OK 140961 1 0.0 1.0234375 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv CHANGED
@@ -1 +1,3 @@
1
  model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir
 
 
 
1
  model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir
2
+ QWEN3_235B echo SELECTED 1.03125 1.029296875 0.0 132927 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125/rep1
3
+ QWEN3_30B echo SELECTED 2.00 1.05859375 0.0 133539 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1
tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/results.tsv CHANGED
@@ -1,3 +1,8 @@
1
  config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
2
- QWEN3_235B echo matched 16 8192 8 2.00 none 1 OK 438.47 435.70 141943 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1
3
- QWEN3_235B echo matched 16 8192 8 1.25 none 1 OK 442.79 439.30 135097 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25/rep1
 
 
 
 
 
 
1
  config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
2
+ QWEN3_235B echo matched 16 8192 8 2.00 none 1 OK 444.56 440.90 141943 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1
3
+ QWEN3_235B echo matched 16 8192 8 1.03125 none 1 OK 434.67 425.80 132927 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125/rep1
4
+ QWEN3_235B echo matched 16 8192 8 1.03125 none+forcebal 1 OK 419.26 407.60 132939 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125__forcebal/rep1
5
+ QWEN3_30B echo matched 16 4096 8 2.00 none 1 OK 965.97 915.30 133539 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1
6
+ QWEN3_30B echo matched 16 4096 8 1.05859375 none 1 OK 1018.41 969.70 125063 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p05859375/rep1
7
+ QWEN3_30B echo matched 16 4096 8 2.00 none+forcebal 1 OK 1021.91 944.00 133539 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1
8
+ DSV2_LITE_L24 echo matched 8 8192 8 2.00 none 1 OK 841.21 834.60 140961 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1
tierb_study_20260826_144644/hf_watcher.log CHANGED
@@ -55,3 +55,6 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
55
  Already logged in to Hugging Face as: AbhishekGhosh1998
56
  Watching /workspace/tierb_study_20260826_144644 -> HF every 600s (kill to stop)
57
  [2026-08-26T17:09:12+00:00] backup pass
 
 
 
 
55
  Already logged in to Hugging Face as: AbhishekGhosh1998
56
  Watching /workspace/tierb_study_20260826_144644 -> HF every 600s (kill to stop)
57
  [2026-08-26T17:09:12+00:00] backup pass
58
+ uploaded: /workspace/tierb_study_20260826_144644 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tierb_study_20260826_144644
59
+ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
60
+ [2026-08-26T17:19:17+00:00] backup pass
tierb_study_20260826_144644/study.log CHANGED
@@ -1378,3 +1378,47 @@ driver_repo: /workspace/blackwell_run_scripts @ 6553011
1378
  === STAGE 7 -- Tier B probe and zero-drop capacity selection
1379
 
1380
  === STEP echo__tierbprobe__gbs8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1378
  === STAGE 7 -- Tier B probe and zero-drop capacity selection
1379
 
1380
  === STEP echo__tierbprobe__gbs8
1381
+ ==============================================================
1382
+ CAMPAIGN: echo backend=echo phase=tierbprobe
1383
+ Megatron: /workspace/Megatron-MoE-Echo @ token-drop-analysis
1384
+ GBS=8 seq=4096 repeats=3 nsys=1
1385
+ results -> /workspace/tierb_study_20260826_144644/echo/tierbprobe
1386
+ ==============================================================
1387
+ --------------------------------------------------------------------------
1388
+ [17:09:26] CELL QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00 (slots=16)
1389
+ [guard] iter 11/12 (70s)
1390
+ [guard] iter 12/12 (80s)
1391
+ [17:10:47] QWEN3_235B echo: measured required=1.03125 at C=2.00 -> validating C=1.03125
1392
+ --------------------------------------------------------------------------
1393
+ [17:10:47] CELL QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125 (slots=16)
1394
+ [guard] iter 10/12 (70s)
1395
+ [guard] iter 12/12 (80s)
1396
+ --------------------------------------------------------------------------
1397
+ [17:12:07] CELL QWEN3_235B__echo__matched__seq8192__gbs8__cap1p03125__forcebal (slots=16)
1398
+ [guard] iter 12/12 (70s)
1399
+ [guard] tearing down process group (outcome=DONE)
1400
+ --------------------------------------------------------------------------
1401
+ [17:13:24] CELL QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00 (slots=16)
1402
+ [guard] iter 5/12 (70s)
1403
+ [guard] iter 12/12 (80s)
1404
+ [guard] tearing down process group (outcome=DONE)
1405
+ [17:14:49] QWEN3_30B echo: measured required=1.05859375 at C=2.00 -> validating C=1.05859375
1406
+ --------------------------------------------------------------------------
1407
+ [17:14:49] CELL QWEN3_30B__echo__matched__seq4096__gbs8__cap1p05859375 (slots=16)
1408
+ [guard] iter 3/12 (70s)
1409
+ [guard] iter 12/12 (80s)
1410
+ [guard] tearing down process group (outcome=DONE)
1411
+ [17:16:16] QWEN3_30B echo: C=1.05859375 dropped; keeping C=2.00
1412
+ --------------------------------------------------------------------------
1413
+ [17:16:16] CELL QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal (slots=16)
1414
+ [guard] iter 5/12 (70s)
1415
+ [guard] iter 12/12 (80s)
1416
+ [guard] tearing down process group (outcome=DONE)
1417
+ --------------------------------------------------------------------------
1418
+ [17:17:41] CELL DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00 (slots=8)
1419
+ [guard] iter 6/12 (70s)
1420
+ [guard] iter 12/12 (80s)
1421
+ [guard] tearing down process group (outcome=DONE)
1422
+ [17:19:04] DSV2_LITE_L24 echo: measured required=1.0234375 at C=2.00 -> validating C=1.0234375
1423
+ --------------------------------------------------------------------------
1424
+ [17:19:04] CELL DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0234375 (slots=8)