AbhishekGhosh1998 commited on
Commit
67e34a7
·
verified ·
1 Parent(s): 9201371

granularity results moonep/tierbprobe @ 2026-08-26T17:59:23Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0

Browse files
Files changed (26) hide show
  1. moonep/tierbprobe/manifest.txt +19 -0
  2. moonep/tierbprobe/pip_freeze.txt +317 -0
  3. moonep/tierbprobe/provenance.txt +90 -0
  4. moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/mem_samples.csv +232 -0
  5. moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/run.env +61 -0
  6. moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/train.log +0 -0
  7. moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/mem_samples.csv +232 -0
  8. moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/run.env +61 -0
  9. moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/train.log +0 -0
  10. moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv +192 -0
  11. moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/run.env +61 -0
  12. moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log +0 -0
  13. moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/mem_samples.csv +192 -0
  14. moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/run.env +61 -0
  15. moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/train.log +0 -0
  16. moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1/mem_samples.csv +192 -0
  17. moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1/run.env +61 -0
  18. moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1/train.log +0 -0
  19. moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv +184 -0
  20. moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/run.env +61 -0
  21. moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log +0 -0
  22. moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv +7 -0
  23. moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv +5 -0
  24. moonep/tierbprobe/tierbprobe_gbs8/probe_summary.json +454 -0
  25. moonep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv +5 -0
  26. moonep/tierbprobe/tierbprobe_gbs8/results.tsv +7 -0
moonep/tierbprobe/manifest.txt ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ timestamp: 2026-08-26T16:52:04+00:00
2
+ host: sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0
3
+ gpu: NVIDIA B200 x8
4
+ driver: 580.95.05
5
+ study_scripts_branch: granularity-moonep
6
+ study_scripts_commit: cc725d8
7
+ campaign: moonep
8
+ backend: moonep
9
+ phase: tierbprobe
10
+ gbs: 8
11
+ seq: 4096
12
+ configs: G1 G2 G3 G4
13
+ capacity: 1.25
14
+ repeats: 3
15
+ nsys: 1
16
+ megatron_repo: /workspace/Megatron-MoE-Echo
17
+ megatron_branch_required: moonep_backend_token_routing_analysis
18
+ megatron_branch_actual: moonep_backend_token_routing_analysis
19
+ megatron_commit: 35ae6e7f
moonep/tierbprobe/pip_freeze.txt ADDED
@@ -0,0 +1,317 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ absl-py==2.3.0
2
+ aiohappyeyeballs @ file:///rapids/aiohappyeyeballs-2.6.1-py3-none-any.whl#sha256=f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8
3
+ aiohttp @ file:///rapids/aiohttp-3.12.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=94f98e0e5a49f89b252e115844f756c04fc8050f38252a32a3dd994ce8121f10
4
+ aiosignal @ file:///rapids/aiosignal-1.3.2-py2.py3-none-any.whl#sha256=45cde58e409a301715980c2b01d0c28bdde3770d8290b5eb2173759d9acb31a5
5
+ annotated-types==0.7.0
6
+ anyio==4.9.0
7
+ apex @ file:///opt/pytorch/apex/dist/apex-0.1-cp312-cp312-linux_x86_64.whl#sha256=a1508e4a386362e9c8f46c92ccada9110e421212b964c694235d0fd5a224b170
8
+ argon2-cffi==25.1.0
9
+ argon2-cffi-bindings==21.2.0
10
+ arrow==1.3.0
11
+ asciitree @ file:///rapids/asciitree-0.3.3-py3-none-any.whl#sha256=3e507ccfaeb417823158091fdb9d26b0205f6a59ba680b479a71db65c73d01d4
12
+ asttokens==3.0.0
13
+ astunparse==1.6.3
14
+ async-lru==2.0.5
15
+ attrs==25.3.0
16
+ audioread==3.0.1
17
+ babel==2.17.0
18
+ beautifulsoup4==4.13.4
19
+ black==25.1.0
20
+ bleach==6.2.0
21
+ blis==0.7.11
22
+ cachetools @ file:///rapids/cachetools-6.0.0-py3-none-any.whl#sha256=82e73ba88f7b30228b5507dce1a1f878498fc669d972aef2dde4f3a3c24f103e
23
+ catalogue==2.0.10
24
+ certifi==2025.4.26
25
+ cffi==1.17.1
26
+ charset-normalizer==3.4.2
27
+ click==8.5.0
28
+ cloudpathlib==0.21.1
29
+ cloudpickle @ file:///rapids/cloudpickle-3.1.1-py3-none-any.whl#sha256=c8c5a44295039331ee9dad40ba100a9c7297b6f988e50e87ccdf3765a668350e
30
+ cmake==3.31.6
31
+ comm==0.2.2
32
+ confection==0.1.5
33
+ contourpy==1.3.2
34
+ cuda-bindings @ file:///rapids/cuda_bindings-12.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=ff0e28d1e34758654b9c961e1f55e4786e49aee6a4dbceaf3cc24c46c672df7e
35
+ cuda-python @ file:///rapids/cuda_python-12.9.0-py3-none-any.whl#sha256=926acba49b2c0a0374c61b7c98f337c085199cf51cdfe4d6423c4129c20547a7
36
+ cudf @ file:///rapids/cudf-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b779a8e8c0e8577b9b560895958496da211bfac49cdc4f11a49e8d18cb82e348
37
+ cudf-polars @ file:///rapids/cudf_polars-25.4.0-py3-none-any.whl#sha256=227b6c5147e93ea86282c8f7219118f916769fa2ca0cdd931e97a148aa66b869
38
+ cugraph @ file:///rapids/cugraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b837c2dc2f7b235c702e4f8df64479b8fcbb342f848516d5faf0cdc4b5f31f29
39
+ cugraph-service-client @ file:///rapids/cugraph_service_client-25.4.0-py3-none-any.whl#sha256=7e568c3e41154b9752f5173555ffcc282ac52d72343feeb79653b2e2b1232470
40
+ cugraph-service-server @ file:///rapids/cugraph_service_server-25.4.0-py3-none-any.whl#sha256=a2830dcffafcccb16e12ebf85b98f8f247869d38c01ab4dd8664a1e7439d31be
41
+ cuml @ file:///rapids/cuml-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=1e90a464a18b094e24797f1676688c5d7696c4cf97c934504214a53d5aa09aa5
42
+ cupy-cuda12x @ file:///rapids/cupy_cuda12x-13.3.0-cp312-cp312-linux_x86_64.whl#sha256=6a2dfa0aa1e8189a10d1534276c08f1d2f73b747fd28bfbe7b9f4ef66d9af73f
43
+ cuvs @ file:///rapids/cuvs-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b12f6b9f80e053a3b38542c2947d2516e468c02cad54a13d8b1a4a61d5aa7914
44
+ cycler==0.12.1
45
+ cymem==2.0.11
46
+ Cython==3.1.1
47
+ dask @ file:///rapids/dask-2025.2.0-py3-none-any.whl#sha256=f0fdeef6ceb0a06569d456c9e704f220f7f54e80f3a6ea42ab98cea6bc642b6e
48
+ dask-cuda @ file:///rapids/dask_cuda-25.4.0-py3-none-any.whl#sha256=442d87da0a95d00ca2a262b64c76e785a32ad905c82000fa2227932f81390e19
49
+ dask-cudf @ file:///rapids/dask_cudf-25.4.0-py3-none-any.whl#sha256=5e7cea0b26daea17da6e0bb457435e24460f76ba740358a3a18e239d6015c95c
50
+ datasets==5.0.1
51
+ debugpy==1.8.14
52
+ decorator==5.2.1
53
+ -e git+https://github.com/deepseek-ai/DeepEP.git@34152ae28f80bcc3ee38d7a12cb2ad87cfd4ea72#egg=deep_ep
54
+ defusedxml==0.7.1
55
+ dill==0.4.1
56
+ distributed @ file:///rapids/distributed-2025.2.0-py3-none-any.whl#sha256=368462084be47092d0a835f6c64c9b852da64dc88302b76d5f93bd5131228c67
57
+ distributed-ucxx @ file:///rapids/distributed_ucxx-0.43.0-py3-none-any.whl#sha256=94f60d0ccb0c94ba8dbfa1eaf455874d18aaab17e2ae218c38590e9fdc2d721d
58
+ dm-tree==0.1.9
59
+ einops==0.8.1
60
+ execnet==2.1.1
61
+ executing==2.2.0
62
+ expecttest==0.3.0
63
+ fasteners @ file:///rapids/fasteners-0.19-py3-none-any.whl#sha256=758819cb5d94cdedf4e836988b74de396ceacb8e2794d21f82d131fd9ee77237
64
+ fastjsonschema==2.21.1
65
+ fastrlock @ file:///rapids/fastrlock-0.8.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl#sha256=5f13ec08f1adb1aa916c384b05ecb7dbebb8df9ea81abd045f60941c6283a670
66
+ filelock==3.18.0
67
+ flash_attn @ file:///opt/pytorch/flash_attn-2.7.4.post1-cp312-cp312-linux_x86_64.whl#sha256=a10455d6c56b387785b7413fd0bbcb392b42b3d4f4b88fda5d86cc8a9ce9182a
68
+ fonttools==4.58.1
69
+ fqdn==1.5.1
70
+ frozenlist @ file:///rapids/frozenlist-1.6.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=4e1be9111cb6756868ac242b3c2bd1f09d9aea09846e4f5c23715e7afb647103
71
+ fsspec==2025.5.1
72
+ gast==0.6.0
73
+ grpcio==1.62.1
74
+ h11==0.16.0
75
+ hf-xet==1.6.0
76
+ httpcore==1.0.9
77
+ httpx==0.28.1
78
+ huggingface_hub==1.28.0
79
+ hypothesis==6.130.8
80
+ idna==3.10
81
+ importlib_metadata==8.7.0
82
+ iniconfig==2.1.0
83
+ intel-openmp==2021.4.0
84
+ ipykernel==6.29.5
85
+ ipython==9.3.0
86
+ ipython_pygments_lexers==1.1.1
87
+ isoduration==20.11.0
88
+ isort==6.0.1
89
+ jedi==0.19.2
90
+ Jinja2==3.1.6
91
+ joblib==1.5.1
92
+ json5==0.12.0
93
+ jsonpointer==3.0.0
94
+ jsonschema==4.24.0
95
+ jsonschema-specifications==2025.4.1
96
+ jupyter-events==0.12.0
97
+ jupyter-lsp==2.2.5
98
+ jupyter_client==8.6.3
99
+ jupyter_core==5.8.1
100
+ jupyter_server==2.16.0
101
+ jupyter_server_terminals==0.5.3
102
+ jupyterlab==4.4.3
103
+ jupyterlab_code_formatter==3.0.2
104
+ jupyterlab_pygments==0.3.0
105
+ jupyterlab_server==2.27.3
106
+ jupyterlab_tensorboard_pro @ file:///builder/jupyterlab_tensorboard_pro-4.0.0-py2.py3-none-any.whl#sha256=6bfcb5ced3358eb72150f4b2ab91312faef2f0ca14d845e8379d8fa52df6902e
107
+ jupytext==1.17.2
108
+ kiwisolver==1.4.8
109
+ kvikio @ file:///rapids/kvikio-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=8b40fd8ea9e4a2974c04b50b7fc26b06094ac807f9a233281d0bafbaa71b1d85
110
+ langcodes==3.5.0
111
+ language_data==1.3.0
112
+ lazy_loader==0.4
113
+ libcudf @ file:///rapids/libcudf-25.4.0-py3-none-linux_x86_64.whl#sha256=238320b4958c1737bba2a609673e2539a78216b811b5498a68e00395d9b021a4
114
+ libcugraph @ file:///rapids/libcugraph-25.4.0-py3-none-linux_x86_64.whl#sha256=29085e901b2f7569ff2653bb1355fe6fad20f2fcaa585f5649da9867d8df766d
115
+ libcuml @ file:///rapids/libcuml-25.4.0-py3-none-linux_x86_64.whl#sha256=8e2e444c72b0e995e15a48da37ff12fb410a47a2baa69ba86ccc9b1a3db91e8c
116
+ libcuvs @ file:///rapids/libcuvs-25.4.0-py3-none-linux_x86_64.whl#sha256=e2f8db61c92b769c590dd752f43439ddd62b2f6592a3c337362b6c1ef220c3b9
117
+ libkvikio @ file:///rapids/libkvikio-25.4.0-py3-none-linux_x86_64.whl#sha256=da8a5713b05fc478bf40529a3e0fb61a2d605bb5a70ca4d461f96a6005dbe8b7
118
+ libraft @ file:///rapids/libraft-25.4.0-py3-none-linux_x86_64.whl#sha256=b921dec005aa4a321dd156776a18376feb47c96a5b67eec125c8a8e19edd1e24
119
+ librmm @ file:///rapids/librmm-25.4.0-py3-none-linux_x86_64.whl#sha256=ff951811bda729d4fa1109ab906225ae1f489a6ec2eeab5d1b674cc34bd4422f
120
+ librmm-cu12 @ file:///rapids/librmm_cu12-25.4.0-py3-none-any.whl#sha256=29296ef2b0e90679a0db280e86015be12d6ab9f4e62a181aa085f62dbadd06b3
121
+ librosa==0.11.0
122
+ libucxx @ file:///rapids/libucxx-0.43.0-py3-none-linux_x86_64.whl#sha256=f15849d7ac2fb68f0fd8998838a13321ddcfb0c594f7e6cbc401a3bcc4750566
123
+ lightning-thunder @ file:///opt/pytorch/lightning-thunder
124
+ lightning-utilities==0.14.3
125
+ lintrunner==0.12.7
126
+ llvmlite==0.42.0
127
+ locket @ file:///rapids/locket-1.0.0-py2.py3-none-any.whl#sha256=b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3
128
+ looseversion==1.3.0
129
+ marisa-trie==1.2.1
130
+ Markdown==3.8
131
+ markdown-it-py==3.0.0
132
+ MarkupSafe==3.0.2
133
+ matplotlib==3.10.3
134
+ matplotlib-inline==0.1.7
135
+ mdit-py-plugins==0.4.2
136
+ mdurl==0.1.2
137
+ -e git+ssh://git@github.com/sriram-17-17/my-Megatron-MoE-Echo.git@35ae6e7f533e3257c4320b6fd256cb34add3c3fd#egg=megatron_core
138
+ mistune==3.1.3
139
+ mkl==2021.1.1
140
+ mkl-devel==2021.1.1
141
+ mkl-include==2021.1.1
142
+ ml_dtypes==0.6.0
143
+ mock==5.2.0
144
+ -e git+ssh://git@github.com/sriram-17-17/MoonEP.git@c27d35da15f25c112218795894504a65c16f4879#egg=moonep
145
+ mpmath==1.3.0
146
+ msgpack==1.1.0
147
+ multidict @ file:///rapids/multidict-6.4.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=a920f9cf2abdf6e493c519492d892c362007f113c94da4c239ae88429835bad1
148
+ multiprocess==0.70.19
149
+ murmurhash==1.0.13
150
+ mypy_extensions==1.1.0
151
+ nbclient==0.10.2
152
+ nbconvert==7.16.6
153
+ nbformat==5.10.4
154
+ nest-asyncio==1.6.0
155
+ networkx==3.5
156
+ ninja==1.11.1.4
157
+ notebook==7.4.3
158
+ notebook_shim==0.2.4
159
+ numba==0.59.1
160
+ numba-cuda @ file:///rapids/numba_cuda-0.4.0-py3-none-any.whl#sha256=5b95a5f42068771e37142944d44fe844fa87f1b2f89051461a4cb0af91bcc28a
161
+ numcodecs @ file:///rapids/numcodecs-0.13.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=da2230484e6102e5fa3cc1a5dd37ca1f92dfbd183d91662074d6f7574e3e8f53
162
+ numpy==1.26.4
163
+ nvdlfw_inspect @ git+https://github.com/NVIDIA/nvidia-dlfw-inspect.git@6b60eb0e675606fb2fbbfcb12667ecbec75eaf87
164
+ nvfuser==0.2.27a0+9bf5aca
165
+ nvidia-cuda-cccl-cu12==12.9.27
166
+ nvidia-cudnn-frontend @ file:///opt/pytorch/lightning-thunder/tmp/cudnn_frontend
167
+ nvidia-cutlass-dsl==4.4.2
168
+ nvidia-cutlass-dsl-libs-base==4.4.2
169
+ nvidia-dali-cuda120==1.50.0
170
+ nvidia-ml-py @ file:///rapids/nvidia_ml_py-12.575.51-py3-none-any.whl#sha256=eb8641800d98ce40a22f479873f34b482e214a7e80349c63be51c3919845446e
171
+ nvidia-modelopt==0.29.0
172
+ nvidia-modelopt-core==0.29.0
173
+ nvidia-nvcomp-cu12==4.2.0.14
174
+ nvidia-nvimgcodec-cu12==0.5.0.13
175
+ nvidia-nvjpeg-cu12==12.4.0.16
176
+ nvidia-nvjpeg2k-cu12==0.8.1.40
177
+ nvidia-nvshmem-cu12==3.7.2
178
+ nvidia-nvtiff-cu12==0.5.0.67
179
+ nvidia-resiliency-ext==0.4.0
180
+ nvtx @ file:///rapids/nvtx-0.2.11-cp312-cp312-linux_x86_64.whl#sha256=73521e37cba42b4db0b3cd68fc5eb814e55c500a61eeed6edcdf6746315ac53e
181
+ nx-cugraph @ file:///rapids/nx_cugraph-25.4.0-py3-none-any.whl#sha256=e625444486cc899a77bd48e7da76a2edd95c0006dd95fbf323745feae7f33111
182
+ onnx @ file:///opt/pytorch/pytorch/third_party/onnx
183
+ onnx-ir==1.0.0
184
+ onnxscript==0.7.1
185
+ opt_einsum==3.4.0
186
+ optree==0.16.0
187
+ overrides==7.7.0
188
+ packaging==23.2
189
+ pandas @ file:///rapids/pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319
190
+ pandocfilters==1.5.1
191
+ parso==0.8.4
192
+ partd @ file:///rapids/partd-1.4.2-py3-none-any.whl#sha256=978e4ac767ec4ba5b86c6eaa52e5a2a3bc748a2ca839e8cc798f1cc6ce6efb0f
193
+ pathspec==0.12.1
194
+ pexpect==4.9.0
195
+ pillow==11.2.1
196
+ platformdirs==4.3.8
197
+ pluggy==1.6.0
198
+ ply @ file:///rapids/ply-3.11-py2.py3-none-any.whl#sha256=096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce
199
+ polars @ file:///rapids/polars-1.25.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=f7fcbb4f476784384ccda48757fca4e8c2e2c5a0a3aef3717aaf56aee4e30e09
200
+ polygraphy==0.49.20
201
+ pooch==1.8.2
202
+ preshed==3.0.10
203
+ prometheus_client==0.22.1
204
+ prompt_toolkit==3.0.51
205
+ propcache @ file:///rapids/propcache-0.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=563f9d8c03ad645597b8d010ef4e9eab359faeb11a0a2ac9f7b4bc8c28ebef25
206
+ protobuf==4.24.4
207
+ psutil==7.0.0
208
+ ptyprocess==0.7.0
209
+ PuLP==3.2.1
210
+ pure_eval==0.2.3
211
+ pyarrow==25.0.1
212
+ pybind11==2.13.6
213
+ pybind11_global==2.13.6
214
+ pycocotools @ git+https://github.com/nvidia/cocoapi.git@08350a157b41f0755a536441b5abec89ab934c5b#subdirectory=PythonAPI
215
+ pycparser==2.22
216
+ pydantic==2.11.5
217
+ pydantic_core==2.33.2
218
+ Pygments==2.19.1
219
+ pylibcudf @ file:///rapids/pylibcudf-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=401b8bfd022d06f84e120779ca7c3d01a28b68762aaa8f7873b10d6b1975cf6a
220
+ pylibcugraph @ file:///rapids/pylibcugraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=061c7389a88c815b54fd5d3a3584fb87bf693e5602f5be335260d0ba575ba557
221
+ pylibcugraphops @ file:///rapids/pylibcugraphops-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=a25ef4b22166c80718d5c7e5bddb755e81a2fe340fa5becd0ac36ec3edfb208a
222
+ pylibraft @ file:///rapids/pylibraft-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=f2ecd23fe51f9c2caa87bbbe85cb32fcaff4ecfb52ceb9a9991bc50f82782c2e
223
+ pylibwholegraph @ file:///rapids/pylibwholegraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b82913d1e38bda76d8c272fbd8cc9072d987209d3d47cc93a464a37c2925aca2
224
+ pynvjitlink @ file:///rapids/pynvjitlink-0.3.0-cp312-cp312-linux_x86_64.whl#sha256=066caf1314730643d3b48c9847bd65789defdd6a7c78993176eb3d8bbe3f4919
225
+ pynvml @ file:///rapids/pynvml-12.0.0-py3-none-any.whl#sha256=fdff84b62a27dbe98e08e1a647eb77342bef1aebe0878bcd15e99a83fcbecb9e
226
+ pyparsing==3.2.3
227
+ pytest==8.1.1
228
+ pytest-flakefinder==1.1.0
229
+ pytest-rerunfailures==15.1
230
+ pytest-shard==0.1.2
231
+ pytest-xdist==3.7.0
232
+ python-dateutil==2.9.0.post0
233
+ python-hostlist==2.2.1
234
+ python-json-logger==3.3.0
235
+ pytorch-triton @ file:///tmp/dist/pytorch_triton-3.3.0%2Bgit96316ce52.nvinternal-cp312-cp312-linux_x86_64.whl#sha256=6d6921772b9dfd0243c055e0d054de42a113e92cabcfdd1d3e80a5ad9c08cf7d
236
+ pytz @ file:///rapids/pytz-2023.4-py2.py3-none-any.whl#sha256=f90ef520d95e7c46951105338d918664ebfd6f1d995bd7d153127ce90efafa6a
237
+ PyYAML==6.0.2
238
+ pyzmq==26.4.0
239
+ raft-dask @ file:///rapids/raft_dask-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=36cf37ba6ce35454c4729c2cb597a360f22780eefb11d514a16a07137da5eb6c
240
+ rapids-dask-dependency @ file:///rapids/rapids_dask_dependency-25.4.0a0-py3-none-any.whl#sha256=484faca626e7760112a7d8ad3426f5bdd720b609006d8dca87c2607076a348bd
241
+ rapids-logger @ file:///rapids/rapids_logger-0.1.18-py3-none-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=3617b2d86291ac6cb957542895f20fc9737ace29773ca90337e6fb04c73bd120
242
+ referencing==0.36.2
243
+ regex==2024.11.6
244
+ requests==2.32.3
245
+ rfc3339-validator==0.1.4
246
+ rfc3986-validator==0.1.1
247
+ rich==14.0.0
248
+ rmm @ file:///rapids/rmm-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=1824bcab3836296666980a997ddcb9c4bca24dfb873012cfa0a7ecea583ca712
249
+ rpds-py==0.25.1
250
+ safetensors==0.5.3
251
+ scikit-learn==1.6.1
252
+ scipy==1.15.3
253
+ Send2Trash==1.8.3
254
+ setuptools==78.1.1
255
+ shellingham==1.5.4
256
+ six==1.16.0
257
+ smart-open==7.1.0
258
+ sniffio==1.3.1
259
+ sortedcontainers==2.4.0
260
+ soundfile==0.13.1
261
+ soupsieve==2.7
262
+ soxr==0.5.0.post1
263
+ spacy==3.7.5
264
+ spacy-legacy==3.0.12
265
+ spacy-loggers==1.0.5
266
+ srsly==2.5.1
267
+ stack-data==0.6.3
268
+ sympy==1.14.0
269
+ tabulate==0.9.0
270
+ tbb==2021.13.1
271
+ tblib @ file:///rapids/tblib-3.1.0-py3-none-any.whl#sha256=670bb4582578134b3d81a84afa1b016128b429f3d48e6cbbaecc9d15675e984e
272
+ tensorboard==2.16.2
273
+ tensorboard-data-server==0.7.2
274
+ tensorrt @ file:///workspace/TensorRT-10.11.0.33/python/tensorrt-10.11.0.33-cp312-none-linux_x86_64.whl#sha256=c2f18f72758a3d54a71901495c8fda9e7094a8bcdd2a5b6b58967fbf05733b3a
275
+ terminado==0.18.1
276
+ thinc==8.2.5
277
+ threadpoolctl==3.6.0
278
+ thriftpy2 @ file:///rapids/thriftpy2-0.5.2-cp312-cp312-linux_x86_64.whl#sha256=c0c7d939bdc5ccf7739154959938445d5fdf72f1d6e12bd563947a36f67e2147
279
+ tinycss2==1.4.0
280
+ toolz @ file:///rapids/toolz-1.0.0-py3-none-any.whl#sha256=292c8f1c4e7516bf9086f8850935c799a874039c8bcf959d47b600e4c44a6236
281
+ torch @ file:///opt/transfer/torch-2.8.0a0%2B5228986c39.nv25.6-29946474-cp312-cp312-linux_x86_64.whl#sha256=7a3c9ff5288d8102a31296b3d62491f3f47c8820a5c9aaefc7dc0fe20e1e8ad5
282
+ torch_tensorrt @ file:///opt/pytorch/torch_tensorrt/dist/torch_tensorrt-2.8.0a0-cp312-cp312-linux_x86_64.whl#sha256=26f4d7a04e7e6e017a5b917b8b98b83720bb2bb0195f48d25f52d567d286bfe1
283
+ torchao @ file:///opt/pytorch/ao
284
+ torchprofile==0.0.4
285
+ torchvision @ file:///opt/transfer/torchvision-0.22.0a0%2B95f10a4e-29946474-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=746a3c674cc61680a86f4a14b374b21af1b5c21f629c42765b40c9494f6b234d
286
+ tornado==6.5.1
287
+ tqdm==4.67.1
288
+ traitlets==5.14.3
289
+ transformer_engine @ file:///pvcdata/aiinfra/b-aghosh/wheels/transformer_engine-2.10.0.dev0%2B10f6128d-cp312-cp312-linux_x86_64.whl#sha256=35cf5f4b83b2baad864e12e512bb422e0929409ac05610812fa521d4942d0189
290
+ treelite @ file:///rapids/treelite-4.4.1-py3-none-linux_x86_64.whl#sha256=174741724205540a1a5b37da81483aeff55d89548f42b3461cf07acac7ae96eb
291
+ typer==0.16.0
292
+ types-dataclasses==0.6.6
293
+ types-python-dateutil==2.9.0.20250516
294
+ typing-inspection==0.4.1
295
+ typing_extensions==4.14.0
296
+ tzdata @ file:///rapids/tzdata-2025.2-py2.py3-none-any.whl#sha256=1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8
297
+ ucx-py @ file:///rapids/ucx_py-0.43.0-cp312-cp312-linux_x86_64.whl#sha256=4a452fd8d8c983283ada3d49086e3657dc2ee94fbad08efdd708d15ab6e0b73a
298
+ ucxx @ file:///rapids/ucxx-0.43.0-cp312-cp312-linux_x86_64.whl#sha256=f1b4e55cb45c66fe88a2d8f9c2b87751165f3cfec21fbdc9c47c4ddff75e8a5f
299
+ -e git+https://github.com/Dots-Infra/UltraEP.git@94cab099b44fffa99a82fea99e7c12d89cf65e4f#egg=ultra_ep
300
+ uri-template==1.3.0
301
+ urllib3==2.0.7
302
+ wasabi==1.1.3
303
+ wcwidth==0.2.13
304
+ weasel==0.4.1
305
+ webcolors==24.11.1
306
+ webencodings==0.5.1
307
+ websocket-client==1.8.0
308
+ Werkzeug==3.1.3
309
+ wheel==0.45.1
310
+ wrapt==1.17.2
311
+ xdoctest==1.0.2
312
+ xgboost @ file:///rapids/xgboost-2.1.4-py3-none-manylinux_2_39_x86_64.whl#sha256=25eeca27f4048fc491940b3a0b4b5782ee48c33cb3aec059cef71f8da542930b
313
+ xxhash==4.0.1
314
+ yarl @ file:///rapids/yarl-1.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=bf099e2432131093cc611623e0b0bcc399b8cddd9a91eded8bfb50402ec35018
315
+ zarr @ file:///rapids/zarr-2.18.7-py3-none-any.whl#sha256=ac3dc4033e9ae4e9d7b5e27c97ea3eaf1003cc0a07f010bd83d5134bf8c4b223
316
+ zict @ file:///rapids/zict-3.0.0-py2.py3-none-any.whl#sha256=5796e36bd0e0cc8cf0fbc1ace6a68912611c1dbd74750a3f3026b9b9d6a327ae
317
+ zipp==3.22.0
moonep/tierbprobe/provenance.txt ADDED
@@ -0,0 +1,90 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ==============================================================
2
+ PROVENANCE: moonep__tierbprobe__gbs8
3
+ ==============================================================
4
+ captured_at : 2026-08-26T16:51:49+00:00
5
+ hostname : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0
6
+ kernel : 6.8.0-1043-nvidia
7
+ cwd : /workspace/blackwell_run_scripts
8
+ command : run_ladder.sh --phase tierbprobe --gbs 8 --out /workspace/tierb_study_20260826_144644/moonep/tierbprobe --nsys
9
+
10
+ ---- GPU / driver --------------------------------------------
11
+ 0, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
12
+ 1, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
13
+ 2, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
14
+ 3, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
15
+ 4, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
16
+ 5, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
17
+ 6, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
18
+ 7, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
19
+ persistence/clocks:
20
+ 0, 120 MHz, 1965 MHz, 1000.00 W, 29
21
+ 1, 150 MHz, 1965 MHz, 1000.00 W, 34
22
+ 2, 120 MHz, 1965 MHz, 1000.00 W, 34
23
+ 3, 120 MHz, 1965 MHz, 1000.00 W, 30
24
+ 4, 120 MHz, 1965 MHz, 1000.00 W, 30
25
+ 5, 120 MHz, 1965 MHz, 1000.00 W, 34
26
+ 6, 120 MHz, 1965 MHz, 1000.00 W, 34
27
+ 7, 120 MHz, 1965 MHz, 1000.00 W, 30
28
+
29
+ NVLink topology:
30
+ GPU0 GPU1 GPU2 GPU3 GPU4 GPU5 GPU6 GPU7 NIC0 NIC1 NIC2 NIC3 NIC4 NIC5 NIC6 NIC7 NIC8 CPU Affinity NUMA Affinity GPU NUMA ID
31
+ GPU0 X NV18 NV18 NV18 NV18 NV18 NV18 NV18 PIX NODE SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A
32
+ GPU1 NV18 X NV18 NV18 NV18 NV18 NV18 NV18 NODE PIX SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A
33
+ GPU2 NV18 NV18 X NV18 NV18 NV18 NV18 NV18 SYS SYS PIX NODE SYS SYS SYS SYS SYS 28-55 1 N/A
34
+ GPU3 NV18 NV18 NV18 X NV18 NV18 NV18 NV18 SYS SYS NODE PIX SYS SYS SYS SYS SYS 28-55 1 N/A
35
+ GPU4 NV18 NV18 NV18 NV18 X NV18 NV18 NV18 SYS SYS SYS SYS PIX NODE SYS SYS SYS 56-83 2 N/A
36
+ GPU5 NV18 NV18 NV18 NV18 NV18 X NV18 NV18 SYS SYS SYS SYS NODE PIX SYS SYS SYS 56-83 2 N/A
37
+ GPU6 NV18 NV18 NV18 NV18 NV18 NV18 X NV18 SYS SYS SYS SYS SYS SYS PIX NODE SYS 84-111 3 N/A
38
+ GPU7 NV18 NV18 NV18 NV18 NV18 NV18 NV18 X SYS SYS SYS SYS SYS SYS NODE PIX SYS 84-111 3 N/A
39
+ NIC0 PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS SYS NODE
40
+ NIC1 NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS SYS NODE
41
+ NIC2 SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS
42
+ NIC3 SYS SYS NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS
43
+ NIC4 SYS SYS SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS
44
+ unavailable
45
+
46
+ ---- toolchain -----------------------------------------------
47
+ nvcc : release 12.9, V12.9.86
48
+ gcc : gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0
49
+ python : 3.12.3
50
+ torch : 2.8.0a0+5228986c39.nv25.06 cuda=12.9 nccl=2.27.3
51
+ torch arch : ['sm_75', 'sm_80', 'sm_86', 'sm_90', 'sm_100', 'sm_120', 'compute_120']
52
+ transformer_engine: 2.10.0.dev0+10f6128d [/usr/local/lib/python3.12/dist-packages/transformer_engine/__init__.py]
53
+ deep_ep : 1.2.1+34152ae [/workspace/DeepEP/deep_ep/__init__.py]
54
+ ultra_ep : 1.0.0 [/workspace/UltraEP/ultra_ep/__init__.py]
55
+ moonep : 0.0.1 [/workspace/MoonEP/moonep/__init__.py]
56
+ megatron : 0.14.0rc5 [None]
57
+ flash_attn : 2.7.4.post1 [/usr/local/lib/python3.12/dist-packages/flash_attn/__init__.py]
58
+ numpy : 1.26.4 [/usr/local/lib/python3.12/dist-packages/numpy/__init__.py]
59
+ huggingface_hub: 1.28.0 [/usr/local/lib/python3.12/dist-packages/huggingface_hub/__init__.py]
60
+
61
+ ---- repositories --------------------------------------------
62
+ Megatron-MoE-Echo moonep_backend_token_routing_analysis 35ae6e7f533e
63
+ Megatron-UltraEP ultraep-token-drop-analysis eaf829e7ad21
64
+ UltraEP main 94cab099b44f
65
+ DeepEP HEAD 34152ae28f80 (+3 untracked: build output)
66
+ MoonEP megatron-integration c27d35da15f2
67
+ blackwell_run_scripts granularity-moonep cc725d8fcc79
68
+
69
+ ---- environment ---------------------------------------------
70
+ CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0
71
+ CUDA_DRIVER_VERSION=575.57.08
72
+ CUDA_HOME=/usr/local/cuda
73
+ CUDA_MODULE_LOADING=LAZY
74
+ CUDA_VERSION=12.9.1.010
75
+ HF_REMOTE_ROOT=tierb_study_20260826_144644
76
+ NCCL_VERSION=2.27.3
77
+ NVSHMEM_VERSION=3.2.5
78
+ PYTORCH_BUILD_NUMBER=0
79
+ PYTORCH_BUILD_VERSION=2.8.0a0+5228986
80
+ PYTORCH_HOME=/opt/pytorch/pytorch
81
+ PYTORCH_VERSION=2.8.0a0+5228986
82
+ TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1
83
+ TORCH_CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0+PTX
84
+ TORCH_NCCL_USE_COMM_NONBLOCKING=0
85
+
86
+ ---- shared library resolution -------------------------------
87
+ libnvshmem_host.so: not in ldconfig
88
+ libnccl.so: /lib/x86_64-linux-gnu/libnccl.so.2
89
+ lrwxrwxrwx 1 root root 20 Aug 26 14:37 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so -> libnvshmem_host.so.3
90
+ -rw-r--r-- 1 root root 179045744 Aug 26 14:09 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so.3
moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/mem_samples.csv ADDED
@@ -0,0 +1,232 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 1
41
+ 0, 4
42
+ 1, 4
43
+ 2, 4
44
+ 3, 4
45
+ 4, 4
46
+ 5, 4
47
+ 6, 4
48
+ 7, 4
49
+ 0, 4
50
+ 1, 629
51
+ 2, 629
52
+ 3, 629
53
+ 4, 629
54
+ 5, 629
55
+ 6, 629
56
+ 7, 629
57
+ 0, 943
58
+ 1, 961
59
+ 2, 987
60
+ 3, 959
61
+ 4, 953
62
+ 5, 943
63
+ 6, 957
64
+ 7, 949
65
+ 0, 1955
66
+ 1, 1955
67
+ 2, 1955
68
+ 3, 1955
69
+ 4, 1955
70
+ 5, 1955
71
+ 6, 1955
72
+ 7, 1955
73
+ 0, 3955
74
+ 1, 3977
75
+ 2, 3953
76
+ 3, 3957
77
+ 4, 3977
78
+ 5, 3967
79
+ 6, 3961
80
+ 7, 3979
81
+ 0, 34565
82
+ 1, 34441
83
+ 2, 34493
84
+ 3, 34631
85
+ 4, 34417
86
+ 5, 34419
87
+ 6, 34585
88
+ 7, 34441
89
+ 0, 35867
90
+ 1, 35991
91
+ 2, 35861
92
+ 3, 35921
93
+ 4, 35873
94
+ 5, 35921
95
+ 6, 35909
96
+ 7, 35991
97
+ 0, 45899
98
+ 1, 46019
99
+ 2, 45893
100
+ 3, 45951
101
+ 4, 45903
102
+ 5, 45949
103
+ 6, 45941
104
+ 7, 46019
105
+ 0, 46289
106
+ 1, 46409
107
+ 2, 46283
108
+ 3, 46343
109
+ 4, 46295
110
+ 5, 46339
111
+ 6, 46331
112
+ 7, 46409
113
+ 0, 46293
114
+ 1, 46415
115
+ 2, 46287
116
+ 3, 46347
117
+ 4, 46299
118
+ 5, 46345
119
+ 6, 46335
120
+ 7, 46415
121
+ 0, 116085
122
+ 1, 116077
123
+ 2, 116079
124
+ 3, 116073
125
+ 4, 116089
126
+ 5, 116071
127
+ 6, 116063
128
+ 7, 116063
129
+ 0, 139413
130
+ 1, 139405
131
+ 2, 139407
132
+ 3, 139401
133
+ 4, 139417
134
+ 5, 139399
135
+ 6, 140177
136
+ 7, 139405
137
+ 0, 139769
138
+ 1, 139733
139
+ 2, 139721
140
+ 3, 139721
141
+ 4, 139755
142
+ 5, 139761
143
+ 6, 140477
144
+ 7, 139727
145
+ 0, 139801
146
+ 1, 139793
147
+ 2, 139795
148
+ 3, 139791
149
+ 4, 139807
150
+ 5, 139787
151
+ 6, 140565
152
+ 7, 139793
153
+ 0, 142165
154
+ 1, 142187
155
+ 2, 142189
156
+ 3, 142185
157
+ 4, 142191
158
+ 5, 142181
159
+ 6, 142969
160
+ 7, 142197
161
+ 0, 143123
162
+ 1, 143133
163
+ 2, 143151
164
+ 3, 143095
165
+ 4, 143125
166
+ 5, 143139
167
+ 6, 143885
168
+ 7, 143133
169
+ 0, 157149
170
+ 1, 157133
171
+ 2, 157143
172
+ 3, 157135
173
+ 4, 157151
174
+ 5, 157127
175
+ 6, 156339
176
+ 7, 157133
177
+ 0, 157149
178
+ 1, 157133
179
+ 2, 157143
180
+ 3, 157135
181
+ 4, 157151
182
+ 5, 157127
183
+ 6, 156339
184
+ 7, 157133
185
+ 0, 157149
186
+ 1, 157133
187
+ 2, 157143
188
+ 3, 157135
189
+ 4, 157151
190
+ 5, 157127
191
+ 6, 156339
192
+ 7, 157133
193
+ 0, 157149
194
+ 1, 157133
195
+ 2, 157143
196
+ 3, 157135
197
+ 4, 157151
198
+ 5, 157127
199
+ 6, 156339
200
+ 7, 157133
201
+ 0, 157085
202
+ 1, 156871
203
+ 2, 157143
204
+ 3, 156809
205
+ 4, 156569
206
+ 5, 156807
207
+ 6, 156339
208
+ 7, 157133
209
+ 0, 155267
210
+ 1, 154989
211
+ 2, 155261
212
+ 3, 154927
213
+ 4, 154687
214
+ 5, 155245
215
+ 6, 154519
216
+ 7, 154669
217
+ 0, 153587
218
+ 1, 153245
219
+ 2, 152935
220
+ 3, 152927
221
+ 4, 152555
222
+ 5, 152973
223
+ 6, 152385
224
+ 7, 153259
225
+ 0, 143044
226
+ 1, 4
227
+ 2, 4
228
+ 3, 4
229
+ 4, 4
230
+ 5, 4
231
+ 6, 4
232
+ 7, 4
moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=moonep
2
+ config=DSV2_LITE_L24
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=moonep_backend_token_routing_analysis
6
+ commit=35ae6e7f
7
+ num_layers=24
8
+ hidden=2048
9
+ ffn_hidden=10944
10
+ moe_ffn_hidden=1408
11
+ num_experts=64
12
+ topk=6
13
+ experts_per_rank=8
14
+ expert_params=8650752
15
+ heads=16
16
+ kv_groups=0
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=2816
20
+ shared_expert_overlap=0
21
+ mla=1
22
+ dense_first_layers=1
23
+ moe_layers=23
24
+ rotary_base=10000
25
+ rotary_scaling=40
26
+ qkv_bias=0
27
+ router_balancing=seq_aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=1
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=8192
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=49152
39
+ mean_tokens_per_expert=6144
40
+ slot_policy=matched
41
+ slots_per_rank=8
42
+ capacity_mode=none
43
+ capacity=none
44
+ cuda_graph=0
45
+ probe_mode=1
46
+ async_grad_reduce=0
47
+ branch_role=probe
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/mem_samples.csv ADDED
@@ -0,0 +1,232 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 4
43
+ 2, 4
44
+ 3, 4
45
+ 4, 4
46
+ 5, 629
47
+ 6, 629
48
+ 7, 4
49
+ 0, 845
50
+ 1, 837
51
+ 2, 835
52
+ 3, 843
53
+ 4, 847
54
+ 5, 845
55
+ 6, 845
56
+ 7, 845
57
+ 0, 1955
58
+ 1, 1955
59
+ 2, 1955
60
+ 3, 1955
61
+ 4, 1955
62
+ 5, 1955
63
+ 6, 1955
64
+ 7, 1955
65
+ 0, 3759
66
+ 1, 3761
67
+ 2, 3757
68
+ 3, 3771
69
+ 4, 3761
70
+ 5, 3761
71
+ 6, 3769
72
+ 7, 3761
73
+ 0, 32465
74
+ 1, 32619
75
+ 2, 32531
76
+ 3, 32509
77
+ 4, 32575
78
+ 5, 32575
79
+ 6, 32575
80
+ 7, 32597
81
+ 0, 35965
82
+ 1, 36055
83
+ 2, 35925
84
+ 3, 35985
85
+ 4, 35937
86
+ 5, 35985
87
+ 6, 35973
88
+ 7, 36055
89
+ 0, 45899
90
+ 1, 46011
91
+ 2, 45877
92
+ 3, 45949
93
+ 4, 45903
94
+ 5, 45947
95
+ 6, 45929
96
+ 7, 46019
97
+ 0, 46289
98
+ 1, 46409
99
+ 2, 46283
100
+ 3, 46343
101
+ 4, 46295
102
+ 5, 46339
103
+ 6, 46331
104
+ 7, 46409
105
+ 0, 46293
106
+ 1, 46415
107
+ 2, 46287
108
+ 3, 46347
109
+ 4, 46299
110
+ 5, 46345
111
+ 6, 46335
112
+ 7, 46415
113
+ 0, 116085
114
+ 1, 116077
115
+ 2, 116079
116
+ 3, 116073
117
+ 4, 116089
118
+ 5, 116071
119
+ 6, 116063
120
+ 7, 116063
121
+ 0, 139413
122
+ 1, 139405
123
+ 2, 139407
124
+ 3, 139401
125
+ 4, 139417
126
+ 5, 139399
127
+ 6, 140177
128
+ 7, 139405
129
+ 0, 139745
130
+ 1, 139713
131
+ 2, 139755
132
+ 3, 139761
133
+ 4, 139737
134
+ 5, 139721
135
+ 6, 140499
136
+ 7, 139749
137
+ 0, 139801
138
+ 1, 139793
139
+ 2, 139795
140
+ 3, 139791
141
+ 4, 139807
142
+ 5, 139787
143
+ 6, 140565
144
+ 7, 139793
145
+ 0, 142111
146
+ 1, 142103
147
+ 2, 142105
148
+ 3, 142101
149
+ 4, 142117
150
+ 5, 142095
151
+ 6, 142875
152
+ 7, 142103
153
+ 0, 143093
154
+ 1, 143109
155
+ 2, 143157
156
+ 3, 143169
157
+ 4, 143129
158
+ 5, 143121
159
+ 6, 143881
160
+ 7, 143155
161
+ 0, 157149
162
+ 1, 157133
163
+ 2, 157143
164
+ 3, 157135
165
+ 4, 157151
166
+ 5, 157127
167
+ 6, 156339
168
+ 7, 157133
169
+ 0, 157149
170
+ 1, 157133
171
+ 2, 157143
172
+ 3, 157135
173
+ 4, 157151
174
+ 5, 157127
175
+ 6, 156339
176
+ 7, 157133
177
+ 0, 157149
178
+ 1, 157133
179
+ 2, 157143
180
+ 3, 157135
181
+ 4, 157151
182
+ 5, 157127
183
+ 6, 156339
184
+ 7, 157133
185
+ 0, 157149
186
+ 1, 157133
187
+ 2, 157143
188
+ 3, 157135
189
+ 4, 157151
190
+ 5, 157127
191
+ 6, 156339
192
+ 7, 157133
193
+ 0, 157085
194
+ 1, 156871
195
+ 2, 157143
196
+ 3, 156873
197
+ 4, 156831
198
+ 5, 157127
199
+ 6, 156339
200
+ 7, 157133
201
+ 0, 155267
202
+ 1, 154989
203
+ 2, 155261
204
+ 3, 155379
205
+ 4, 155657
206
+ 5, 155245
207
+ 6, 154751
208
+ 7, 154989
209
+ 0, 152553
210
+ 1, 152989
211
+ 2, 153581
212
+ 3, 153375
213
+ 4, 154235
214
+ 5, 153623
215
+ 6, 152777
216
+ 7, 153245
217
+ 0, 4
218
+ 1, 4
219
+ 2, 143043
220
+ 3, 4
221
+ 4, 143447
222
+ 5, 4
223
+ 6, 142239
224
+ 7, 4
225
+ 0, 0
226
+ 1, 0
227
+ 2, 0
228
+ 3, 0
229
+ 4, 0
230
+ 5, 0
231
+ 6, 0
232
+ 7, 0
moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=moonep
2
+ config=DSV2_LITE_L24
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=moonep_backend_token_routing_analysis
6
+ commit=35ae6e7f
7
+ num_layers=24
8
+ hidden=2048
9
+ ffn_hidden=10944
10
+ moe_ffn_hidden=1408
11
+ num_experts=64
12
+ topk=6
13
+ experts_per_rank=8
14
+ expert_params=8650752
15
+ heads=16
16
+ kv_groups=0
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=2816
20
+ shared_expert_overlap=0
21
+ mla=1
22
+ dense_first_layers=1
23
+ moe_layers=23
24
+ rotary_base=10000
25
+ rotary_scaling=40
26
+ qkv_bias=0
27
+ router_balancing=seq_aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=1
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=8192
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=49152
39
+ mean_tokens_per_expert=6144
40
+ slot_policy=matched
41
+ slots_per_rank=8
42
+ capacity_mode=none
43
+ capacity=none
44
+ cuda_graph=0
45
+ probe_mode=1
46
+ async_grad_reduce=0
47
+ branch_role=probe
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=1
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv ADDED
@@ -0,0 +1,192 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 4
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 725
42
+ 1, 725
43
+ 2, 725
44
+ 3, 725
45
+ 4, 725
46
+ 5, 725
47
+ 6, 725
48
+ 7, 725
49
+ 0, 1083
50
+ 1, 1081
51
+ 2, 1083
52
+ 3, 1083
53
+ 4, 1081
54
+ 5, 1083
55
+ 6, 1081
56
+ 7, 1081
57
+ 0, 1863
58
+ 1, 1863
59
+ 2, 1863
60
+ 3, 1863
61
+ 4, 1863
62
+ 5, 1863
63
+ 6, 1863
64
+ 7, 1863
65
+ 0, 6795
66
+ 1, 6791
67
+ 2, 6801
68
+ 3, 6811
69
+ 4, 6813
70
+ 5, 6789
71
+ 6, 6779
72
+ 7, 6797
73
+ 0, 58379
74
+ 1, 57837
75
+ 2, 57837
76
+ 3, 58517
77
+ 4, 58565
78
+ 5, 58493
79
+ 6, 58581
80
+ 7, 58535
81
+ 0, 59427
82
+ 1, 58873
83
+ 2, 58869
84
+ 3, 59577
85
+ 4, 59581
86
+ 5, 59577
87
+ 6, 59577
88
+ 7, 59587
89
+ 0, 59719
90
+ 1, 59151
91
+ 2, 59151
92
+ 3, 59853
93
+ 4, 59851
94
+ 5, 59851
95
+ 6, 59851
96
+ 7, 59851
97
+ 0, 60421
98
+ 1, 59851
99
+ 2, 59851
100
+ 3, 60551
101
+ 4, 60551
102
+ 5, 60551
103
+ 6, 60551
104
+ 7, 60551
105
+ 0, 87081
106
+ 1, 86879
107
+ 2, 90725
108
+ 3, 87037
109
+ 4, 88613
110
+ 5, 88575
111
+ 6, 89863
112
+ 7, 86317
113
+ 0, 117979
114
+ 1, 118017
115
+ 2, 117995
116
+ 3, 118023
117
+ 4, 118031
118
+ 5, 118033
119
+ 6, 118031
120
+ 7, 118041
121
+ 0, 118939
122
+ 1, 118985
123
+ 2, 118967
124
+ 3, 119001
125
+ 4, 119001
126
+ 5, 119001
127
+ 6, 119001
128
+ 7, 119001
129
+ 0, 118939
130
+ 1, 118985
131
+ 2, 118967
132
+ 3, 119001
133
+ 4, 119001
134
+ 5, 119001
135
+ 6, 119001
136
+ 7, 119001
137
+ 0, 119213
138
+ 1, 119257
139
+ 2, 119239
140
+ 3, 119273
141
+ 4, 119273
142
+ 5, 119273
143
+ 6, 119273
144
+ 7, 119273
145
+ 0, 128731
146
+ 1, 128367
147
+ 2, 128533
148
+ 3, 128585
149
+ 4, 128531
150
+ 5, 128497
151
+ 6, 128519
152
+ 7, 128517
153
+ 0, 141327
154
+ 1, 141555
155
+ 2, 141529
156
+ 3, 141355
157
+ 4, 141355
158
+ 5, 141355
159
+ 6, 141355
160
+ 7, 141355
161
+ 0, 141327
162
+ 1, 141555
163
+ 2, 141529
164
+ 3, 141355
165
+ 4, 141355
166
+ 5, 141355
167
+ 6, 141355
168
+ 7, 141355
169
+ 0, 140745
170
+ 1, 141555
171
+ 2, 141225
172
+ 3, 140931
173
+ 4, 140819
174
+ 5, 140709
175
+ 6, 140709
176
+ 7, 140709
177
+ 0, 139407
178
+ 1, 139697
179
+ 2, 138963
180
+ 3, 139177
181
+ 4, 139177
182
+ 5, 139177
183
+ 6, 138789
184
+ 7, 138853
185
+ 0, 12601
186
+ 1, 12604
187
+ 2, 12604
188
+ 3, 12732
189
+ 4, 138787
190
+ 5, 13244
191
+ 6, 12604
192
+ 7, 12604
moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=moonep
2
+ config=QWEN2_57B
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=moonep_backend_token_routing_analysis
6
+ commit=35ae6e7f
7
+ num_layers=10
8
+ hidden=3584
9
+ ffn_hidden=2560
10
+ moe_ffn_hidden=2560
11
+ num_experts=64
12
+ topk=8
13
+ experts_per_rank=8
14
+ expert_params=27525120
15
+ heads=28
16
+ kv_groups=4
17
+ kv_channels=
18
+ qk_layernorm=0
19
+ shared_ffn=20480
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=10
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=1
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-2
29
+ router_pre_softmax=1
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=4096
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=32768
39
+ mean_tokens_per_expert=4096
40
+ slot_policy=matched
41
+ slots_per_rank=8
42
+ capacity_mode=none
43
+ capacity=none
44
+ cuda_graph=0
45
+ probe_mode=1
46
+ async_grad_reduce=0
47
+ branch_role=probe
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/mem_samples.csv ADDED
@@ -0,0 +1,192 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 47
43
+ 2, 629
44
+ 3, 629
45
+ 4, 629
46
+ 5, 47
47
+ 6, 629
48
+ 7, 629
49
+ 0, 927
50
+ 1, 903
51
+ 2, 913
52
+ 3, 903
53
+ 4, 907
54
+ 5, 913
55
+ 6, 929
56
+ 7, 919
57
+ 0, 1803
58
+ 1, 1803
59
+ 2, 1803
60
+ 3, 1803
61
+ 4, 1803
62
+ 5, 1803
63
+ 6, 1803
64
+ 7, 1803
65
+ 0, 6769
66
+ 1, 6769
67
+ 2, 6763
68
+ 3, 6779
69
+ 4, 6785
70
+ 5, 6771
71
+ 6, 6749
72
+ 7, 6769
73
+ 0, 58457
74
+ 1, 57867
75
+ 2, 57895
76
+ 3, 58575
77
+ 4, 58571
78
+ 5, 58555
79
+ 6, 58549
80
+ 7, 58621
81
+ 0, 59439
82
+ 1, 58873
83
+ 2, 58871
84
+ 3, 59571
85
+ 4, 59591
86
+ 5, 59585
87
+ 6, 59581
88
+ 7, 59591
89
+ 0, 59719
90
+ 1, 59151
91
+ 2, 59151
92
+ 3, 59851
93
+ 4, 59851
94
+ 5, 59851
95
+ 6, 59851
96
+ 7, 59851
97
+ 0, 60421
98
+ 1, 59851
99
+ 2, 59851
100
+ 3, 60551
101
+ 4, 60551
102
+ 5, 60551
103
+ 6, 60551
104
+ 7, 60551
105
+ 0, 89393
106
+ 1, 88729
107
+ 2, 84787
108
+ 3, 86505
109
+ 4, 89453
110
+ 5, 86055
111
+ 6, 86483
112
+ 7, 86565
113
+ 0, 118011
114
+ 1, 118087
115
+ 2, 118059
116
+ 3, 118071
117
+ 4, 118061
118
+ 5, 118093
119
+ 6, 118087
120
+ 7, 118101
121
+ 0, 118939
122
+ 1, 118985
123
+ 2, 118967
124
+ 3, 119001
125
+ 4, 119001
126
+ 5, 119001
127
+ 6, 119001
128
+ 7, 119001
129
+ 0, 118939
130
+ 1, 118985
131
+ 2, 118967
132
+ 3, 119001
133
+ 4, 119001
134
+ 5, 119001
135
+ 6, 119001
136
+ 7, 119001
137
+ 0, 119601
138
+ 1, 119645
139
+ 2, 119629
140
+ 3, 119663
141
+ 4, 119663
142
+ 5, 119663
143
+ 6, 119663
144
+ 7, 119663
145
+ 0, 128675
146
+ 1, 128377
147
+ 2, 128485
148
+ 3, 128489
149
+ 4, 128499
150
+ 5, 128543
151
+ 6, 128461
152
+ 7, 128469
153
+ 0, 141327
154
+ 1, 141555
155
+ 2, 141531
156
+ 3, 141357
157
+ 4, 141357
158
+ 5, 141357
159
+ 6, 141357
160
+ 7, 141357
161
+ 0, 141327
162
+ 1, 141555
163
+ 2, 141531
164
+ 3, 141357
165
+ 4, 141357
166
+ 5, 141357
167
+ 6, 141357
168
+ 7, 141357
169
+ 0, 140681
170
+ 1, 140909
171
+ 2, 140885
172
+ 3, 140711
173
+ 4, 140711
174
+ 5, 140711
175
+ 6, 140711
176
+ 7, 140711
177
+ 0, 138373
178
+ 1, 138989
179
+ 2, 139353
180
+ 3, 138791
181
+ 4, 138403
182
+ 5, 138791
183
+ 6, 138403
184
+ 7, 138791
185
+ 0, 12601
186
+ 1, 12604
187
+ 2, 12732
188
+ 3, 138789
189
+ 4, 13244
190
+ 5, 12604
191
+ 6, 12604
192
+ 7, 12604
moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=moonep
2
+ config=QWEN2_57B
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=moonep_backend_token_routing_analysis
6
+ commit=35ae6e7f
7
+ num_layers=10
8
+ hidden=3584
9
+ ffn_hidden=2560
10
+ moe_ffn_hidden=2560
11
+ num_experts=64
12
+ topk=8
13
+ experts_per_rank=8
14
+ expert_params=27525120
15
+ heads=28
16
+ kv_groups=4
17
+ kv_channels=
18
+ qk_layernorm=0
19
+ shared_ffn=20480
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=10
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=1
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-2
29
+ router_pre_softmax=1
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=4096
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=32768
39
+ mean_tokens_per_expert=4096
40
+ slot_policy=matched
41
+ slots_per_rank=8
42
+ capacity_mode=none
43
+ capacity=none
44
+ cuda_graph=0
45
+ probe_mode=1
46
+ async_grad_reduce=0
47
+ branch_role=probe
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=1
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1/mem_samples.csv ADDED
@@ -0,0 +1,192 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 4
43
+ 2, 4
44
+ 3, 4
45
+ 4, 4
46
+ 5, 4
47
+ 6, 4
48
+ 7, 4
49
+ 0, 821
50
+ 1, 819
51
+ 2, 819
52
+ 3, 819
53
+ 4, 819
54
+ 5, 821
55
+ 6, 819
56
+ 7, 819
57
+ 0, 1537
58
+ 1, 1537
59
+ 2, 1537
60
+ 3, 1537
61
+ 4, 1537
62
+ 5, 1537
63
+ 6, 1537
64
+ 7, 1537
65
+ 0, 2551
66
+ 1, 2551
67
+ 2, 2551
68
+ 3, 2551
69
+ 4, 2551
70
+ 5, 2551
71
+ 6, 2551
72
+ 7, 2551
73
+ 0, 14899
74
+ 1, 14901
75
+ 2, 14901
76
+ 3, 14901
77
+ 4, 14901
78
+ 5, 14901
79
+ 6, 14901
80
+ 7, 14901
81
+ 0, 70421
82
+ 1, 70395
83
+ 2, 70425
84
+ 3, 70451
85
+ 4, 70369
86
+ 5, 70445
87
+ 6, 70397
88
+ 7, 70417
89
+ 0, 72743
90
+ 1, 72743
91
+ 2, 72743
92
+ 3, 72743
93
+ 4, 72743
94
+ 5, 72743
95
+ 6, 72743
96
+ 7, 72743
97
+ 0, 73289
98
+ 1, 73289
99
+ 2, 73289
100
+ 3, 73289
101
+ 4, 73289
102
+ 5, 73289
103
+ 6, 73289
104
+ 7, 73289
105
+ 0, 73291
106
+ 1, 73291
107
+ 2, 73291
108
+ 3, 73291
109
+ 4, 73291
110
+ 5, 73291
111
+ 6, 73291
112
+ 7, 73291
113
+ 0, 94717
114
+ 1, 94815
115
+ 2, 94405
116
+ 3, 106753
117
+ 4, 94629
118
+ 5, 94827
119
+ 6, 108453
120
+ 7, 95571
121
+ 0, 131701
122
+ 1, 134425
123
+ 2, 131723
124
+ 3, 140655
125
+ 4, 131747
126
+ 5, 134239
127
+ 6, 140757
128
+ 7, 135495
129
+ 0, 160705
130
+ 1, 160761
131
+ 2, 160735
132
+ 3, 160765
133
+ 4, 160745
134
+ 5, 160763
135
+ 6, 160743
136
+ 7, 160771
137
+ 0, 161649
138
+ 1, 161685
139
+ 2, 161685
140
+ 3, 161685
141
+ 4, 161697
142
+ 5, 161685
143
+ 6, 161685
144
+ 7, 161685
145
+ 0, 161649
146
+ 1, 161685
147
+ 2, 161685
148
+ 3, 161685
149
+ 4, 161697
150
+ 5, 161685
151
+ 6, 161685
152
+ 7, 161685
153
+ 0, 170517
154
+ 1, 170649
155
+ 2, 170457
156
+ 3, 170649
157
+ 4, 170637
158
+ 5, 170265
159
+ 6, 170649
160
+ 7, 170649
161
+ 0, 171217
162
+ 1, 171279
163
+ 2, 171359
164
+ 3, 171329
165
+ 4, 171299
166
+ 5, 171331
167
+ 6, 171335
168
+ 7, 171295
169
+ 0, 182495
170
+ 1, 182575
171
+ 2, 182575
172
+ 3, 182575
173
+ 4, 182589
174
+ 5, 182575
175
+ 6, 182575
176
+ 7, 182575
177
+ 0, 182495
178
+ 1, 182575
179
+ 2, 182575
180
+ 3, 182575
181
+ 4, 182589
182
+ 5, 182575
183
+ 6, 182575
184
+ 7, 182575
185
+ 0, 182495
186
+ 1, 182575
187
+ 2, 182575
188
+ 3, 182575
189
+ 4, 182589
190
+ 5, 182575
191
+ 6, 182575
192
+ 7, 182575
moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=moonep
2
+ config=QWEN3_235B
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=moonep_backend_token_routing_analysis
6
+ commit=35ae6e7f
7
+ num_layers=12
8
+ hidden=4096
9
+ ffn_hidden=12288
10
+ moe_ffn_hidden=1536
11
+ num_experts=128
12
+ topk=8
13
+ experts_per_rank=16
14
+ expert_params=18874368
15
+ heads=64
16
+ kv_groups=4
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=0
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=12
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=0
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=0
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=8192
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=65536
39
+ mean_tokens_per_expert=4096
40
+ slot_policy=matched
41
+ slots_per_rank=16
42
+ capacity_mode=none
43
+ capacity=none
44
+ cuda_graph=0
45
+ probe_mode=1
46
+ async_grad_reduce=0
47
+ branch_role=probe
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv ADDED
@@ -0,0 +1,184 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 629
43
+ 2, 629
44
+ 3, 629
45
+ 4, 629
46
+ 5, 629
47
+ 6, 629
48
+ 7, 629
49
+ 0, 943
50
+ 1, 937
51
+ 2, 935
52
+ 3, 975
53
+ 4, 947
54
+ 5, 945
55
+ 6, 941
56
+ 7, 931
57
+ 0, 1831
58
+ 1, 1831
59
+ 2, 1831
60
+ 3, 1831
61
+ 4, 1831
62
+ 5, 1831
63
+ 6, 1831
64
+ 7, 1831
65
+ 0, 3789
66
+ 1, 3793
67
+ 2, 3789
68
+ 3, 3781
69
+ 4, 3785
70
+ 5, 3803
71
+ 6, 3809
72
+ 7, 3787
73
+ 0, 15563
74
+ 1, 15565
75
+ 2, 15565
76
+ 3, 15565
77
+ 4, 15565
78
+ 5, 15565
79
+ 6, 15565
80
+ 7, 15565
81
+ 0, 57141
82
+ 1, 57099
83
+ 2, 57365
84
+ 3, 57003
85
+ 4, 57267
86
+ 5, 57177
87
+ 6, 57159
88
+ 7, 57365
89
+ 0, 67479
90
+ 1, 67481
91
+ 2, 67459
92
+ 3, 67409
93
+ 4, 67481
94
+ 5, 67459
95
+ 6, 67481
96
+ 7, 67495
97
+ 0, 68527
98
+ 1, 68565
99
+ 2, 68543
100
+ 3, 68493
101
+ 4, 68529
102
+ 5, 68543
103
+ 6, 68565
104
+ 7, 68543
105
+ 0, 68673
106
+ 1, 68711
107
+ 2, 68689
108
+ 3, 68639
109
+ 4, 68675
110
+ 5, 68689
111
+ 6, 68711
112
+ 7, 68689
113
+ 0, 68675
114
+ 1, 68713
115
+ 2, 68691
116
+ 3, 68641
117
+ 4, 68677
118
+ 5, 68691
119
+ 6, 68713
120
+ 7, 68691
121
+ 0, 87971
122
+ 1, 87467
123
+ 2, 84739
124
+ 3, 87193
125
+ 4, 85637
126
+ 5, 86851
127
+ 6, 85601
128
+ 7, 86233
129
+ 0, 125825
130
+ 1, 125839
131
+ 2, 125529
132
+ 3, 125837
133
+ 4, 125833
134
+ 5, 125825
135
+ 6, 125839
136
+ 7, 125825
137
+ 0, 155829
138
+ 1, 155843
139
+ 2, 155761
140
+ 3, 155841
141
+ 4, 155835
142
+ 5, 155829
143
+ 6, 155843
144
+ 7, 155829
145
+ 0, 156475
146
+ 1, 156489
147
+ 2, 156475
148
+ 3, 156487
149
+ 4, 156483
150
+ 5, 156475
151
+ 6, 156489
152
+ 7, 156475
153
+ 0, 156865
154
+ 1, 156879
155
+ 2, 156865
156
+ 3, 156877
157
+ 4, 156873
158
+ 5, 156865
159
+ 6, 156879
160
+ 7, 156865
161
+ 0, 156879
162
+ 1, 156893
163
+ 2, 156879
164
+ 3, 156891
165
+ 4, 156887
166
+ 5, 156879
167
+ 6, 156893
168
+ 7, 156879
169
+ 0, 167327
170
+ 1, 167239
171
+ 2, 167349
172
+ 3, 167187
173
+ 4, 167269
174
+ 5, 167333
175
+ 6, 167479
176
+ 7, 167391
177
+ 0, 169195
178
+ 1, 168833
179
+ 2, 168939
180
+ 3, 168923
181
+ 4, 168847
182
+ 5, 168939
183
+ 6, 169109
184
+ 7, 169083
moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=moonep
2
+ config=QWEN3_30B
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=moonep_backend_token_routing_analysis
6
+ commit=35ae6e7f
7
+ num_layers=48
8
+ hidden=2048
9
+ ffn_hidden=6144
10
+ moe_ffn_hidden=768
11
+ num_experts=128
12
+ topk=8
13
+ experts_per_rank=16
14
+ expert_params=4718592
15
+ heads=32
16
+ kv_groups=4
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=0
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=48
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=0
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=0
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=4096
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=32768
39
+ mean_tokens_per_expert=2048
40
+ slot_policy=matched
41
+ slots_per_rank=16
42
+ capacity_mode=none
43
+ capacity=none
44
+ cuda_graph=0
45
+ probe_mode=1
46
+ async_grad_reduce=0
47
+ branch_role=probe
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=12
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir
2
+ QWEN3_235B moonep natural none OOM 182589 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1
3
+ QWEN3_30B moonep natural none OOM 169195 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1
4
+ DSV2_LITE_L24 moonep natural none OK 157151 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1
5
+ DSV2_LITE_L24 moonep forced none OK 157151 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1
6
+ QWEN2_57B moonep natural none OK 141555 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1
7
+ QWEN2_57B moonep forced none OK 141555 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1
moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir
2
+ QWEN3_235B moonep DOES_NOT_FIT 182589 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1
3
+ QWEN3_30B moonep DOES_NOT_FIT 169195 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1
4
+ DSV2_LITE_L24 moonep SELECTED none 0 157151 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1
5
+ QWEN2_57B moonep SELECTED none 0 141555 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1
moonep/tierbprobe/tierbprobe_gbs8/probe_summary.json ADDED
@@ -0,0 +1,454 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "arm": "natural",
4
+ "aux_loss_coeff": "1e-3",
5
+ "backend": "moonep",
6
+ "branch": "moonep_backend_token_routing_analysis",
7
+ "capacity": "none",
8
+ "capacity_mode": "none",
9
+ "cold_expert_fraction_max": 0.578125,
10
+ "cold_expert_fraction_mean": 0.15625,
11
+ "cold_expert_fraction_min": 0.0,
12
+ "cold_expert_fraction_n": 230,
13
+ "commit": "35ae6e7f",
14
+ "config": "DSV2_LITE_L24",
15
+ "cuda_graph": "0",
16
+ "dense_first_layers": "1",
17
+ "experts_per_rank": "8",
18
+ "force_balance": "0",
19
+ "global_batch_size": "8",
20
+ "global_snapshots": 230,
21
+ "hottest_expert_token_fraction_max": 0.999969482421875,
22
+ "hottest_expert_token_fraction_mean": 0.7797577567722487,
23
+ "hottest_expert_token_fraction_min": 0.162750244140625,
24
+ "hottest_expert_token_fraction_n": 230,
25
+ "incomplete_snapshots": 0,
26
+ "malformed_records": 0,
27
+ "moe_layers": "23",
28
+ "moonep_bytes_global_snapshots": 230,
29
+ "moonep_bytes_incomplete_snapshots": 0,
30
+ "moonep_bytes_malformed_records": 0,
31
+ "moonep_global_combine_payload_bytes_max": 2592899072.0,
32
+ "moonep_global_combine_payload_bytes_mean": 2282338571.130435,
33
+ "moonep_global_combine_payload_bytes_min": 2026799104.0,
34
+ "moonep_global_combine_payload_bytes_n": 230,
35
+ "moonep_global_dispatch_payload_bytes_max": 2592899072.0,
36
+ "moonep_global_dispatch_payload_bytes_mean": 2282338571.130435,
37
+ "moonep_global_dispatch_payload_bytes_min": 2026799104.0,
38
+ "moonep_global_dispatch_payload_bytes_n": 230,
39
+ "moonep_global_prefetch_bytes_max": 346030080.0,
40
+ "moonep_global_prefetch_bytes_mean": 252601958.4,
41
+ "moonep_global_prefetch_bytes_min": 242221056.0,
42
+ "moonep_global_prefetch_bytes_n": 230,
43
+ "moonep_global_reduce_bytes_max": 346030080.0,
44
+ "moonep_global_reduce_bytes_mean": 252601958.4,
45
+ "moonep_global_reduce_bytes_min": 242221056.0,
46
+ "moonep_global_reduce_bytes_n": 230,
47
+ "moonep_global_token_bytes_max": 5188482536.0,
48
+ "moonep_global_token_bytes_mean": 4567339392.869565,
49
+ "moonep_global_token_bytes_min": 4056064752.0,
50
+ "moonep_global_token_bytes_n": 230,
51
+ "moonep_global_total_nvlink_bytes_max": 5672924648.0,
52
+ "moonep_global_total_nvlink_bytes_mean": 5072543309.669565,
53
+ "moonep_global_total_nvlink_bytes_min": 4540506864.0,
54
+ "moonep_global_total_nvlink_bytes_n": 230,
55
+ "moonep_global_weight_movement_bytes_max": 692060160.0,
56
+ "moonep_global_weight_movement_bytes_mean": 505203916.8,
57
+ "moonep_global_weight_movement_bytes_min": 484442112.0,
58
+ "moonep_global_weight_movement_bytes_n": 230,
59
+ "moonep_global_weight_to_token_ratio_max": 0.14851639788299403,
60
+ "moonep_global_weight_to_token_ratio_mean": 0.11096021713514241,
61
+ "moonep_global_weight_to_token_ratio_min": 0.09336874676530664,
62
+ "moonep_global_weight_to_token_ratio_n": 230,
63
+ "moonep_remote_demand_max": 3.0,
64
+ "moonep_remote_demand_mean": 0.9125,
65
+ "moonep_remote_demand_min": 0.0,
66
+ "moonep_remote_demand_n": 1840,
67
+ "moonep_slot_utilization_max": 0.375,
68
+ "moonep_slot_utilization_mean": 0.1140625,
69
+ "moonep_slot_utilization_min": 0.0,
70
+ "moonep_slot_utilization_n": 1840,
71
+ "moonep_unused_prefetch_slots_max": 16.0,
72
+ "moonep_unused_prefetch_slots_mean": 14.175,
73
+ "moonep_unused_prefetch_slots_min": 10.0,
74
+ "moonep_unused_prefetch_slots_n": 1840,
75
+ "num_experts": "64",
76
+ "num_layers": "24",
77
+ "post_physical_rank_cv_max": 0.24644338469771185,
78
+ "post_physical_rank_cv_mean": 0.0866314272142518,
79
+ "post_physical_rank_cv_min": 0.016159622971429973,
80
+ "post_physical_rank_cv_n": 230,
81
+ "post_physical_rank_imax_max": 1.2001146488570558,
82
+ "post_physical_rank_imax_mean": 1.1121655106296537,
83
+ "post_physical_rank_imax_min": 1.0188653892666146,
84
+ "post_physical_rank_imax_n": 230,
85
+ "pre_home_rank_cv_max": 1.3626457501059894,
86
+ "pre_home_rank_cv_mean": 0.6592086328754161,
87
+ "pre_home_rank_cv_min": 0.06474057713157837,
88
+ "pre_home_rank_cv_n": 230,
89
+ "pre_home_rank_imax_max": 4.328043619791667,
90
+ "pre_home_rank_imax_mean": 2.2370494897814766,
91
+ "pre_home_rank_imax_min": 1.1056925455729167,
92
+ "pre_home_rank_imax_n": 230,
93
+ "raw_expert_cv_max": 3.0868156557140782,
94
+ "raw_expert_cv_mean": 1.9632045302354437,
95
+ "raw_expert_cv_min": 0.37852206422070017,
96
+ "raw_expert_cv_n": 230,
97
+ "raw_expert_imax_ceiling": 10.666666666666666,
98
+ "raw_expert_imax_hit_ceiling": 1,
99
+ "raw_expert_imax_max": 10.666341145833334,
100
+ "raw_expert_imax_mean": 8.317416072237318,
101
+ "raw_expert_imax_min": 1.7360026041666667,
102
+ "raw_expert_imax_n": 230,
103
+ "records": {
104
+ "moonep_bytes": 1840,
105
+ "moonep_routing": 1840,
106
+ "moonep_summary": 80,
107
+ "moonep_weights": 1840
108
+ },
109
+ "router_balancing": "seq_aux_loss",
110
+ "run_dir": "DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1",
111
+ "seq_length": "8192",
112
+ "shared_expert_overlap": "0",
113
+ "topk": "6"
114
+ },
115
+ {
116
+ "arm": "forced",
117
+ "aux_loss_coeff": "1e-3",
118
+ "backend": "moonep",
119
+ "branch": "moonep_backend_token_routing_analysis",
120
+ "capacity": "none",
121
+ "capacity_mode": "none",
122
+ "cold_expert_fraction_max": 0.0,
123
+ "cold_expert_fraction_mean": 0.0,
124
+ "cold_expert_fraction_min": 0.0,
125
+ "cold_expert_fraction_n": 230,
126
+ "commit": "35ae6e7f",
127
+ "config": "DSV2_LITE_L24",
128
+ "cuda_graph": "0",
129
+ "dense_first_layers": "1",
130
+ "experts_per_rank": "8",
131
+ "force_balance": "1",
132
+ "global_batch_size": "8",
133
+ "global_snapshots": 230,
134
+ "hottest_expert_token_fraction_max": 0.0982818603515625,
135
+ "hottest_expert_token_fraction_mean": 0.09643501613451087,
136
+ "hottest_expert_token_fraction_min": 0.09539794921875,
137
+ "hottest_expert_token_fraction_n": 230,
138
+ "incomplete_snapshots": 0,
139
+ "malformed_records": 0,
140
+ "moe_layers": "23",
141
+ "moonep_bytes_global_snapshots": 230,
142
+ "moonep_bytes_incomplete_snapshots": 0,
143
+ "moonep_bytes_malformed_records": 0,
144
+ "moonep_global_combine_payload_bytes_max": 2133762048.0,
145
+ "moonep_global_combine_payload_bytes_mean": 2129465023.4434783,
146
+ "moonep_global_combine_payload_bytes_min": 2124537856.0,
147
+ "moonep_global_combine_payload_bytes_n": 230,
148
+ "moonep_global_dispatch_payload_bytes_max": 2133762048.0,
149
+ "moonep_global_dispatch_payload_bytes_mean": 2129465023.4434783,
150
+ "moonep_global_dispatch_payload_bytes_min": 2124537856.0,
151
+ "moonep_global_dispatch_payload_bytes_n": 230,
152
+ "moonep_global_prefetch_bytes_max": 242221056.0,
153
+ "moonep_global_prefetch_bytes_mean": 238309411.61739132,
154
+ "moonep_global_prefetch_bytes_min": 207618048.0,
155
+ "moonep_global_prefetch_bytes_n": 230,
156
+ "moonep_global_reduce_bytes_max": 242221056.0,
157
+ "moonep_global_reduce_bytes_mean": 238309411.61739132,
158
+ "moonep_global_reduce_bytes_min": 207618048.0,
159
+ "moonep_global_reduce_bytes_n": 230,
160
+ "moonep_global_token_bytes_max": 4270279408.0,
161
+ "moonep_global_token_bytes_mean": 4261681018.2608695,
162
+ "moonep_global_token_bytes_min": 4251824656.0,
163
+ "moonep_global_token_bytes_n": 230,
164
+ "moonep_global_total_nvlink_bytes_max": 4754721520.0,
165
+ "moonep_global_total_nvlink_bytes_mean": 4738299841.495652,
166
+ "moonep_global_total_nvlink_bytes_min": 4667434440.0,
167
+ "moonep_global_total_nvlink_bytes_n": 230,
168
+ "moonep_global_weight_movement_bytes_max": 484442112.0,
169
+ "moonep_global_weight_movement_bytes_mean": 476618823.23478264,
170
+ "moonep_global_weight_movement_bytes_min": 415236096.0,
171
+ "moonep_global_weight_movement_bytes_n": 230,
172
+ "moonep_global_weight_to_token_ratio_max": 0.11393746242954192,
173
+ "moonep_global_weight_to_token_ratio_mean": 0.11183841293995254,
174
+ "moonep_global_weight_to_token_ratio_min": 0.09728537714366937,
175
+ "moonep_global_weight_to_token_ratio_n": 230,
176
+ "moonep_remote_demand_max": 1.0,
177
+ "moonep_remote_demand_mean": 0.8608695652173913,
178
+ "moonep_remote_demand_min": 0.0,
179
+ "moonep_remote_demand_n": 1840,
180
+ "moonep_slot_utilization_max": 0.125,
181
+ "moonep_slot_utilization_mean": 0.10760869565217392,
182
+ "moonep_slot_utilization_min": 0.0,
183
+ "moonep_slot_utilization_n": 1840,
184
+ "moonep_unused_prefetch_slots_max": 16.0,
185
+ "moonep_unused_prefetch_slots_mean": 14.278260869565218,
186
+ "moonep_unused_prefetch_slots_min": 14.0,
187
+ "moonep_unused_prefetch_slots_n": 1840,
188
+ "num_experts": "64",
189
+ "num_layers": "24",
190
+ "post_physical_rank_cv_max": 0.0031667036922078573,
191
+ "post_physical_rank_cv_mean": 0.0016501228109707747,
192
+ "post_physical_rank_cv_min": 0.0006791931370339787,
193
+ "post_physical_rank_cv_n": 230,
194
+ "post_physical_rank_imax_max": 1.006201201251668,
195
+ "post_physical_rank_imax_mean": 1.0026117364992109,
196
+ "post_physical_rank_imax_min": 1.0009196904719393,
197
+ "post_physical_rank_imax_n": 230,
198
+ "pre_home_rank_cv_max": 0.006673533732490181,
199
+ "pre_home_rank_cv_mean": 0.003923942735502633,
200
+ "pre_home_rank_cv_min": 0.0014708009315538319,
201
+ "pre_home_rank_cv_n": 230,
202
+ "pre_home_rank_imax_max": 1.0122884114583333,
203
+ "pre_home_rank_imax_mean": 1.0061054616734602,
204
+ "pre_home_rank_imax_min": 1.00213623046875,
205
+ "pre_home_rank_imax_n": 230,
206
+ "raw_expert_cv_max": 0.015123587213954862,
207
+ "raw_expert_cv_mean": 0.012068912140234289,
208
+ "raw_expert_cv_min": 0.008171875787738593,
209
+ "raw_expert_cv_n": 230,
210
+ "raw_expert_imax_ceiling": 10.666666666666666,
211
+ "raw_expert_imax_hit_ceiling": 0,
212
+ "raw_expert_imax_max": 1.04833984375,
213
+ "raw_expert_imax_mean": 1.0286401721014493,
214
+ "raw_expert_imax_min": 1.017578125,
215
+ "raw_expert_imax_n": 230,
216
+ "records": {
217
+ "moonep_bytes": 1840,
218
+ "moonep_routing": 1840,
219
+ "moonep_summary": 80,
220
+ "moonep_weights": 1840
221
+ },
222
+ "router_balancing": "seq_aux_loss",
223
+ "run_dir": "DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1",
224
+ "seq_length": "8192",
225
+ "shared_expert_overlap": "0",
226
+ "topk": "6"
227
+ },
228
+ {
229
+ "arm": "natural",
230
+ "aux_loss_coeff": "1e-2",
231
+ "backend": "moonep",
232
+ "branch": "moonep_backend_token_routing_analysis",
233
+ "capacity": "none",
234
+ "capacity_mode": "none",
235
+ "cold_expert_fraction_max": 0.796875,
236
+ "cold_expert_fraction_mean": 0.3940625,
237
+ "cold_expert_fraction_min": 0.0,
238
+ "cold_expert_fraction_n": 100,
239
+ "commit": "35ae6e7f",
240
+ "config": "QWEN2_57B",
241
+ "cuda_graph": "0",
242
+ "dense_first_layers": "0",
243
+ "experts_per_rank": "8",
244
+ "force_balance": "0",
245
+ "global_batch_size": "8",
246
+ "global_snapshots": 100,
247
+ "hottest_expert_token_fraction_max": 1.0,
248
+ "hottest_expert_token_fraction_mean": 0.923837890625,
249
+ "hottest_expert_token_fraction_min": 0.238525390625,
250
+ "hottest_expert_token_fraction_n": 100,
251
+ "incomplete_snapshots": 0,
252
+ "malformed_records": 0,
253
+ "moe_layers": "10",
254
+ "moonep_bytes_global_snapshots": 100,
255
+ "moonep_bytes_incomplete_snapshots": 0,
256
+ "moonep_bytes_malformed_records": 0,
257
+ "moonep_global_combine_payload_bytes_max": 3287474176.0,
258
+ "moonep_global_combine_payload_bytes_mean": 2792054415.36,
259
+ "moonep_global_combine_payload_bytes_min": 2200748032.0,
260
+ "moonep_global_combine_payload_bytes_n": 100,
261
+ "moonep_global_dispatch_payload_bytes_max": 3287474176.0,
262
+ "moonep_global_dispatch_payload_bytes_mean": 2792054415.36,
263
+ "moonep_global_dispatch_payload_bytes_min": 2200748032.0,
264
+ "moonep_global_dispatch_payload_bytes_n": 100,
265
+ "moonep_global_prefetch_bytes_max": 990904320.0,
266
+ "moonep_global_prefetch_bytes_mean": 826854604.8,
267
+ "moonep_global_prefetch_bytes_min": 660602880.0,
268
+ "moonep_global_prefetch_bytes_n": 100,
269
+ "moonep_global_reduce_bytes_max": 990904320.0,
270
+ "moonep_global_reduce_bytes_mean": 826854604.8,
271
+ "moonep_global_reduce_bytes_min": 660602880.0,
272
+ "moonep_global_reduce_bytes_n": 100,
273
+ "moonep_global_token_bytes_max": 6576783336.0,
274
+ "moonep_global_token_bytes_mean": 5585911141.52,
275
+ "moonep_global_token_bytes_min": 4403305176.0,
276
+ "moonep_global_token_bytes_n": 100,
277
+ "moonep_global_total_nvlink_bytes_max": 8345010168.0,
278
+ "moonep_global_total_nvlink_bytes_mean": 7239620351.12,
279
+ "moonep_global_total_nvlink_bytes_min": 5944711896.0,
280
+ "moonep_global_total_nvlink_bytes_n": 100,
281
+ "moonep_global_weight_movement_bytes_max": 1981808640.0,
282
+ "moonep_global_weight_movement_bytes_mean": 1653709209.6,
283
+ "moonep_global_weight_movement_bytes_min": 1321205760.0,
284
+ "moonep_global_weight_movement_bytes_n": 100,
285
+ "moonep_global_weight_to_token_ratio_max": 0.4264504160968184,
286
+ "moonep_global_weight_to_token_ratio_mean": 0.3001723215454669,
287
+ "moonep_global_weight_to_token_ratio_min": 0.20088935464362695,
288
+ "moonep_global_weight_to_token_ratio_n": 100,
289
+ "moonep_remote_demand_max": 2.0,
290
+ "moonep_remote_demand_mean": 0.93875,
291
+ "moonep_remote_demand_min": 0.0,
292
+ "moonep_remote_demand_n": 800,
293
+ "moonep_slot_utilization_max": 0.25,
294
+ "moonep_slot_utilization_mean": 0.11734375,
295
+ "moonep_slot_utilization_min": 0.0,
296
+ "moonep_slot_utilization_n": 800,
297
+ "moonep_unused_prefetch_slots_max": 16.0,
298
+ "moonep_unused_prefetch_slots_mean": 14.1225,
299
+ "moonep_unused_prefetch_slots_min": 12.0,
300
+ "moonep_unused_prefetch_slots_n": 800,
301
+ "num_experts": "64",
302
+ "num_layers": "10",
303
+ "post_physical_rank_cv_max": 0.19752362049364244,
304
+ "post_physical_rank_cv_mean": 0.08253319288939637,
305
+ "post_physical_rank_cv_min": 0.0001808697752907129,
306
+ "post_physical_rank_cv_n": 100,
307
+ "post_physical_rank_imax_max": 1.2480128708509701,
308
+ "post_physical_rank_imax_mean": 1.1014610244938725,
309
+ "post_physical_rank_imax_min": 1.0002403836981697,
310
+ "post_physical_rank_imax_n": 100,
311
+ "pre_home_rank_cv_max": 1.3115485671633254,
312
+ "pre_home_rank_cv_mean": 0.6864519759847463,
313
+ "pre_home_rank_cv_min": 0.05302373213438137,
314
+ "pre_home_rank_cv_n": 100,
315
+ "pre_home_rank_imax_max": 3.8150634765625,
316
+ "pre_home_rank_imax_mean": 2.2216781616210937,
317
+ "pre_home_rank_imax_min": 1.113861083984375,
318
+ "pre_home_rank_imax_n": 100,
319
+ "raw_expert_cv_max": 2.6451862448859087,
320
+ "raw_expert_cv_mean": 2.182940410315874,
321
+ "raw_expert_cv_min": 0.5018423789492765,
322
+ "raw_expert_cv_n": 100,
323
+ "raw_expert_imax_ceiling": 8.0,
324
+ "raw_expert_imax_hit_ceiling": 1,
325
+ "raw_expert_imax_max": 8.0,
326
+ "raw_expert_imax_mean": 7.390703125,
327
+ "raw_expert_imax_min": 1.908203125,
328
+ "raw_expert_imax_n": 100,
329
+ "records": {
330
+ "moonep_bytes": 800,
331
+ "moonep_routing": 800,
332
+ "moonep_summary": 80,
333
+ "moonep_weights": 800
334
+ },
335
+ "router_balancing": "aux_loss",
336
+ "run_dir": "QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1",
337
+ "seq_length": "4096",
338
+ "shared_expert_overlap": "0",
339
+ "topk": "8"
340
+ },
341
+ {
342
+ "arm": "forced",
343
+ "aux_loss_coeff": "1e-2",
344
+ "backend": "moonep",
345
+ "branch": "moonep_backend_token_routing_analysis",
346
+ "capacity": "none",
347
+ "capacity_mode": "none",
348
+ "cold_expert_fraction_max": 0.0,
349
+ "cold_expert_fraction_mean": 0.0,
350
+ "cold_expert_fraction_min": 0.0,
351
+ "cold_expert_fraction_n": 100,
352
+ "commit": "35ae6e7f",
353
+ "config": "QWEN2_57B",
354
+ "cuda_graph": "0",
355
+ "dense_first_layers": "0",
356
+ "experts_per_rank": "8",
357
+ "force_balance": "1",
358
+ "global_batch_size": "8",
359
+ "global_snapshots": 100,
360
+ "hottest_expert_token_fraction_max": 0.13189697265625,
361
+ "hottest_expert_token_fraction_mean": 0.1293536376953125,
362
+ "hottest_expert_token_fraction_min": 0.1279296875,
363
+ "hottest_expert_token_fraction_n": 100,
364
+ "incomplete_snapshots": 0,
365
+ "malformed_records": 0,
366
+ "moe_layers": "10",
367
+ "moonep_bytes_global_snapshots": 100,
368
+ "moonep_bytes_incomplete_snapshots": 0,
369
+ "moonep_bytes_malformed_records": 0,
370
+ "moonep_global_combine_payload_bytes_max": 2236903424.0,
371
+ "moonep_global_combine_payload_bytes_mean": 2231590215.68,
372
+ "moonep_global_combine_payload_bytes_min": 2225391616.0,
373
+ "moonep_global_combine_payload_bytes_n": 100,
374
+ "moonep_global_dispatch_payload_bytes_max": 2236903424.0,
375
+ "moonep_global_dispatch_payload_bytes_mean": 2231590215.68,
376
+ "moonep_global_dispatch_payload_bytes_min": 2225391616.0,
377
+ "moonep_global_dispatch_payload_bytes_n": 100,
378
+ "moonep_global_prefetch_bytes_max": 770703360.0,
379
+ "moonep_global_prefetch_bytes_mean": 761895321.6,
380
+ "moonep_global_prefetch_bytes_min": 550502400.0,
381
+ "moonep_global_prefetch_bytes_n": 100,
382
+ "moonep_global_reduce_bytes_max": 770703360.0,
383
+ "moonep_global_reduce_bytes_mean": 761895321.6,
384
+ "moonep_global_reduce_bytes_min": 550502400.0,
385
+ "moonep_global_reduce_bytes_n": 100,
386
+ "moonep_global_token_bytes_max": 4475642264.0,
387
+ "moonep_global_token_bytes_mean": 4465014321.2,
388
+ "moonep_global_token_bytes_min": 4452613880.0,
389
+ "moonep_global_token_bytes_n": 100,
390
+ "moonep_global_total_nvlink_bytes_max": 6017048984.0,
391
+ "moonep_global_total_nvlink_bytes_mean": 5988804964.4,
392
+ "moonep_global_total_nvlink_bytes_min": 5568474336.0,
393
+ "moonep_global_total_nvlink_bytes_n": 100,
394
+ "moonep_global_weight_movement_bytes_max": 1541406720.0,
395
+ "moonep_global_weight_movement_bytes_mean": 1523790643.2,
396
+ "moonep_global_weight_movement_bytes_min": 1101004800.0,
397
+ "moonep_global_weight_movement_bytes_n": 100,
398
+ "moonep_global_weight_to_token_ratio_max": 0.34618019023019353,
399
+ "moonep_global_weight_to_token_ratio_mean": 0.3412749588086127,
400
+ "moonep_global_weight_to_token_ratio_min": 0.2464493134486592,
401
+ "moonep_global_weight_to_token_ratio_n": 100,
402
+ "moonep_remote_demand_max": 1.0,
403
+ "moonep_remote_demand_mean": 0.865,
404
+ "moonep_remote_demand_min": 0.0,
405
+ "moonep_remote_demand_n": 800,
406
+ "moonep_slot_utilization_max": 0.125,
407
+ "moonep_slot_utilization_mean": 0.108125,
408
+ "moonep_slot_utilization_min": 0.0,
409
+ "moonep_slot_utilization_n": 800,
410
+ "moonep_unused_prefetch_slots_max": 16.0,
411
+ "moonep_unused_prefetch_slots_mean": 14.27,
412
+ "moonep_unused_prefetch_slots_min": 14.0,
413
+ "moonep_unused_prefetch_slots_n": 800,
414
+ "num_experts": "64",
415
+ "num_layers": "10",
416
+ "post_physical_rank_cv_max": 0.004151405986167901,
417
+ "post_physical_rank_cv_mean": 0.002179447588207395,
418
+ "post_physical_rank_cv_min": 0.0007203705850357968,
419
+ "post_physical_rank_cv_n": 100,
420
+ "post_physical_rank_imax_max": 1.0062412574645103,
421
+ "post_physical_rank_imax_mean": 1.0033121152297226,
422
+ "post_physical_rank_imax_min": 1.0013429305103698,
423
+ "post_physical_rank_imax_n": 100,
424
+ "pre_home_rank_cv_max": 0.009250589744190636,
425
+ "pre_home_rank_cv_mean": 0.004711613912774553,
426
+ "pre_home_rank_cv_min": 0.002294812301292466,
427
+ "pre_home_rank_cv_n": 100,
428
+ "pre_home_rank_imax_max": 1.015838623046875,
429
+ "pre_home_rank_imax_mean": 1.00729736328125,
430
+ "pre_home_rank_imax_min": 1.002899169921875,
431
+ "pre_home_rank_imax_n": 100,
432
+ "raw_expert_cv_max": 0.018080370892918116,
433
+ "raw_expert_cv_mean": 0.014541988425652842,
434
+ "raw_expert_cv_min": 0.011942398269672016,
435
+ "raw_expert_cv_n": 100,
436
+ "raw_expert_imax_ceiling": 8.0,
437
+ "raw_expert_imax_hit_ceiling": 0,
438
+ "raw_expert_imax_max": 1.05517578125,
439
+ "raw_expert_imax_mean": 1.0348291015625,
440
+ "raw_expert_imax_min": 1.0234375,
441
+ "raw_expert_imax_n": 100,
442
+ "records": {
443
+ "moonep_bytes": 800,
444
+ "moonep_routing": 800,
445
+ "moonep_summary": 80,
446
+ "moonep_weights": 800
447
+ },
448
+ "router_balancing": "aux_loss",
449
+ "run_dir": "QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1",
450
+ "seq_length": "4096",
451
+ "shared_expert_overlap": "0",
452
+ "topk": "8"
453
+ }
454
+ ]
moonep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ run_dir arm config backend num_experts topk experts_per_rank num_layers moe_layers dense_first_layers seq_length global_batch_size capacity_mode capacity force_balance cuda_graph branch commit router_balancing aux_loss_coeff shared_expert_overlap cold_expert_fraction_max cold_expert_fraction_mean cold_expert_fraction_min cold_expert_fraction_n global_snapshots hottest_expert_token_fraction_max hottest_expert_token_fraction_mean hottest_expert_token_fraction_min hottest_expert_token_fraction_n incomplete_snapshots malformed_records moonep_bytes_global_snapshots moonep_bytes_incomplete_snapshots moonep_bytes_malformed_records moonep_global_combine_payload_bytes_max moonep_global_combine_payload_bytes_mean moonep_global_combine_payload_bytes_min moonep_global_combine_payload_bytes_n moonep_global_dispatch_payload_bytes_max moonep_global_dispatch_payload_bytes_mean moonep_global_dispatch_payload_bytes_min moonep_global_dispatch_payload_bytes_n moonep_global_prefetch_bytes_max moonep_global_prefetch_bytes_mean moonep_global_prefetch_bytes_min moonep_global_prefetch_bytes_n moonep_global_reduce_bytes_max moonep_global_reduce_bytes_mean moonep_global_reduce_bytes_min moonep_global_reduce_bytes_n moonep_global_token_bytes_max moonep_global_token_bytes_mean moonep_global_token_bytes_min moonep_global_token_bytes_n moonep_global_total_nvlink_bytes_max moonep_global_total_nvlink_bytes_mean moonep_global_total_nvlink_bytes_min moonep_global_total_nvlink_bytes_n moonep_global_weight_movement_bytes_max moonep_global_weight_movement_bytes_mean moonep_global_weight_movement_bytes_min moonep_global_weight_movement_bytes_n moonep_global_weight_to_token_ratio_max moonep_global_weight_to_token_ratio_mean moonep_global_weight_to_token_ratio_min moonep_global_weight_to_token_ratio_n moonep_remote_demand_max moonep_remote_demand_mean moonep_remote_demand_min moonep_remote_demand_n moonep_slot_utilization_max moonep_slot_utilization_mean moonep_slot_utilization_min moonep_slot_utilization_n moonep_unused_prefetch_slots_max moonep_unused_prefetch_slots_mean moonep_unused_prefetch_slots_min moonep_unused_prefetch_slots_n post_physical_rank_cv_max post_physical_rank_cv_mean post_physical_rank_cv_min post_physical_rank_cv_n post_physical_rank_imax_max post_physical_rank_imax_mean post_physical_rank_imax_min post_physical_rank_imax_n pre_home_rank_cv_max pre_home_rank_cv_mean pre_home_rank_cv_min pre_home_rank_cv_n pre_home_rank_imax_max pre_home_rank_imax_mean pre_home_rank_imax_min pre_home_rank_imax_n raw_expert_cv_max raw_expert_cv_mean raw_expert_cv_min raw_expert_cv_n raw_expert_imax_ceiling raw_expert_imax_hit_ceiling raw_expert_imax_max raw_expert_imax_mean raw_expert_imax_min raw_expert_imax_n
2
+ DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1 natural DSV2_LITE_L24 moonep 64 6 8 24 23 1 8192 8 none none 0 0 moonep_backend_token_routing_analysis 35ae6e7f seq_aux_loss 1e-3 0 0.578125 0.15625 0.0 230 230 0.999969482421875 0.7797577567722487 0.162750244140625 230 0 0 230 0 0 2592899072.0 2282338571.130435 2026799104.0 230 2592899072.0 2282338571.130435 2026799104.0 230 346030080.0 252601958.4 242221056.0 230 346030080.0 252601958.4 242221056.0 230 5188482536.0 4567339392.869565 4056064752.0 230 5672924648.0 5072543309.669565 4540506864.0 230 692060160.0 505203916.8 484442112.0 230 0.14851639788299403 0.11096021713514241 0.09336874676530664 230 3.0 0.9125 0.0 1840 0.375 0.1140625 0.0 1840 16.0 14.175 10.0 1840 0.24644338469771185 0.0866314272142518 0.016159622971429973 230 1.2001146488570558 1.1121655106296537 1.0188653892666146 230 1.3626457501059894 0.6592086328754161 0.06474057713157837 230 4.328043619791667 2.2370494897814766 1.1056925455729167 230 3.0868156557140782 1.9632045302354437 0.37852206422070017 230 10.666666666666666 1 10.666341145833334 8.317416072237318 1.7360026041666667 230
3
+ DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1 forced DSV2_LITE_L24 moonep 64 6 8 24 23 1 8192 8 none none 1 0 moonep_backend_token_routing_analysis 35ae6e7f seq_aux_loss 1e-3 0 0.0 0.0 0.0 230 230 0.0982818603515625 0.09643501613451087 0.09539794921875 230 0 0 230 0 0 2133762048.0 2129465023.4434783 2124537856.0 230 2133762048.0 2129465023.4434783 2124537856.0 230 242221056.0 238309411.61739132 207618048.0 230 242221056.0 238309411.61739132 207618048.0 230 4270279408.0 4261681018.2608695 4251824656.0 230 4754721520.0 4738299841.495652 4667434440.0 230 484442112.0 476618823.23478264 415236096.0 230 0.11393746242954192 0.11183841293995254 0.09728537714366937 230 1.0 0.8608695652173913 0.0 1840 0.125 0.10760869565217392 0.0 1840 16.0 14.278260869565218 14.0 1840 0.0031667036922078573 0.0016501228109707747 0.0006791931370339787 230 1.006201201251668 1.0026117364992109 1.0009196904719393 230 0.006673533732490181 0.003923942735502633 0.0014708009315538319 230 1.0122884114583333 1.0061054616734602 1.00213623046875 230 0.015123587213954862 0.012068912140234289 0.008171875787738593 230 10.666666666666666 0 1.04833984375 1.0286401721014493 1.017578125 230
4
+ QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1 natural QWEN2_57B moonep 64 8 8 10 10 0 4096 8 none none 0 0 moonep_backend_token_routing_analysis 35ae6e7f aux_loss 1e-2 0 0.796875 0.3940625 0.0 100 100 1.0 0.923837890625 0.238525390625 100 0 0 100 0 0 3287474176.0 2792054415.36 2200748032.0 100 3287474176.0 2792054415.36 2200748032.0 100 990904320.0 826854604.8 660602880.0 100 990904320.0 826854604.8 660602880.0 100 6576783336.0 5585911141.52 4403305176.0 100 8345010168.0 7239620351.12 5944711896.0 100 1981808640.0 1653709209.6 1321205760.0 100 0.4264504160968184 0.3001723215454669 0.20088935464362695 100 2.0 0.93875 0.0 800 0.25 0.11734375 0.0 800 16.0 14.1225 12.0 800 0.19752362049364244 0.08253319288939637 0.0001808697752907129 100 1.2480128708509701 1.1014610244938725 1.0002403836981697 100 1.3115485671633254 0.6864519759847463 0.05302373213438137 100 3.8150634765625 2.2216781616210937 1.113861083984375 100 2.6451862448859087 2.182940410315874 0.5018423789492765 100 8.0 1 8.0 7.390703125 1.908203125 100
5
+ QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1 forced QWEN2_57B moonep 64 8 8 10 10 0 4096 8 none none 1 0 moonep_backend_token_routing_analysis 35ae6e7f aux_loss 1e-2 0 0.0 0.0 0.0 100 100 0.13189697265625 0.1293536376953125 0.1279296875 100 0 0 100 0 0 2236903424.0 2231590215.68 2225391616.0 100 2236903424.0 2231590215.68 2225391616.0 100 770703360.0 761895321.6 550502400.0 100 770703360.0 761895321.6 550502400.0 100 4475642264.0 4465014321.2 4452613880.0 100 6017048984.0 5988804964.4 5568474336.0 100 1541406720.0 1523790643.2 1101004800.0 100 0.34618019023019353 0.3412749588086127 0.2464493134486592 100 1.0 0.865 0.0 800 0.125 0.108125 0.0 800 16.0 14.27 14.0 800 0.004151405986167901 0.002179447588207395 0.0007203705850357968 100 1.0062412574645103 1.0033121152297226 1.0013429305103698 100 0.009250589744190636 0.004711613912774553 0.002294812301292466 100 1.015838623046875 1.00729736328125 1.002899169921875 100 0.018080370892918116 0.014541988425652842 0.011942398269672016 100 8.0 0 1.05517578125 1.0348291015625 1.0234375 100
moonep/tierbprobe/tierbprobe_gbs8/results.tsv ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
2
+ QWEN3_235B moonep matched 16 8192 8 none none 1 OOM NA NA 182589 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1
3
+ QWEN3_30B moonep matched 16 4096 8 none none 1 OOM NA NA 169195 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1
4
+ DSV2_LITE_L24 moonep matched 8 8192 8 none none 1 OK 798.26 793.60 157151 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1
5
+ DSV2_LITE_L24 moonep matched 8 8192 8 none none+forcebal 1 OK 796.77 790.80 157151 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1
6
+ QWEN2_57B moonep matched 8 4096 8 none none 1 OK 413.31 366.10 141555 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1
7
+ QWEN2_57B moonep matched 8 4096 8 none none+forcebal 1 OK 408.20 357.00 141555 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1