AbhishekGhosh1998 commited on
Commit
73cd01e
·
verified ·
1 Parent(s): b077d7b

granularity results granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe @ 2026-09-21T08:32:58Z from sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0

Browse files
Files changed (14) hide show
  1. granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/manifest.txt +19 -0
  2. granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/pip_freeze.txt +317 -0
  3. granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/provenance.txt +90 -0
  4. granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/rep1/mem_samples.csv +192 -0
  5. granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/rep1/run.env +105 -0
  6. granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/rep1/train.log +0 -0
  7. granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/mem_samples.csv +184 -0
  8. granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/run.env +105 -0
  9. granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/train.log +0 -0
  10. granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv +3 -0
  11. granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv +2 -0
  12. granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.json +228 -0
  13. granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv +3 -0
  14. granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/results.tsv +3 -0
granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/manifest.txt ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ timestamp: 2026-09-21T08:30:47+00:00
2
+ host: sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0
3
+ gpu: NVIDIA B200 x8
4
+ driver: 580.95.05
5
+ study_scripts_branch: granularity-moonep
6
+ study_scripts_commit: d29f657
7
+ campaign: moonep
8
+ backend: moonep
9
+ phase: tierbprobe
10
+ gbs: 8
11
+ seq: 4096
12
+ configs: G1 G2 G3 G4
13
+ capacity: 1.25
14
+ repeats: 3
15
+ nsys: 1
16
+ megatron_repo: /workspace/Megatron-MoE-Echo
17
+ megatron_branch_required: moonep_backend_async_grad_reduce_token_routing_analysis
18
+ megatron_branch_actual: moonep_backend_async_grad_reduce_token_routing_analysis
19
+ megatron_commit: a6a1a221
granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/pip_freeze.txt ADDED
@@ -0,0 +1,317 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ absl-py==2.3.0
2
+ aiohappyeyeballs @ file:///rapids/aiohappyeyeballs-2.6.1-py3-none-any.whl#sha256=f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8
3
+ aiohttp @ file:///rapids/aiohttp-3.12.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=94f98e0e5a49f89b252e115844f756c04fc8050f38252a32a3dd994ce8121f10
4
+ aiosignal @ file:///rapids/aiosignal-1.3.2-py2.py3-none-any.whl#sha256=45cde58e409a301715980c2b01d0c28bdde3770d8290b5eb2173759d9acb31a5
5
+ annotated-types==0.7.0
6
+ anyio==4.9.0
7
+ apex @ file:///opt/pytorch/apex/dist/apex-0.1-cp312-cp312-linux_x86_64.whl#sha256=a1508e4a386362e9c8f46c92ccada9110e421212b964c694235d0fd5a224b170
8
+ argon2-cffi==25.1.0
9
+ argon2-cffi-bindings==21.2.0
10
+ arrow==1.3.0
11
+ asciitree @ file:///rapids/asciitree-0.3.3-py3-none-any.whl#sha256=3e507ccfaeb417823158091fdb9d26b0205f6a59ba680b479a71db65c73d01d4
12
+ asttokens==3.0.0
13
+ astunparse==1.6.3
14
+ async-lru==2.0.5
15
+ attrs==25.3.0
16
+ audioread==3.0.1
17
+ babel==2.17.0
18
+ beautifulsoup4==4.13.4
19
+ black==25.1.0
20
+ bleach==6.2.0
21
+ blis==0.7.11
22
+ cachetools @ file:///rapids/cachetools-6.0.0-py3-none-any.whl#sha256=82e73ba88f7b30228b5507dce1a1f878498fc669d972aef2dde4f3a3c24f103e
23
+ catalogue==2.0.10
24
+ certifi==2025.4.26
25
+ cffi==1.17.1
26
+ charset-normalizer==3.4.2
27
+ click==8.5.0
28
+ cloudpathlib==0.21.1
29
+ cloudpickle @ file:///rapids/cloudpickle-3.1.1-py3-none-any.whl#sha256=c8c5a44295039331ee9dad40ba100a9c7297b6f988e50e87ccdf3765a668350e
30
+ cmake==3.31.6
31
+ comm==0.2.2
32
+ confection==0.1.5
33
+ contourpy==1.3.2
34
+ cuda-bindings @ file:///rapids/cuda_bindings-12.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=ff0e28d1e34758654b9c961e1f55e4786e49aee6a4dbceaf3cc24c46c672df7e
35
+ cuda-python @ file:///rapids/cuda_python-12.9.0-py3-none-any.whl#sha256=926acba49b2c0a0374c61b7c98f337c085199cf51cdfe4d6423c4129c20547a7
36
+ cudf @ file:///rapids/cudf-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b779a8e8c0e8577b9b560895958496da211bfac49cdc4f11a49e8d18cb82e348
37
+ cudf-polars @ file:///rapids/cudf_polars-25.4.0-py3-none-any.whl#sha256=227b6c5147e93ea86282c8f7219118f916769fa2ca0cdd931e97a148aa66b869
38
+ cugraph @ file:///rapids/cugraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b837c2dc2f7b235c702e4f8df64479b8fcbb342f848516d5faf0cdc4b5f31f29
39
+ cugraph-service-client @ file:///rapids/cugraph_service_client-25.4.0-py3-none-any.whl#sha256=7e568c3e41154b9752f5173555ffcc282ac52d72343feeb79653b2e2b1232470
40
+ cugraph-service-server @ file:///rapids/cugraph_service_server-25.4.0-py3-none-any.whl#sha256=a2830dcffafcccb16e12ebf85b98f8f247869d38c01ab4dd8664a1e7439d31be
41
+ cuml @ file:///rapids/cuml-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=1e90a464a18b094e24797f1676688c5d7696c4cf97c934504214a53d5aa09aa5
42
+ cupy-cuda12x @ file:///rapids/cupy_cuda12x-13.3.0-cp312-cp312-linux_x86_64.whl#sha256=6a2dfa0aa1e8189a10d1534276c08f1d2f73b747fd28bfbe7b9f4ef66d9af73f
43
+ cuvs @ file:///rapids/cuvs-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b12f6b9f80e053a3b38542c2947d2516e468c02cad54a13d8b1a4a61d5aa7914
44
+ cycler==0.12.1
45
+ cymem==2.0.11
46
+ Cython==3.1.1
47
+ dask @ file:///rapids/dask-2025.2.0-py3-none-any.whl#sha256=f0fdeef6ceb0a06569d456c9e704f220f7f54e80f3a6ea42ab98cea6bc642b6e
48
+ dask-cuda @ file:///rapids/dask_cuda-25.4.0-py3-none-any.whl#sha256=442d87da0a95d00ca2a262b64c76e785a32ad905c82000fa2227932f81390e19
49
+ dask-cudf @ file:///rapids/dask_cudf-25.4.0-py3-none-any.whl#sha256=5e7cea0b26daea17da6e0bb457435e24460f76ba740358a3a18e239d6015c95c
50
+ datasets==5.0.1
51
+ debugpy==1.8.14
52
+ decorator==5.2.1
53
+ -e git+https://github.com/deepseek-ai/DeepEP.git@34152ae28f80bcc3ee38d7a12cb2ad87cfd4ea72#egg=deep_ep
54
+ defusedxml==0.7.1
55
+ dill==0.4.1
56
+ distributed @ file:///rapids/distributed-2025.2.0-py3-none-any.whl#sha256=368462084be47092d0a835f6c64c9b852da64dc88302b76d5f93bd5131228c67
57
+ distributed-ucxx @ file:///rapids/distributed_ucxx-0.43.0-py3-none-any.whl#sha256=94f60d0ccb0c94ba8dbfa1eaf455874d18aaab17e2ae218c38590e9fdc2d721d
58
+ dm-tree==0.1.9
59
+ einops==0.8.1
60
+ execnet==2.1.1
61
+ executing==2.2.0
62
+ expecttest==0.3.0
63
+ fasteners @ file:///rapids/fasteners-0.19-py3-none-any.whl#sha256=758819cb5d94cdedf4e836988b74de396ceacb8e2794d21f82d131fd9ee77237
64
+ fastjsonschema==2.21.1
65
+ fastrlock @ file:///rapids/fastrlock-0.8.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl#sha256=5f13ec08f1adb1aa916c384b05ecb7dbebb8df9ea81abd045f60941c6283a670
66
+ filelock==3.18.0
67
+ flash_attn @ file:///opt/pytorch/flash_attn-2.7.4.post1-cp312-cp312-linux_x86_64.whl#sha256=a10455d6c56b387785b7413fd0bbcb392b42b3d4f4b88fda5d86cc8a9ce9182a
68
+ fonttools==4.58.1
69
+ fqdn==1.5.1
70
+ frozenlist @ file:///rapids/frozenlist-1.6.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=4e1be9111cb6756868ac242b3c2bd1f09d9aea09846e4f5c23715e7afb647103
71
+ fsspec==2025.5.1
72
+ gast==0.6.0
73
+ grpcio==1.62.1
74
+ h11==0.16.0
75
+ hf-xet==1.6.0
76
+ httpcore==1.0.9
77
+ httpx==0.28.1
78
+ huggingface_hub==1.32.0
79
+ hypothesis==6.130.8
80
+ idna==3.10
81
+ importlib_metadata==8.7.0
82
+ iniconfig==2.1.0
83
+ intel-openmp==2021.4.0
84
+ ipykernel==6.29.5
85
+ ipython==9.3.0
86
+ ipython_pygments_lexers==1.1.1
87
+ isoduration==20.11.0
88
+ isort==6.0.1
89
+ jedi==0.19.2
90
+ Jinja2==3.1.6
91
+ joblib==1.5.1
92
+ json5==0.12.0
93
+ jsonpointer==3.0.0
94
+ jsonschema==4.24.0
95
+ jsonschema-specifications==2025.4.1
96
+ jupyter-events==0.12.0
97
+ jupyter-lsp==2.2.5
98
+ jupyter_client==8.6.3
99
+ jupyter_core==5.8.1
100
+ jupyter_server==2.16.0
101
+ jupyter_server_terminals==0.5.3
102
+ jupyterlab==4.4.3
103
+ jupyterlab_code_formatter==3.0.2
104
+ jupyterlab_pygments==0.3.0
105
+ jupyterlab_server==2.27.3
106
+ jupyterlab_tensorboard_pro @ file:///builder/jupyterlab_tensorboard_pro-4.0.0-py2.py3-none-any.whl#sha256=6bfcb5ced3358eb72150f4b2ab91312faef2f0ca14d845e8379d8fa52df6902e
107
+ jupytext==1.17.2
108
+ kiwisolver==1.4.8
109
+ kvikio @ file:///rapids/kvikio-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=8b40fd8ea9e4a2974c04b50b7fc26b06094ac807f9a233281d0bafbaa71b1d85
110
+ langcodes==3.5.0
111
+ language_data==1.3.0
112
+ lazy_loader==0.4
113
+ libcudf @ file:///rapids/libcudf-25.4.0-py3-none-linux_x86_64.whl#sha256=238320b4958c1737bba2a609673e2539a78216b811b5498a68e00395d9b021a4
114
+ libcugraph @ file:///rapids/libcugraph-25.4.0-py3-none-linux_x86_64.whl#sha256=29085e901b2f7569ff2653bb1355fe6fad20f2fcaa585f5649da9867d8df766d
115
+ libcuml @ file:///rapids/libcuml-25.4.0-py3-none-linux_x86_64.whl#sha256=8e2e444c72b0e995e15a48da37ff12fb410a47a2baa69ba86ccc9b1a3db91e8c
116
+ libcuvs @ file:///rapids/libcuvs-25.4.0-py3-none-linux_x86_64.whl#sha256=e2f8db61c92b769c590dd752f43439ddd62b2f6592a3c337362b6c1ef220c3b9
117
+ libkvikio @ file:///rapids/libkvikio-25.4.0-py3-none-linux_x86_64.whl#sha256=da8a5713b05fc478bf40529a3e0fb61a2d605bb5a70ca4d461f96a6005dbe8b7
118
+ libraft @ file:///rapids/libraft-25.4.0-py3-none-linux_x86_64.whl#sha256=b921dec005aa4a321dd156776a18376feb47c96a5b67eec125c8a8e19edd1e24
119
+ librmm @ file:///rapids/librmm-25.4.0-py3-none-linux_x86_64.whl#sha256=ff951811bda729d4fa1109ab906225ae1f489a6ec2eeab5d1b674cc34bd4422f
120
+ librmm-cu12 @ file:///rapids/librmm_cu12-25.4.0-py3-none-any.whl#sha256=29296ef2b0e90679a0db280e86015be12d6ab9f4e62a181aa085f62dbadd06b3
121
+ librosa==0.11.0
122
+ libucxx @ file:///rapids/libucxx-0.43.0-py3-none-linux_x86_64.whl#sha256=f15849d7ac2fb68f0fd8998838a13321ddcfb0c594f7e6cbc401a3bcc4750566
123
+ lightning-thunder @ file:///opt/pytorch/lightning-thunder
124
+ lightning-utilities==0.14.3
125
+ lintrunner==0.12.7
126
+ llvmlite==0.42.0
127
+ locket @ file:///rapids/locket-1.0.0-py2.py3-none-any.whl#sha256=b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3
128
+ looseversion==1.3.0
129
+ marisa-trie==1.2.1
130
+ Markdown==3.8
131
+ markdown-it-py==3.0.0
132
+ MarkupSafe==3.0.2
133
+ matplotlib==3.10.3
134
+ matplotlib-inline==0.1.7
135
+ mdit-py-plugins==0.4.2
136
+ mdurl==0.1.2
137
+ -e git+ssh://git@github.com/sriram-17-17/my-Megatron-MoE-Echo.git@a6a1a2210babdf423efb969705f84584d06e97e0#egg=megatron_core
138
+ mistune==3.1.3
139
+ mkl==2021.1.1
140
+ mkl-devel==2021.1.1
141
+ mkl-include==2021.1.1
142
+ ml_dtypes==0.6.0
143
+ mock==5.2.0
144
+ -e git+ssh://git@github.com/sriram-17-17/MoonEP.git@d8bc6667fad7177e6e09b637d773fa67f08bae93#egg=moonep
145
+ mpmath==1.3.0
146
+ msgpack==1.1.0
147
+ multidict @ file:///rapids/multidict-6.4.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=a920f9cf2abdf6e493c519492d892c362007f113c94da4c239ae88429835bad1
148
+ multiprocess==0.70.19
149
+ murmurhash==1.0.13
150
+ mypy_extensions==1.1.0
151
+ nbclient==0.10.2
152
+ nbconvert==7.16.6
153
+ nbformat==5.10.4
154
+ nest-asyncio==1.6.0
155
+ networkx==3.5
156
+ ninja==1.11.1.4
157
+ notebook==7.4.3
158
+ notebook_shim==0.2.4
159
+ numba==0.59.1
160
+ numba-cuda @ file:///rapids/numba_cuda-0.4.0-py3-none-any.whl#sha256=5b95a5f42068771e37142944d44fe844fa87f1b2f89051461a4cb0af91bcc28a
161
+ numcodecs @ file:///rapids/numcodecs-0.13.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=da2230484e6102e5fa3cc1a5dd37ca1f92dfbd183d91662074d6f7574e3e8f53
162
+ numpy==1.26.4
163
+ nvdlfw_inspect @ git+https://github.com/NVIDIA/nvidia-dlfw-inspect.git@6b60eb0e675606fb2fbbfcb12667ecbec75eaf87
164
+ nvfuser==0.2.27a0+9bf5aca
165
+ nvidia-cuda-cccl-cu12==12.9.27
166
+ nvidia-cudnn-frontend @ file:///opt/pytorch/lightning-thunder/tmp/cudnn_frontend
167
+ nvidia-cutlass-dsl==4.4.2
168
+ nvidia-cutlass-dsl-libs-base==4.4.2
169
+ nvidia-dali-cuda120==1.50.0
170
+ nvidia-ml-py @ file:///rapids/nvidia_ml_py-12.575.51-py3-none-any.whl#sha256=eb8641800d98ce40a22f479873f34b482e214a7e80349c63be51c3919845446e
171
+ nvidia-modelopt==0.29.0
172
+ nvidia-modelopt-core==0.29.0
173
+ nvidia-nvcomp-cu12==4.2.0.14
174
+ nvidia-nvimgcodec-cu12==0.5.0.13
175
+ nvidia-nvjpeg-cu12==12.4.0.16
176
+ nvidia-nvjpeg2k-cu12==0.8.1.40
177
+ nvidia-nvshmem-cu12==3.7.2
178
+ nvidia-nvtiff-cu12==0.5.0.67
179
+ nvidia-resiliency-ext==0.4.0
180
+ nvtx @ file:///rapids/nvtx-0.2.11-cp312-cp312-linux_x86_64.whl#sha256=73521e37cba42b4db0b3cd68fc5eb814e55c500a61eeed6edcdf6746315ac53e
181
+ nx-cugraph @ file:///rapids/nx_cugraph-25.4.0-py3-none-any.whl#sha256=e625444486cc899a77bd48e7da76a2edd95c0006dd95fbf323745feae7f33111
182
+ onnx @ file:///opt/pytorch/pytorch/third_party/onnx
183
+ onnx-ir==1.0.0
184
+ onnxscript==0.7.2
185
+ opt_einsum==3.4.0
186
+ optree==0.16.0
187
+ overrides==7.7.0
188
+ packaging==23.2
189
+ pandas @ file:///rapids/pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319
190
+ pandocfilters==1.5.1
191
+ parso==0.8.4
192
+ partd @ file:///rapids/partd-1.4.2-py3-none-any.whl#sha256=978e4ac767ec4ba5b86c6eaa52e5a2a3bc748a2ca839e8cc798f1cc6ce6efb0f
193
+ pathspec==0.12.1
194
+ pexpect==4.9.0
195
+ pillow==11.2.1
196
+ platformdirs==4.3.8
197
+ pluggy==1.6.0
198
+ ply @ file:///rapids/ply-3.11-py2.py3-none-any.whl#sha256=096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce
199
+ polars @ file:///rapids/polars-1.25.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=f7fcbb4f476784384ccda48757fca4e8c2e2c5a0a3aef3717aaf56aee4e30e09
200
+ polygraphy==0.49.20
201
+ pooch==1.8.2
202
+ preshed==3.0.10
203
+ prometheus_client==0.22.1
204
+ prompt_toolkit==3.0.51
205
+ propcache @ file:///rapids/propcache-0.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=563f9d8c03ad645597b8d010ef4e9eab359faeb11a0a2ac9f7b4bc8c28ebef25
206
+ protobuf==4.24.4
207
+ psutil==7.0.0
208
+ ptyprocess==0.7.0
209
+ PuLP==3.2.1
210
+ pure_eval==0.2.3
211
+ pyarrow==25.0.1
212
+ pybind11==2.13.6
213
+ pybind11_global==2.13.6
214
+ pycocotools @ git+https://github.com/nvidia/cocoapi.git@08350a157b41f0755a536441b5abec89ab934c5b#subdirectory=PythonAPI
215
+ pycparser==2.22
216
+ pydantic==2.11.5
217
+ pydantic_core==2.33.2
218
+ Pygments==2.19.1
219
+ pylibcudf @ file:///rapids/pylibcudf-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=401b8bfd022d06f84e120779ca7c3d01a28b68762aaa8f7873b10d6b1975cf6a
220
+ pylibcugraph @ file:///rapids/pylibcugraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=061c7389a88c815b54fd5d3a3584fb87bf693e5602f5be335260d0ba575ba557
221
+ pylibcugraphops @ file:///rapids/pylibcugraphops-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=a25ef4b22166c80718d5c7e5bddb755e81a2fe340fa5becd0ac36ec3edfb208a
222
+ pylibraft @ file:///rapids/pylibraft-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=f2ecd23fe51f9c2caa87bbbe85cb32fcaff4ecfb52ceb9a9991bc50f82782c2e
223
+ pylibwholegraph @ file:///rapids/pylibwholegraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b82913d1e38bda76d8c272fbd8cc9072d987209d3d47cc93a464a37c2925aca2
224
+ pynvjitlink @ file:///rapids/pynvjitlink-0.3.0-cp312-cp312-linux_x86_64.whl#sha256=066caf1314730643d3b48c9847bd65789defdd6a7c78993176eb3d8bbe3f4919
225
+ pynvml @ file:///rapids/pynvml-12.0.0-py3-none-any.whl#sha256=fdff84b62a27dbe98e08e1a647eb77342bef1aebe0878bcd15e99a83fcbecb9e
226
+ pyparsing==3.2.3
227
+ pytest==8.1.1
228
+ pytest-flakefinder==1.1.0
229
+ pytest-rerunfailures==15.1
230
+ pytest-shard==0.1.2
231
+ pytest-xdist==3.7.0
232
+ python-dateutil==2.9.0.post0
233
+ python-hostlist==2.2.1
234
+ python-json-logger==3.3.0
235
+ pytorch-triton @ file:///tmp/dist/pytorch_triton-3.3.0%2Bgit96316ce52.nvinternal-cp312-cp312-linux_x86_64.whl#sha256=6d6921772b9dfd0243c055e0d054de42a113e92cabcfdd1d3e80a5ad9c08cf7d
236
+ pytz @ file:///rapids/pytz-2023.4-py2.py3-none-any.whl#sha256=f90ef520d95e7c46951105338d918664ebfd6f1d995bd7d153127ce90efafa6a
237
+ PyYAML==6.0.2
238
+ pyzmq==26.4.0
239
+ raft-dask @ file:///rapids/raft_dask-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=36cf37ba6ce35454c4729c2cb597a360f22780eefb11d514a16a07137da5eb6c
240
+ rapids-dask-dependency @ file:///rapids/rapids_dask_dependency-25.4.0a0-py3-none-any.whl#sha256=484faca626e7760112a7d8ad3426f5bdd720b609006d8dca87c2607076a348bd
241
+ rapids-logger @ file:///rapids/rapids_logger-0.1.18-py3-none-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=3617b2d86291ac6cb957542895f20fc9737ace29773ca90337e6fb04c73bd120
242
+ referencing==0.36.2
243
+ regex==2024.11.6
244
+ requests==2.32.3
245
+ rfc3339-validator==0.1.4
246
+ rfc3986-validator==0.1.1
247
+ rich==14.0.0
248
+ rmm @ file:///rapids/rmm-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=1824bcab3836296666980a997ddcb9c4bca24dfb873012cfa0a7ecea583ca712
249
+ rpds-py==0.25.1
250
+ safetensors==0.5.3
251
+ scikit-learn==1.6.1
252
+ scipy==1.15.3
253
+ Send2Trash==1.8.3
254
+ setuptools==78.1.1
255
+ shellingham==1.5.4
256
+ six==1.16.0
257
+ smart-open==7.1.0
258
+ sniffio==1.3.1
259
+ sortedcontainers==2.4.0
260
+ soundfile==0.13.1
261
+ soupsieve==2.7
262
+ soxr==0.5.0.post1
263
+ spacy==3.7.5
264
+ spacy-legacy==3.0.12
265
+ spacy-loggers==1.0.5
266
+ srsly==2.5.1
267
+ stack-data==0.6.3
268
+ sympy==1.14.0
269
+ tabulate==0.9.0
270
+ tbb==2021.13.1
271
+ tblib @ file:///rapids/tblib-3.1.0-py3-none-any.whl#sha256=670bb4582578134b3d81a84afa1b016128b429f3d48e6cbbaecc9d15675e984e
272
+ tensorboard==2.16.2
273
+ tensorboard-data-server==0.7.2
274
+ tensorrt @ file:///workspace/TensorRT-10.11.0.33/python/tensorrt-10.11.0.33-cp312-none-linux_x86_64.whl#sha256=c2f18f72758a3d54a71901495c8fda9e7094a8bcdd2a5b6b58967fbf05733b3a
275
+ terminado==0.18.1
276
+ thinc==8.2.5
277
+ threadpoolctl==3.6.0
278
+ thriftpy2 @ file:///rapids/thriftpy2-0.5.2-cp312-cp312-linux_x86_64.whl#sha256=c0c7d939bdc5ccf7739154959938445d5fdf72f1d6e12bd563947a36f67e2147
279
+ tinycss2==1.4.0
280
+ toolz @ file:///rapids/toolz-1.0.0-py3-none-any.whl#sha256=292c8f1c4e7516bf9086f8850935c799a874039c8bcf959d47b600e4c44a6236
281
+ torch @ file:///opt/transfer/torch-2.8.0a0%2B5228986c39.nv25.6-29946474-cp312-cp312-linux_x86_64.whl#sha256=7a3c9ff5288d8102a31296b3d62491f3f47c8820a5c9aaefc7dc0fe20e1e8ad5
282
+ torch_tensorrt @ file:///opt/pytorch/torch_tensorrt/dist/torch_tensorrt-2.8.0a0-cp312-cp312-linux_x86_64.whl#sha256=26f4d7a04e7e6e017a5b917b8b98b83720bb2bb0195f48d25f52d567d286bfe1
283
+ torchao @ file:///opt/pytorch/ao
284
+ torchprofile==0.0.4
285
+ torchvision @ file:///opt/transfer/torchvision-0.22.0a0%2B95f10a4e-29946474-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=746a3c674cc61680a86f4a14b374b21af1b5c21f629c42765b40c9494f6b234d
286
+ tornado==6.5.1
287
+ tqdm==4.67.1
288
+ traitlets==5.14.3
289
+ transformer_engine @ file:///pvcdata/aiinfra/b-aghosh/wheels/transformer_engine-2.10.0.dev0%2B10f6128d-cp312-cp312-linux_x86_64.whl#sha256=35cf5f4b83b2baad864e12e512bb422e0929409ac05610812fa521d4942d0189
290
+ treelite @ file:///rapids/treelite-4.4.1-py3-none-linux_x86_64.whl#sha256=174741724205540a1a5b37da81483aeff55d89548f42b3461cf07acac7ae96eb
291
+ typer==0.16.0
292
+ types-dataclasses==0.6.6
293
+ types-python-dateutil==2.9.0.20250516
294
+ typing-inspection==0.4.1
295
+ typing_extensions==4.14.0
296
+ tzdata @ file:///rapids/tzdata-2025.2-py2.py3-none-any.whl#sha256=1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8
297
+ ucx-py @ file:///rapids/ucx_py-0.43.0-cp312-cp312-linux_x86_64.whl#sha256=4a452fd8d8c983283ada3d49086e3657dc2ee94fbad08efdd708d15ab6e0b73a
298
+ ucxx @ file:///rapids/ucxx-0.43.0-cp312-cp312-linux_x86_64.whl#sha256=f1b4e55cb45c66fe88a2d8f9c2b87751165f3cfec21fbdc9c47c4ddff75e8a5f
299
+ -e git+https://github.com/Dots-Infra/UltraEP.git@94cab099b44fffa99a82fea99e7c12d89cf65e4f#egg=ultra_ep
300
+ uri-template==1.3.0
301
+ urllib3==2.0.7
302
+ wasabi==1.1.3
303
+ wcwidth==0.2.13
304
+ weasel==0.4.1
305
+ webcolors==24.11.1
306
+ webencodings==0.5.1
307
+ websocket-client==1.8.0
308
+ Werkzeug==3.1.3
309
+ wheel==0.45.1
310
+ wrapt==1.17.2
311
+ xdoctest==1.0.2
312
+ xgboost @ file:///rapids/xgboost-2.1.4-py3-none-manylinux_2_39_x86_64.whl#sha256=25eeca27f4048fc491940b3a0b4b5782ee48c33cb3aec059cef71f8da542930b
313
+ xxhash==4.0.1
314
+ yarl @ file:///rapids/yarl-1.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=bf099e2432131093cc611623e0b0bcc399b8cddd9a91eded8bfb50402ec35018
315
+ zarr @ file:///rapids/zarr-2.18.7-py3-none-any.whl#sha256=ac3dc4033e9ae4e9d7b5e27c97ea3eaf1003cc0a07f010bd83d5134bf8c4b223
316
+ zict @ file:///rapids/zict-3.0.0-py2.py3-none-any.whl#sha256=5796e36bd0e0cc8cf0fbc1ace6a68912611c1dbd74750a3f3026b9b9d6a327ae
317
+ zipp==3.22.0
granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/provenance.txt ADDED
@@ -0,0 +1,90 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ==============================================================
2
+ PROVENANCE: moonep__tierbprobe__gbs8
3
+ ==============================================================
4
+ captured_at : 2026-09-21T08:30:33+00:00
5
+ hostname : sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0
6
+ kernel : 6.8.0-1043-nvidia
7
+ cwd : /workspace/blackwell_run_scripts
8
+ command : run_ladder.sh --phase tierbprobe --gbs 8 --out /workspace/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe --nsys
9
+
10
+ ---- GPU / driver --------------------------------------------
11
+ 0, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
12
+ 1, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
13
+ 2, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
14
+ 3, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
15
+ 4, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
16
+ 5, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
17
+ 6, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
18
+ 7, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
19
+ persistence/clocks:
20
+ 0, 120 MHz, 1965 MHz, 1000.00 W, 29
21
+ 1, 1965 MHz, 1965 MHz, 1000.00 W, 35
22
+ 2, 120 MHz, 1965 MHz, 1000.00 W, 34
23
+ 3, 120 MHz, 1965 MHz, 1000.00 W, 30
24
+ 4, 1965 MHz, 1965 MHz, 1000.00 W, 30
25
+ 5, 120 MHz, 1965 MHz, 1000.00 W, 34
26
+ 6, 120 MHz, 1965 MHz, 1000.00 W, 34
27
+ 7, 120 MHz, 1965 MHz, 1000.00 W, 29
28
+
29
+ NVLink topology:
30
+ GPU0 GPU1 GPU2 GPU3 GPU4 GPU5 GPU6 GPU7 NIC0 NIC1 NIC2 NIC3 NIC4 NIC5 NIC6 NIC7 NIC8 CPU Affinity NUMA Affinity GPU NUMA ID
31
+ GPU0 X NV18 NV18 NV18 NV18 NV18 NV18 NV18 PIX NODE SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A
32
+ GPU1 NV18 X NV18 NV18 NV18 NV18 NV18 NV18 NODE PIX SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A
33
+ GPU2 NV18 NV18 X NV18 NV18 NV18 NV18 NV18 SYS SYS PIX NODE SYS SYS SYS SYS SYS 28-55 1 N/A
34
+ GPU3 NV18 NV18 NV18 X NV18 NV18 NV18 NV18 SYS SYS NODE PIX SYS SYS SYS SYS SYS 28-55 1 N/A
35
+ GPU4 NV18 NV18 NV18 NV18 X NV18 NV18 NV18 SYS SYS SYS SYS PIX NODE SYS SYS SYS 56-83 2 N/A
36
+ GPU5 NV18 NV18 NV18 NV18 NV18 X NV18 NV18 SYS SYS SYS SYS NODE PIX SYS SYS SYS 56-83 2 N/A
37
+ GPU6 NV18 NV18 NV18 NV18 NV18 NV18 X NV18 SYS SYS SYS SYS SYS SYS PIX NODE SYS 84-111 3 N/A
38
+ GPU7 NV18 NV18 NV18 NV18 NV18 NV18 NV18 X SYS SYS SYS SYS SYS SYS NODE PIX SYS 84-111 3 N/A
39
+ NIC0 PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS SYS NODE
40
+ NIC1 NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS SYS NODE
41
+ NIC2 SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS
42
+ NIC3 SYS SYS NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS
43
+ NIC4 SYS SYS SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS
44
+ unavailable
45
+
46
+ ---- toolchain -----------------------------------------------
47
+ nvcc : release 12.9, V12.9.86
48
+ gcc : gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0
49
+ python : 3.12.3
50
+ torch : 2.8.0a0+5228986c39.nv25.06 cuda=12.9 nccl=2.27.3
51
+ torch arch : ['sm_75', 'sm_80', 'sm_86', 'sm_90', 'sm_100', 'sm_120', 'compute_120']
52
+ transformer_engine: 2.10.0.dev0+10f6128d [/usr/local/lib/python3.12/dist-packages/transformer_engine/__init__.py]
53
+ deep_ep : 1.2.1+34152ae [/workspace/DeepEP/deep_ep/__init__.py]
54
+ ultra_ep : 1.0.0 [/workspace/UltraEP/ultra_ep/__init__.py]
55
+ moonep : 0.0.1 [/workspace/MoonEP/moonep/__init__.py]
56
+ megatron : 0.14.0rc5 [None]
57
+ flash_attn : 2.7.4.post1 [/usr/local/lib/python3.12/dist-packages/flash_attn/__init__.py]
58
+ numpy : 1.26.4 [/usr/local/lib/python3.12/dist-packages/numpy/__init__.py]
59
+ huggingface_hub: 1.32.0 [/usr/local/lib/python3.12/dist-packages/huggingface_hub/__init__.py]
60
+
61
+ ---- repositories --------------------------------------------
62
+ Megatron-MoE-Echo moonep_backend_async_grad_reduce_token_routing_analysis a6a1a2210bab
63
+ Megatron-UltraEP ultraep-token-drop-analysis eaf829e7ad21
64
+ UltraEP main 94cab099b44f
65
+ DeepEP HEAD 34152ae28f80 (+3 untracked: build output)
66
+ MoonEP megatron-ddp-symmetric-params d8bc6667fad7
67
+ blackwell_run_scripts granularity-moonep d29f6570cdce
68
+
69
+ ---- environment ---------------------------------------------
70
+ CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0
71
+ CUDA_DRIVER_VERSION=575.57.08
72
+ CUDA_HOME=/usr/local/cuda
73
+ CUDA_MODULE_LOADING=LAZY
74
+ CUDA_VERSION=12.9.1.010
75
+ HF_REMOTE_ROOT=granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16
76
+ NCCL_VERSION=2.27.3
77
+ NVSHMEM_VERSION=3.2.5
78
+ PYTORCH_BUILD_NUMBER=0
79
+ PYTORCH_BUILD_VERSION=2.8.0a0+5228986
80
+ PYTORCH_HOME=/opt/pytorch/pytorch
81
+ PYTORCH_VERSION=2.8.0a0+5228986
82
+ TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1
83
+ TORCH_CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0+PTX
84
+ TORCH_NCCL_USE_COMM_NONBLOCKING=0
85
+
86
+ ---- shared library resolution -------------------------------
87
+ libnvshmem_host.so: not in ldconfig
88
+ libnccl.so: /lib/x86_64-linux-gnu/libnccl.so.2
89
+ lrwxrwxrwx 1 root root 20 Sep 21 04:57 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so -> libnvshmem_host.so.3
90
+ -rw-r--r-- 1 root root 179045744 Sep 21 04:54 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so.3
granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/rep1/mem_samples.csv ADDED
@@ -0,0 +1,192 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 4
10
+ 1, 4
11
+ 2, 4
12
+ 3, 1
13
+ 4, 1
14
+ 5, 1
15
+ 6, 1
16
+ 7, 1
17
+ 0, 4
18
+ 1, 4
19
+ 2, 4
20
+ 3, 4
21
+ 4, 4
22
+ 5, 4
23
+ 6, 4
24
+ 7, 1
25
+ 0, 1
26
+ 1, 1
27
+ 2, 1
28
+ 3, 1
29
+ 4, 3
30
+ 5, 4
31
+ 6, 4
32
+ 7, 4
33
+ 0, 0
34
+ 1, 0
35
+ 2, 0
36
+ 3, 0
37
+ 4, 0
38
+ 5, 0
39
+ 6, 0
40
+ 7, 0
41
+ 0, 1
42
+ 1, 1
43
+ 2, 1
44
+ 3, 1
45
+ 4, 1
46
+ 5, 1
47
+ 6, 1
48
+ 7, 1
49
+ 0, 4
50
+ 1, 4
51
+ 2, 4
52
+ 3, 4
53
+ 4, 1
54
+ 5, 1
55
+ 6, 1
56
+ 7, 1
57
+ 0, 4
58
+ 1, 4
59
+ 2, 4
60
+ 3, 4
61
+ 4, 4
62
+ 5, 4
63
+ 6, 4
64
+ 7, 4
65
+ 0, 725
66
+ 1, 725
67
+ 2, 725
68
+ 3, 725
69
+ 4, 725
70
+ 5, 725
71
+ 6, 725
72
+ 7, 725
73
+ 0, 977
74
+ 1, 979
75
+ 2, 999
76
+ 3, 975
77
+ 4, 1009
78
+ 5, 975
79
+ 6, 973
80
+ 7, 1001
81
+ 0, 1803
82
+ 1, 1803
83
+ 2, 1803
84
+ 3, 1803
85
+ 4, 1803
86
+ 5, 1803
87
+ 6, 1803
88
+ 7, 1803
89
+ 0, 13245
90
+ 1, 13243
91
+ 2, 13249
92
+ 3, 13257
93
+ 4, 13275
94
+ 5, 13263
95
+ 6, 13267
96
+ 7, 13259
97
+ 0, 14275
98
+ 1, 14313
99
+ 2, 14297
100
+ 3, 14297
101
+ 4, 14339
102
+ 5, 14363
103
+ 6, 14311
104
+ 7, 14335
105
+ 0, 15071
106
+ 1, 15077
107
+ 2, 15077
108
+ 3, 15079
109
+ 4, 15069
110
+ 5, 15079
111
+ 6, 15079
112
+ 7, 15069
113
+ 0, 15597
114
+ 1, 15603
115
+ 2, 15603
116
+ 3, 15605
117
+ 4, 15595
118
+ 5, 15605
119
+ 6, 15605
120
+ 7, 15595
121
+ 0, 15877
122
+ 1, 15883
123
+ 2, 15883
124
+ 3, 15885
125
+ 4, 15875
126
+ 5, 15883
127
+ 6, 15883
128
+ 7, 15875
129
+ 0, 26363
130
+ 1, 26355
131
+ 2, 26363
132
+ 3, 26383
133
+ 4, 26403
134
+ 5, 26391
135
+ 6, 26387
136
+ 7, 26403
137
+ 0, 26775
138
+ 1, 26781
139
+ 2, 26801
140
+ 3, 26783
141
+ 4, 26793
142
+ 5, 26781
143
+ 6, 26783
144
+ 7, 26793
145
+ 0, 27089
146
+ 1, 27067
147
+ 2, 27171
148
+ 3, 27179
149
+ 4, 27161
150
+ 5, 27141
151
+ 6, 27141
152
+ 7, 27143
153
+ 0, 27975
154
+ 1, 28011
155
+ 2, 28027
156
+ 3, 28039
157
+ 4, 28025
158
+ 5, 28037
159
+ 6, 28029
160
+ 7, 28041
161
+ 0, 35739
162
+ 1, 35739
163
+ 2, 35727
164
+ 3, 35745
165
+ 4, 35751
166
+ 5, 35743
167
+ 6, 35743
168
+ 7, 35751
169
+ 0, 35739
170
+ 1, 35739
171
+ 2, 35727
172
+ 3, 35745
173
+ 4, 35751
174
+ 5, 35743
175
+ 6, 35743
176
+ 7, 35751
177
+ 0, 35739
178
+ 1, 35739
179
+ 2, 6798
180
+ 3, 35099
181
+ 4, 6492
182
+ 5, 35375
183
+ 6, 35743
184
+ 7, 35709
185
+ 0, 0
186
+ 1, 0
187
+ 2, 0
188
+ 3, 0
189
+ 4, 0
190
+ 5, 0
191
+ 6, 0
192
+ 7, 0
granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/rep1/run.env ADDED
@@ -0,0 +1,105 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=moonep
2
+ config=OLMOE_1B_7B_L16
3
+ family=tierB
4
+ source_model=OLMoE-1B-7B-0924
5
+ standard_num_layers=16
6
+ base_rung=
7
+ repo=/workspace/Megatron-MoE-Echo
8
+ branch=moonep_backend_async_grad_reduce_token_routing_analysis
9
+ commit=a6a1a221
10
+ moonep_source_dir=/workspace/MoonEP
11
+ moonep_source_branch=megatron-ddp-symmetric-params
12
+ moonep_source_commit=d8bc666
13
+ num_layers=16
14
+ hidden=2048
15
+ ffn_hidden=1024
16
+ moe_ffn_hidden=1024
17
+ num_experts=64
18
+ topk=8
19
+ experts_per_rank=8
20
+ expert_params=6291456
21
+ heads=16
22
+ kv_groups=16
23
+ kv_channels=128
24
+ qk_layernorm=0
25
+ shared_ffn=0
26
+ shared_expert_overlap=0
27
+ moe_token_dispatcher=flex
28
+ mla=0
29
+ q_lora_rank=
30
+ kv_lora_rank=512
31
+ qk_head_dim=128
32
+ qk_pos_emb_head_dim=64
33
+ v_head_dim=128
34
+ dense_first_layers=0
35
+ moe_layers=16
36
+ rotary_base=10000
37
+ rotary_percent=1.0
38
+ rotary_scaling=
39
+ mscale=
40
+ mscale_all_dim=
41
+ qkv_bias=0
42
+ router_balancing=aux_loss
43
+ aux_loss_coeff=1e-2
44
+ router_pre_softmax=1
45
+ router_fusion=0
46
+ router_dtype=fp32
47
+ router_groups=
48
+ router_group_topk=
49
+ router_score_function=softmax
50
+ router_expert_bias=0
51
+ router_bias_update_rate=
52
+ router_topk_scaling_factor=
53
+ shared_expert_gate=0
54
+ apply_layernorm_1p=0
55
+ attention_output_gate=0
56
+ linear_attention_type=
57
+ linear_attention_freq=
58
+ linear_conv_kernel_dim=
59
+ linear_key_head_dim=
60
+ linear_value_head_dim=
61
+ linear_num_key_heads=
62
+ linear_num_value_heads=
63
+ mtp_num_layers=0
64
+ mtp_loss_scaling_factor=0.1
65
+ no_weight_decay_cond_type=
66
+ max_position_embeddings=4096
67
+ init_method_std=0.02
68
+ device_initiated_grouped_gemm=1
69
+ norm_epsilon=1e-5
70
+ seq_length=4096
71
+ micro_batch_size=1
72
+ global_batch_size=8
73
+ num_microbatches=1
74
+ assignments_per_rank=32768
75
+ mean_tokens_per_expert=4096
76
+ slot_policy=matched
77
+ slots_per_rank=8
78
+ capacity_mode=none
79
+ capacity=none
80
+ cuda_graph=0
81
+ probe_mode=1
82
+ async_grad_reduce=1
83
+ branch_role=probe
84
+ memory_dump_mode=none
85
+ memory_snapshot_path=
86
+ cuda_graph_warmup_steps=3
87
+ recompute=none
88
+ force_balance=0
89
+ train_iters=12
90
+ ep_size=8
91
+ echo_dispatch_overlap=1
92
+ echo_dispatch_recompute=0
93
+ ultraep_quota_min_tokens=
94
+ ultraep_weight_sync_join=
95
+ profile_enable=0
96
+ kernel_trace=0
97
+ fp8_mode=mxfp8
98
+ fp8_recipe=mxfp8
99
+ training_seed=1234
100
+ learning_rate=1e-4
101
+ min_learning_rate=1.0e-5
102
+ lr_warmup_iters=5
103
+ state_dump_enable=0
104
+ state_dump_dir=
105
+ moonep_assert_no_pending_reduce=1
granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/mem_samples.csv ADDED
@@ -0,0 +1,184 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 4
10
+ 1, 4
11
+ 2, 4
12
+ 3, 1
13
+ 4, 1
14
+ 5, 1
15
+ 6, 1
16
+ 7, 1
17
+ 0, 4
18
+ 1, 4
19
+ 2, 4
20
+ 3, 4
21
+ 4, 4
22
+ 5, 4
23
+ 6, 4
24
+ 7, 1
25
+ 0, 1
26
+ 1, 1
27
+ 2, 1
28
+ 3, 1
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 4
33
+ 0, 0
34
+ 1, 0
35
+ 2, 0
36
+ 3, 0
37
+ 4, 0
38
+ 5, 0
39
+ 6, 0
40
+ 7, 0
41
+ 0, 4
42
+ 1, 2
43
+ 2, 1
44
+ 3, 1
45
+ 4, 1
46
+ 5, 1
47
+ 6, 1
48
+ 7, 1
49
+ 0, 4
50
+ 1, 4
51
+ 2, 4
52
+ 3, 4
53
+ 4, 4
54
+ 5, 1
55
+ 6, 1
56
+ 7, 1
57
+ 0, 4
58
+ 1, 4
59
+ 2, 4
60
+ 3, 4
61
+ 4, 4
62
+ 5, 4
63
+ 6, 4
64
+ 7, 4
65
+ 0, 819
66
+ 1, 803
67
+ 2, 815
68
+ 3, 803
69
+ 4, 813
70
+ 5, 801
71
+ 6, 815
72
+ 7, 817
73
+ 0, 1079
74
+ 1, 1079
75
+ 2, 1079
76
+ 3, 1079
77
+ 4, 1079
78
+ 5, 1079
79
+ 6, 1079
80
+ 7, 1079
81
+ 0, 1823
82
+ 1, 1823
83
+ 2, 1823
84
+ 3, 1823
85
+ 4, 1823
86
+ 5, 1823
87
+ 6, 1823
88
+ 7, 1823
89
+ 0, 13295
90
+ 1, 13273
91
+ 2, 13291
92
+ 3, 13279
93
+ 4, 13299
94
+ 5, 13293
95
+ 6, 13303
96
+ 7, 13303
97
+ 0, 14301
98
+ 1, 14273
99
+ 2, 14295
100
+ 3, 14295
101
+ 4, 14295
102
+ 5, 14311
103
+ 6, 14349
104
+ 7, 14323
105
+ 0, 14945
106
+ 1, 14935
107
+ 2, 14959
108
+ 3, 14967
109
+ 4, 14979
110
+ 5, 14975
111
+ 6, 14979
112
+ 7, 14951
113
+ 0, 15599
114
+ 1, 15603
115
+ 2, 15603
116
+ 3, 15607
117
+ 4, 15597
118
+ 5, 15605
119
+ 6, 15605
120
+ 7, 15595
121
+ 0, 15877
122
+ 1, 15883
123
+ 2, 15883
124
+ 3, 15885
125
+ 4, 15875
126
+ 5, 15883
127
+ 6, 15883
128
+ 7, 15875
129
+ 0, 26363
130
+ 1, 26341
131
+ 2, 26373
132
+ 3, 26351
133
+ 4, 26369
134
+ 5, 26365
135
+ 6, 26349
136
+ 7, 26403
137
+ 0, 26775
138
+ 1, 26781
139
+ 2, 26801
140
+ 3, 26783
141
+ 4, 26793
142
+ 5, 26781
143
+ 6, 26783
144
+ 7, 26793
145
+ 0, 27067
146
+ 1, 27097
147
+ 2, 27137
148
+ 3, 27079
149
+ 4, 27087
150
+ 5, 27115
151
+ 6, 27091
152
+ 7, 27159
153
+ 0, 28023
154
+ 1, 28041
155
+ 2, 28051
156
+ 3, 28031
157
+ 4, 28059
158
+ 5, 28027
159
+ 6, 28041
160
+ 7, 28061
161
+ 0, 35741
162
+ 1, 35739
163
+ 2, 35727
164
+ 3, 35745
165
+ 4, 35751
166
+ 5, 35743
167
+ 6, 35745
168
+ 7, 35751
169
+ 0, 35741
170
+ 1, 35739
171
+ 2, 35727
172
+ 3, 35745
173
+ 4, 35751
174
+ 5, 35743
175
+ 6, 35745
176
+ 7, 35751
177
+ 0, 4942
178
+ 1, 4942
179
+ 2, 4942
180
+ 3, 5262
181
+ 4, 35367
182
+ 5, 6542
183
+ 6, 4942
184
+ 7, 4942
granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/run.env ADDED
@@ -0,0 +1,105 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=moonep
2
+ config=OLMOE_1B_7B_L16
3
+ family=tierB
4
+ source_model=OLMoE-1B-7B-0924
5
+ standard_num_layers=16
6
+ base_rung=
7
+ repo=/workspace/Megatron-MoE-Echo
8
+ branch=moonep_backend_async_grad_reduce_token_routing_analysis
9
+ commit=a6a1a221
10
+ moonep_source_dir=/workspace/MoonEP
11
+ moonep_source_branch=megatron-ddp-symmetric-params
12
+ moonep_source_commit=d8bc666
13
+ num_layers=16
14
+ hidden=2048
15
+ ffn_hidden=1024
16
+ moe_ffn_hidden=1024
17
+ num_experts=64
18
+ topk=8
19
+ experts_per_rank=8
20
+ expert_params=6291456
21
+ heads=16
22
+ kv_groups=16
23
+ kv_channels=128
24
+ qk_layernorm=0
25
+ shared_ffn=0
26
+ shared_expert_overlap=0
27
+ moe_token_dispatcher=flex
28
+ mla=0
29
+ q_lora_rank=
30
+ kv_lora_rank=512
31
+ qk_head_dim=128
32
+ qk_pos_emb_head_dim=64
33
+ v_head_dim=128
34
+ dense_first_layers=0
35
+ moe_layers=16
36
+ rotary_base=10000
37
+ rotary_percent=1.0
38
+ rotary_scaling=
39
+ mscale=
40
+ mscale_all_dim=
41
+ qkv_bias=0
42
+ router_balancing=aux_loss
43
+ aux_loss_coeff=1e-2
44
+ router_pre_softmax=1
45
+ router_fusion=0
46
+ router_dtype=fp32
47
+ router_groups=
48
+ router_group_topk=
49
+ router_score_function=softmax
50
+ router_expert_bias=0
51
+ router_bias_update_rate=
52
+ router_topk_scaling_factor=
53
+ shared_expert_gate=0
54
+ apply_layernorm_1p=0
55
+ attention_output_gate=0
56
+ linear_attention_type=
57
+ linear_attention_freq=
58
+ linear_conv_kernel_dim=
59
+ linear_key_head_dim=
60
+ linear_value_head_dim=
61
+ linear_num_key_heads=
62
+ linear_num_value_heads=
63
+ mtp_num_layers=0
64
+ mtp_loss_scaling_factor=0.1
65
+ no_weight_decay_cond_type=
66
+ max_position_embeddings=4096
67
+ init_method_std=0.02
68
+ device_initiated_grouped_gemm=1
69
+ norm_epsilon=1e-5
70
+ seq_length=4096
71
+ micro_batch_size=1
72
+ global_batch_size=8
73
+ num_microbatches=1
74
+ assignments_per_rank=32768
75
+ mean_tokens_per_expert=4096
76
+ slot_policy=matched
77
+ slots_per_rank=8
78
+ capacity_mode=none
79
+ capacity=none
80
+ cuda_graph=0
81
+ probe_mode=1
82
+ async_grad_reduce=1
83
+ branch_role=probe
84
+ memory_dump_mode=none
85
+ memory_snapshot_path=
86
+ cuda_graph_warmup_steps=3
87
+ recompute=none
88
+ force_balance=1
89
+ train_iters=12
90
+ ep_size=8
91
+ echo_dispatch_overlap=1
92
+ echo_dispatch_recompute=0
93
+ ultraep_quota_min_tokens=
94
+ ultraep_weight_sync_join=
95
+ profile_enable=0
96
+ kernel_trace=0
97
+ fp8_mode=mxfp8
98
+ fp8_recipe=mxfp8
99
+ training_seed=1234
100
+ learning_rate=1e-4
101
+ min_learning_rate=1.0e-5
102
+ lr_warmup_iters=5
103
+ state_dump_enable=0
104
+ state_dump_dir=
105
+ moonep_assert_no_pending_reduce=1
granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir
2
+ OLMOE_1B_7B_L16 moonep natural none OK 35751 1 0 /workspace/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/rep1
3
+ OLMOE_1B_7B_L16 moonep forced none OK 35751 1 0 /workspace/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1
granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir
2
+ OLMOE_1B_7B_L16 moonep SELECTED none 0 35751 /workspace/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/rep1
granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.json ADDED
@@ -0,0 +1,228 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "arm": "natural",
4
+ "aux_loss_coeff": "1e-2",
5
+ "backend": "moonep",
6
+ "branch": "moonep_backend_async_grad_reduce_token_routing_analysis",
7
+ "capacity": "none",
8
+ "capacity_mode": "none",
9
+ "cold_expert_fraction_max": 0.484375,
10
+ "cold_expert_fraction_mean": 0.2111328125,
11
+ "cold_expert_fraction_min": 0.0,
12
+ "cold_expert_fraction_n": 160,
13
+ "commit": "a6a1a221",
14
+ "config": "OLMOE_1B_7B_L16",
15
+ "cuda_graph": "0",
16
+ "dense_first_layers": "0",
17
+ "experts_per_rank": "8",
18
+ "force_balance": "0",
19
+ "global_batch_size": "8",
20
+ "global_snapshots": 160,
21
+ "hottest_expert_token_fraction_max": 0.99993896484375,
22
+ "hottest_expert_token_fraction_mean": 0.9486982345581054,
23
+ "hottest_expert_token_fraction_min": 0.390716552734375,
24
+ "hottest_expert_token_fraction_n": 160,
25
+ "incomplete_snapshots": 0,
26
+ "malformed_records": 0,
27
+ "moe_layers": "16",
28
+ "moonep_bytes_global_snapshots": 160,
29
+ "moonep_bytes_incomplete_snapshots": 0,
30
+ "moonep_bytes_malformed_records": 0,
31
+ "moonep_global_combine_payload_bytes_max": 1867161600.0,
32
+ "moonep_global_combine_payload_bytes_mean": 1569235660.8,
33
+ "moonep_global_combine_payload_bytes_min": 1295032320.0,
34
+ "moonep_global_combine_payload_bytes_n": 160,
35
+ "moonep_global_dispatch_payload_bytes_max": 1867161600.0,
36
+ "moonep_global_dispatch_payload_bytes_mean": 1569235660.8,
37
+ "moonep_global_dispatch_payload_bytes_min": 1295032320.0,
38
+ "moonep_global_dispatch_payload_bytes_n": 160,
39
+ "moonep_global_prefetch_bytes_max": 251658240.0,
40
+ "moonep_global_prefetch_bytes_mean": 193934131.2,
41
+ "moonep_global_prefetch_bytes_min": 176160768.0,
42
+ "moonep_global_prefetch_bytes_n": 160,
43
+ "moonep_global_reduce_bytes_max": 251658240.0,
44
+ "moonep_global_reduce_bytes_mean": 193934131.2,
45
+ "moonep_global_reduce_bytes_min": 176160768.0,
46
+ "moonep_global_reduce_bytes_n": 160,
47
+ "moonep_global_token_bytes_max": 3736155704.0,
48
+ "moonep_global_token_bytes_mean": 3140265240.85,
49
+ "moonep_global_token_bytes_min": 2591880616.0,
50
+ "moonep_global_token_bytes_n": 160,
51
+ "moonep_global_total_nvlink_bytes_max": 4138808888.0,
52
+ "moonep_global_total_nvlink_bytes_mean": 3528133503.25,
53
+ "moonep_global_total_nvlink_bytes_min": 2944202152.0,
54
+ "moonep_global_total_nvlink_bytes_n": 160,
55
+ "moonep_global_weight_movement_bytes_max": 503316480.0,
56
+ "moonep_global_weight_movement_bytes_mean": 387868262.4,
57
+ "moonep_global_weight_movement_bytes_min": 352321536.0,
58
+ "moonep_global_weight_movement_bytes_n": 160,
59
+ "moonep_global_weight_to_token_ratio_max": 0.1934270379594032,
60
+ "moonep_global_weight_to_token_ratio_mean": 0.12484131355395116,
61
+ "moonep_global_weight_to_token_ratio_min": 0.09462042088977833,
62
+ "moonep_global_weight_to_token_ratio_n": 160,
63
+ "moonep_remote_demand_max": 3.0,
64
+ "moonep_remote_demand_mean": 0.96328125,
65
+ "moonep_remote_demand_min": 0.0,
66
+ "moonep_remote_demand_n": 1280,
67
+ "moonep_slot_utilization_max": 0.375,
68
+ "moonep_slot_utilization_mean": 0.12041015625,
69
+ "moonep_slot_utilization_min": 0.0,
70
+ "moonep_slot_utilization_n": 1280,
71
+ "moonep_unused_prefetch_slots_max": 16.0,
72
+ "moonep_unused_prefetch_slots_mean": 14.0734375,
73
+ "moonep_unused_prefetch_slots_min": 10.0,
74
+ "moonep_unused_prefetch_slots_n": 1280,
75
+ "num_experts": "64",
76
+ "num_layers": "16",
77
+ "post_physical_rank_cv_max": 0.22561764409106666,
78
+ "post_physical_rank_cv_mean": 0.09623031930856718,
79
+ "post_physical_rank_cv_min": 0.004191678422331892,
80
+ "post_physical_rank_cv_n": 160,
81
+ "post_physical_rank_imax_max": 1.3098809189892138,
82
+ "post_physical_rank_imax_mean": 1.1295311977570006,
83
+ "post_physical_rank_imax_min": 1.0052335128996117,
84
+ "post_physical_rank_imax_n": 160,
85
+ "pre_home_rank_cv_max": 1.304032801068936,
86
+ "pre_home_rank_cv_mean": 0.7290410654343018,
87
+ "pre_home_rank_cv_min": 0.23067981190173764,
88
+ "pre_home_rank_cv_n": 160,
89
+ "pre_home_rank_imax_max": 3.941009521484375,
90
+ "pre_home_rank_imax_mean": 2.289567756652832,
91
+ "pre_home_rank_imax_min": 1.311187744140625,
92
+ "pre_home_rank_imax_n": 160,
93
+ "raw_expert_cv_max": 2.6256940664188297,
94
+ "raw_expert_cv_mean": 2.1930980791488244,
95
+ "raw_expert_cv_min": 0.8856034659168336,
96
+ "raw_expert_cv_n": 160,
97
+ "raw_expert_imax_ceiling": 8.0,
98
+ "raw_expert_imax_hit_ceiling": 1,
99
+ "raw_expert_imax_max": 7.99951171875,
100
+ "raw_expert_imax_mean": 7.589585876464843,
101
+ "raw_expert_imax_min": 3.125732421875,
102
+ "raw_expert_imax_n": 160,
103
+ "records": {
104
+ "moonep_bytes": 1280,
105
+ "moonep_routing": 1280,
106
+ "moonep_summary": 80,
107
+ "moonep_weights": 1280
108
+ },
109
+ "router_balancing": "aux_loss",
110
+ "run_dir": "OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/rep1",
111
+ "seq_length": "4096",
112
+ "shared_expert_overlap": "0",
113
+ "topk": "8"
114
+ },
115
+ {
116
+ "arm": "forced",
117
+ "aux_loss_coeff": "1e-2",
118
+ "backend": "moonep",
119
+ "branch": "moonep_backend_async_grad_reduce_token_routing_analysis",
120
+ "capacity": "none",
121
+ "capacity_mode": "none",
122
+ "cold_expert_fraction_max": 0.0,
123
+ "cold_expert_fraction_mean": 0.0,
124
+ "cold_expert_fraction_min": 0.0,
125
+ "cold_expert_fraction_n": 160,
126
+ "commit": "a6a1a221",
127
+ "config": "OLMOE_1B_7B_L16",
128
+ "cuda_graph": "0",
129
+ "dense_first_layers": "0",
130
+ "experts_per_rank": "8",
131
+ "force_balance": "1",
132
+ "global_batch_size": "8",
133
+ "global_snapshots": 160,
134
+ "hottest_expert_token_fraction_max": 0.13189697265625,
135
+ "hottest_expert_token_fraction_mean": 0.1294095993041992,
136
+ "hottest_expert_token_fraction_min": 0.1278076171875,
137
+ "hottest_expert_token_fraction_n": 160,
138
+ "incomplete_snapshots": 0,
139
+ "malformed_records": 0,
140
+ "moe_layers": "16",
141
+ "moonep_bytes_global_snapshots": 160,
142
+ "moonep_bytes_incomplete_snapshots": 0,
143
+ "moonep_bytes_malformed_records": 0,
144
+ "moonep_global_combine_payload_bytes_max": 1278230528.0,
145
+ "moonep_global_combine_payload_bytes_mean": 1275283558.4,
146
+ "moonep_global_combine_payload_bytes_min": 1271652352.0,
147
+ "moonep_global_combine_payload_bytes_n": 160,
148
+ "moonep_global_dispatch_payload_bytes_max": 1278230528.0,
149
+ "moonep_global_dispatch_payload_bytes_mean": 1275283558.4,
150
+ "moonep_global_dispatch_payload_bytes_min": 1271652352.0,
151
+ "moonep_global_dispatch_payload_bytes_n": 160,
152
+ "moonep_global_prefetch_bytes_max": 176160768.0,
153
+ "moonep_global_prefetch_bytes_mean": 173486899.2,
154
+ "moonep_global_prefetch_bytes_min": 125829120.0,
155
+ "moonep_global_prefetch_bytes_n": 160,
156
+ "moonep_global_reduce_bytes_max": 176160768.0,
157
+ "moonep_global_reduce_bytes_mean": 173486899.2,
158
+ "moonep_global_reduce_bytes_min": 125829120.0,
159
+ "moonep_global_reduce_bytes_n": 160,
160
+ "moonep_global_token_bytes_max": 2558296472.0,
161
+ "moonep_global_token_bytes_mean": 2552401001.25,
162
+ "moonep_global_token_bytes_min": 2545135352.0,
163
+ "moonep_global_token_bytes_n": 160,
164
+ "moonep_global_total_nvlink_bytes_max": 2910618008.0,
165
+ "moonep_global_total_nvlink_bytes_mean": 2899374799.65,
166
+ "moonep_global_total_nvlink_bytes_min": 2805284064.0,
167
+ "moonep_global_total_nvlink_bytes_n": 160,
168
+ "moonep_global_weight_movement_bytes_max": 352321536.0,
169
+ "moonep_global_weight_movement_bytes_mean": 346973798.4,
170
+ "moonep_global_weight_movement_bytes_min": 251658240.0,
171
+ "moonep_global_weight_movement_bytes_n": 160,
172
+ "moonep_global_weight_to_token_ratio_max": 0.1384293906896312,
173
+ "moonep_global_weight_to_token_ratio_mean": 0.1359406755873575,
174
+ "moonep_global_weight_to_token_ratio_min": 0.09854937933146465,
175
+ "moonep_global_weight_to_token_ratio_n": 160,
176
+ "moonep_remote_demand_max": 1.0,
177
+ "moonep_remote_demand_mean": 0.86171875,
178
+ "moonep_remote_demand_min": 0.0,
179
+ "moonep_remote_demand_n": 1280,
180
+ "moonep_slot_utilization_max": 0.125,
181
+ "moonep_slot_utilization_mean": 0.10771484375,
182
+ "moonep_slot_utilization_min": 0.0,
183
+ "moonep_slot_utilization_n": 1280,
184
+ "moonep_unused_prefetch_slots_max": 16.0,
185
+ "moonep_unused_prefetch_slots_mean": 14.2765625,
186
+ "moonep_unused_prefetch_slots_min": 14.0,
187
+ "moonep_unused_prefetch_slots_n": 1280,
188
+ "num_experts": "64",
189
+ "num_layers": "16",
190
+ "post_physical_rank_cv_max": 0.004151405986167901,
191
+ "post_physical_rank_cv_mean": 0.0021700897212804954,
192
+ "post_physical_rank_cv_min": 0.0007203705850357968,
193
+ "post_physical_rank_cv_n": 160,
194
+ "post_physical_rank_imax_max": 1.0083050780744813,
195
+ "post_physical_rank_imax_mean": 1.0033887240372406,
196
+ "post_physical_rank_imax_min": 1.0013429305103698,
197
+ "post_physical_rank_imax_n": 160,
198
+ "pre_home_rank_cv_max": 0.009250589744190636,
199
+ "pre_home_rank_cv_mean": 0.004649780487546198,
200
+ "pre_home_rank_cv_min": 0.001428470427059137,
201
+ "pre_home_rank_cv_n": 160,
202
+ "pre_home_rank_imax_max": 1.015838623046875,
203
+ "pre_home_rank_imax_mean": 1.007100486755371,
204
+ "pre_home_rank_imax_min": 1.002044677734375,
205
+ "pre_home_rank_imax_n": 160,
206
+ "raw_expert_cv_max": 0.017802554164692116,
207
+ "raw_expert_cv_mean": 0.014583830753424936,
208
+ "raw_expert_cv_min": 0.010177326341354839,
209
+ "raw_expert_cv_n": 160,
210
+ "raw_expert_imax_ceiling": 8.0,
211
+ "raw_expert_imax_hit_ceiling": 0,
212
+ "raw_expert_imax_max": 1.05517578125,
213
+ "raw_expert_imax_mean": 1.0352767944335937,
214
+ "raw_expert_imax_min": 1.0224609375,
215
+ "raw_expert_imax_n": 160,
216
+ "records": {
217
+ "moonep_bytes": 1280,
218
+ "moonep_routing": 1280,
219
+ "moonep_summary": 80,
220
+ "moonep_weights": 1280
221
+ },
222
+ "router_balancing": "aux_loss",
223
+ "run_dir": "OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1",
224
+ "seq_length": "4096",
225
+ "shared_expert_overlap": "0",
226
+ "topk": "8"
227
+ }
228
+ ]
granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ run_dir arm config backend num_experts topk experts_per_rank num_layers moe_layers dense_first_layers seq_length global_batch_size capacity_mode capacity force_balance cuda_graph branch commit router_balancing aux_loss_coeff shared_expert_overlap cold_expert_fraction_max cold_expert_fraction_mean cold_expert_fraction_min cold_expert_fraction_n global_snapshots hottest_expert_token_fraction_max hottest_expert_token_fraction_mean hottest_expert_token_fraction_min hottest_expert_token_fraction_n incomplete_snapshots malformed_records moonep_bytes_global_snapshots moonep_bytes_incomplete_snapshots moonep_bytes_malformed_records moonep_global_combine_payload_bytes_max moonep_global_combine_payload_bytes_mean moonep_global_combine_payload_bytes_min moonep_global_combine_payload_bytes_n moonep_global_dispatch_payload_bytes_max moonep_global_dispatch_payload_bytes_mean moonep_global_dispatch_payload_bytes_min moonep_global_dispatch_payload_bytes_n moonep_global_prefetch_bytes_max moonep_global_prefetch_bytes_mean moonep_global_prefetch_bytes_min moonep_global_prefetch_bytes_n moonep_global_reduce_bytes_max moonep_global_reduce_bytes_mean moonep_global_reduce_bytes_min moonep_global_reduce_bytes_n moonep_global_token_bytes_max moonep_global_token_bytes_mean moonep_global_token_bytes_min moonep_global_token_bytes_n moonep_global_total_nvlink_bytes_max moonep_global_total_nvlink_bytes_mean moonep_global_total_nvlink_bytes_min moonep_global_total_nvlink_bytes_n moonep_global_weight_movement_bytes_max moonep_global_weight_movement_bytes_mean moonep_global_weight_movement_bytes_min moonep_global_weight_movement_bytes_n moonep_global_weight_to_token_ratio_max moonep_global_weight_to_token_ratio_mean moonep_global_weight_to_token_ratio_min moonep_global_weight_to_token_ratio_n moonep_remote_demand_max moonep_remote_demand_mean moonep_remote_demand_min moonep_remote_demand_n moonep_slot_utilization_max moonep_slot_utilization_mean moonep_slot_utilization_min moonep_slot_utilization_n moonep_unused_prefetch_slots_max moonep_unused_prefetch_slots_mean moonep_unused_prefetch_slots_min moonep_unused_prefetch_slots_n post_physical_rank_cv_max post_physical_rank_cv_mean post_physical_rank_cv_min post_physical_rank_cv_n post_physical_rank_imax_max post_physical_rank_imax_mean post_physical_rank_imax_min post_physical_rank_imax_n pre_home_rank_cv_max pre_home_rank_cv_mean pre_home_rank_cv_min pre_home_rank_cv_n pre_home_rank_imax_max pre_home_rank_imax_mean pre_home_rank_imax_min pre_home_rank_imax_n raw_expert_cv_max raw_expert_cv_mean raw_expert_cv_min raw_expert_cv_n raw_expert_imax_ceiling raw_expert_imax_hit_ceiling raw_expert_imax_max raw_expert_imax_mean raw_expert_imax_min raw_expert_imax_n
2
+ OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/rep1 natural OLMOE_1B_7B_L16 moonep 64 8 8 16 16 0 4096 8 none none 0 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 aux_loss 1e-2 0 0.484375 0.2111328125 0.0 160 160 0.99993896484375 0.9486982345581054 0.390716552734375 160 0 0 160 0 0 1867161600.0 1569235660.8 1295032320.0 160 1867161600.0 1569235660.8 1295032320.0 160 251658240.0 193934131.2 176160768.0 160 251658240.0 193934131.2 176160768.0 160 3736155704.0 3140265240.85 2591880616.0 160 4138808888.0 3528133503.25 2944202152.0 160 503316480.0 387868262.4 352321536.0 160 0.1934270379594032 0.12484131355395116 0.09462042088977833 160 3.0 0.96328125 0.0 1280 0.375 0.12041015625 0.0 1280 16.0 14.0734375 10.0 1280 0.22561764409106666 0.09623031930856718 0.004191678422331892 160 1.3098809189892138 1.1295311977570006 1.0052335128996117 160 1.304032801068936 0.7290410654343018 0.23067981190173764 160 3.941009521484375 2.289567756652832 1.311187744140625 160 2.6256940664188297 2.1930980791488244 0.8856034659168336 160 8.0 1 7.99951171875 7.589585876464843 3.125732421875 160
3
+ OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1 forced OLMOE_1B_7B_L16 moonep 64 8 8 16 16 0 4096 8 none none 1 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 aux_loss 1e-2 0 0.0 0.0 0.0 160 160 0.13189697265625 0.1294095993041992 0.1278076171875 160 0 0 160 0 0 1278230528.0 1275283558.4 1271652352.0 160 1278230528.0 1275283558.4 1271652352.0 160 176160768.0 173486899.2 125829120.0 160 176160768.0 173486899.2 125829120.0 160 2558296472.0 2552401001.25 2545135352.0 160 2910618008.0 2899374799.65 2805284064.0 160 352321536.0 346973798.4 251658240.0 160 0.1384293906896312 0.1359406755873575 0.09854937933146465 160 1.0 0.86171875 0.0 1280 0.125 0.10771484375 0.0 1280 16.0 14.2765625 14.0 1280 0.004151405986167901 0.0021700897212804954 0.0007203705850357968 160 1.0083050780744813 1.0033887240372406 1.0013429305103698 160 0.009250589744190636 0.004649780487546198 0.001428470427059137 160 1.015838623046875 1.007100486755371 1.002044677734375 160 0.017802554164692116 0.014583830753424936 0.010177326341354839 160 8.0 0 1.05517578125 1.0352767944335937 1.0224609375 160
granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/results.tsv ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
2
+ OLMOE_1B_7B_L16 moonep matched 8 4096 8 none none 1 OK 220.81 215.30 35751 /workspace/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/rep1
3
+ OLMOE_1B_7B_L16 moonep matched 8 4096 8 none none+forcebal 1 OK 212.30 210.50 35751 /workspace/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1