AbhishekGhosh1998 commited on
Commit
2bbfb21
·
verified ·
1 Parent(s): 7ff5ea2

granularity results granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb @ 2026-09-21T07:50:43Z from sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0

Browse files
Files changed (13) hide show
  1. granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/manifest.txt +19 -0
  2. granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/pip_freeze.txt +317 -0
  3. granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/provenance.txt +90 -0
  4. granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep1/mem_samples.csv +200 -0
  5. granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep1/run.env +102 -0
  6. granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep1/train.log +1073 -0
  7. granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep2/mem_samples.csv +192 -0
  8. granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep2/run.env +102 -0
  9. granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep2/train.log +1072 -0
  10. granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep3/mem_samples.csv +192 -0
  11. granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep3/run.env +102 -0
  12. granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep3/train.log +1072 -0
  13. granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/results.tsv +4 -0
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/manifest.txt ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ timestamp: 2026-09-21T07:47:26+00:00
2
+ host: sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0
3
+ gpu: NVIDIA B200 x8
4
+ driver: 580.95.05
5
+ study_scripts_branch: granularity-ultraep
6
+ study_scripts_commit: 05dd2cb
7
+ campaign: ultraep
8
+ backend: ultraep
9
+ phase: tierb
10
+ gbs: 8
11
+ seq: 4096
12
+ configs: G1 G2 G3 G4
13
+ capacity: 1.25
14
+ repeats: 3
15
+ nsys: 1
16
+ megatron_repo: /workspace/Megatron-UltraEP
17
+ megatron_branch_required: ultraep-bracing-regions-through-kernels
18
+ megatron_branch_actual: ultraep-bracing-regions-through-kernels
19
+ megatron_commit: 03d57ef7
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/pip_freeze.txt ADDED
@@ -0,0 +1,317 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ absl-py==2.3.0
2
+ aiohappyeyeballs @ file:///rapids/aiohappyeyeballs-2.6.1-py3-none-any.whl#sha256=f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8
3
+ aiohttp @ file:///rapids/aiohttp-3.12.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=94f98e0e5a49f89b252e115844f756c04fc8050f38252a32a3dd994ce8121f10
4
+ aiosignal @ file:///rapids/aiosignal-1.3.2-py2.py3-none-any.whl#sha256=45cde58e409a301715980c2b01d0c28bdde3770d8290b5eb2173759d9acb31a5
5
+ annotated-types==0.7.0
6
+ anyio==4.9.0
7
+ apex @ file:///opt/pytorch/apex/dist/apex-0.1-cp312-cp312-linux_x86_64.whl#sha256=a1508e4a386362e9c8f46c92ccada9110e421212b964c694235d0fd5a224b170
8
+ argon2-cffi==25.1.0
9
+ argon2-cffi-bindings==21.2.0
10
+ arrow==1.3.0
11
+ asciitree @ file:///rapids/asciitree-0.3.3-py3-none-any.whl#sha256=3e507ccfaeb417823158091fdb9d26b0205f6a59ba680b479a71db65c73d01d4
12
+ asttokens==3.0.0
13
+ astunparse==1.6.3
14
+ async-lru==2.0.5
15
+ attrs==25.3.0
16
+ audioread==3.0.1
17
+ babel==2.17.0
18
+ beautifulsoup4==4.13.4
19
+ black==25.1.0
20
+ bleach==6.2.0
21
+ blis==0.7.11
22
+ cachetools @ file:///rapids/cachetools-6.0.0-py3-none-any.whl#sha256=82e73ba88f7b30228b5507dce1a1f878498fc669d972aef2dde4f3a3c24f103e
23
+ catalogue==2.0.10
24
+ certifi==2025.4.26
25
+ cffi==1.17.1
26
+ charset-normalizer==3.4.2
27
+ click==8.5.0
28
+ cloudpathlib==0.21.1
29
+ cloudpickle @ file:///rapids/cloudpickle-3.1.1-py3-none-any.whl#sha256=c8c5a44295039331ee9dad40ba100a9c7297b6f988e50e87ccdf3765a668350e
30
+ cmake==3.31.6
31
+ comm==0.2.2
32
+ confection==0.1.5
33
+ contourpy==1.3.2
34
+ cuda-bindings @ file:///rapids/cuda_bindings-12.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=ff0e28d1e34758654b9c961e1f55e4786e49aee6a4dbceaf3cc24c46c672df7e
35
+ cuda-python @ file:///rapids/cuda_python-12.9.0-py3-none-any.whl#sha256=926acba49b2c0a0374c61b7c98f337c085199cf51cdfe4d6423c4129c20547a7
36
+ cudf @ file:///rapids/cudf-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b779a8e8c0e8577b9b560895958496da211bfac49cdc4f11a49e8d18cb82e348
37
+ cudf-polars @ file:///rapids/cudf_polars-25.4.0-py3-none-any.whl#sha256=227b6c5147e93ea86282c8f7219118f916769fa2ca0cdd931e97a148aa66b869
38
+ cugraph @ file:///rapids/cugraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b837c2dc2f7b235c702e4f8df64479b8fcbb342f848516d5faf0cdc4b5f31f29
39
+ cugraph-service-client @ file:///rapids/cugraph_service_client-25.4.0-py3-none-any.whl#sha256=7e568c3e41154b9752f5173555ffcc282ac52d72343feeb79653b2e2b1232470
40
+ cugraph-service-server @ file:///rapids/cugraph_service_server-25.4.0-py3-none-any.whl#sha256=a2830dcffafcccb16e12ebf85b98f8f247869d38c01ab4dd8664a1e7439d31be
41
+ cuml @ file:///rapids/cuml-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=1e90a464a18b094e24797f1676688c5d7696c4cf97c934504214a53d5aa09aa5
42
+ cupy-cuda12x @ file:///rapids/cupy_cuda12x-13.3.0-cp312-cp312-linux_x86_64.whl#sha256=6a2dfa0aa1e8189a10d1534276c08f1d2f73b747fd28bfbe7b9f4ef66d9af73f
43
+ cuvs @ file:///rapids/cuvs-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b12f6b9f80e053a3b38542c2947d2516e468c02cad54a13d8b1a4a61d5aa7914
44
+ cycler==0.12.1
45
+ cymem==2.0.11
46
+ Cython==3.1.1
47
+ dask @ file:///rapids/dask-2025.2.0-py3-none-any.whl#sha256=f0fdeef6ceb0a06569d456c9e704f220f7f54e80f3a6ea42ab98cea6bc642b6e
48
+ dask-cuda @ file:///rapids/dask_cuda-25.4.0-py3-none-any.whl#sha256=442d87da0a95d00ca2a262b64c76e785a32ad905c82000fa2227932f81390e19
49
+ dask-cudf @ file:///rapids/dask_cudf-25.4.0-py3-none-any.whl#sha256=5e7cea0b26daea17da6e0bb457435e24460f76ba740358a3a18e239d6015c95c
50
+ datasets==5.0.1
51
+ debugpy==1.8.14
52
+ decorator==5.2.1
53
+ -e git+https://github.com/deepseek-ai/DeepEP.git@34152ae28f80bcc3ee38d7a12cb2ad87cfd4ea72#egg=deep_ep
54
+ defusedxml==0.7.1
55
+ dill==0.4.1
56
+ distributed @ file:///rapids/distributed-2025.2.0-py3-none-any.whl#sha256=368462084be47092d0a835f6c64c9b852da64dc88302b76d5f93bd5131228c67
57
+ distributed-ucxx @ file:///rapids/distributed_ucxx-0.43.0-py3-none-any.whl#sha256=94f60d0ccb0c94ba8dbfa1eaf455874d18aaab17e2ae218c38590e9fdc2d721d
58
+ dm-tree==0.1.9
59
+ einops==0.8.1
60
+ execnet==2.1.1
61
+ executing==2.2.0
62
+ expecttest==0.3.0
63
+ fasteners @ file:///rapids/fasteners-0.19-py3-none-any.whl#sha256=758819cb5d94cdedf4e836988b74de396ceacb8e2794d21f82d131fd9ee77237
64
+ fastjsonschema==2.21.1
65
+ fastrlock @ file:///rapids/fastrlock-0.8.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl#sha256=5f13ec08f1adb1aa916c384b05ecb7dbebb8df9ea81abd045f60941c6283a670
66
+ filelock==3.18.0
67
+ flash_attn @ file:///opt/pytorch/flash_attn-2.7.4.post1-cp312-cp312-linux_x86_64.whl#sha256=a10455d6c56b387785b7413fd0bbcb392b42b3d4f4b88fda5d86cc8a9ce9182a
68
+ fonttools==4.58.1
69
+ fqdn==1.5.1
70
+ frozenlist @ file:///rapids/frozenlist-1.6.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=4e1be9111cb6756868ac242b3c2bd1f09d9aea09846e4f5c23715e7afb647103
71
+ fsspec==2025.5.1
72
+ gast==0.6.0
73
+ grpcio==1.62.1
74
+ h11==0.16.0
75
+ hf-xet==1.6.0
76
+ httpcore==1.0.9
77
+ httpx==0.28.1
78
+ huggingface_hub==1.32.0
79
+ hypothesis==6.130.8
80
+ idna==3.10
81
+ importlib_metadata==8.7.0
82
+ iniconfig==2.1.0
83
+ intel-openmp==2021.4.0
84
+ ipykernel==6.29.5
85
+ ipython==9.3.0
86
+ ipython_pygments_lexers==1.1.1
87
+ isoduration==20.11.0
88
+ isort==6.0.1
89
+ jedi==0.19.2
90
+ Jinja2==3.1.6
91
+ joblib==1.5.1
92
+ json5==0.12.0
93
+ jsonpointer==3.0.0
94
+ jsonschema==4.24.0
95
+ jsonschema-specifications==2025.4.1
96
+ jupyter-events==0.12.0
97
+ jupyter-lsp==2.2.5
98
+ jupyter_client==8.6.3
99
+ jupyter_core==5.8.1
100
+ jupyter_server==2.16.0
101
+ jupyter_server_terminals==0.5.3
102
+ jupyterlab==4.4.3
103
+ jupyterlab_code_formatter==3.0.2
104
+ jupyterlab_pygments==0.3.0
105
+ jupyterlab_server==2.27.3
106
+ jupyterlab_tensorboard_pro @ file:///builder/jupyterlab_tensorboard_pro-4.0.0-py2.py3-none-any.whl#sha256=6bfcb5ced3358eb72150f4b2ab91312faef2f0ca14d845e8379d8fa52df6902e
107
+ jupytext==1.17.2
108
+ kiwisolver==1.4.8
109
+ kvikio @ file:///rapids/kvikio-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=8b40fd8ea9e4a2974c04b50b7fc26b06094ac807f9a233281d0bafbaa71b1d85
110
+ langcodes==3.5.0
111
+ language_data==1.3.0
112
+ lazy_loader==0.4
113
+ libcudf @ file:///rapids/libcudf-25.4.0-py3-none-linux_x86_64.whl#sha256=238320b4958c1737bba2a609673e2539a78216b811b5498a68e00395d9b021a4
114
+ libcugraph @ file:///rapids/libcugraph-25.4.0-py3-none-linux_x86_64.whl#sha256=29085e901b2f7569ff2653bb1355fe6fad20f2fcaa585f5649da9867d8df766d
115
+ libcuml @ file:///rapids/libcuml-25.4.0-py3-none-linux_x86_64.whl#sha256=8e2e444c72b0e995e15a48da37ff12fb410a47a2baa69ba86ccc9b1a3db91e8c
116
+ libcuvs @ file:///rapids/libcuvs-25.4.0-py3-none-linux_x86_64.whl#sha256=e2f8db61c92b769c590dd752f43439ddd62b2f6592a3c337362b6c1ef220c3b9
117
+ libkvikio @ file:///rapids/libkvikio-25.4.0-py3-none-linux_x86_64.whl#sha256=da8a5713b05fc478bf40529a3e0fb61a2d605bb5a70ca4d461f96a6005dbe8b7
118
+ libraft @ file:///rapids/libraft-25.4.0-py3-none-linux_x86_64.whl#sha256=b921dec005aa4a321dd156776a18376feb47c96a5b67eec125c8a8e19edd1e24
119
+ librmm @ file:///rapids/librmm-25.4.0-py3-none-linux_x86_64.whl#sha256=ff951811bda729d4fa1109ab906225ae1f489a6ec2eeab5d1b674cc34bd4422f
120
+ librmm-cu12 @ file:///rapids/librmm_cu12-25.4.0-py3-none-any.whl#sha256=29296ef2b0e90679a0db280e86015be12d6ab9f4e62a181aa085f62dbadd06b3
121
+ librosa==0.11.0
122
+ libucxx @ file:///rapids/libucxx-0.43.0-py3-none-linux_x86_64.whl#sha256=f15849d7ac2fb68f0fd8998838a13321ddcfb0c594f7e6cbc401a3bcc4750566
123
+ lightning-thunder @ file:///opt/pytorch/lightning-thunder
124
+ lightning-utilities==0.14.3
125
+ lintrunner==0.12.7
126
+ llvmlite==0.42.0
127
+ locket @ file:///rapids/locket-1.0.0-py2.py3-none-any.whl#sha256=b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3
128
+ looseversion==1.3.0
129
+ marisa-trie==1.2.1
130
+ Markdown==3.8
131
+ markdown-it-py==3.0.0
132
+ MarkupSafe==3.0.2
133
+ matplotlib==3.10.3
134
+ matplotlib-inline==0.1.7
135
+ mdit-py-plugins==0.4.2
136
+ mdurl==0.1.2
137
+ -e git+ssh://git@github.com/sriram-17-17/my-Megatron-MoE-Echo.git@9a23679390d7ab1fe462bc61fa31d90d393db49c#egg=megatron_core
138
+ mistune==3.1.3
139
+ mkl==2021.1.1
140
+ mkl-devel==2021.1.1
141
+ mkl-include==2021.1.1
142
+ ml_dtypes==0.6.0
143
+ mock==5.2.0
144
+ -e git+ssh://git@github.com/sriram-17-17/MoonEP.git@d8bc6667fad7177e6e09b637d773fa67f08bae93#egg=moonep
145
+ mpmath==1.3.0
146
+ msgpack==1.1.0
147
+ multidict @ file:///rapids/multidict-6.4.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=a920f9cf2abdf6e493c519492d892c362007f113c94da4c239ae88429835bad1
148
+ multiprocess==0.70.19
149
+ murmurhash==1.0.13
150
+ mypy_extensions==1.1.0
151
+ nbclient==0.10.2
152
+ nbconvert==7.16.6
153
+ nbformat==5.10.4
154
+ nest-asyncio==1.6.0
155
+ networkx==3.5
156
+ ninja==1.11.1.4
157
+ notebook==7.4.3
158
+ notebook_shim==0.2.4
159
+ numba==0.59.1
160
+ numba-cuda @ file:///rapids/numba_cuda-0.4.0-py3-none-any.whl#sha256=5b95a5f42068771e37142944d44fe844fa87f1b2f89051461a4cb0af91bcc28a
161
+ numcodecs @ file:///rapids/numcodecs-0.13.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=da2230484e6102e5fa3cc1a5dd37ca1f92dfbd183d91662074d6f7574e3e8f53
162
+ numpy==1.26.4
163
+ nvdlfw_inspect @ git+https://github.com/NVIDIA/nvidia-dlfw-inspect.git@6b60eb0e675606fb2fbbfcb12667ecbec75eaf87
164
+ nvfuser==0.2.27a0+9bf5aca
165
+ nvidia-cuda-cccl-cu12==12.9.27
166
+ nvidia-cudnn-frontend @ file:///opt/pytorch/lightning-thunder/tmp/cudnn_frontend
167
+ nvidia-cutlass-dsl==4.4.2
168
+ nvidia-cutlass-dsl-libs-base==4.4.2
169
+ nvidia-dali-cuda120==1.50.0
170
+ nvidia-ml-py @ file:///rapids/nvidia_ml_py-12.575.51-py3-none-any.whl#sha256=eb8641800d98ce40a22f479873f34b482e214a7e80349c63be51c3919845446e
171
+ nvidia-modelopt==0.29.0
172
+ nvidia-modelopt-core==0.29.0
173
+ nvidia-nvcomp-cu12==4.2.0.14
174
+ nvidia-nvimgcodec-cu12==0.5.0.13
175
+ nvidia-nvjpeg-cu12==12.4.0.16
176
+ nvidia-nvjpeg2k-cu12==0.8.1.40
177
+ nvidia-nvshmem-cu12==3.7.2
178
+ nvidia-nvtiff-cu12==0.5.0.67
179
+ nvidia-resiliency-ext==0.4.0
180
+ nvtx @ file:///rapids/nvtx-0.2.11-cp312-cp312-linux_x86_64.whl#sha256=73521e37cba42b4db0b3cd68fc5eb814e55c500a61eeed6edcdf6746315ac53e
181
+ nx-cugraph @ file:///rapids/nx_cugraph-25.4.0-py3-none-any.whl#sha256=e625444486cc899a77bd48e7da76a2edd95c0006dd95fbf323745feae7f33111
182
+ onnx @ file:///opt/pytorch/pytorch/third_party/onnx
183
+ onnx-ir==1.0.0
184
+ onnxscript==0.7.2
185
+ opt_einsum==3.4.0
186
+ optree==0.16.0
187
+ overrides==7.7.0
188
+ packaging==23.2
189
+ pandas @ file:///rapids/pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319
190
+ pandocfilters==1.5.1
191
+ parso==0.8.4
192
+ partd @ file:///rapids/partd-1.4.2-py3-none-any.whl#sha256=978e4ac767ec4ba5b86c6eaa52e5a2a3bc748a2ca839e8cc798f1cc6ce6efb0f
193
+ pathspec==0.12.1
194
+ pexpect==4.9.0
195
+ pillow==11.2.1
196
+ platformdirs==4.3.8
197
+ pluggy==1.6.0
198
+ ply @ file:///rapids/ply-3.11-py2.py3-none-any.whl#sha256=096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce
199
+ polars @ file:///rapids/polars-1.25.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=f7fcbb4f476784384ccda48757fca4e8c2e2c5a0a3aef3717aaf56aee4e30e09
200
+ polygraphy==0.49.20
201
+ pooch==1.8.2
202
+ preshed==3.0.10
203
+ prometheus_client==0.22.1
204
+ prompt_toolkit==3.0.51
205
+ propcache @ file:///rapids/propcache-0.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=563f9d8c03ad645597b8d010ef4e9eab359faeb11a0a2ac9f7b4bc8c28ebef25
206
+ protobuf==4.24.4
207
+ psutil==7.0.0
208
+ ptyprocess==0.7.0
209
+ PuLP==3.2.1
210
+ pure_eval==0.2.3
211
+ pyarrow==25.0.1
212
+ pybind11==2.13.6
213
+ pybind11_global==2.13.6
214
+ pycocotools @ git+https://github.com/nvidia/cocoapi.git@08350a157b41f0755a536441b5abec89ab934c5b#subdirectory=PythonAPI
215
+ pycparser==2.22
216
+ pydantic==2.11.5
217
+ pydantic_core==2.33.2
218
+ Pygments==2.19.1
219
+ pylibcudf @ file:///rapids/pylibcudf-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=401b8bfd022d06f84e120779ca7c3d01a28b68762aaa8f7873b10d6b1975cf6a
220
+ pylibcugraph @ file:///rapids/pylibcugraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=061c7389a88c815b54fd5d3a3584fb87bf693e5602f5be335260d0ba575ba557
221
+ pylibcugraphops @ file:///rapids/pylibcugraphops-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=a25ef4b22166c80718d5c7e5bddb755e81a2fe340fa5becd0ac36ec3edfb208a
222
+ pylibraft @ file:///rapids/pylibraft-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=f2ecd23fe51f9c2caa87bbbe85cb32fcaff4ecfb52ceb9a9991bc50f82782c2e
223
+ pylibwholegraph @ file:///rapids/pylibwholegraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b82913d1e38bda76d8c272fbd8cc9072d987209d3d47cc93a464a37c2925aca2
224
+ pynvjitlink @ file:///rapids/pynvjitlink-0.3.0-cp312-cp312-linux_x86_64.whl#sha256=066caf1314730643d3b48c9847bd65789defdd6a7c78993176eb3d8bbe3f4919
225
+ pynvml @ file:///rapids/pynvml-12.0.0-py3-none-any.whl#sha256=fdff84b62a27dbe98e08e1a647eb77342bef1aebe0878bcd15e99a83fcbecb9e
226
+ pyparsing==3.2.3
227
+ pytest==8.1.1
228
+ pytest-flakefinder==1.1.0
229
+ pytest-rerunfailures==15.1
230
+ pytest-shard==0.1.2
231
+ pytest-xdist==3.7.0
232
+ python-dateutil==2.9.0.post0
233
+ python-hostlist==2.2.1
234
+ python-json-logger==3.3.0
235
+ pytorch-triton @ file:///tmp/dist/pytorch_triton-3.3.0%2Bgit96316ce52.nvinternal-cp312-cp312-linux_x86_64.whl#sha256=6d6921772b9dfd0243c055e0d054de42a113e92cabcfdd1d3e80a5ad9c08cf7d
236
+ pytz @ file:///rapids/pytz-2023.4-py2.py3-none-any.whl#sha256=f90ef520d95e7c46951105338d918664ebfd6f1d995bd7d153127ce90efafa6a
237
+ PyYAML==6.0.2
238
+ pyzmq==26.4.0
239
+ raft-dask @ file:///rapids/raft_dask-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=36cf37ba6ce35454c4729c2cb597a360f22780eefb11d514a16a07137da5eb6c
240
+ rapids-dask-dependency @ file:///rapids/rapids_dask_dependency-25.4.0a0-py3-none-any.whl#sha256=484faca626e7760112a7d8ad3426f5bdd720b609006d8dca87c2607076a348bd
241
+ rapids-logger @ file:///rapids/rapids_logger-0.1.18-py3-none-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=3617b2d86291ac6cb957542895f20fc9737ace29773ca90337e6fb04c73bd120
242
+ referencing==0.36.2
243
+ regex==2024.11.6
244
+ requests==2.32.3
245
+ rfc3339-validator==0.1.4
246
+ rfc3986-validator==0.1.1
247
+ rich==14.0.0
248
+ rmm @ file:///rapids/rmm-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=1824bcab3836296666980a997ddcb9c4bca24dfb873012cfa0a7ecea583ca712
249
+ rpds-py==0.25.1
250
+ safetensors==0.5.3
251
+ scikit-learn==1.6.1
252
+ scipy==1.15.3
253
+ Send2Trash==1.8.3
254
+ setuptools==78.1.1
255
+ shellingham==1.5.4
256
+ six==1.16.0
257
+ smart-open==7.1.0
258
+ sniffio==1.3.1
259
+ sortedcontainers==2.4.0
260
+ soundfile==0.13.1
261
+ soupsieve==2.7
262
+ soxr==0.5.0.post1
263
+ spacy==3.7.5
264
+ spacy-legacy==3.0.12
265
+ spacy-loggers==1.0.5
266
+ srsly==2.5.1
267
+ stack-data==0.6.3
268
+ sympy==1.14.0
269
+ tabulate==0.9.0
270
+ tbb==2021.13.1
271
+ tblib @ file:///rapids/tblib-3.1.0-py3-none-any.whl#sha256=670bb4582578134b3d81a84afa1b016128b429f3d48e6cbbaecc9d15675e984e
272
+ tensorboard==2.16.2
273
+ tensorboard-data-server==0.7.2
274
+ tensorrt @ file:///workspace/TensorRT-10.11.0.33/python/tensorrt-10.11.0.33-cp312-none-linux_x86_64.whl#sha256=c2f18f72758a3d54a71901495c8fda9e7094a8bcdd2a5b6b58967fbf05733b3a
275
+ terminado==0.18.1
276
+ thinc==8.2.5
277
+ threadpoolctl==3.6.0
278
+ thriftpy2 @ file:///rapids/thriftpy2-0.5.2-cp312-cp312-linux_x86_64.whl#sha256=c0c7d939bdc5ccf7739154959938445d5fdf72f1d6e12bd563947a36f67e2147
279
+ tinycss2==1.4.0
280
+ toolz @ file:///rapids/toolz-1.0.0-py3-none-any.whl#sha256=292c8f1c4e7516bf9086f8850935c799a874039c8bcf959d47b600e4c44a6236
281
+ torch @ file:///opt/transfer/torch-2.8.0a0%2B5228986c39.nv25.6-29946474-cp312-cp312-linux_x86_64.whl#sha256=7a3c9ff5288d8102a31296b3d62491f3f47c8820a5c9aaefc7dc0fe20e1e8ad5
282
+ torch_tensorrt @ file:///opt/pytorch/torch_tensorrt/dist/torch_tensorrt-2.8.0a0-cp312-cp312-linux_x86_64.whl#sha256=26f4d7a04e7e6e017a5b917b8b98b83720bb2bb0195f48d25f52d567d286bfe1
283
+ torchao @ file:///opt/pytorch/ao
284
+ torchprofile==0.0.4
285
+ torchvision @ file:///opt/transfer/torchvision-0.22.0a0%2B95f10a4e-29946474-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=746a3c674cc61680a86f4a14b374b21af1b5c21f629c42765b40c9494f6b234d
286
+ tornado==6.5.1
287
+ tqdm==4.67.1
288
+ traitlets==5.14.3
289
+ transformer_engine @ file:///pvcdata/aiinfra/b-aghosh/wheels/transformer_engine-2.10.0.dev0%2B10f6128d-cp312-cp312-linux_x86_64.whl#sha256=35cf5f4b83b2baad864e12e512bb422e0929409ac05610812fa521d4942d0189
290
+ treelite @ file:///rapids/treelite-4.4.1-py3-none-linux_x86_64.whl#sha256=174741724205540a1a5b37da81483aeff55d89548f42b3461cf07acac7ae96eb
291
+ typer==0.16.0
292
+ types-dataclasses==0.6.6
293
+ types-python-dateutil==2.9.0.20250516
294
+ typing-inspection==0.4.1
295
+ typing_extensions==4.14.0
296
+ tzdata @ file:///rapids/tzdata-2025.2-py2.py3-none-any.whl#sha256=1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8
297
+ ucx-py @ file:///rapids/ucx_py-0.43.0-cp312-cp312-linux_x86_64.whl#sha256=4a452fd8d8c983283ada3d49086e3657dc2ee94fbad08efdd708d15ab6e0b73a
298
+ ucxx @ file:///rapids/ucxx-0.43.0-cp312-cp312-linux_x86_64.whl#sha256=f1b4e55cb45c66fe88a2d8f9c2b87751165f3cfec21fbdc9c47c4ddff75e8a5f
299
+ -e git+https://github.com/Dots-Infra/UltraEP.git@94cab099b44fffa99a82fea99e7c12d89cf65e4f#egg=ultra_ep
300
+ uri-template==1.3.0
301
+ urllib3==2.0.7
302
+ wasabi==1.1.3
303
+ wcwidth==0.2.13
304
+ weasel==0.4.1
305
+ webcolors==24.11.1
306
+ webencodings==0.5.1
307
+ websocket-client==1.8.0
308
+ Werkzeug==3.1.3
309
+ wheel==0.45.1
310
+ wrapt==1.17.2
311
+ xdoctest==1.0.2
312
+ xgboost @ file:///rapids/xgboost-2.1.4-py3-none-manylinux_2_39_x86_64.whl#sha256=25eeca27f4048fc491940b3a0b4b5782ee48c33cb3aec059cef71f8da542930b
313
+ xxhash==4.0.1
314
+ yarl @ file:///rapids/yarl-1.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=bf099e2432131093cc611623e0b0bcc399b8cddd9a91eded8bfb50402ec35018
315
+ zarr @ file:///rapids/zarr-2.18.7-py3-none-any.whl#sha256=ac3dc4033e9ae4e9d7b5e27c97ea3eaf1003cc0a07f010bd83d5134bf8c4b223
316
+ zict @ file:///rapids/zict-3.0.0-py2.py3-none-any.whl#sha256=5796e36bd0e0cc8cf0fbc1ace6a68912611c1dbd74750a3f3026b9b9d6a327ae
317
+ zipp==3.22.0
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/provenance.txt ADDED
@@ -0,0 +1,90 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ==============================================================
2
+ PROVENANCE: ultraep__tierb__gbs8
3
+ ==============================================================
4
+ captured_at : 2026-09-21T07:47:12+00:00
5
+ hostname : sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0
6
+ kernel : 6.8.0-1043-nvidia
7
+ cwd : /workspace/blackwell_run_scripts
8
+ command : run_ladder.sh --phase tierb --gbs 8 --out /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb --nsys
9
+
10
+ ---- GPU / driver --------------------------------------------
11
+ 0, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
12
+ 1, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
13
+ 2, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
14
+ 3, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
15
+ 4, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
16
+ 5, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
17
+ 6, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
18
+ 7, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
19
+ persistence/clocks:
20
+ 0, 120 MHz, 1965 MHz, 1000.00 W, 29
21
+ 1, 1965 MHz, 1965 MHz, 1000.00 W, 35
22
+ 2, 120 MHz, 1965 MHz, 1000.00 W, 35
23
+ 3, 262 MHz, 1965 MHz, 1000.00 W, 30
24
+ 4, 1965 MHz, 1965 MHz, 1000.00 W, 30
25
+ 5, 120 MHz, 1965 MHz, 1000.00 W, 34
26
+ 6, 165 MHz, 1965 MHz, 1000.00 W, 34
27
+ 7, 120 MHz, 1965 MHz, 1000.00 W, 30
28
+
29
+ NVLink topology:
30
+ GPU0 GPU1 GPU2 GPU3 GPU4 GPU5 GPU6 GPU7 NIC0 NIC1 NIC2 NIC3 NIC4 NIC5 NIC6 NIC7 NIC8 CPU Affinity NUMA Affinity GPU NUMA ID
31
+ GPU0 X NV18 NV18 NV18 NV18 NV18 NV18 NV18 PIX NODE SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A
32
+ GPU1 NV18 X NV18 NV18 NV18 NV18 NV18 NV18 NODE PIX SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A
33
+ GPU2 NV18 NV18 X NV18 NV18 NV18 NV18 NV18 SYS SYS PIX NODE SYS SYS SYS SYS SYS 28-55 1 N/A
34
+ GPU3 NV18 NV18 NV18 X NV18 NV18 NV18 NV18 SYS SYS NODE PIX SYS SYS SYS SYS SYS 28-55 1 N/A
35
+ GPU4 NV18 NV18 NV18 NV18 X NV18 NV18 NV18 SYS SYS SYS SYS PIX NODE SYS SYS SYS 56-83 2 N/A
36
+ GPU5 NV18 NV18 NV18 NV18 NV18 X NV18 NV18 SYS SYS SYS SYS NODE PIX SYS SYS SYS 56-83 2 N/A
37
+ GPU6 NV18 NV18 NV18 NV18 NV18 NV18 X NV18 SYS SYS SYS SYS SYS SYS PIX NODE SYS 84-111 3 N/A
38
+ GPU7 NV18 NV18 NV18 NV18 NV18 NV18 NV18 X SYS SYS SYS SYS SYS SYS NODE PIX SYS 84-111 3 N/A
39
+ NIC0 PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS SYS NODE
40
+ NIC1 NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS SYS NODE
41
+ NIC2 SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS
42
+ NIC3 SYS SYS NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS
43
+ NIC4 SYS SYS SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS
44
+ unavailable
45
+
46
+ ---- toolchain -----------------------------------------------
47
+ nvcc : release 12.9, V12.9.86
48
+ gcc : gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0
49
+ python : 3.12.3
50
+ torch : 2.8.0a0+5228986c39.nv25.06 cuda=12.9 nccl=2.27.3
51
+ torch arch : ['sm_75', 'sm_80', 'sm_86', 'sm_90', 'sm_100', 'sm_120', 'compute_120']
52
+ transformer_engine: 2.10.0.dev0+10f6128d [/usr/local/lib/python3.12/dist-packages/transformer_engine/__init__.py]
53
+ deep_ep : 1.2.1+34152ae [/workspace/DeepEP/deep_ep/__init__.py]
54
+ ultra_ep : 1.0.0 [/workspace/UltraEP/ultra_ep/__init__.py]
55
+ moonep : 0.0.1 [/workspace/MoonEP/moonep/__init__.py]
56
+ megatron : 0.14.0rc5 [None]
57
+ flash_attn : 2.7.4.post1 [/usr/local/lib/python3.12/dist-packages/flash_attn/__init__.py]
58
+ numpy : 1.26.4 [/usr/local/lib/python3.12/dist-packages/numpy/__init__.py]
59
+ huggingface_hub: 1.32.0 [/usr/local/lib/python3.12/dist-packages/huggingface_hub/__init__.py]
60
+
61
+ ---- repositories --------------------------------------------
62
+ Megatron-MoE-Echo bracing-regions-through-kernels 9a23679390d7
63
+ Megatron-UltraEP ultraep-bracing-regions-through-kernels 03d57ef7d476
64
+ UltraEP main 94cab099b44f
65
+ DeepEP HEAD 34152ae28f80 (+3 untracked: build output)
66
+ MoonEP megatron-ddp-symmetric-params d8bc6667fad7
67
+ blackwell_run_scripts granularity-ultraep 05dd2cbfb1eb
68
+
69
+ ---- environment ---------------------------------------------
70
+ CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0
71
+ CUDA_DRIVER_VERSION=575.57.08
72
+ CUDA_HOME=/usr/local/cuda
73
+ CUDA_MODULE_LOADING=LAZY
74
+ CUDA_VERSION=12.9.1.010
75
+ HF_REMOTE_ROOT=granularity_next_b200/tierb_models/attempts/KIMI_K2_L2
76
+ NCCL_VERSION=2.27.3
77
+ NVSHMEM_VERSION=3.2.5
78
+ PYTORCH_BUILD_NUMBER=0
79
+ PYTORCH_BUILD_VERSION=2.8.0a0+5228986
80
+ PYTORCH_HOME=/opt/pytorch/pytorch
81
+ PYTORCH_VERSION=2.8.0a0+5228986
82
+ TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1
83
+ TORCH_CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0+PTX
84
+ TORCH_NCCL_USE_COMM_NONBLOCKING=0
85
+
86
+ ---- shared library resolution -------------------------------
87
+ libnvshmem_host.so: not in ldconfig
88
+ libnccl.so: /lib/x86_64-linux-gnu/libnccl.so.2
89
+ lrwxrwxrwx 1 root root 20 Sep 21 04:57 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so -> libnvshmem_host.so.3
90
+ -rw-r--r-- 1 root root 179045744 Sep 21 04:54 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so.3
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep1/mem_samples.csv ADDED
@@ -0,0 +1,200 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 629
43
+ 2, 629
44
+ 3, 629
45
+ 4, 629
46
+ 5, 629
47
+ 6, 629
48
+ 7, 629
49
+ 0, 887
50
+ 1, 885
51
+ 2, 915
52
+ 3, 881
53
+ 4, 885
54
+ 5, 905
55
+ 6, 909
56
+ 7, 903
57
+ 0, 1927
58
+ 1, 1927
59
+ 2, 1927
60
+ 3, 1927
61
+ 4, 1927
62
+ 5, 1927
63
+ 6, 1927
64
+ 7, 1927
65
+ 0, 3783
66
+ 1, 3789
67
+ 2, 3803
68
+ 3, 3815
69
+ 4, 3807
70
+ 5, 3803
71
+ 6, 3811
72
+ 7, 3809
73
+ 0, 50559
74
+ 1, 50559
75
+ 2, 50559
76
+ 3, 50559
77
+ 4, 50559
78
+ 5, 50559
79
+ 6, 50559
80
+ 7, 50559
81
+ 0, 55555
82
+ 1, 55555
83
+ 2, 55555
84
+ 3, 55555
85
+ 4, 55555
86
+ 5, 55555
87
+ 6, 55555
88
+ 7, 55555
89
+ 0, 60691
90
+ 1, 60693
91
+ 2, 60691
92
+ 3, 60691
93
+ 4, 60691
94
+ 5, 60691
95
+ 6, 60691
96
+ 7, 60691
97
+ 0, 61267
98
+ 1, 61269
99
+ 2, 61267
100
+ 3, 61267
101
+ 4, 61267
102
+ 5, 61267
103
+ 6, 61267
104
+ 7, 61267
105
+ 0, 61267
106
+ 1, 61269
107
+ 2, 61267
108
+ 3, 61267
109
+ 4, 61267
110
+ 5, 61267
111
+ 6, 61267
112
+ 7, 61267
113
+ 0, 64465
114
+ 1, 64467
115
+ 2, 64465
116
+ 3, 64465
117
+ 4, 64465
118
+ 5, 64465
119
+ 6, 64465
120
+ 7, 64465
121
+ 0, 64465
122
+ 1, 64467
123
+ 2, 64465
124
+ 3, 64465
125
+ 4, 64465
126
+ 5, 64465
127
+ 6, 64465
128
+ 7, 64465
129
+ 0, 65963
130
+ 1, 65965
131
+ 2, 65963
132
+ 3, 65963
133
+ 4, 65963
134
+ 5, 65963
135
+ 6, 65963
136
+ 7, 65963
137
+ 0, 65963
138
+ 1, 65965
139
+ 2, 65963
140
+ 3, 65963
141
+ 4, 65963
142
+ 5, 65963
143
+ 6, 65963
144
+ 7, 65963
145
+ 0, 65969
146
+ 1, 65969
147
+ 2, 65969
148
+ 3, 65969
149
+ 4, 65969
150
+ 5, 65969
151
+ 6, 65969
152
+ 7, 65969
153
+ 0, 68251
154
+ 1, 68233
155
+ 2, 68277
156
+ 3, 68259
157
+ 4, 68265
158
+ 5, 68251
159
+ 6, 68297
160
+ 7, 68303
161
+ 0, 69055
162
+ 1, 69055
163
+ 2, 69055
164
+ 3, 69055
165
+ 4, 69055
166
+ 5, 69055
167
+ 6, 69055
168
+ 7, 69055
169
+ 0, 75415
170
+ 1, 75387
171
+ 2, 75419
172
+ 3, 75397
173
+ 4, 75471
174
+ 5, 75487
175
+ 6, 75405
176
+ 7, 75459
177
+ 0, 88919
178
+ 1, 89733
179
+ 2, 86735
180
+ 3, 89731
181
+ 4, 85699
182
+ 5, 88975
183
+ 6, 89745
184
+ 7, 85223
185
+ 0, 90291
186
+ 1, 90297
187
+ 2, 90291
188
+ 3, 90347
189
+ 4, 90347
190
+ 5, 90347
191
+ 6, 90375
192
+ 7, 90347
193
+ 0, 13828
194
+ 1, 13828
195
+ 2, 14148
196
+ 3, 89125
197
+ 4, 15428
198
+ 5, 13828
199
+ 6, 13828
200
+ 7, 13828
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep1/run.env ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=ultraep
2
+ config=KIMI_K2_L2
3
+ family=tierB
4
+ source_model=Kimi-K2
5
+ standard_num_layers=61
6
+ base_rung=
7
+ repo=/workspace/Megatron-UltraEP
8
+ branch=ultraep-bracing-regions-through-kernels
9
+ commit=03d57ef7
10
+ num_layers=2
11
+ hidden=7168
12
+ ffn_hidden=18432
13
+ moe_ffn_hidden=2048
14
+ num_experts=384
15
+ topk=8
16
+ experts_per_rank=48
17
+ expert_params=44040192
18
+ heads=64
19
+ kv_groups=64
20
+ kv_channels=64
21
+ qk_layernorm=1
22
+ shared_ffn=2048
23
+ shared_expert_overlap=0
24
+ moe_token_dispatcher=flex
25
+ mla=1
26
+ q_lora_rank=1536
27
+ kv_lora_rank=512
28
+ qk_head_dim=128
29
+ qk_pos_emb_head_dim=64
30
+ v_head_dim=128
31
+ dense_first_layers=1
32
+ moe_layers=1
33
+ rotary_base=50000
34
+ rotary_percent=1.0
35
+ rotary_scaling=32
36
+ mscale=1.0
37
+ mscale_all_dim=1.0
38
+ qkv_bias=0
39
+ router_balancing=seq_aux_loss
40
+ aux_loss_coeff=0
41
+ router_pre_softmax=0
42
+ router_fusion=0
43
+ router_dtype=fp32
44
+ router_groups=1
45
+ router_group_topk=1
46
+ router_score_function=sigmoid
47
+ router_expert_bias=1
48
+ router_bias_update_rate=1e-3
49
+ router_topk_scaling_factor=2.827
50
+ shared_expert_gate=0
51
+ apply_layernorm_1p=0
52
+ attention_output_gate=0
53
+ linear_attention_type=
54
+ linear_attention_freq=
55
+ linear_conv_kernel_dim=
56
+ linear_key_head_dim=
57
+ linear_value_head_dim=
58
+ linear_num_key_heads=
59
+ linear_num_value_heads=
60
+ mtp_num_layers=0
61
+ mtp_loss_scaling_factor=0.1
62
+ no_weight_decay_cond_type=
63
+ max_position_embeddings=4096
64
+ init_method_std=0.02
65
+ device_initiated_grouped_gemm=1
66
+ norm_epsilon=1e-6
67
+ seq_length=4096
68
+ micro_batch_size=1
69
+ global_batch_size=8
70
+ num_microbatches=1
71
+ assignments_per_rank=32768
72
+ mean_tokens_per_expert=682
73
+ slot_policy=matched
74
+ slots_per_rank=48
75
+ capacity_mode=static
76
+ capacity=1.17578125
77
+ cuda_graph=1
78
+ probe_mode=0
79
+ async_grad_reduce=0
80
+ branch_role=timing
81
+ memory_dump_mode=none
82
+ memory_snapshot_path=
83
+ cuda_graph_warmup_steps=3
84
+ recompute=none
85
+ force_balance=0
86
+ train_iters=12
87
+ ep_size=8
88
+ echo_dispatch_overlap=1
89
+ echo_dispatch_recompute=0
90
+ ultraep_quota_min_tokens=85
91
+ ultraep_weight_sync_join=
92
+ profile_enable=0
93
+ kernel_trace=0
94
+ fp8_mode=mxfp8
95
+ fp8_recipe=mxfp8
96
+ training_seed=1234
97
+ learning_rate=1e-4
98
+ min_learning_rate=1.0e-5
99
+ lr_warmup_iters=5
100
+ state_dump_enable=0
101
+ state_dump_dir=
102
+ moonep_assert_no_pending_reduce=0
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep1/train.log ADDED
@@ -0,0 +1,1073 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [07:47:26] WARN: ultraep-bracing-regions-through-kernels lacks --moe-router-fusion; KIMI_K2_L2 runs unfused (timing only, routing identical)
2
+ ==============================================================
3
+ KIMI_K2_L2 / ultraep (slots/rank=48 policy=matched)
4
+ E=384 K=8 H'=2048 m=48
5
+ L=2 seq=4096 gbs=8 (1 microbatches)
6
+ cap_mode=static cap=1.17578125 cg=1 recompute=none
7
+ memory_dump=none snapshots=/workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep1/memory_snapshots/snapshot.pickle
8
+ repo=/workspace/Megatron-UltraEP @ 03d57ef7
9
+ ==============================================================
10
+ W0921 07:47:28.271000 519691 torch/distributed/run.py:766]
11
+ W0921 07:47:28.271000 519691 torch/distributed/run.py:766] *****************************************
12
+ W0921 07:47:28.271000 519691 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
13
+ W0921 07:47:28.271000 519691 torch/distributed/run.py:766] *****************************************
14
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
15
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
16
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
17
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
18
+ using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1
19
+ WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer
20
+ Number of virtual stages per pipeline stage: None
21
+ accumulate and all-reduce gradients in fp32 for bfloat16 data type.
22
+ using torch.bfloat16 for parameters ...
23
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
24
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
25
+ ------------------------ arguments ------------------------
26
+ account_for_embedding_in_pipeline_split ......... False
27
+ account_for_loss_in_pipeline_split .............. False
28
+ accumulate_allreduce_grads_in_fp32 .............. True
29
+ adam_beta1 ...................................... 0.9
30
+ adam_beta2 ...................................... 0.999
31
+ adam_eps ........................................ 1e-08
32
+ add_bias_linear ................................. False
33
+ add_position_embedding .......................... False
34
+ add_qkv_bias .................................... False
35
+ adlr_autoresume ................................. False
36
+ adlr_autoresume_interval ........................ 1000
37
+ align_grad_reduce ............................... True
38
+ align_param_gather .............................. False
39
+ app_tag_run_name ................................ None
40
+ app_tag_run_version ............................. 0.0.0
41
+ apply_layernorm_1p .............................. False
42
+ apply_query_key_layer_scaling ................... False
43
+ apply_residual_connection_post_layernorm ........ False
44
+ apply_rope_fusion ............................... False
45
+ async_save ...................................... None
46
+ async_tensor_model_parallel_allreduce ........... True
47
+ attention_backend ............................... AttnBackend.auto
48
+ attention_dropout ............................... 0.0
49
+ attention_softmax_in_fp32 ....................... False
50
+ auto_detect_ckpt_format ......................... False
51
+ barrier_with_L1_time ............................ True
52
+ bert_binary_head ................................ True
53
+ bert_embedder_type .............................. megatron
54
+ bert_load ....................................... None
55
+ bf16 ............................................ True
56
+ bias_dropout_fusion ............................. True
57
+ bias_gelu_fusion ................................ False
58
+ bias_swiglu_fusion .............................. True
59
+ biencoder_projection_dim ........................ 0
60
+ biencoder_shared_query_context_model ............ False
61
+ block_data_path ................................. None
62
+ calc_ft_timeouts ................................ False
63
+ calculate_per_token_loss ........................ False
64
+ check_for_large_grads ........................... False
65
+ check_for_nan_in_loss_and_grad .................. False
66
+ check_for_spiky_loss ............................ False
67
+ check_weight_hash_across_dp_replicas_interval ... None
68
+ ckpt_assume_constant_structure .................. False
69
+ ckpt_convert_format ............................. None
70
+ ckpt_convert_save ............................... None
71
+ ckpt_convert_update_legacy_dist_opt_format ...... False
72
+ ckpt_format ..................................... torch_dist
73
+ ckpt_fully_parallel_load ........................ False
74
+ ckpt_fully_parallel_save ........................ True
75
+ ckpt_fully_parallel_save_deprecated ............. False
76
+ ckpt_step ....................................... None
77
+ classes_fraction ................................ 1.0
78
+ clip_grad ....................................... 1.0
79
+ clone_scatter_output_in_embedding ............... True
80
+ config_logger_dir ...............................
81
+ consumed_train_samples .......................... 0
82
+ consumed_valid_samples .......................... 0
83
+ context_parallel_size ........................... 1
84
+ cp_comm_type .................................... ['p2p']
85
+ create_attention_mask_in_dataloader ............. True
86
+ cross_entropy_fusion_impl ....................... native
87
+ cross_entropy_loss_fusion ....................... False
88
+ cuda_graph_scope ................................ full_iteration
89
+ cuda_graph_warmup_steps ......................... 3
90
+ data_args_path .................................. None
91
+ data_cache_path ................................. None
92
+ data_parallel_random_init ....................... False
93
+ data_parallel_sharding_strategy ................. no_shard
94
+ data_parallel_size .............................. 8
95
+ data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document']
96
+ data_per_class_fraction ......................... 1.0
97
+ data_sharding ................................... True
98
+ dataloader_type ................................. single
99
+ ddp_average_in_collective ....................... False
100
+ ddp_bucket_size ................................. None
101
+ ddp_num_buckets ................................. None
102
+ ddp_pad_buckets_for_high_nccl_busbw ............. False
103
+ decoder_first_pipeline_num_layers ............... None
104
+ decoder_last_pipeline_num_layers ................ None
105
+ decoder_num_layers .............................. None
106
+ decoder_seq_length .............................. None
107
+ decoupled_lr .................................... None
108
+ decoupled_min_lr ................................ None
109
+ decrease_batch_size_if_needed ................... False
110
+ defer_embedding_wgrad_compute ................... False
111
+ delay_wgrad_compute ............................. False
112
+ deprecated_use_mcore_models ..................... True
113
+ deterministic_mode .............................. False
114
+ dino_bottleneck_size ............................ 256
115
+ dino_freeze_last_layer .......................... 1
116
+ dino_head_hidden_size ........................... 2048
117
+ dino_local_crops_number ......................... 10
118
+ dino_local_img_size ............................. 96
119
+ dino_norm_last_layer ............................ False
120
+ dino_teacher_temp ............................... 0.07
121
+ dino_warmup_teacher_temp ........................ 0.04
122
+ dino_warmup_teacher_temp_epochs ................. 30
123
+ disable_bf16_reduced_precision_matmul ........... False
124
+ disable_mamba_mem_eff_path ...................... False
125
+ disable_straggler_on_startup .................... False
126
+ dist_ckpt_format_deprecated ..................... None
127
+ dist_ckpt_strictness ............................ assume_ok_unexpected
128
+ distribute_saved_activations .................... False
129
+ distributed_backend ............................. nccl
130
+ distributed_timeout_minutes ..................... 10
131
+ embedding_init_method_std ....................... None
132
+ embedding_path .................................. None
133
+ empty_unused_memory_level ....................... 0
134
+ enable_cuda_graph ............................... True
135
+ enable_experimental ............................. False
136
+ enable_ft_package ............................... False
137
+ enable_gloo_process_groups ...................... True
138
+ enable_msc ...................................... True
139
+ enable_one_logger ............................... True
140
+ encoder_num_layers .............................. 2
141
+ encoder_seq_length .............................. 4096
142
+ end_weight_decay ................................ 0.1
143
+ eod_mask_loss ................................... False
144
+ error_injection_rate ............................ 0
145
+ error_injection_type ............................ transient_error
146
+ eval_interval ................................... 1000000
147
+ eval_iters ...................................... 0
148
+ evidence_data_path .............................. None
149
+ exit_duration_in_mins ........................... None
150
+ exit_interval ................................... None
151
+ exit_on_missing_checkpoint ...................... False
152
+ exit_signal_handler ............................. False
153
+ exp_avg_dtype ................................... torch.float32
154
+ exp_avg_sq_dtype ................................ torch.float32
155
+ expert_model_parallel_size ...................... 8
156
+ expert_tensor_parallel_size ..................... 1
157
+ export_force_local_attention .................... False
158
+ export_kd_cfg ................................... None
159
+ export_kd_teacher_ckpt_format ................... None
160
+ export_kd_teacher_load .......................... None
161
+ export_kv_cache_quant ........................... False
162
+ export_legacy_megatron .......................... False
163
+ export_model_type ............................... GPTModel
164
+ export_moe_apply_probs_on_input ................. False
165
+ export_qk_l2_norm ............................... False
166
+ export_quant_cfg ................................ None
167
+ export_real_quant_cfg ........................... None
168
+ export_te_mcore_model ........................... False
169
+ external_cuda_graph ............................. False
170
+ ffn_hidden_size ................................. 18432
171
+ finetune ........................................ False
172
+ finetune_data_split ............................. train
173
+ finetune_hf_dataset ............................. None
174
+ first_last_layers_bf16 .......................... False
175
+ flash_decode .................................... False
176
+ fp16 ............................................ False
177
+ fp16_lm_cross_entropy ........................... False
178
+ fp32_residual_connection ........................ False
179
+ fp8 ............................................. e4m3
180
+ fp8_amax_compute_algo ........................... most_recent
181
+ fp8_amax_history_len ............................ 1
182
+ fp8_interval .................................... 1
183
+ fp8_margin ...................................... 0
184
+ fp8_param_gather ................................ False
185
+ fp8_recipe ...................................... mxfp8
186
+ fp8_wgrad ....................................... True
187
+ fsdp_double_buffer .............................. False
188
+ global_batch_size ............................... 8
189
+ grad_reduce_in_bf16 ............................. False
190
+ gradient_accumulation_fusion .................... True
191
+ gradient_reduce_div_fusion ...................... True
192
+ group_query_attention ........................... False
193
+ head_lr_mult .................................... 1.0
194
+ heterogeneous_layers_config_encoded_json ........ None
195
+ heterogeneous_layers_config_path ................ None
196
+ hidden_dropout .................................. 0.0
197
+ hidden_size ..................................... 7168
198
+ hierarchical_context_parallel_sizes ............. None
199
+ high_priority_stream_groups ..................... []
200
+ hybrid_attention_ratio .......................... 0.0
201
+ hybrid_mlp_ratio ................................ 0.0
202
+ hybrid_override_pattern ......................... None
203
+ hysteresis ...................................... 2
204
+ ict_head_size ................................... None
205
+ ict_load ........................................ None
206
+ img_h ........................................... 224
207
+ img_w ........................................... 224
208
+ indexer_batch_size .............................. 128
209
+ indexer_log_interval ............................ 1000
210
+ inference_batch_times_seqlen_threshold .......... -1
211
+ inference_dynamic_batching ...................... False
212
+ inference_dynamic_batching_buffer_guaranteed_fraction 0.2
213
+ inference_dynamic_batching_buffer_overflow_factor None
214
+ inference_dynamic_batching_buffer_size_gb ....... 40.0
215
+ inference_dynamic_batching_chunk_size ........... 256
216
+ inference_dynamic_batching_max_requests_override None
217
+ inference_dynamic_batching_max_tokens_override .. None
218
+ inference_dynamic_batching_num_cuda_graphs ...... 16
219
+ inference_max_batch_size ........................ 8
220
+ inference_max_seq_length ........................ 2560
221
+ inference_rng_tracker ........................... False
222
+ init_method_std ................................. 0.02
223
+ init_method_xavier_uniform ...................... False
224
+ init_model_with_meta_device ..................... False
225
+ initial_loss_scale .............................. 4294967296
226
+ inprocess_active_world_size ..................... 8
227
+ inprocess_barrier_timeout ....................... 120
228
+ inprocess_completion_timeout .................... 120
229
+ inprocess_empty_cuda_cache ...................... False
230
+ inprocess_granularity ........................... node
231
+ inprocess_hard_timeout .......................... 90
232
+ inprocess_heartbeat_interval .................... 30
233
+ inprocess_heartbeat_timeout ..................... 60
234
+ inprocess_last_call_wait ........................ 1
235
+ inprocess_max_iterations ........................ None
236
+ inprocess_monitor_process_interval .............. 1.0
237
+ inprocess_monitor_thread_interval ............... 1.0
238
+ inprocess_progress_watchdog_interval ............ 1.0
239
+ inprocess_restart ............................... False
240
+ inprocess_soft_timeout .......................... 60
241
+ inprocess_termination_grace_time ................ 1
242
+ is_hybrid_model ................................. False
243
+ iter_per_epoch .................................. 1250
244
+ iterations_to_skip .............................. []
245
+ keep_fp8_transpose_cache_when_using_custom_fsdp . False
246
+ kitchen_config_file ............................. None
247
+ kitchen_recipe_number ........................... None
248
+ kv_channels ..................................... 64
249
+ kv_lora_rank .................................... 512
250
+ lazy_mpu_init ................................... None
251
+ load ............................................ None
252
+ load_main_params_from_ckpt ...................... None
253
+ load_model_opt_format ........................... False
254
+ local_rank ...................................... 0
255
+ log_energy ...................................... False
256
+ log_interval .................................... 1
257
+ log_loss_scale_to_tensorboard ................... True
258
+ log_memory_to_tensorboard ....................... False
259
+ log_num_zeros_in_grad ........................... False
260
+ log_params_norm ................................. False
261
+ log_progress .................................... False
262
+ log_straggler ................................... False
263
+ log_throughput .................................. True
264
+ log_timers_to_tensorboard ....................... False
265
+ log_validation_ppl_to_tensorboard ............... False
266
+ log_world_size_to_tensorboard ................... False
267
+ logging_level ................................... None
268
+ loss_scale ...................................... None
269
+ loss_scale_window ............................... 1000
270
+ lr .............................................. 0.0001
271
+ lr_decay_iters .................................. 8
272
+ lr_decay_samples ................................ None
273
+ lr_decay_style .................................. cosine
274
+ lr_warmup_fraction .............................. None
275
+ lr_warmup_init .................................. 0.0
276
+ lr_warmup_iters ................................. 5
277
+ lr_warmup_samples ............................... 0
278
+ lr_wsd_decay_iters .............................. None
279
+ lr_wsd_decay_samples ............................ None
280
+ lr_wsd_decay_style .............................. exponential
281
+ main_grads_dtype ................................ torch.float32
282
+ main_params_dtype ............................... torch.float32
283
+ make_vocab_size_divisible_by .................... 128
284
+ mamba_head_dim .................................. 64
285
+ mamba_num_groups ................................ 8
286
+ mamba_num_heads ................................. None
287
+ mamba_state_dim ................................. 128
288
+ manual_gc ....................................... False
289
+ manual_gc_eval .................................. True
290
+ manual_gc_interval .............................. 0
291
+ mask_factor ..................................... 1.0
292
+ mask_prob ....................................... 0.15
293
+ mask_type ....................................... random
294
+ masked_softmax_fusion ........................... False
295
+ max_position_embeddings ......................... 4096
296
+ max_tokens_to_oom ............................... 12000
297
+ memory_snapshot_path ............................ snapshot.pickle
298
+ merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt
299
+ micro_batch_size ................................ 1
300
+ microbatch_group_size_per_vp_stage .............. None
301
+ mid_level_dataset_surplus ....................... 0.005
302
+ min_loss_scale .................................. 1.0
303
+ min_lr .......................................... 1e-05
304
+ mlp_chunks_for_prefill .......................... 1
305
+ mmap_bin_files .................................. True
306
+ mock_data ....................................... False
307
+ moe_apply_probs_on_input ........................ False
308
+ moe_aux_loss_coeff .............................. 0.0
309
+ moe_capacity_monitor ............................ True
310
+ moe_deepep_num_sms .............................. 20
311
+ moe_echo_enable_random_offloading ............... False
312
+ moe_echo_expert_dispatch_overlap ................ False
313
+ moe_echo_expert_dispatcher_type ................. hybridep
314
+ moe_echo_recompute_expert_dispatch .............. False
315
+ moe_enable_deepep ............................... False
316
+ moe_enable_echo ................................. False
317
+ moe_enable_hybridep ............................. True
318
+ moe_enable_ultraep .............................. True
319
+ moe_expert_capacity_factor ...................... None
320
+ moe_extended_tp ................................. False
321
+ moe_ffn_hidden_size ............................. 2048
322
+ moe_grouped_gemm ................................ True
323
+ moe_hybridep_num_sms ............................ 16
324
+ moe_input_jitter_eps ............................ None
325
+ moe_layer_freq .................................. [0, 1]
326
+ moe_layer_recompute ............................. False
327
+ moe_num_echo_experts ............................ None
328
+ moe_num_redundant_experts_per_rank .............. 48
329
+ moe_pad_expert_input_to_capacity ................ False
330
+ moe_per_layer_logging ........................... False
331
+ moe_permute_fusion .............................. False
332
+ moe_received_token_capacity ..................... 1.17578125
333
+ moe_router_bias_update_rate ..................... 0.001
334
+ moe_router_dtype ................................ fp32
335
+ moe_router_enable_expert_bias ................... True
336
+ moe_router_force_load_balancing ................. False
337
+ moe_router_group_topk ........................... 1
338
+ moe_router_load_balancing_type .................. seq_aux_loss
339
+ moe_router_num_groups ........................... 1
340
+ moe_router_padding_for_fp8 ...................... False
341
+ moe_router_pre_softmax .......................... False
342
+ moe_router_score_function ....................... sigmoid
343
+ moe_router_topk ................................. 8
344
+ moe_router_topk_scaling_factor .................. 2.827
345
+ moe_shared_expert_intermediate_size ............. 2048
346
+ moe_shared_expert_overlap ....................... False
347
+ moe_token_dispatcher_type ....................... flex
348
+ moe_token_drop_policy ........................... probs
349
+ moe_ultraep_max_microbatches .................... None
350
+ moe_ultraep_overlap_weight_sync ................. True
351
+ moe_ultraep_weight_sync_join .................... before_dispatch
352
+ moe_upcycling_granularity ....................... 1
353
+ moe_use_device_initiated_grouped_gemm ........... True
354
+ moe_use_legacy_grouped_gemm ..................... False
355
+ moe_use_upcycling ............................... False
356
+ moe_z_loss_coeff ................................ None
357
+ mrope_section ................................... None
358
+ mscale .......................................... 1.0
359
+ mscale_all_dim .................................. 1.0
360
+ mtp_loss_scaling_factor ......................... 0.1
361
+ mtp_num_layers .................................. None
362
+ multi_latent_attention .......................... True
363
+ nccl_all_reduce_for_prefill ..................... False
364
+ nccl_communicator_config_path ................... None
365
+ nccl_ub ......................................... False
366
+ no_load_optim ................................... None
367
+ no_load_rng ..................................... None
368
+ no_persist_layer_norm ........................... False
369
+ no_rope_freq .................................... None
370
+ no_save_optim ................................... None
371
+ no_save_rng ..................................... None
372
+ non_persistent_ckpt_type ........................ None
373
+ non_persistent_global_ckpt_dir .................. None
374
+ non_persistent_local_ckpt_algo .................. fully_parallel
375
+ non_persistent_local_ckpt_dir ................... None
376
+ non_persistent_save_interval .................... None
377
+ norm_epsilon .................................... 1e-06
378
+ normalization ................................... RMSNorm
379
+ num_attention_heads ............................. 64
380
+ num_channels .................................... 3
381
+ num_classes ..................................... 1000
382
+ num_dataset_builder_threads ..................... 1
383
+ num_distributed_optimizer_instances ............. 1
384
+ num_experts ..................................... 384
385
+ num_layers ...................................... 2
386
+ num_layers_at_end_in_bf16 ....................... 1
387
+ num_layers_at_start_in_bf16 ..................... 1
388
+ num_layers_per_virtual_pipeline_stage ........... None
389
+ num_query_groups ................................ 1
390
+ num_virtual_stages_per_pipeline_rank ............ None
391
+ num_workers ..................................... 2
392
+ object_storage_cache_path ....................... None
393
+ one_logger_async ................................ False
394
+ one_logger_project .............................. megatron-lm
395
+ one_logger_run_name ............................. None
396
+ onnx_safe ....................................... None
397
+ openai_gelu ..................................... False
398
+ optimizer ....................................... adam
399
+ optimizer_cpu_offload ........................... False
400
+ optimizer_offload_fraction ...................... 1.0
401
+ output_bert_embeddings .......................... False
402
+ overlap_cpu_optimizer_d2h_h2d ................... False
403
+ overlap_grad_reduce ............................. True
404
+ overlap_moe_expert_parallel_comm ................ False
405
+ overlap_p2p_comm ................................ False
406
+ overlap_p2p_comm_warmup_flush ................... False
407
+ overlap_param_gather ............................ True
408
+ overlap_param_gather_with_optimizer_step ........ False
409
+ override_opt_param_scheduler .................... False
410
+ params_dtype .................................... torch.bfloat16
411
+ patch_dim ....................................... 16
412
+ per_split_data_args_path ........................ None
413
+ perform_initialization .......................... True
414
+ pin_cpu_grads ................................... True
415
+ pin_cpu_params .................................. True
416
+ pipeline_model_parallel_comm_backend ............ None
417
+ pipeline_model_parallel_layout .................. None
418
+ pipeline_model_parallel_size .................... 1
419
+ position_embedding_type ......................... rope
420
+ pretrained_checkpoint ........................... None
421
+ profile ......................................... False
422
+ profile_ranks ................................... [0]
423
+ profile_step_end ................................ 12
424
+ profile_step_start .............................. 10
425
+ q_lora_rank ..................................... 1536
426
+ qk_head_dim ..................................... 128
427
+ qk_l2_norm ...................................... False
428
+ qk_layernorm .................................... True
429
+ qk_pos_emb_head_dim ............................. 64
430
+ query_in_block_prob ............................. 0.1
431
+ rampup_batch_size ............................... None
432
+ rank ............................................ 0
433
+ recompute_granularity ........................... None
434
+ recompute_method ................................ None
435
+ recompute_modules ............................... None
436
+ recompute_num_layers ............................ None
437
+ record_memory_history ........................... False
438
+ relative_attention_max_distance ................. 128
439
+ relative_attention_num_buckets .................. 32
440
+ replication ..................................... False
441
+ replication_factor .............................. 2
442
+ replication_jump ................................ None
443
+ rerun_mode ...................................... validate_results
444
+ reset_attention_mask ............................ False
445
+ reset_position_ids .............................. False
446
+ result_rejected_tracker_filename ................ None
447
+ retriever_report_topk_accuracies ................ []
448
+ retriever_score_scaling ......................... False
449
+ retriever_seq_length ............................ 256
450
+ retro_add_retriever ............................. False
451
+ retro_attention_gate ............................ 1
452
+ retro_cyclic_train_iters ........................ None
453
+ retro_encoder_attention_dropout ................. 0.1
454
+ retro_encoder_hidden_dropout .................... 0.1
455
+ retro_encoder_layers ............................ 2
456
+ retro_num_neighbors ............................. 2
457
+ retro_num_retrieved_chunks ...................... 2
458
+ retro_project_dir ............................... None
459
+ retro_verify_neighbor_count ..................... True
460
+ reuse_grad_buf_for_mxfp8_param_ag ............... False
461
+ rope_scaling_factor ............................. 8.0
462
+ rope_type ....................................... None
463
+ rotary_base ..................................... 50000
464
+ rotary_interleaved .............................. False
465
+ rotary_percent .................................. 1.0
466
+ rotary_scaling_factor ........................... 32.0
467
+ rotary_seq_len_interpolation_factor ............. None
468
+ run_workload_inspector_server ................... False
469
+ sample_rate ..................................... 1.0
470
+ save ............................................ None
471
+ save_interval ................................... None
472
+ save_retain_interval ............................ None
473
+ scatter_gather_tensors_in_pipeline .............. True
474
+ seed ............................................ 1234
475
+ seq_length ...................................... 4096
476
+ sequence_parallel ............................... False
477
+ sft ............................................. False
478
+ sft_tokenizer_prompt_format ..................... nemotron-h-aligned
479
+ sgd_momentum .................................... 0.9
480
+ short_seq_prob .................................. 0.1
481
+ skip_train ...................................... False
482
+ skipped_train_samples ........................... 0
483
+ spec ............................................ None
484
+ split ........................................... 100,0,0
485
+ squared_relu .................................... False
486
+ start_weight_decay .............................. 0.1
487
+ straggler_ctrlr_port ............................ 65535
488
+ straggler_minmax_count .......................... 1
489
+ suggested_communication_unit_size ............... None
490
+ swiglu .......................................... True
491
+ swin_backbone_type .............................. tiny
492
+ symmetric_ar_type ............................... None
493
+ te_rng_tracker .................................. True
494
+ tensor_model_parallel_size ...................... 1
495
+ tensorboard_dir ................................. None
496
+ tensorboard_log_interval ........................ 1
497
+ tensorboard_queue_size .......................... 1000
498
+ test_data_path .................................. None
499
+ test_mode ....................................... False
500
+ tiktoken_num_special_tokens ..................... 1000
501
+ tiktoken_pattern ................................ None
502
+ tiktoken_special_tokens ......................... None
503
+ timing_log_level ................................ 0
504
+ timing_log_option ............................... minmax
505
+ titles_data_path ................................ None
506
+ tokenizer_model ................................. None
507
+ tokenizer_type .................................. GPT2BPETokenizer
508
+ torch_fsdp2_reshard_after_forward ............... True
509
+ tp_comm_bootstrap_backend ....................... nccl
510
+ tp_comm_bulk_dgrad .............................. True
511
+ tp_comm_bulk_wgrad .............................. True
512
+ tp_comm_overlap ................................. False
513
+ tp_comm_overlap_ag .............................. True
514
+ tp_comm_overlap_cfg ............................. None
515
+ tp_comm_overlap_rs .............................. True
516
+ tp_comm_overlap_rs_dgrad ........................ False
517
+ tp_comm_split_ag ................................ True
518
+ tp_comm_split_rs ................................ True
519
+ train_data_path ................................. None
520
+ train_iters ..................................... 12
521
+ train_samples ................................... None
522
+ train_sync_interval ............................. None
523
+ transformer_impl ................................ transformer_engine
524
+ transformer_pipeline_model_parallel_size ........ 1
525
+ untie_embeddings_and_output_weights ............. True
526
+ use_checkpoint_args ............................. False
527
+ use_checkpoint_opt_param_scheduler .............. False
528
+ use_cpu_initialization .......................... None
529
+ use_custom_fsdp ................................. False
530
+ use_dist_ckpt ................................... True
531
+ use_dist_ckpt_deprecated ........................ False
532
+ use_distributed_optimizer ....................... True
533
+ use_flash_attn .................................. False
534
+ use_legacy_models ............................... False
535
+ use_mp_args_from_checkpoint_args ................ False
536
+ use_one_sent_docs ............................... False
537
+ use_persistent_ckpt_worker ...................... False
538
+ use_precision_aware_optimizer ................... False
539
+ use_pytorch_profiler ............................ False
540
+ use_ring_exchange_p2p ........................... False
541
+ use_rope_scaling ................................ False
542
+ use_rotary_position_embeddings .................. False
543
+ use_sharp ....................................... False
544
+ use_tokenizer_model_from_checkpoint_args ........ True
545
+ use_torch_fsdp2 ................................. False
546
+ use_torch_optimizer_for_cpu_offload ............. False
547
+ use_tp_pp_dp_mapping ............................ False
548
+ v_head_dim ...................................... 128
549
+ valid_data_path ................................. None
550
+ variable_seq_lengths ............................ False
551
+ virtual_pipeline_model_parallel_size ............ None
552
+ vision_backbone_type ............................ vit
553
+ vision_pretraining .............................. False
554
+ vision_pretraining_type ......................... classify
555
+ vocab_extra_ids ................................. 0
556
+ vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json
557
+ vocab_size ...................................... None
558
+ wandb_exp_name ..................................
559
+ wandb_project ...................................
560
+ wandb_save_dir ..................................
561
+ weight_decay .................................... 0.1
562
+ weight_decay_incr_style ......................... constant
563
+ wgrad_deferral_limit ............................ 0
564
+ world_size ...................................... 8
565
+ yaml_cfg ........................................ None
566
+ -------------------- end of arguments ---------------------
567
+ INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1
568
+ > building GPT2BPETokenizer tokenizer ...
569
+ > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304)
570
+ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS
571
+ > initializing torch distributed ...
572
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
573
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
574
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
575
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
576
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
577
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
578
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
579
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
580
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
581
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
582
+ WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it
583
+ [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
584
+ [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank
585
+ 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
586
+ [Gloo] Rank [Gloo] Rank 7 is connected to 75[Gloo] Rank peer ranks. is connected to 2Expected number of connected peer ranks is : 7 is connected to 7 peer ranks. 7
587
+ Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is :
588
+ 7
589
+ [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
590
+ [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
591
+ [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
592
+ [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
593
+ [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
594
+ [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
595
+ [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
596
+ [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
597
+ [Gloo] Rank [Gloo] Rank 7 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is :
598
+ 7
599
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
600
+ > initialized tensor model parallel with size 1
601
+ > initialized pipeline model parallel with size 1
602
+ > setting random seeds to 1234 ...
603
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
604
+ [Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00
605
+
606
+ [Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00
607
+
608
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
609
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
610
+ [rank5]:[W921 07:47:45.193572765 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
611
+ [rank2]:[W921 07:47:45.193626668 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
612
+ [rank7]:[W921 07:47:45.195412548 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
613
+ [rank4]:[W921 07:47:45.195476768 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
614
+ [rank3]:[W921 07:47:45.195480188 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
615
+ [rank1]:[W921 07:47:45.195505767 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
616
+ [rank6]:[W921 07:47:45.195651183 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
617
+ > compiling dataset index builder ...
618
+ make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets'
619
+ make: Nothing to be done for 'default'.
620
+ make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets'
621
+ >>> done with dataset index builder. Compilation time: 0.035 seconds
622
+ WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations.
623
+ > compiling and loading fused kernels ...
624
+ [rank0]:[W921 07:47:45.354250232 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
625
+ >>> done with compiling and loading fused kernels. Compilation time: 3.644 seconds
626
+ time to initialize megatron (seconds): 6.952
627
+ [after megatron is initialized] datetime: 2026-09-21 07:47:51
628
+ building GPT model ...
629
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
630
+ warnings.warn(
631
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
632
+ warnings.warn(
633
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
634
+ warnings.warn(
635
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
636
+ warnings.warn(
637
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
638
+ warnings.warn(
639
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
640
+ warnings.warn(
641
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
642
+ warnings.warn(
643
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
644
+ warnings.warn(
645
+ [INFO] Use MNNVL fabric: False
646
+ [WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8.
647
+ NVSHMEM v3.7.2
648
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
649
+
650
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
651
+
652
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
653
+
654
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
655
+
656
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
657
+
658
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
659
+
660
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
661
+
662
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
663
+
664
+ UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive).
665
+ > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 5594455040
666
+ INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False)
667
+ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 9
668
+ Params for bucket 1 (360579072 elements, 360579072 padded size):
669
+ module.output_layer.weight
670
+ Params for bucket 2 (44047360 elements, 44047360 padded size):
671
+ module.decoder.final_layernorm.weight
672
+ module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight
673
+ module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight
674
+ Params for bucket 3 (45163520 elements, 45163520 padded size):
675
+ module.decoder.layers.1.mlp.router.weight
676
+ module.decoder.layers.1.self_attention.linear_q_up_proj.layer_norm_weight
677
+ module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight
678
+ module.decoder.layers.1.self_attention.linear_kv_down_proj.weight
679
+ module.decoder.layers.1.self_attention.linear_kv_up_proj.weight
680
+ module.decoder.layers.1.pre_mlp_layernorm.weight
681
+ module.decoder.layers.1.self_attention.linear_q_up_proj.weight
682
+ module.decoder.layers.1.self_attention.linear_q_down_proj.weight
683
+ Params for bucket 4 (58720256 elements, 58720256 padded size):
684
+ module.decoder.layers.1.self_attention.linear_proj.weight
685
+ Params for bucket 5 (132127744 elements, 132127744 padded size):
686
+ module.decoder.layers.1.input_layernorm.weight
687
+ module.decoder.layers.0.mlp.linear_fc2.weight
688
+ Params for bucket 6 (264241152 elements, 264241152 padded size):
689
+ module.decoder.layers.0.mlp.linear_fc1.weight
690
+ Params for bucket 7 (42411008 elements, 42411008 padded size):
691
+ module.decoder.layers.0.self_attention.linear_kv_up_proj.weight
692
+ module.decoder.layers.0.self_attention.linear_q_up_proj.layer_norm_weight
693
+ module.decoder.layers.0.self_attention.linear_q_down_proj.weight
694
+ module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight
695
+ module.decoder.layers.0.self_attention.linear_kv_down_proj.weight
696
+ module.decoder.layers.0.self_attention.linear_q_up_proj.weight
697
+ module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight
698
+ Params for bucket 8 (58720256 elements, 58720256 padded size):
699
+ module.decoder.layers.0.self_attention.linear_proj.weight
700
+ Params for bucket 9 (360586240 elements, 360586240 padded size):
701
+ module.decoder.layers.0.input_layernorm.weight
702
+ module.embedding.word_embeddings.weight
703
+ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40
704
+ Params for bucket 1 (44040192 elements, 44040192 padded size):
705
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight47
706
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight45
707
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight46
708
+ Params for bucket 2 (44040192 elements, 44040192 padded size):
709
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight44
710
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight42
711
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight43
712
+ Params for bucket 3 (44040192 elements, 44040192 padded size):
713
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight41
714
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight39
715
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight40
716
+ Params for bucket 4 (44040192 elements, 44040192 padded size):
717
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight38
718
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight36
719
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight37
720
+ Params for bucket 5 (44040192 elements, 44040192 padded size):
721
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight35
722
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight33
723
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight34
724
+ Params for bucket 6 (44040192 elements, 44040192 padded size):
725
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight32
726
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight31
727
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight30
728
+ Params for bucket 7 (44040192 elements, 44040192 padded size):
729
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight29
730
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight28
731
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight27
732
+ Params for bucket 8 (44040192 elements, 44040192 padded size):
733
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight26
734
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight25
735
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight24
736
+ Params for bucket 9 (44040192 elements, 44040192 padded size):
737
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight23
738
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight22
739
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight21
740
+ Params for bucket 10 (44040192 elements, 44040192 padded size):
741
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight20
742
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight19
743
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight18
744
+ Params for bucket 11 (44040192 elements, 44040192 padded size):
745
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight17
746
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight16
747
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight15
748
+ Params for bucket 12 (44040192 elements, 44040192 padded size):
749
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight14
750
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight13
751
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight12
752
+ Params for bucket 13 (44040192 elements, 44040192 padded size):
753
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight11
754
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight9
755
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight10
756
+ Params for bucket 14 (44040192 elements, 44040192 padded size):
757
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight8
758
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight6
759
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight7
760
+ Params for bucket 15 (44040192 elements, 44040192 padded size):
761
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight5
762
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight3
763
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight4
764
+ Params for bucket 16 (44040192 elements, 44040192 padded size):
765
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight2
766
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight0
767
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight1
768
+ Params for bucket 17 (58720256 elements, 58720256 padded size):
769
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight47
770
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight46
771
+ Params for bucket 18 (58720256 elements, 58720256 padded size):
772
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight45
773
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight44
774
+ Params for bucket 19 (58720256 elements, 58720256 padded size):
775
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight43
776
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight42
777
+ Params for bucket 20 (58720256 elements, 58720256 padded size):
778
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight41
779
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight40
780
+ Params for bucket 21 (58720256 elements, 58720256 padded size):
781
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight39
782
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight38
783
+ Params for bucket 22 (58720256 elements, 58720256 padded size):
784
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight37
785
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight36
786
+ Params for bucket 23 (58720256 elements, 58720256 padded size):
787
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight35
788
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight34
789
+ Params for bucket 24 (58720256 elements, 58720256 padded size):
790
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight33
791
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight32
792
+ Params for bucket 25 (58720256 elements, 58720256 padded size):
793
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight31
794
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight30
795
+ Params for bucket 26 (58720256 elements, 58720256 padded size):
796
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight29
797
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight28
798
+ Params for bucket 27 (58720256 elements, 58720256 padded size):
799
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight27
800
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight26
801
+ Params for bucket 28 (58720256 elements, 58720256 padded size):
802
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight25
803
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight24
804
+ Params for bucket 29 (58720256 elements, 58720256 padded size):
805
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight23
806
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight22
807
+ Params for bucket 30 (58720256 elements, 58720256 padded size):
808
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight21
809
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight20
810
+ Params for bucket 31 (58720256 elements, 58720256 padded size):
811
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight19
812
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight18
813
+ Params for bucket 32 (58720256 elements, 58720256 padded size):
814
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight17
815
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight16
816
+ Params for bucket 33 (58720256 elements, 58720256 padded size):
817
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight15
818
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight14
819
+ Params for bucket 34 (58720256 elements, 58720256 padded size):
820
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight13
821
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight12
822
+ Params for bucket 35 (58720256 elements, 58720256 padded size):
823
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight11
824
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight10
825
+ Params for bucket 36 (58720256 elements, 58720256 padded size):
826
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight9
827
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight8
828
+ Params for bucket 37 (58720256 elements, 58720256 padded size):
829
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight7
830
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight6
831
+ Params for bucket 38 (58720256 elements, 58720256 padded size):
832
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight5
833
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight4
834
+ Params for bucket 39 (58720256 elements, 58720256 padded size):
835
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight3
836
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight2
837
+ Params for bucket 40 (58720256 elements, 58720256 padded size):
838
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight1
839
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight0
840
+ INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=<megatron.core.timers.Timers object at 0x72c41e8fea20>, config_logger_dir='')
841
+ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine
842
+ [after model, optimizer, and learning rate scheduler are built] datetime: 2026-09-21 07:47:54
843
+ > building train, validation, and test datasets ...
844
+ > datasets target sizes (minimum size):
845
+ train: 96
846
+ validation: 0
847
+ test: 0
848
+ INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None]
849
+ > building train, validation, and test datasets for GPT ...
850
+ INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=<megatron.training.tokenizer.tokenizer._GPT2BPETokenizer object at 0x72c41edc2000>, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None)
851
+ INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx
852
+ INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths
853
+ INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers
854
+ INFO:megatron.core.datasets.indexed_dataset: Extract the document indices
855
+ INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029
856
+ INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350
857
+ INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices
858
+ INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy
859
+ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy
860
+ INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy
861
+ INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073
862
+ > finished creating GPT datasets ...
863
+ [after dataloaders are built] datetime: 2026-09-21 07:47:54
864
+ done with setup ...
865
+ (min, max) time across ranks (ms):
866
+ model-and-optimizer-setup ......................: (3380.12, 3384.53)
867
+ train/valid/test-data-iterators-setup ..........: (72.90, 94.49)
868
+ training ...
869
+ Overwriting rerun_state_machine.current_iteration from -1 to 0...
870
+ [before the start of training step] datetime: 2026-09-21 07:47:54
871
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
872
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
873
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
874
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
875
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
876
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
877
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
878
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
879
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
880
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
881
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
882
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
883
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
884
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
885
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
886
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
887
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
888
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
889
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
890
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
891
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
892
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
893
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
894
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
895
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
896
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
897
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
898
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
899
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
900
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
901
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
902
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
903
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
904
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
905
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
906
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
907
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
908
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
909
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
910
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
911
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
912
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
913
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
914
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
915
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
916
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
917
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
918
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
919
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
920
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
921
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
922
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
923
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
924
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
925
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
926
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
927
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
928
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
929
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
930
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
931
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
932
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
933
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
934
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
935
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
936
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
937
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
938
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
939
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
940
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
941
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
942
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
943
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
944
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
945
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
946
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
947
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
948
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
949
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
950
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
951
+ [2026-09-21 07:48:22] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 28037.9 | throughput per GPU (TFLOP/s/GPU): 1.3 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.227924E+01 | loss scale: 1.0 | grad norm: 12.617 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36224 | min safe capacity: 1.1055 |
952
+ Number of parameters in transformer block in billions: 17.55
953
+ Number of parameters in embedding layers in billions: 0.72
954
+ Total number of parameters in billions: 18.28
955
+ Number of parameters in most loaded shard in billions: 18.2753
956
+ Theoretical memory footprints: weight and optimizer=130715.23 MB
957
+ [Rank 0] (after 1 iterations) memory (MB) | allocated: 49136.10400390625 | max allocated: 49136.1044921875 | reserved: 54394.0 | max reserved: 54394.0
958
+ /workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
959
+ warnings.warn(
960
+ /workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
961
+ warnings.warn(
962
+ /workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
963
+ warnings.warn(
964
+ /workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
965
+ warnings.warn(
966
+ /workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
967
+ warnings.warn(
968
+ /workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
969
+ warnings.warn(
970
+ /workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
971
+ warnings.warn(
972
+ /workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
973
+ warnings.warn(
974
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
975
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
976
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
977
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
978
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
979
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
980
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
981
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
982
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
983
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
984
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
985
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
986
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
987
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
988
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
989
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
990
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
991
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
992
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
993
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
994
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
995
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
996
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
997
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
998
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
999
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1000
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1001
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1002
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1003
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1004
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1005
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1006
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1007
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1008
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1009
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1010
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1011
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1012
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1013
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1014
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1015
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1016
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1017
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1018
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1019
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1020
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1021
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1022
+ [2026-09-21 07:48:22] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 163.2 | throughput per GPU (TFLOP/s/GPU): 216.7 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.227088E+01 | loss scale: 1.0 | grad norm: 12.534 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36224 | min safe capacity: 1.1055 |
1023
+ [2026-09-21 07:48:22] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 111.8 | throughput per GPU (TFLOP/s/GPU): 316.4 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.088328E+01 | loss scale: 1.0 | grad norm: 11.023 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36224 | min safe capacity: 1.1055 |
1024
+ Capture CUDA graph for training!!!
1025
+ CUDA graph capture done!!!
1026
+ [2026-09-21 07:48:24] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 1313.7 | throughput per GPU (TFLOP/s/GPU): 26.9 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.448181E+01 | loss scale: 1.0 | grad norm: 72.911 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36608 | min safe capacity: 1.1172 |
1027
+ [2026-09-21 07:48:24] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 110.4 | throughput per GPU (TFLOP/s/GPU): 320.3 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.808459E+01 | loss scale: 1.0 | grad norm: 108.268 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
1028
+ [2026-09-21 07:48:24] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 111.9 | throughput per GPU (TFLOP/s/GPU): 316.2 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.491727E+01 | loss scale: 1.0 | grad norm: 83.473 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
1029
+ [2026-09-21 07:48:24] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 114.9 | throughput per GPU (TFLOP/s/GPU): 307.9 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.143662E+01 | loss scale: 1.0 | grad norm: 26.627 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
1030
+ [2026-09-21 07:48:24] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 108.6 | throughput per GPU (TFLOP/s/GPU): 325.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.166943E+01 | loss scale: 1.0 | grad norm: 12.800 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
1031
+ [2026-09-21 07:48:24] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 109.1 | throughput per GPU (TFLOP/s/GPU): 324.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.131635E+01 | loss scale: 1.0 | grad norm: 10.441 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
1032
+ [2026-09-21 07:48:24] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 108.9 | throughput per GPU (TFLOP/s/GPU): 324.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.096943E+01 | loss scale: 1.0 | grad norm: 9.147 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
1033
+ [2026-09-21 07:48:24] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 108.2 | throughput per GPU (TFLOP/s/GPU): 327.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.073911E+01 | loss scale: 1.0 | grad norm: 9.755 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
1034
+ [2026-09-21 07:48:25] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 108.8 | throughput per GPU (TFLOP/s/GPU): 325.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.048286E+01 | loss scale: 1.0 | grad norm: 8.843 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
1035
+ [after training is done] datetime: 2026-09-21 07:48:25
1036
+ [rank0]:[W921 07:48:26.297949340 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1037
+ W0921 07:48:26.937000 519691 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers
1038
+ W0921 07:48:26.938000 519691 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 519990 closing signal SIGTERM
1039
+ W0921 07:48:26.938000 519691 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 519991 closing signal SIGTERM
1040
+ W0921 07:48:26.939000 519691 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 519993 closing signal SIGTERM
1041
+ W0921 07:48:26.940000 519691 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 519994 closing signal SIGTERM
1042
+ W0921 07:48:26.941000 519691 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 519996 closing signal SIGTERM
1043
+ W0921 07:48:26.942000 519691 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 519997 closing signal SIGTERM
1044
+ W0921 07:48:26.943000 519691 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 519998 closing signal SIGTERM
1045
+ W0921 07:48:26.944000 519691 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 519999 closing signal SIGTERM
1046
+ Traceback (most recent call last):
1047
+ File "/usr/local/bin/torchrun", line 8, in <module>
1048
+ sys.exit(main())
1049
+ ^^^^^^
1050
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
1051
+ return f(*args, **kwargs)
1052
+ ^^^^^^^^^^^^^^^^^^
1053
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main
1054
+ run(args)
1055
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run
1056
+ elastic_launch(
1057
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__
1058
+ return launch_agent(self._config, self._entrypoint, list(args))
1059
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1060
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent
1061
+ result = agent.run()
1062
+ ^^^^^^^^^^^
1063
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
1064
+ result = f(*args, **kwargs)
1065
+ ^^^^^^^^^^^^^^^^^^
1066
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run
1067
+ result = self._invoke_run(role)
1068
+ ^^^^^^^^^^^^^^^^^^^^^^
1069
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run
1070
+ time.sleep(monitor_interval)
1071
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler
1072
+ raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
1073
+ torch.distributed.elastic.multiprocessing.api.SignalException: Process 519691 got signal: 15
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep2/mem_samples.csv ADDED
@@ -0,0 +1,192 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 4
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 629
43
+ 2, 4
44
+ 3, 4
45
+ 4, 4
46
+ 5, 4
47
+ 6, 4
48
+ 7, 4
49
+ 0, 821
50
+ 1, 821
51
+ 2, 821
52
+ 3, 821
53
+ 4, 821
54
+ 5, 821
55
+ 6, 821
56
+ 7, 821
57
+ 0, 1927
58
+ 1, 1927
59
+ 2, 1927
60
+ 3, 1927
61
+ 4, 1927
62
+ 5, 1927
63
+ 6, 1927
64
+ 7, 1927
65
+ 0, 3783
66
+ 1, 3767
67
+ 2, 3769
68
+ 3, 3777
69
+ 4, 3795
70
+ 5, 3779
71
+ 6, 3769
72
+ 7, 3765
73
+ 0, 46859
74
+ 1, 46859
75
+ 2, 46859
76
+ 3, 46859
77
+ 4, 46859
78
+ 5, 46859
79
+ 6, 46859
80
+ 7, 46859
81
+ 0, 55553
82
+ 1, 55553
83
+ 2, 55553
84
+ 3, 55553
85
+ 4, 55553
86
+ 5, 55553
87
+ 6, 55553
88
+ 7, 55553
89
+ 0, 60691
90
+ 1, 60693
91
+ 2, 60691
92
+ 3, 60691
93
+ 4, 60691
94
+ 5, 60691
95
+ 6, 60691
96
+ 7, 60691
97
+ 0, 61267
98
+ 1, 61269
99
+ 2, 61267
100
+ 3, 61267
101
+ 4, 61267
102
+ 5, 61267
103
+ 6, 61267
104
+ 7, 61267
105
+ 0, 61267
106
+ 1, 61269
107
+ 2, 61267
108
+ 3, 61267
109
+ 4, 61267
110
+ 5, 61267
111
+ 6, 61267
112
+ 7, 61267
113
+ 0, 63677
114
+ 1, 64467
115
+ 2, 63851
116
+ 3, 64465
117
+ 4, 64465
118
+ 5, 64423
119
+ 6, 63679
120
+ 7, 64465
121
+ 0, 64465
122
+ 1, 64467
123
+ 2, 64465
124
+ 3, 64465
125
+ 4, 64465
126
+ 5, 64465
127
+ 6, 64465
128
+ 7, 64465
129
+ 0, 65963
130
+ 1, 65965
131
+ 2, 65963
132
+ 3, 65963
133
+ 4, 65963
134
+ 5, 65963
135
+ 6, 65963
136
+ 7, 65963
137
+ 0, 65963
138
+ 1, 65965
139
+ 2, 65963
140
+ 3, 65963
141
+ 4, 65963
142
+ 5, 65963
143
+ 6, 65963
144
+ 7, 65963
145
+ 0, 65969
146
+ 1, 65969
147
+ 2, 65969
148
+ 3, 65969
149
+ 4, 65969
150
+ 5, 65969
151
+ 6, 65969
152
+ 7, 65969
153
+ 0, 68223
154
+ 1, 68257
155
+ 2, 68253
156
+ 3, 68345
157
+ 4, 68283
158
+ 5, 68271
159
+ 6, 68275
160
+ 7, 68315
161
+ 0, 69183
162
+ 1, 69199
163
+ 2, 69199
164
+ 3, 69213
165
+ 4, 69239
166
+ 5, 69175
167
+ 6, 69191
168
+ 7, 69187
169
+ 0, 75761
170
+ 1, 75727
171
+ 2, 75751
172
+ 3, 75785
173
+ 4, 75801
174
+ 5, 75763
175
+ 6, 75819
176
+ 7, 75791
177
+ 0, 90291
178
+ 1, 90297
179
+ 2, 90291
180
+ 3, 90347
181
+ 4, 90347
182
+ 5, 90347
183
+ 6, 90375
184
+ 7, 90347
185
+ 0, 14406
186
+ 1, 14406
187
+ 2, 14406
188
+ 3, 14790
189
+ 4, 90347
190
+ 5, 16326
191
+ 6, 14406
192
+ 7, 14406
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep2/run.env ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=ultraep
2
+ config=KIMI_K2_L2
3
+ family=tierB
4
+ source_model=Kimi-K2
5
+ standard_num_layers=61
6
+ base_rung=
7
+ repo=/workspace/Megatron-UltraEP
8
+ branch=ultraep-bracing-regions-through-kernels
9
+ commit=03d57ef7
10
+ num_layers=2
11
+ hidden=7168
12
+ ffn_hidden=18432
13
+ moe_ffn_hidden=2048
14
+ num_experts=384
15
+ topk=8
16
+ experts_per_rank=48
17
+ expert_params=44040192
18
+ heads=64
19
+ kv_groups=64
20
+ kv_channels=64
21
+ qk_layernorm=1
22
+ shared_ffn=2048
23
+ shared_expert_overlap=0
24
+ moe_token_dispatcher=flex
25
+ mla=1
26
+ q_lora_rank=1536
27
+ kv_lora_rank=512
28
+ qk_head_dim=128
29
+ qk_pos_emb_head_dim=64
30
+ v_head_dim=128
31
+ dense_first_layers=1
32
+ moe_layers=1
33
+ rotary_base=50000
34
+ rotary_percent=1.0
35
+ rotary_scaling=32
36
+ mscale=1.0
37
+ mscale_all_dim=1.0
38
+ qkv_bias=0
39
+ router_balancing=seq_aux_loss
40
+ aux_loss_coeff=0
41
+ router_pre_softmax=0
42
+ router_fusion=0
43
+ router_dtype=fp32
44
+ router_groups=1
45
+ router_group_topk=1
46
+ router_score_function=sigmoid
47
+ router_expert_bias=1
48
+ router_bias_update_rate=1e-3
49
+ router_topk_scaling_factor=2.827
50
+ shared_expert_gate=0
51
+ apply_layernorm_1p=0
52
+ attention_output_gate=0
53
+ linear_attention_type=
54
+ linear_attention_freq=
55
+ linear_conv_kernel_dim=
56
+ linear_key_head_dim=
57
+ linear_value_head_dim=
58
+ linear_num_key_heads=
59
+ linear_num_value_heads=
60
+ mtp_num_layers=0
61
+ mtp_loss_scaling_factor=0.1
62
+ no_weight_decay_cond_type=
63
+ max_position_embeddings=4096
64
+ init_method_std=0.02
65
+ device_initiated_grouped_gemm=1
66
+ norm_epsilon=1e-6
67
+ seq_length=4096
68
+ micro_batch_size=1
69
+ global_batch_size=8
70
+ num_microbatches=1
71
+ assignments_per_rank=32768
72
+ mean_tokens_per_expert=682
73
+ slot_policy=matched
74
+ slots_per_rank=48
75
+ capacity_mode=static
76
+ capacity=1.17578125
77
+ cuda_graph=1
78
+ probe_mode=0
79
+ async_grad_reduce=0
80
+ branch_role=timing
81
+ memory_dump_mode=none
82
+ memory_snapshot_path=
83
+ cuda_graph_warmup_steps=3
84
+ recompute=none
85
+ force_balance=0
86
+ train_iters=12
87
+ ep_size=8
88
+ echo_dispatch_overlap=1
89
+ echo_dispatch_recompute=0
90
+ ultraep_quota_min_tokens=85
91
+ ultraep_weight_sync_join=
92
+ profile_enable=0
93
+ kernel_trace=0
94
+ fp8_mode=mxfp8
95
+ fp8_recipe=mxfp8
96
+ training_seed=1234
97
+ learning_rate=1e-4
98
+ min_learning_rate=1.0e-5
99
+ lr_warmup_iters=5
100
+ state_dump_enable=0
101
+ state_dump_dir=
102
+ moonep_assert_no_pending_reduce=0
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep2/train.log ADDED
@@ -0,0 +1,1072 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [07:48:32] WARN: ultraep-bracing-regions-through-kernels lacks --moe-router-fusion; KIMI_K2_L2 runs unfused (timing only, routing identical)
2
+ ==============================================================
3
+ KIMI_K2_L2 / ultraep (slots/rank=48 policy=matched)
4
+ E=384 K=8 H'=2048 m=48
5
+ L=2 seq=4096 gbs=8 (1 microbatches)
6
+ cap_mode=static cap=1.17578125 cg=1 recompute=none
7
+ memory_dump=none snapshots=/workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep2/memory_snapshots/snapshot.pickle
8
+ repo=/workspace/Megatron-UltraEP @ 03d57ef7
9
+ ==============================================================
10
+ W0921 07:48:33.484000 524467 torch/distributed/run.py:766]
11
+ W0921 07:48:33.484000 524467 torch/distributed/run.py:766] *****************************************
12
+ W0921 07:48:33.484000 524467 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
13
+ W0921 07:48:33.484000 524467 torch/distributed/run.py:766] *****************************************
14
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
15
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
16
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
17
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
18
+ using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1
19
+ WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer
20
+ Number of virtual stages per pipeline stage: None
21
+ accumulate and all-reduce gradients in fp32 for bfloat16 data type.
22
+ using torch.bfloat16 for parameters ...
23
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
24
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
25
+ ------------------------ arguments ------------------------
26
+ account_for_embedding_in_pipeline_split ......... False
27
+ account_for_loss_in_pipeline_split .............. False
28
+ accumulate_allreduce_grads_in_fp32 .............. True
29
+ adam_beta1 ...................................... 0.9
30
+ adam_beta2 ...................................... 0.999
31
+ adam_eps ........................................ 1e-08
32
+ add_bias_linear ................................. False
33
+ add_position_embedding .......................... False
34
+ add_qkv_bias .................................... False
35
+ adlr_autoresume ................................. False
36
+ adlr_autoresume_interval ........................ 1000
37
+ align_grad_reduce ............................... True
38
+ align_param_gather .............................. False
39
+ app_tag_run_name ................................ None
40
+ app_tag_run_version ............................. 0.0.0
41
+ apply_layernorm_1p .............................. False
42
+ apply_query_key_layer_scaling ................... False
43
+ apply_residual_connection_post_layernorm ........ False
44
+ apply_rope_fusion ............................... False
45
+ async_save ...................................... None
46
+ async_tensor_model_parallel_allreduce ........... True
47
+ attention_backend ............................... AttnBackend.auto
48
+ attention_dropout ............................... 0.0
49
+ attention_softmax_in_fp32 ....................... False
50
+ auto_detect_ckpt_format ......................... False
51
+ barrier_with_L1_time ............................ True
52
+ bert_binary_head ................................ True
53
+ bert_embedder_type .............................. megatron
54
+ bert_load ....................................... None
55
+ bf16 ............................................ True
56
+ bias_dropout_fusion ............................. True
57
+ bias_gelu_fusion ................................ False
58
+ bias_swiglu_fusion .............................. True
59
+ biencoder_projection_dim ........................ 0
60
+ biencoder_shared_query_context_model ............ False
61
+ block_data_path ................................. None
62
+ calc_ft_timeouts ................................ False
63
+ calculate_per_token_loss ........................ False
64
+ check_for_large_grads ........................... False
65
+ check_for_nan_in_loss_and_grad .................. False
66
+ check_for_spiky_loss ............................ False
67
+ check_weight_hash_across_dp_replicas_interval ... None
68
+ ckpt_assume_constant_structure .................. False
69
+ ckpt_convert_format ............................. None
70
+ ckpt_convert_save ............................... None
71
+ ckpt_convert_update_legacy_dist_opt_format ...... False
72
+ ckpt_format ..................................... torch_dist
73
+ ckpt_fully_parallel_load ........................ False
74
+ ckpt_fully_parallel_save ........................ True
75
+ ckpt_fully_parallel_save_deprecated ............. False
76
+ ckpt_step ....................................... None
77
+ classes_fraction ................................ 1.0
78
+ clip_grad ....................................... 1.0
79
+ clone_scatter_output_in_embedding ............... True
80
+ config_logger_dir ...............................
81
+ consumed_train_samples .......................... 0
82
+ consumed_valid_samples .......................... 0
83
+ context_parallel_size ........................... 1
84
+ cp_comm_type .................................... ['p2p']
85
+ create_attention_mask_in_dataloader ............. True
86
+ cross_entropy_fusion_impl ....................... native
87
+ cross_entropy_loss_fusion ....................... False
88
+ cuda_graph_scope ................................ full_iteration
89
+ cuda_graph_warmup_steps ......................... 3
90
+ data_args_path .................................. None
91
+ data_cache_path ................................. None
92
+ data_parallel_random_init ....................... False
93
+ data_parallel_sharding_strategy ................. no_shard
94
+ data_parallel_size .............................. 8
95
+ data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document']
96
+ data_per_class_fraction ......................... 1.0
97
+ data_sharding ................................... True
98
+ dataloader_type ................................. single
99
+ ddp_average_in_collective ....................... False
100
+ ddp_bucket_size ................................. None
101
+ ddp_num_buckets ................................. None
102
+ ddp_pad_buckets_for_high_nccl_busbw ............. False
103
+ decoder_first_pipeline_num_layers ............... None
104
+ decoder_last_pipeline_num_layers ................ None
105
+ decoder_num_layers .............................. None
106
+ decoder_seq_length .............................. None
107
+ decoupled_lr .................................... None
108
+ decoupled_min_lr ................................ None
109
+ decrease_batch_size_if_needed ................... False
110
+ defer_embedding_wgrad_compute ................... False
111
+ delay_wgrad_compute ............................. False
112
+ deprecated_use_mcore_models ..................... True
113
+ deterministic_mode .............................. False
114
+ dino_bottleneck_size ............................ 256
115
+ dino_freeze_last_layer .......................... 1
116
+ dino_head_hidden_size ........................... 2048
117
+ dino_local_crops_number ......................... 10
118
+ dino_local_img_size ............................. 96
119
+ dino_norm_last_layer ............................ False
120
+ dino_teacher_temp ............................... 0.07
121
+ dino_warmup_teacher_temp ........................ 0.04
122
+ dino_warmup_teacher_temp_epochs ................. 30
123
+ disable_bf16_reduced_precision_matmul ........... False
124
+ disable_mamba_mem_eff_path ...................... False
125
+ disable_straggler_on_startup .................... False
126
+ dist_ckpt_format_deprecated ..................... None
127
+ dist_ckpt_strictness ............................ assume_ok_unexpected
128
+ distribute_saved_activations .................... False
129
+ distributed_backend ............................. nccl
130
+ distributed_timeout_minutes ..................... 10
131
+ embedding_init_method_std ....................... None
132
+ embedding_path .................................. None
133
+ empty_unused_memory_level ....................... 0
134
+ enable_cuda_graph ............................... True
135
+ enable_experimental ............................. False
136
+ enable_ft_package ............................... False
137
+ enable_gloo_process_groups ...................... True
138
+ enable_msc ...................................... True
139
+ enable_one_logger ............................... True
140
+ encoder_num_layers .............................. 2
141
+ encoder_seq_length .............................. 4096
142
+ end_weight_decay ................................ 0.1
143
+ eod_mask_loss ................................... False
144
+ error_injection_rate ............................ 0
145
+ error_injection_type ............................ transient_error
146
+ eval_interval ................................... 1000000
147
+ eval_iters ...................................... 0
148
+ evidence_data_path .............................. None
149
+ exit_duration_in_mins ........................... None
150
+ exit_interval ................................... None
151
+ exit_on_missing_checkpoint ...................... False
152
+ exit_signal_handler ............................. False
153
+ exp_avg_dtype ................................... torch.float32
154
+ exp_avg_sq_dtype ................................ torch.float32
155
+ expert_model_parallel_size ...................... 8
156
+ expert_tensor_parallel_size ..................... 1
157
+ export_force_local_attention .................... False
158
+ export_kd_cfg ................................... None
159
+ export_kd_teacher_ckpt_format ................... None
160
+ export_kd_teacher_load .......................... None
161
+ export_kv_cache_quant ........................... False
162
+ export_legacy_megatron .......................... False
163
+ export_model_type ............................... GPTModel
164
+ export_moe_apply_probs_on_input ................. False
165
+ export_qk_l2_norm ............................... False
166
+ export_quant_cfg ................................ None
167
+ export_real_quant_cfg ........................... None
168
+ export_te_mcore_model ........................... False
169
+ external_cuda_graph ............................. False
170
+ ffn_hidden_size ................................. 18432
171
+ finetune ........................................ False
172
+ finetune_data_split ............................. train
173
+ finetune_hf_dataset ............................. None
174
+ first_last_layers_bf16 .......................... False
175
+ flash_decode .................................... False
176
+ fp16 ............................................ False
177
+ fp16_lm_cross_entropy ........................... False
178
+ fp32_residual_connection ........................ False
179
+ fp8 ............................................. e4m3
180
+ fp8_amax_compute_algo ........................... most_recent
181
+ fp8_amax_history_len ............................ 1
182
+ fp8_interval .................................... 1
183
+ fp8_margin ...................................... 0
184
+ fp8_param_gather ................................ False
185
+ fp8_recipe ...................................... mxfp8
186
+ fp8_wgrad ....................................... True
187
+ fsdp_double_buffer .............................. False
188
+ global_batch_size ............................... 8
189
+ grad_reduce_in_bf16 ............................. False
190
+ gradient_accumulation_fusion .................... True
191
+ gradient_reduce_div_fusion ...................... True
192
+ group_query_attention ........................... False
193
+ head_lr_mult .................................... 1.0
194
+ heterogeneous_layers_config_encoded_json ........ None
195
+ heterogeneous_layers_config_path ................ None
196
+ hidden_dropout .................................. 0.0
197
+ hidden_size ..................................... 7168
198
+ hierarchical_context_parallel_sizes ............. None
199
+ high_priority_stream_groups ..................... []
200
+ hybrid_attention_ratio .......................... 0.0
201
+ hybrid_mlp_ratio ................................ 0.0
202
+ hybrid_override_pattern ......................... None
203
+ hysteresis ...................................... 2
204
+ ict_head_size ................................... None
205
+ ict_load ........................................ None
206
+ img_h ........................................... 224
207
+ img_w ........................................... 224
208
+ indexer_batch_size .............................. 128
209
+ indexer_log_interval ............................ 1000
210
+ inference_batch_times_seqlen_threshold .......... -1
211
+ inference_dynamic_batching ...................... False
212
+ inference_dynamic_batching_buffer_guaranteed_fraction 0.2
213
+ inference_dynamic_batching_buffer_overflow_factor None
214
+ inference_dynamic_batching_buffer_size_gb ....... 40.0
215
+ inference_dynamic_batching_chunk_size ........... 256
216
+ inference_dynamic_batching_max_requests_override None
217
+ inference_dynamic_batching_max_tokens_override .. None
218
+ inference_dynamic_batching_num_cuda_graphs ...... 16
219
+ inference_max_batch_size ........................ 8
220
+ inference_max_seq_length ........................ 2560
221
+ inference_rng_tracker ........................... False
222
+ init_method_std ................................. 0.02
223
+ init_method_xavier_uniform ...................... False
224
+ init_model_with_meta_device ..................... False
225
+ initial_loss_scale .............................. 4294967296
226
+ inprocess_active_world_size ..................... 8
227
+ inprocess_barrier_timeout ....................... 120
228
+ inprocess_completion_timeout .................... 120
229
+ inprocess_empty_cuda_cache ...................... False
230
+ inprocess_granularity ........................... node
231
+ inprocess_hard_timeout .......................... 90
232
+ inprocess_heartbeat_interval .................... 30
233
+ inprocess_heartbeat_timeout ..................... 60
234
+ inprocess_last_call_wait ........................ 1
235
+ inprocess_max_iterations ........................ None
236
+ inprocess_monitor_process_interval .............. 1.0
237
+ inprocess_monitor_thread_interval ............... 1.0
238
+ inprocess_progress_watchdog_interval ............ 1.0
239
+ inprocess_restart ............................... False
240
+ inprocess_soft_timeout .......................... 60
241
+ inprocess_termination_grace_time ................ 1
242
+ is_hybrid_model ................................. False
243
+ iter_per_epoch .................................. 1250
244
+ iterations_to_skip .............................. []
245
+ keep_fp8_transpose_cache_when_using_custom_fsdp . False
246
+ kitchen_config_file ............................. None
247
+ kitchen_recipe_number ........................... None
248
+ kv_channels ..................................... 64
249
+ kv_lora_rank .................................... 512
250
+ lazy_mpu_init ................................... None
251
+ load ............................................ None
252
+ load_main_params_from_ckpt ...................... None
253
+ load_model_opt_format ........................... False
254
+ local_rank ...................................... 0
255
+ log_energy ...................................... False
256
+ log_interval .................................... 1
257
+ log_loss_scale_to_tensorboard ................... True
258
+ log_memory_to_tensorboard ....................... False
259
+ log_num_zeros_in_grad ........................... False
260
+ log_params_norm ................................. False
261
+ log_progress .................................... False
262
+ log_straggler ................................... False
263
+ log_throughput .................................. True
264
+ log_timers_to_tensorboard ....................... False
265
+ log_validation_ppl_to_tensorboard ............... False
266
+ log_world_size_to_tensorboard ................... False
267
+ logging_level ................................... None
268
+ loss_scale ...................................... None
269
+ loss_scale_window ............................... 1000
270
+ lr .............................................. 0.0001
271
+ lr_decay_iters .................................. 8
272
+ lr_decay_samples ................................ None
273
+ lr_decay_style .................................. cosine
274
+ lr_warmup_fraction .............................. None
275
+ lr_warmup_init .................................. 0.0
276
+ lr_warmup_iters ................................. 5
277
+ lr_warmup_samples ............................... 0
278
+ lr_wsd_decay_iters .............................. None
279
+ lr_wsd_decay_samples ............................ None
280
+ lr_wsd_decay_style .............................. exponential
281
+ main_grads_dtype ................................ torch.float32
282
+ main_params_dtype ............................... torch.float32
283
+ make_vocab_size_divisible_by .................... 128
284
+ mamba_head_dim .................................. 64
285
+ mamba_num_groups ................................ 8
286
+ mamba_num_heads ................................. None
287
+ mamba_state_dim ................................. 128
288
+ manual_gc ....................................... False
289
+ manual_gc_eval .................................. True
290
+ manual_gc_interval .............................. 0
291
+ mask_factor ..................................... 1.0
292
+ mask_prob ....................................... 0.15
293
+ mask_type ....................................... random
294
+ masked_softmax_fusion ........................... False
295
+ max_position_embeddings ......................... 4096
296
+ max_tokens_to_oom ............................... 12000
297
+ memory_snapshot_path ............................ snapshot.pickle
298
+ merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt
299
+ micro_batch_size ................................ 1
300
+ microbatch_group_size_per_vp_stage .............. None
301
+ mid_level_dataset_surplus ....................... 0.005
302
+ min_loss_scale .................................. 1.0
303
+ min_lr .......................................... 1e-05
304
+ mlp_chunks_for_prefill .......................... 1
305
+ mmap_bin_files .................................. True
306
+ mock_data ....................................... False
307
+ moe_apply_probs_on_input ........................ False
308
+ moe_aux_loss_coeff .............................. 0.0
309
+ moe_capacity_monitor ............................ True
310
+ moe_deepep_num_sms .............................. 20
311
+ moe_echo_enable_random_offloading ............... False
312
+ moe_echo_expert_dispatch_overlap ................ False
313
+ moe_echo_expert_dispatcher_type ................. hybridep
314
+ moe_echo_recompute_expert_dispatch .............. False
315
+ moe_enable_deepep ............................... False
316
+ moe_enable_echo ................................. False
317
+ moe_enable_hybridep ............................. True
318
+ moe_enable_ultraep .............................. True
319
+ moe_expert_capacity_factor ...................... None
320
+ moe_extended_tp ................................. False
321
+ moe_ffn_hidden_size ............................. 2048
322
+ moe_grouped_gemm ................................ True
323
+ moe_hybridep_num_sms ............................ 16
324
+ moe_input_jitter_eps ............................ None
325
+ moe_layer_freq .................................. [0, 1]
326
+ moe_layer_recompute ............................. False
327
+ moe_num_echo_experts ............................ None
328
+ moe_num_redundant_experts_per_rank .............. 48
329
+ moe_pad_expert_input_to_capacity ................ False
330
+ moe_per_layer_logging ........................... False
331
+ moe_permute_fusion .............................. False
332
+ moe_received_token_capacity ..................... 1.17578125
333
+ moe_router_bias_update_rate ..................... 0.001
334
+ moe_router_dtype ................................ fp32
335
+ moe_router_enable_expert_bias ................... True
336
+ moe_router_force_load_balancing ................. False
337
+ moe_router_group_topk ........................... 1
338
+ moe_router_load_balancing_type .................. seq_aux_loss
339
+ moe_router_num_groups ........................... 1
340
+ moe_router_padding_for_fp8 ...................... False
341
+ moe_router_pre_softmax .......................... False
342
+ moe_router_score_function ....................... sigmoid
343
+ moe_router_topk ................................. 8
344
+ moe_router_topk_scaling_factor .................. 2.827
345
+ moe_shared_expert_intermediate_size ............. 2048
346
+ moe_shared_expert_overlap ....................... False
347
+ moe_token_dispatcher_type ....................... flex
348
+ moe_token_drop_policy ........................... probs
349
+ moe_ultraep_max_microbatches .................... None
350
+ moe_ultraep_overlap_weight_sync ................. True
351
+ moe_ultraep_weight_sync_join .................... before_dispatch
352
+ moe_upcycling_granularity ....................... 1
353
+ moe_use_device_initiated_grouped_gemm ........... True
354
+ moe_use_legacy_grouped_gemm ..................... False
355
+ moe_use_upcycling ............................... False
356
+ moe_z_loss_coeff ................................ None
357
+ mrope_section ................................... None
358
+ mscale .......................................... 1.0
359
+ mscale_all_dim .................................. 1.0
360
+ mtp_loss_scaling_factor ......................... 0.1
361
+ mtp_num_layers .................................. None
362
+ multi_latent_attention .......................... True
363
+ nccl_all_reduce_for_prefill ..................... False
364
+ nccl_communicator_config_path ................... None
365
+ nccl_ub ......................................... False
366
+ no_load_optim ................................... None
367
+ no_load_rng ..................................... None
368
+ no_persist_layer_norm ........................... False
369
+ no_rope_freq .................................... None
370
+ no_save_optim ................................... None
371
+ no_save_rng ..................................... None
372
+ non_persistent_ckpt_type ........................ None
373
+ non_persistent_global_ckpt_dir .................. None
374
+ non_persistent_local_ckpt_algo .................. fully_parallel
375
+ non_persistent_local_ckpt_dir ................... None
376
+ non_persistent_save_interval .................... None
377
+ norm_epsilon .................................... 1e-06
378
+ normalization ................................... RMSNorm
379
+ num_attention_heads ............................. 64
380
+ num_channels .................................... 3
381
+ num_classes ..................................... 1000
382
+ num_dataset_builder_threads ..................... 1
383
+ num_distributed_optimizer_instances ............. 1
384
+ num_experts ..................................... 384
385
+ num_layers ...................................... 2
386
+ num_layers_at_end_in_bf16 ....................... 1
387
+ num_layers_at_start_in_bf16 ..................... 1
388
+ num_layers_per_virtual_pipeline_stage ........... None
389
+ num_query_groups ................................ 1
390
+ num_virtual_stages_per_pipeline_rank ............ None
391
+ num_workers ..................................... 2
392
+ object_storage_cache_path ....................... None
393
+ one_logger_async ................................ False
394
+ one_logger_project .............................. megatron-lm
395
+ one_logger_run_name ............................. None
396
+ onnx_safe ....................................... None
397
+ openai_gelu ..................................... False
398
+ optimizer ....................................... adam
399
+ optimizer_cpu_offload ........................... False
400
+ optimizer_offload_fraction ...................... 1.0
401
+ output_bert_embeddings .......................... False
402
+ overlap_cpu_optimizer_d2h_h2d ................... False
403
+ overlap_grad_reduce ............................. True
404
+ overlap_moe_expert_parallel_comm ................ False
405
+ overlap_p2p_comm ................................ False
406
+ overlap_p2p_comm_warmup_flush ................... False
407
+ overlap_param_gather ............................ True
408
+ overlap_param_gather_with_optimizer_step ........ False
409
+ override_opt_param_scheduler .................... False
410
+ params_dtype .................................... torch.bfloat16
411
+ patch_dim ....................................... 16
412
+ per_split_data_args_path ........................ None
413
+ perform_initialization .......................... True
414
+ pin_cpu_grads ................................... True
415
+ pin_cpu_params .................................. True
416
+ pipeline_model_parallel_comm_backend ............ None
417
+ pipeline_model_parallel_layout .................. None
418
+ pipeline_model_parallel_size .................... 1
419
+ position_embedding_type ......................... rope
420
+ pretrained_checkpoint ........................... None
421
+ profile ......................................... False
422
+ profile_ranks ................................... [0]
423
+ profile_step_end ................................ 12
424
+ profile_step_start .............................. 10
425
+ q_lora_rank ..................................... 1536
426
+ qk_head_dim ..................................... 128
427
+ qk_l2_norm ...................................... False
428
+ qk_layernorm .................................... True
429
+ qk_pos_emb_head_dim ............................. 64
430
+ query_in_block_prob ............................. 0.1
431
+ rampup_batch_size ............................... None
432
+ rank ............................................ 0
433
+ recompute_granularity ........................... None
434
+ recompute_method ................................ None
435
+ recompute_modules ............................... None
436
+ recompute_num_layers ............................ None
437
+ record_memory_history ........................... False
438
+ relative_attention_max_distance ................. 128
439
+ relative_attention_num_buckets .................. 32
440
+ replication ..................................... False
441
+ replication_factor .............................. 2
442
+ replication_jump ................................ None
443
+ rerun_mode ...................................... validate_results
444
+ reset_attention_mask ............................ False
445
+ reset_position_ids .............................. False
446
+ result_rejected_tracker_filename ................ None
447
+ retriever_report_topk_accuracies ................ []
448
+ retriever_score_scaling ......................... False
449
+ retriever_seq_length ............................ 256
450
+ retro_add_retriever ............................. False
451
+ retro_attention_gate ............................ 1
452
+ retro_cyclic_train_iters ........................ None
453
+ retro_encoder_attention_dropout ................. 0.1
454
+ retro_encoder_hidden_dropout .................... 0.1
455
+ retro_encoder_layers ............................ 2
456
+ retro_num_neighbors ............................. 2
457
+ retro_num_retrieved_chunks ...................... 2
458
+ retro_project_dir ............................... None
459
+ retro_verify_neighbor_count ..................... True
460
+ reuse_grad_buf_for_mxfp8_param_ag ............... False
461
+ rope_scaling_factor ............................. 8.0
462
+ rope_type ....................................... None
463
+ rotary_base ..................................... 50000
464
+ rotary_interleaved .............................. False
465
+ rotary_percent .................................. 1.0
466
+ rotary_scaling_factor ........................... 32.0
467
+ rotary_seq_len_interpolation_factor ............. None
468
+ run_workload_inspector_server ................... False
469
+ sample_rate ..................................... 1.0
470
+ save ............................................ None
471
+ save_interval ................................... None
472
+ save_retain_interval ............................ None
473
+ scatter_gather_tensors_in_pipeline .............. True
474
+ seed ............................................ 1234
475
+ seq_length ...................................... 4096
476
+ sequence_parallel ............................... False
477
+ sft ............................................. False
478
+ sft_tokenizer_prompt_format ..................... nemotron-h-aligned
479
+ sgd_momentum .................................... 0.9
480
+ short_seq_prob .................................. 0.1
481
+ skip_train ...................................... False
482
+ skipped_train_samples ........................... 0
483
+ spec ............................................ None
484
+ split ........................................... 100,0,0
485
+ squared_relu .................................... False
486
+ start_weight_decay .............................. 0.1
487
+ straggler_ctrlr_port ............................ 65535
488
+ straggler_minmax_count .......................... 1
489
+ suggested_communication_unit_size ............... None
490
+ swiglu .......................................... True
491
+ swin_backbone_type .............................. tiny
492
+ symmetric_ar_type ............................... None
493
+ te_rng_tracker .................................. True
494
+ tensor_model_parallel_size ...................... 1
495
+ tensorboard_dir ................................. None
496
+ tensorboard_log_interval ........................ 1
497
+ tensorboard_queue_size .......................... 1000
498
+ test_data_path .................................. None
499
+ test_mode ....................................... False
500
+ tiktoken_num_special_tokens ..................... 1000
501
+ tiktoken_pattern ................................ None
502
+ tiktoken_special_tokens ......................... None
503
+ timing_log_level ................................ 0
504
+ timing_log_option ............................... minmax
505
+ titles_data_path ................................ None
506
+ tokenizer_model ................................. None
507
+ tokenizer_type .................................. GPT2BPETokenizer
508
+ torch_fsdp2_reshard_after_forward ............... True
509
+ tp_comm_bootstrap_backend ....................... nccl
510
+ tp_comm_bulk_dgrad .............................. True
511
+ tp_comm_bulk_wgrad .............................. True
512
+ tp_comm_overlap ................................. False
513
+ tp_comm_overlap_ag .............................. True
514
+ tp_comm_overlap_cfg ............................. None
515
+ tp_comm_overlap_rs .............................. True
516
+ tp_comm_overlap_rs_dgrad ........................ False
517
+ tp_comm_split_ag ................................ True
518
+ tp_comm_split_rs ................................ True
519
+ train_data_path ................................. None
520
+ train_iters ..................................... 12
521
+ train_samples ................................... None
522
+ train_sync_interval ............................. None
523
+ transformer_impl ................................ transformer_engine
524
+ transformer_pipeline_model_parallel_size ........ 1
525
+ untie_embeddings_and_output_weights ............. True
526
+ use_checkpoint_args ............................. False
527
+ use_checkpoint_opt_param_scheduler .............. False
528
+ use_cpu_initialization .......................... None
529
+ use_custom_fsdp ................................. False
530
+ use_dist_ckpt ................................... True
531
+ use_dist_ckpt_deprecated ........................ False
532
+ use_distributed_optimizer ....................... True
533
+ use_flash_attn .................................. False
534
+ use_legacy_models ............................... False
535
+ use_mp_args_from_checkpoint_args ................ False
536
+ use_one_sent_docs ............................... False
537
+ use_persistent_ckpt_worker ...................... False
538
+ use_precision_aware_optimizer ................... False
539
+ use_pytorch_profiler ............................ False
540
+ use_ring_exchange_p2p ........................... False
541
+ use_rope_scaling ................................ False
542
+ use_rotary_position_embeddings .................. False
543
+ use_sharp ....................................... False
544
+ use_tokenizer_model_from_checkpoint_args ........ True
545
+ use_torch_fsdp2 ................................. False
546
+ use_torch_optimizer_for_cpu_offload ............. False
547
+ use_tp_pp_dp_mapping ............................ False
548
+ v_head_dim ...................................... 128
549
+ valid_data_path ................................. None
550
+ variable_seq_lengths ............................ False
551
+ virtual_pipeline_model_parallel_size ............ None
552
+ vision_backbone_type ............................ vit
553
+ vision_pretraining .............................. False
554
+ vision_pretraining_type ......................... classify
555
+ vocab_extra_ids ................................. 0
556
+ vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json
557
+ vocab_size ...................................... None
558
+ wandb_exp_name ..................................
559
+ wandb_project ...................................
560
+ wandb_save_dir ..................................
561
+ weight_decay .................................... 0.1
562
+ weight_decay_incr_style ......................... constant
563
+ wgrad_deferral_limit ............................ 0
564
+ world_size ...................................... 8
565
+ yaml_cfg ........................................ None
566
+ -------------------- end of arguments ---------------------
567
+ INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1
568
+ > building GPT2BPETokenizer tokenizer ...
569
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
570
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
571
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
572
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
573
+ > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304)
574
+ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS
575
+ > initializing torch distributed ...
576
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
577
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
578
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
579
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
580
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
581
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
582
+ WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it
583
+ [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
584
+ [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
585
+ [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
586
+ [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
587
+ [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
588
+ [Gloo] Rank [Gloo] Rank 5 is connected to 77 is connected to [Gloo] Rank peer ranks. 76Expected number of connected peer ranks is : peer ranks. is connected to 7Expected number of connected peer ranks is : 7
589
+ 7 peer ranks.
590
+ Expected number of connected peer ranks is : 7
591
+ [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
592
+ [Gloo] Rank 0 is connected to [Gloo] Rank 75 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. Expected number of connected peer ranks is :
593
+ 7
594
+ [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
595
+ [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
596
+ [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
597
+ [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
598
+ [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
599
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
600
+ > initialized tensor model parallel with size 1
601
+ > initialized pipeline model parallel with size 1
602
+ > setting random seeds to 1234 ...
603
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
604
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
605
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
606
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
607
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
608
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
609
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
610
+ [rank4]:[W921 07:48:50.518278748 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
611
+ [rank3]:[W921 07:48:50.518509064 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
612
+ [rank2]:[W921 07:48:50.518511490 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
613
+ [rank1]:[W921 07:48:50.518613627 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
614
+ [rank5]:[W921 07:48:50.520159673 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
615
+ [rank6]:[W921 07:48:50.520258096 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
616
+ [rank7]:[W921 07:48:50.520356968 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
617
+ > compiling dataset index builder ...
618
+ make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets'
619
+ make: Nothing to be done for 'default'.
620
+ make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets'
621
+ >>> done with dataset index builder. Compilation time: 0.037 seconds
622
+ WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations.
623
+ > compiling and loading fused kernels ...
624
+ [rank0]:[W921 07:48:50.676485981 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
625
+ >>> done with compiling and loading fused kernels. Compilation time: 4.061 seconds
626
+ time to initialize megatron (seconds): 7.706
627
+ [after megatron is initialized] datetime: 2026-09-21 07:48:56
628
+ building GPT model ...
629
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
630
+ warnings.warn(
631
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
632
+ warnings.warn(
633
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
634
+ warnings.warn(
635
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
636
+ warnings.warn(
637
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
638
+ warnings.warn(
639
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
640
+ warnings.warn(
641
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
642
+ warnings.warn(
643
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
644
+ warnings.warn(
645
+ [INFO] Use MNNVL fabric: False
646
+ [WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8.
647
+ NVSHMEM v3.7.2
648
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
649
+
650
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
651
+
652
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
653
+
654
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
655
+
656
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
657
+
658
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
659
+
660
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
661
+
662
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
663
+
664
+ UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive).
665
+ > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 5594455040
666
+ INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False)
667
+ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 9
668
+ Params for bucket 1 (360579072 elements, 360579072 padded size):
669
+ module.output_layer.weight
670
+ Params for bucket 2 (44047360 elements, 44047360 padded size):
671
+ module.decoder.final_layernorm.weight
672
+ module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight
673
+ module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight
674
+ Params for bucket 3 (45163520 elements, 45163520 padded size):
675
+ module.decoder.layers.1.pre_mlp_layernorm.weight
676
+ module.decoder.layers.1.self_attention.linear_kv_down_proj.weight
677
+ module.decoder.layers.1.self_attention.linear_q_down_proj.weight
678
+ module.decoder.layers.1.mlp.router.weight
679
+ module.decoder.layers.1.self_attention.linear_kv_up_proj.weight
680
+ module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight
681
+ module.decoder.layers.1.self_attention.linear_q_up_proj.weight
682
+ module.decoder.layers.1.self_attention.linear_q_up_proj.layer_norm_weight
683
+ Params for bucket 4 (58720256 elements, 58720256 padded size):
684
+ module.decoder.layers.1.self_attention.linear_proj.weight
685
+ Params for bucket 5 (132127744 elements, 132127744 padded size):
686
+ module.decoder.layers.1.input_layernorm.weight
687
+ module.decoder.layers.0.mlp.linear_fc2.weight
688
+ Params for bucket 6 (264241152 elements, 264241152 padded size):
689
+ module.decoder.layers.0.mlp.linear_fc1.weight
690
+ Params for bucket 7 (42411008 elements, 42411008 padded size):
691
+ module.decoder.layers.0.self_attention.linear_kv_down_proj.weight
692
+ module.decoder.layers.0.self_attention.linear_q_up_proj.layer_norm_weight
693
+ module.decoder.layers.0.self_attention.linear_q_down_proj.weight
694
+ module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight
695
+ module.decoder.layers.0.self_attention.linear_kv_up_proj.weight
696
+ module.decoder.layers.0.self_attention.linear_q_up_proj.weight
697
+ module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight
698
+ Params for bucket 8 (58720256 elements, 58720256 padded size):
699
+ module.decoder.layers.0.self_attention.linear_proj.weight
700
+ Params for bucket 9 (360586240 elements, 360586240 padded size):
701
+ module.decoder.layers.0.input_layernorm.weight
702
+ module.embedding.word_embeddings.weight
703
+ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40
704
+ Params for bucket 1 (44040192 elements, 44040192 padded size):
705
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight47
706
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight45
707
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight46
708
+ Params for bucket 2 (44040192 elements, 44040192 padded size):
709
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight44
710
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight42
711
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight43
712
+ Params for bucket 3 (44040192 elements, 44040192 padded size):
713
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight41
714
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight39
715
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight40
716
+ Params for bucket 4 (44040192 elements, 44040192 padded size):
717
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight38
718
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight37
719
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight36
720
+ Params for bucket 5 (44040192 elements, 44040192 padded size):
721
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight35
722
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight34
723
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight33
724
+ Params for bucket 6 (44040192 elements, 44040192 padded size):
725
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight32
726
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight31
727
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight30
728
+ Params for bucket 7 (44040192 elements, 44040192 padded size):
729
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight29
730
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight28
731
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight27
732
+ Params for bucket 8 (44040192 elements, 44040192 padded size):
733
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight26
734
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight25
735
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight24
736
+ Params for bucket 9 (44040192 elements, 44040192 padded size):
737
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight23
738
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight21
739
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight22
740
+ Params for bucket 10 (44040192 elements, 44040192 padded size):
741
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight20
742
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight18
743
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight19
744
+ Params for bucket 11 (44040192 elements, 44040192 padded size):
745
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight17
746
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight15
747
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight16
748
+ Params for bucket 12 (44040192 elements, 44040192 padded size):
749
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight14
750
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight12
751
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight13
752
+ Params for bucket 13 (44040192 elements, 44040192 padded size):
753
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight11
754
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight9
755
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight10
756
+ Params for bucket 14 (44040192 elements, 44040192 padded size):
757
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight8
758
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight6
759
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight7
760
+ Params for bucket 15 (44040192 elements, 44040192 padded size):
761
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight5
762
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight3
763
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight4
764
+ Params for bucket 16 (44040192 elements, 44040192 padded size):
765
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight2
766
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight0
767
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight1
768
+ Params for bucket 17 (58720256 elements, 58720256 padded size):
769
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight47
770
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight46
771
+ Params for bucket 18 (58720256 elements, 58720256 padded size):
772
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight45
773
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight44
774
+ Params for bucket 19 (58720256 elements, 58720256 padded size):
775
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight43
776
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight42
777
+ Params for bucket 20 (58720256 elements, 58720256 padded size):
778
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight41
779
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight40
780
+ Params for bucket 21 (58720256 elements, 58720256 padded size):
781
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight39
782
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight38
783
+ Params for bucket 22 (58720256 elements, 58720256 padded size):
784
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight37
785
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight36
786
+ Params for bucket 23 (58720256 elements, 58720256 padded size):
787
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight35
788
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight34
789
+ Params for bucket 24 (58720256 elements, 58720256 padded size):
790
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight33
791
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight32
792
+ Params for bucket 25 (58720256 elements, 58720256 padded size):
793
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight31
794
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight30
795
+ Params for bucket 26 (58720256 elements, 58720256 padded size):
796
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight29
797
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight28
798
+ Params for bucket 27 (58720256 elements, 58720256 padded size):
799
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight27
800
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight26
801
+ Params for bucket 28 (58720256 elements, 58720256 padded size):
802
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight25
803
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight24
804
+ Params for bucket 29 (58720256 elements, 58720256 padded size):
805
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight23
806
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight22
807
+ Params for bucket 30 (58720256 elements, 58720256 padded size):
808
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight21
809
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight20
810
+ Params for bucket 31 (58720256 elements, 58720256 padded size):
811
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight19
812
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight18
813
+ Params for bucket 32 (58720256 elements, 58720256 padded size):
814
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight17
815
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight16
816
+ Params for bucket 33 (58720256 elements, 58720256 padded size):
817
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight15
818
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight14
819
+ Params for bucket 34 (58720256 elements, 58720256 padded size):
820
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight13
821
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight12
822
+ Params for bucket 35 (58720256 elements, 58720256 padded size):
823
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight11
824
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight10
825
+ Params for bucket 36 (58720256 elements, 58720256 padded size):
826
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight9
827
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight8
828
+ Params for bucket 37 (58720256 elements, 58720256 padded size):
829
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight7
830
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight6
831
+ Params for bucket 38 (58720256 elements, 58720256 padded size):
832
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight5
833
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight4
834
+ Params for bucket 39 (58720256 elements, 58720256 padded size):
835
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight3
836
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight2
837
+ Params for bucket 40 (58720256 elements, 58720256 padded size):
838
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight1
839
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight0
840
+ INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=<megatron.core.timers.Timers object at 0x71d5f8ed6a50>, config_logger_dir='')
841
+ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine
842
+ [after model, optimizer, and learning rate scheduler are built] datetime: 2026-09-21 07:49:00
843
+ > building train, validation, and test datasets ...
844
+ > datasets target sizes (minimum size):
845
+ train: 96
846
+ validation: 0
847
+ test: 0
848
+ INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None]
849
+ > building train, validation, and test datasets for GPT ...
850
+ INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=<megatron.training.tokenizer.tokenizer._GPT2BPETokenizer object at 0x71d5f93d2ae0>, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None)
851
+ INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx
852
+ INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths
853
+ INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers
854
+ INFO:megatron.core.datasets.indexed_dataset: Extract the document indices
855
+ INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029
856
+ INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350
857
+ INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices
858
+ INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy
859
+ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy
860
+ INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy
861
+ INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073
862
+ > finished creating GPT datasets ...
863
+ [after dataloaders are built] datetime: 2026-09-21 07:49:00
864
+ done with setup ...
865
+ (min, max) time across ranks (ms):
866
+ model-and-optimizer-setup ......................: (3818.24, 3822.76)
867
+ train/valid/test-data-iterators-setup ..........: (72.48, 94.09)
868
+ training ...
869
+ Overwriting rerun_state_machine.current_iteration from -1 to 0...
870
+ [before the start of training step] datetime: 2026-09-21 07:49:00
871
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
872
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
873
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
874
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
875
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
876
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
877
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
878
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
879
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
880
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
881
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
882
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
883
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
884
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
885
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
886
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
887
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
888
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
889
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
890
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
891
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
892
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
893
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
894
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
895
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
896
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
897
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
898
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
899
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
900
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
901
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
902
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
903
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
904
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
905
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
906
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
907
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
908
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
909
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
910
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
911
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
912
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
913
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
914
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
915
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
916
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
917
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
918
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
919
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
920
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
921
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
922
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
923
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
924
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
925
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
926
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
927
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
928
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
929
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
930
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
931
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
932
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
933
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
934
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
935
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
936
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
937
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
938
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
939
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
940
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
941
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
942
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
943
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
944
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
945
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
946
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
947
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
948
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
949
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
950
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
951
+ [2026-09-21 07:49:29] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 28413.5 | throughput per GPU (TFLOP/s/GPU): 1.2 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.227924E+01 | loss scale: 1.0 | grad norm: 12.617 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36224 | min safe capacity: 1.1055 |
952
+ Number of parameters in transformer block in billions: 17.55
953
+ Number of parameters in embedding layers in billions: 0.72
954
+ Total number of parameters in billions: 18.28
955
+ Number of parameters in most loaded shard in billions: 18.2753
956
+ Theoretical memory footprints: weight and optimizer=130715.23 MB
957
+ [Rank 0] (after 1 iterations) memory (MB) | allocated: 49136.10400390625 | max allocated: 49136.1044921875 | reserved: 54394.0 | max reserved: 54394.0
958
+ /workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
959
+ warnings.warn(
960
+ /workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
961
+ warnings.warn(
962
+ /workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
963
+ warnings.warn(
964
+ /workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
965
+ warnings.warn(
966
+ /workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
967
+ warnings.warn(
968
+ /workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
969
+ warnings.warn(
970
+ /workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
971
+ warnings.warn(
972
+ /workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
973
+ warnings.warn(
974
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
975
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
976
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
977
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
978
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
979
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
980
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
981
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
982
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
983
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
984
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
985
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
986
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
987
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
988
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
989
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
990
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
991
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
992
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
993
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
994
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
995
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
996
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
997
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
998
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
999
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1000
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1001
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1002
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1003
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1004
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1005
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1006
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1007
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1008
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1009
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1010
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1011
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1012
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1013
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1014
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1015
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1016
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1017
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1018
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1019
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1020
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1021
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1022
+ [2026-09-21 07:49:29] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 166.7 | throughput per GPU (TFLOP/s/GPU): 212.2 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.227088E+01 | loss scale: 1.0 | grad norm: 12.534 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36224 | min safe capacity: 1.1055 |
1023
+ [2026-09-21 07:49:29] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 111.9 | throughput per GPU (TFLOP/s/GPU): 316.1 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.088328E+01 | loss scale: 1.0 | grad norm: 11.023 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36224 | min safe capacity: 1.1055 |
1024
+ Capture CUDA graph for training!!!
1025
+ CUDA graph capture done!!!
1026
+ [2026-09-21 07:49:30] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 1272.5 | throughput per GPU (TFLOP/s/GPU): 27.8 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.448181E+01 | loss scale: 1.0 | grad norm: 72.911 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36608 | min safe capacity: 1.1172 |
1027
+ [2026-09-21 07:49:30] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 110.5 | throughput per GPU (TFLOP/s/GPU): 320.1 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.808459E+01 | loss scale: 1.0 | grad norm: 108.268 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
1028
+ [2026-09-21 07:49:30] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 111.5 | throughput per GPU (TFLOP/s/GPU): 317.3 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.491727E+01 | loss scale: 1.0 | grad norm: 83.473 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
1029
+ [2026-09-21 07:49:31] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 114.8 | throughput per GPU (TFLOP/s/GPU): 308.1 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.143662E+01 | loss scale: 1.0 | grad norm: 26.627 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
1030
+ [2026-09-21 07:49:31] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 108.9 | throughput per GPU (TFLOP/s/GPU): 324.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.166943E+01 | loss scale: 1.0 | grad norm: 12.800 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
1031
+ [2026-09-21 07:49:31] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 108.9 | throughput per GPU (TFLOP/s/GPU): 324.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.131635E+01 | loss scale: 1.0 | grad norm: 10.441 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
1032
+ [2026-09-21 07:49:31] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 109.1 | throughput per GPU (TFLOP/s/GPU): 324.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.096943E+01 | loss scale: 1.0 | grad norm: 9.147 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
1033
+ [2026-09-21 07:49:31] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 108.6 | throughput per GPU (TFLOP/s/GPU): 325.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.073911E+01 | loss scale: 1.0 | grad norm: 9.755 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
1034
+ [2026-09-21 07:49:31] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 108.9 | throughput per GPU (TFLOP/s/GPU): 324.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.048286E+01 | loss scale: 1.0 | grad norm: 8.843 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
1035
+ [after training is done] datetime: 2026-09-21 07:49:31
1036
+ W0921 07:49:32.152000 524467 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers
1037
+ W0921 07:49:32.152000 524467 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 524570 closing signal SIGTERM
1038
+ W0921 07:49:32.153000 524467 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 524571 closing signal SIGTERM
1039
+ W0921 07:49:32.154000 524467 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 524572 closing signal SIGTERM
1040
+ W0921 07:49:32.155000 524467 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 524573 closing signal SIGTERM
1041
+ W0921 07:49:32.156000 524467 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 524574 closing signal SIGTERM
1042
+ W0921 07:49:32.158000 524467 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 524575 closing signal SIGTERM
1043
+ W0921 07:49:32.159000 524467 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 524576 closing signal SIGTERM
1044
+ W0921 07:49:32.160000 524467 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 524577 closing signal SIGTERM
1045
+ Traceback (most recent call last):
1046
+ File "/usr/local/bin/torchrun", line 8, in <module>
1047
+ sys.exit(main())
1048
+ ^^^^^^
1049
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
1050
+ return f(*args, **kwargs)
1051
+ ^^^^^^^^^^^^^^^^^^
1052
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main
1053
+ run(args)
1054
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run
1055
+ elastic_launch(
1056
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__
1057
+ return launch_agent(self._config, self._entrypoint, list(args))
1058
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1059
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent
1060
+ result = agent.run()
1061
+ ^^^^^^^^^^^
1062
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
1063
+ result = f(*args, **kwargs)
1064
+ ^^^^^^^^^^^^^^^^^^
1065
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run
1066
+ result = self._invoke_run(role)
1067
+ ^^^^^^^^^^^^^^^^^^^^^^
1068
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run
1069
+ time.sleep(monitor_interval)
1070
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler
1071
+ raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
1072
+ torch.distributed.elastic.multiprocessing.api.SignalException: Process 524467 got signal: 15
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep3/mem_samples.csv ADDED
@@ -0,0 +1,192 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 4
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 4
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 4
43
+ 2, 4
44
+ 3, 4
45
+ 4, 4
46
+ 5, 4
47
+ 6, 4
48
+ 7, 35
49
+ 0, 821
50
+ 1, 821
51
+ 2, 821
52
+ 3, 821
53
+ 4, 821
54
+ 5, 821
55
+ 6, 821
56
+ 7, 821
57
+ 0, 1783
58
+ 1, 1783
59
+ 2, 1783
60
+ 3, 1783
61
+ 4, 1783
62
+ 5, 1783
63
+ 6, 1783
64
+ 7, 1783
65
+ 0, 3437
66
+ 1, 3493
67
+ 2, 2681
68
+ 3, 2681
69
+ 4, 2681
70
+ 5, 3493
71
+ 6, 2681
72
+ 7, 3437
73
+ 0, 4289
74
+ 1, 4289
75
+ 2, 4289
76
+ 3, 4289
77
+ 4, 4289
78
+ 5, 4289
79
+ 6, 4289
80
+ 7, 4289
81
+ 0, 50889
82
+ 1, 50831
83
+ 2, 50857
84
+ 3, 50859
85
+ 4, 50863
86
+ 5, 50903
87
+ 6, 50897
88
+ 7, 50915
89
+ 0, 60211
90
+ 1, 60277
91
+ 2, 60233
92
+ 3, 60227
93
+ 4, 60239
94
+ 5, 60233
95
+ 6, 60241
96
+ 7, 60243
97
+ 0, 61267
98
+ 1, 61269
99
+ 2, 61267
100
+ 3, 61267
101
+ 4, 61267
102
+ 5, 61267
103
+ 6, 61267
104
+ 7, 61267
105
+ 0, 61267
106
+ 1, 61269
107
+ 2, 61267
108
+ 3, 61267
109
+ 4, 61267
110
+ 5, 61267
111
+ 6, 61267
112
+ 7, 61267
113
+ 0, 62923
114
+ 1, 62925
115
+ 2, 62823
116
+ 3, 62895
117
+ 4, 62981
118
+ 5, 63097
119
+ 6, 62923
120
+ 7, 62925
121
+ 0, 64465
122
+ 1, 64467
123
+ 2, 64465
124
+ 3, 64465
125
+ 4, 64465
126
+ 5, 64465
127
+ 6, 64465
128
+ 7, 64465
129
+ 0, 65963
130
+ 1, 65965
131
+ 2, 65963
132
+ 3, 65963
133
+ 4, 65963
134
+ 5, 65963
135
+ 6, 65963
136
+ 7, 65963
137
+ 0, 65963
138
+ 1, 65965
139
+ 2, 65963
140
+ 3, 65963
141
+ 4, 65963
142
+ 5, 65963
143
+ 6, 65963
144
+ 7, 65963
145
+ 0, 65969
146
+ 1, 65969
147
+ 2, 65969
148
+ 3, 65969
149
+ 4, 65969
150
+ 5, 65969
151
+ 6, 65969
152
+ 7, 65969
153
+ 0, 68243
154
+ 1, 68215
155
+ 2, 68215
156
+ 3, 68245
157
+ 4, 68263
158
+ 5, 68267
159
+ 6, 68289
160
+ 7, 68279
161
+ 0, 69055
162
+ 1, 69055
163
+ 2, 69055
164
+ 3, 69055
165
+ 4, 69055
166
+ 5, 69055
167
+ 6, 69055
168
+ 7, 69055
169
+ 0, 75835
170
+ 1, 75843
171
+ 2, 75809
172
+ 3, 75865
173
+ 4, 75851
174
+ 5, 75833
175
+ 6, 75893
176
+ 7, 75841
177
+ 0, 90291
178
+ 1, 90297
179
+ 2, 90291
180
+ 3, 90347
181
+ 4, 90347
182
+ 5, 90347
183
+ 6, 90375
184
+ 7, 90347
185
+ 0, 14403
186
+ 1, 14406
187
+ 2, 14406
188
+ 3, 14406
189
+ 4, 14406
190
+ 5, 14790
191
+ 6, 90375
192
+ 7, 16326
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep3/run.env ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=ultraep
2
+ config=KIMI_K2_L2
3
+ family=tierB
4
+ source_model=Kimi-K2
5
+ standard_num_layers=61
6
+ base_rung=
7
+ repo=/workspace/Megatron-UltraEP
8
+ branch=ultraep-bracing-regions-through-kernels
9
+ commit=03d57ef7
10
+ num_layers=2
11
+ hidden=7168
12
+ ffn_hidden=18432
13
+ moe_ffn_hidden=2048
14
+ num_experts=384
15
+ topk=8
16
+ experts_per_rank=48
17
+ expert_params=44040192
18
+ heads=64
19
+ kv_groups=64
20
+ kv_channels=64
21
+ qk_layernorm=1
22
+ shared_ffn=2048
23
+ shared_expert_overlap=0
24
+ moe_token_dispatcher=flex
25
+ mla=1
26
+ q_lora_rank=1536
27
+ kv_lora_rank=512
28
+ qk_head_dim=128
29
+ qk_pos_emb_head_dim=64
30
+ v_head_dim=128
31
+ dense_first_layers=1
32
+ moe_layers=1
33
+ rotary_base=50000
34
+ rotary_percent=1.0
35
+ rotary_scaling=32
36
+ mscale=1.0
37
+ mscale_all_dim=1.0
38
+ qkv_bias=0
39
+ router_balancing=seq_aux_loss
40
+ aux_loss_coeff=0
41
+ router_pre_softmax=0
42
+ router_fusion=0
43
+ router_dtype=fp32
44
+ router_groups=1
45
+ router_group_topk=1
46
+ router_score_function=sigmoid
47
+ router_expert_bias=1
48
+ router_bias_update_rate=1e-3
49
+ router_topk_scaling_factor=2.827
50
+ shared_expert_gate=0
51
+ apply_layernorm_1p=0
52
+ attention_output_gate=0
53
+ linear_attention_type=
54
+ linear_attention_freq=
55
+ linear_conv_kernel_dim=
56
+ linear_key_head_dim=
57
+ linear_value_head_dim=
58
+ linear_num_key_heads=
59
+ linear_num_value_heads=
60
+ mtp_num_layers=0
61
+ mtp_loss_scaling_factor=0.1
62
+ no_weight_decay_cond_type=
63
+ max_position_embeddings=4096
64
+ init_method_std=0.02
65
+ device_initiated_grouped_gemm=1
66
+ norm_epsilon=1e-6
67
+ seq_length=4096
68
+ micro_batch_size=1
69
+ global_batch_size=8
70
+ num_microbatches=1
71
+ assignments_per_rank=32768
72
+ mean_tokens_per_expert=682
73
+ slot_policy=matched
74
+ slots_per_rank=48
75
+ capacity_mode=static
76
+ capacity=1.17578125
77
+ cuda_graph=1
78
+ probe_mode=0
79
+ async_grad_reduce=0
80
+ branch_role=timing
81
+ memory_dump_mode=none
82
+ memory_snapshot_path=
83
+ cuda_graph_warmup_steps=3
84
+ recompute=none
85
+ force_balance=0
86
+ train_iters=12
87
+ ep_size=8
88
+ echo_dispatch_overlap=1
89
+ echo_dispatch_recompute=0
90
+ ultraep_quota_min_tokens=85
91
+ ultraep_weight_sync_join=
92
+ profile_enable=0
93
+ kernel_trace=0
94
+ fp8_mode=mxfp8
95
+ fp8_recipe=mxfp8
96
+ training_seed=1234
97
+ learning_rate=1e-4
98
+ min_learning_rate=1.0e-5
99
+ lr_warmup_iters=5
100
+ state_dump_enable=0
101
+ state_dump_dir=
102
+ moonep_assert_no_pending_reduce=0
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep3/train.log ADDED
@@ -0,0 +1,1072 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [07:49:37] WARN: ultraep-bracing-regions-through-kernels lacks --moe-router-fusion; KIMI_K2_L2 runs unfused (timing only, routing identical)
2
+ ==============================================================
3
+ KIMI_K2_L2 / ultraep (slots/rank=48 policy=matched)
4
+ E=384 K=8 H'=2048 m=48
5
+ L=2 seq=4096 gbs=8 (1 microbatches)
6
+ cap_mode=static cap=1.17578125 cg=1 recompute=none
7
+ memory_dump=none snapshots=/workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep3/memory_snapshots/snapshot.pickle
8
+ repo=/workspace/Megatron-UltraEP @ 03d57ef7
9
+ ==============================================================
10
+ W0921 07:49:38.672000 527457 torch/distributed/run.py:766]
11
+ W0921 07:49:38.672000 527457 torch/distributed/run.py:766] *****************************************
12
+ W0921 07:49:38.672000 527457 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
13
+ W0921 07:49:38.672000 527457 torch/distributed/run.py:766] *****************************************
14
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
15
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
16
+ WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it
17
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
18
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
19
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
20
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
21
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
22
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
23
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
24
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
25
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
26
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
27
+ using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1
28
+ WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer
29
+ Number of virtual stages per pipeline stage: None
30
+ accumulate and all-reduce gradients in fp32 for bfloat16 data type.
31
+ using torch.bfloat16 for parameters ...
32
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
33
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
34
+ ------------------------ arguments ------------------------
35
+ account_for_embedding_in_pipeline_split ......... False
36
+ account_for_loss_in_pipeline_split .............. False
37
+ accumulate_allreduce_grads_in_fp32 .............. True
38
+ adam_beta1 ...................................... 0.9
39
+ adam_beta2 ...................................... 0.999
40
+ adam_eps ........................................ 1e-08
41
+ add_bias_linear ................................. False
42
+ add_position_embedding .......................... False
43
+ add_qkv_bias .................................... False
44
+ adlr_autoresume ................................. False
45
+ adlr_autoresume_interval ........................ 1000
46
+ align_grad_reduce ............................... True
47
+ align_param_gather .............................. False
48
+ app_tag_run_name ................................ None
49
+ app_tag_run_version ............................. 0.0.0
50
+ apply_layernorm_1p .............................. False
51
+ apply_query_key_layer_scaling ................... False
52
+ apply_residual_connection_post_layernorm ........ False
53
+ apply_rope_fusion ............................... False
54
+ async_save ...................................... None
55
+ async_tensor_model_parallel_allreduce ........... True
56
+ attention_backend ............................... AttnBackend.auto
57
+ attention_dropout ............................... 0.0
58
+ attention_softmax_in_fp32 ....................... False
59
+ auto_detect_ckpt_format ......................... False
60
+ barrier_with_L1_time ............................ True
61
+ bert_binary_head ................................ True
62
+ bert_embedder_type .............................. megatron
63
+ bert_load ....................................... None
64
+ bf16 ............................................ True
65
+ bias_dropout_fusion ............................. True
66
+ bias_gelu_fusion ................................ False
67
+ bias_swiglu_fusion .............................. True
68
+ biencoder_projection_dim ........................ 0
69
+ biencoder_shared_query_context_model ............ False
70
+ block_data_path ................................. None
71
+ calc_ft_timeouts ................................ False
72
+ calculate_per_token_loss ........................ False
73
+ check_for_large_grads ........................... False
74
+ check_for_nan_in_loss_and_grad .................. False
75
+ check_for_spiky_loss ............................ False
76
+ check_weight_hash_across_dp_replicas_interval ... None
77
+ ckpt_assume_constant_structure .................. False
78
+ ckpt_convert_format ............................. None
79
+ ckpt_convert_save ............................... None
80
+ ckpt_convert_update_legacy_dist_opt_format ...... False
81
+ ckpt_format ..................................... torch_dist
82
+ ckpt_fully_parallel_load ........................ False
83
+ ckpt_fully_parallel_save ........................ True
84
+ ckpt_fully_parallel_save_deprecated ............. False
85
+ ckpt_step ....................................... None
86
+ classes_fraction ................................ 1.0
87
+ clip_grad ....................................... 1.0
88
+ clone_scatter_output_in_embedding ............... True
89
+ config_logger_dir ...............................
90
+ consumed_train_samples .......................... 0
91
+ consumed_valid_samples .......................... 0
92
+ context_parallel_size ........................... 1
93
+ cp_comm_type .................................... ['p2p']
94
+ create_attention_mask_in_dataloader ............. True
95
+ cross_entropy_fusion_impl ....................... native
96
+ cross_entropy_loss_fusion ....................... False
97
+ cuda_graph_scope ................................ full_iteration
98
+ cuda_graph_warmup_steps ......................... 3
99
+ data_args_path .................................. None
100
+ data_cache_path ................................. None
101
+ data_parallel_random_init ....................... False
102
+ data_parallel_sharding_strategy ................. no_shard
103
+ data_parallel_size .............................. 8
104
+ data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document']
105
+ data_per_class_fraction ......................... 1.0
106
+ data_sharding ................................... True
107
+ dataloader_type ................................. single
108
+ ddp_average_in_collective ....................... False
109
+ ddp_bucket_size ................................. None
110
+ ddp_num_buckets ................................. None
111
+ ddp_pad_buckets_for_high_nccl_busbw ............. False
112
+ decoder_first_pipeline_num_layers ............... None
113
+ decoder_last_pipeline_num_layers ................ None
114
+ decoder_num_layers .............................. None
115
+ decoder_seq_length .............................. None
116
+ decoupled_lr .................................... None
117
+ decoupled_min_lr ................................ None
118
+ decrease_batch_size_if_needed ................... False
119
+ defer_embedding_wgrad_compute ................... False
120
+ delay_wgrad_compute ............................. False
121
+ deprecated_use_mcore_models ..................... True
122
+ deterministic_mode .............................. False
123
+ dino_bottleneck_size ............................ 256
124
+ dino_freeze_last_layer .......................... 1
125
+ dino_head_hidden_size ........................... 2048
126
+ dino_local_crops_number ......................... 10
127
+ dino_local_img_size ............................. 96
128
+ dino_norm_last_layer ............................ False
129
+ dino_teacher_temp ............................... 0.07
130
+ dino_warmup_teacher_temp ........................ 0.04
131
+ dino_warmup_teacher_temp_epochs ................. 30
132
+ disable_bf16_reduced_precision_matmul ........... False
133
+ disable_mamba_mem_eff_path ...................... False
134
+ disable_straggler_on_startup .................... False
135
+ dist_ckpt_format_deprecated ..................... None
136
+ dist_ckpt_strictness ............................ assume_ok_unexpected
137
+ distribute_saved_activations .................... False
138
+ distributed_backend ............................. nccl
139
+ distributed_timeout_minutes ..................... 10
140
+ embedding_init_method_std ....................... None
141
+ embedding_path .................................. None
142
+ empty_unused_memory_level ....................... 0
143
+ enable_cuda_graph ............................... True
144
+ enable_experimental ............................. False
145
+ enable_ft_package ............................... False
146
+ enable_gloo_process_groups ...................... True
147
+ enable_msc ...................................... True
148
+ enable_one_logger ............................... True
149
+ encoder_num_layers .............................. 2
150
+ encoder_seq_length .............................. 4096
151
+ end_weight_decay ................................ 0.1
152
+ eod_mask_loss ................................... False
153
+ error_injection_rate ............................ 0
154
+ error_injection_type ............................ transient_error
155
+ eval_interval ................................... 1000000
156
+ eval_iters ...................................... 0
157
+ evidence_data_path .............................. None
158
+ exit_duration_in_mins ........................... None
159
+ exit_interval ................................... None
160
+ exit_on_missing_checkpoint ...................... False
161
+ exit_signal_handler ............................. False
162
+ exp_avg_dtype ................................... torch.float32
163
+ exp_avg_sq_dtype ................................ torch.float32
164
+ expert_model_parallel_size ...................... 8
165
+ expert_tensor_parallel_size ..................... 1
166
+ export_force_local_attention .................... False
167
+ export_kd_cfg ................................... None
168
+ export_kd_teacher_ckpt_format ................... None
169
+ export_kd_teacher_load .......................... None
170
+ export_kv_cache_quant ........................... False
171
+ export_legacy_megatron .......................... False
172
+ export_model_type ............................... GPTModel
173
+ export_moe_apply_probs_on_input ................. False
174
+ export_qk_l2_norm ............................... False
175
+ export_quant_cfg ................................ None
176
+ export_real_quant_cfg ........................... None
177
+ export_te_mcore_model ........................... False
178
+ external_cuda_graph ............................. False
179
+ ffn_hidden_size ................................. 18432
180
+ finetune ........................................ False
181
+ finetune_data_split ............................. train
182
+ finetune_hf_dataset ............................. None
183
+ first_last_layers_bf16 .......................... False
184
+ flash_decode .................................... False
185
+ fp16 ............................................ False
186
+ fp16_lm_cross_entropy ........................... False
187
+ fp32_residual_connection ........................ False
188
+ fp8 ............................................. e4m3
189
+ fp8_amax_compute_algo ........................... most_recent
190
+ fp8_amax_history_len ............................ 1
191
+ fp8_interval .................................... 1
192
+ fp8_margin ...................................... 0
193
+ fp8_param_gather ................................ False
194
+ fp8_recipe ...................................... mxfp8
195
+ fp8_wgrad ....................................... True
196
+ fsdp_double_buffer .............................. False
197
+ global_batch_size ............................... 8
198
+ grad_reduce_in_bf16 ............................. False
199
+ gradient_accumulation_fusion .................... True
200
+ gradient_reduce_div_fusion ...................... True
201
+ group_query_attention ........................... False
202
+ head_lr_mult .................................... 1.0
203
+ heterogeneous_layers_config_encoded_json ........ None
204
+ heterogeneous_layers_config_path ................ None
205
+ hidden_dropout .................................. 0.0
206
+ hidden_size ..................................... 7168
207
+ hierarchical_context_parallel_sizes ............. None
208
+ high_priority_stream_groups ..................... []
209
+ hybrid_attention_ratio .......................... 0.0
210
+ hybrid_mlp_ratio ................................ 0.0
211
+ hybrid_override_pattern ......................... None
212
+ hysteresis ...................................... 2
213
+ ict_head_size ................................... None
214
+ ict_load ........................................ None
215
+ img_h ........................................... 224
216
+ img_w ........................................... 224
217
+ indexer_batch_size .............................. 128
218
+ indexer_log_interval ............................ 1000
219
+ inference_batch_times_seqlen_threshold .......... -1
220
+ inference_dynamic_batching ...................... False
221
+ inference_dynamic_batching_buffer_guaranteed_fraction 0.2
222
+ inference_dynamic_batching_buffer_overflow_factor None
223
+ inference_dynamic_batching_buffer_size_gb ....... 40.0
224
+ inference_dynamic_batching_chunk_size ........... 256
225
+ inference_dynamic_batching_max_requests_override None
226
+ inference_dynamic_batching_max_tokens_override .. None
227
+ inference_dynamic_batching_num_cuda_graphs ...... 16
228
+ inference_max_batch_size ........................ 8
229
+ inference_max_seq_length ........................ 2560
230
+ inference_rng_tracker ........................... False
231
+ init_method_std ................................. 0.02
232
+ init_method_xavier_uniform ...................... False
233
+ init_model_with_meta_device ..................... False
234
+ initial_loss_scale .............................. 4294967296
235
+ inprocess_active_world_size ..................... 8
236
+ inprocess_barrier_timeout ....................... 120
237
+ inprocess_completion_timeout .................... 120
238
+ inprocess_empty_cuda_cache ...................... False
239
+ inprocess_granularity ........................... node
240
+ inprocess_hard_timeout .......................... 90
241
+ inprocess_heartbeat_interval .................... 30
242
+ inprocess_heartbeat_timeout ..................... 60
243
+ inprocess_last_call_wait ........................ 1
244
+ inprocess_max_iterations ........................ None
245
+ inprocess_monitor_process_interval .............. 1.0
246
+ inprocess_monitor_thread_interval ............... 1.0
247
+ inprocess_progress_watchdog_interval ............ 1.0
248
+ inprocess_restart ............................... False
249
+ inprocess_soft_timeout .......................... 60
250
+ inprocess_termination_grace_time ................ 1
251
+ is_hybrid_model ................................. False
252
+ iter_per_epoch .................................. 1250
253
+ iterations_to_skip .............................. []
254
+ keep_fp8_transpose_cache_when_using_custom_fsdp . False
255
+ kitchen_config_file ............................. None
256
+ kitchen_recipe_number ........................... None
257
+ kv_channels ..................................... 64
258
+ kv_lora_rank .................................... 512
259
+ lazy_mpu_init ................................... None
260
+ load ............................................ None
261
+ load_main_params_from_ckpt ...................... None
262
+ load_model_opt_format ........................... False
263
+ local_rank ...................................... 0
264
+ log_energy ...................................... False
265
+ log_interval .................................... 1
266
+ log_loss_scale_to_tensorboard ................... True
267
+ log_memory_to_tensorboard ....................... False
268
+ log_num_zeros_in_grad ........................... False
269
+ log_params_norm ................................. False
270
+ log_progress .................................... False
271
+ log_straggler ................................... False
272
+ log_throughput .................................. True
273
+ log_timers_to_tensorboard ....................... False
274
+ log_validation_ppl_to_tensorboard ............... False
275
+ log_world_size_to_tensorboard ................... False
276
+ logging_level ................................... None
277
+ loss_scale ...................................... None
278
+ loss_scale_window ............................... 1000
279
+ lr .............................................. 0.0001
280
+ lr_decay_iters .................................. 8
281
+ lr_decay_samples ................................ None
282
+ lr_decay_style .................................. cosine
283
+ lr_warmup_fraction .............................. None
284
+ lr_warmup_init .................................. 0.0
285
+ lr_warmup_iters ................................. 5
286
+ lr_warmup_samples ............................... 0
287
+ lr_wsd_decay_iters .............................. None
288
+ lr_wsd_decay_samples ............................ None
289
+ lr_wsd_decay_style .............................. exponential
290
+ main_grads_dtype ................................ torch.float32
291
+ main_params_dtype ............................... torch.float32
292
+ make_vocab_size_divisible_by .................... 128
293
+ mamba_head_dim .................................. 64
294
+ mamba_num_groups ................................ 8
295
+ mamba_num_heads ................................. None
296
+ mamba_state_dim ................................. 128
297
+ manual_gc ....................................... False
298
+ manual_gc_eval .................................. True
299
+ manual_gc_interval .............................. 0
300
+ mask_factor ..................................... 1.0
301
+ mask_prob ....................................... 0.15
302
+ mask_type ....................................... random
303
+ masked_softmax_fusion ........................... False
304
+ max_position_embeddings ......................... 4096
305
+ max_tokens_to_oom ............................... 12000
306
+ memory_snapshot_path ............................ snapshot.pickle
307
+ merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt
308
+ micro_batch_size ................................ 1
309
+ microbatch_group_size_per_vp_stage .............. None
310
+ mid_level_dataset_surplus ....................... 0.005
311
+ min_loss_scale .................................. 1.0
312
+ min_lr .......................................... 1e-05
313
+ mlp_chunks_for_prefill .......................... 1
314
+ mmap_bin_files .................................. True
315
+ mock_data ....................................... False
316
+ moe_apply_probs_on_input ........................ False
317
+ moe_aux_loss_coeff .............................. 0.0
318
+ moe_capacity_monitor ............................ True
319
+ moe_deepep_num_sms .............................. 20
320
+ moe_echo_enable_random_offloading ............... False
321
+ moe_echo_expert_dispatch_overlap ................ False
322
+ moe_echo_expert_dispatcher_type ................. hybridep
323
+ moe_echo_recompute_expert_dispatch .............. False
324
+ moe_enable_deepep ............................... False
325
+ moe_enable_echo ................................. False
326
+ moe_enable_hybridep ............................. True
327
+ moe_enable_ultraep .............................. True
328
+ moe_expert_capacity_factor ...................... None
329
+ moe_extended_tp ................................. False
330
+ moe_ffn_hidden_size ............................. 2048
331
+ moe_grouped_gemm ................................ True
332
+ moe_hybridep_num_sms ............................ 16
333
+ moe_input_jitter_eps ............................ None
334
+ moe_layer_freq .................................. [0, 1]
335
+ moe_layer_recompute ............................. False
336
+ moe_num_echo_experts ............................ None
337
+ moe_num_redundant_experts_per_rank .............. 48
338
+ moe_pad_expert_input_to_capacity ................ False
339
+ moe_per_layer_logging ........................... False
340
+ moe_permute_fusion .............................. False
341
+ moe_received_token_capacity ..................... 1.17578125
342
+ moe_router_bias_update_rate ..................... 0.001
343
+ moe_router_dtype ................................ fp32
344
+ moe_router_enable_expert_bias ................... True
345
+ moe_router_force_load_balancing ................. False
346
+ moe_router_group_topk ........................... 1
347
+ moe_router_load_balancing_type .................. seq_aux_loss
348
+ moe_router_num_groups ........................... 1
349
+ moe_router_padding_for_fp8 ...................... False
350
+ moe_router_pre_softmax .......................... False
351
+ moe_router_score_function ....................... sigmoid
352
+ moe_router_topk ................................. 8
353
+ moe_router_topk_scaling_factor .................. 2.827
354
+ moe_shared_expert_intermediate_size ............. 2048
355
+ moe_shared_expert_overlap ....................... False
356
+ moe_token_dispatcher_type ....................... flex
357
+ moe_token_drop_policy ........................... probs
358
+ moe_ultraep_max_microbatches .................... None
359
+ moe_ultraep_overlap_weight_sync ................. True
360
+ moe_ultraep_weight_sync_join .................... before_dispatch
361
+ moe_upcycling_granularity ....................... 1
362
+ moe_use_device_initiated_grouped_gemm ........... True
363
+ moe_use_legacy_grouped_gemm ..................... False
364
+ moe_use_upcycling ............................... False
365
+ moe_z_loss_coeff ................................ None
366
+ mrope_section ................................... None
367
+ mscale .......................................... 1.0
368
+ mscale_all_dim .................................. 1.0
369
+ mtp_loss_scaling_factor ......................... 0.1
370
+ mtp_num_layers .................................. None
371
+ multi_latent_attention .......................... True
372
+ nccl_all_reduce_for_prefill ..................... False
373
+ nccl_communicator_config_path ................... None
374
+ nccl_ub ......................................... False
375
+ no_load_optim ................................... None
376
+ no_load_rng ..................................... None
377
+ no_persist_layer_norm ........................... False
378
+ no_rope_freq .................................... None
379
+ no_save_optim ................................... None
380
+ no_save_rng ..................................... None
381
+ non_persistent_ckpt_type ........................ None
382
+ non_persistent_global_ckpt_dir .................. None
383
+ non_persistent_local_ckpt_algo .................. fully_parallel
384
+ non_persistent_local_ckpt_dir ................... None
385
+ non_persistent_save_interval .................... None
386
+ norm_epsilon .................................... 1e-06
387
+ normalization ................................... RMSNorm
388
+ num_attention_heads ............................. 64
389
+ num_channels .................................... 3
390
+ num_classes ..................................... 1000
391
+ num_dataset_builder_threads ..................... 1
392
+ num_distributed_optimizer_instances ............. 1
393
+ num_experts ..................................... 384
394
+ num_layers ...................................... 2
395
+ num_layers_at_end_in_bf16 ....................... 1
396
+ num_layers_at_start_in_bf16 ..................... 1
397
+ num_layers_per_virtual_pipeline_stage ........... None
398
+ num_query_groups ................................ 1
399
+ num_virtual_stages_per_pipeline_rank ............ None
400
+ num_workers ..................................... 2
401
+ object_storage_cache_path ....................... None
402
+ one_logger_async ................................ False
403
+ one_logger_project .............................. megatron-lm
404
+ one_logger_run_name ............................. None
405
+ onnx_safe ....................................... None
406
+ openai_gelu ..................................... False
407
+ optimizer ....................................... adam
408
+ optimizer_cpu_offload ........................... False
409
+ optimizer_offload_fraction ...................... 1.0
410
+ output_bert_embeddings .......................... False
411
+ overlap_cpu_optimizer_d2h_h2d ................... False
412
+ overlap_grad_reduce ............................. True
413
+ overlap_moe_expert_parallel_comm ................ False
414
+ overlap_p2p_comm ................................ False
415
+ overlap_p2p_comm_warmup_flush ................... False
416
+ overlap_param_gather ............................ True
417
+ overlap_param_gather_with_optimizer_step ........ False
418
+ override_opt_param_scheduler .................... False
419
+ params_dtype .................................... torch.bfloat16
420
+ patch_dim ....................................... 16
421
+ per_split_data_args_path ........................ None
422
+ perform_initialization .......................... True
423
+ pin_cpu_grads ................................... True
424
+ pin_cpu_params .................................. True
425
+ pipeline_model_parallel_comm_backend ............ None
426
+ pipeline_model_parallel_layout .................. None
427
+ pipeline_model_parallel_size .................... 1
428
+ position_embedding_type ......................... rope
429
+ pretrained_checkpoint ........................... None
430
+ profile ......................................... False
431
+ profile_ranks ................................... [0]
432
+ profile_step_end ................................ 12
433
+ profile_step_start .............................. 10
434
+ q_lora_rank ..................................... 1536
435
+ qk_head_dim ..................................... 128
436
+ qk_l2_norm ...................................... False
437
+ qk_layernorm .................................... True
438
+ qk_pos_emb_head_dim ............................. 64
439
+ query_in_block_prob ............................. 0.1
440
+ rampup_batch_size ............................... None
441
+ rank ............................................ 0
442
+ recompute_granularity ........................... None
443
+ recompute_method ................................ None
444
+ recompute_modules ............................... None
445
+ recompute_num_layers ............................ None
446
+ record_memory_history ........................... False
447
+ relative_attention_max_distance ................. 128
448
+ relative_attention_num_buckets .................. 32
449
+ replication ..................................... False
450
+ replication_factor .............................. 2
451
+ replication_jump ................................ None
452
+ rerun_mode ...................................... validate_results
453
+ reset_attention_mask ............................ False
454
+ reset_position_ids .............................. False
455
+ result_rejected_tracker_filename ................ None
456
+ retriever_report_topk_accuracies ................ []
457
+ retriever_score_scaling ......................... False
458
+ retriever_seq_length ............................ 256
459
+ retro_add_retriever ............................. False
460
+ retro_attention_gate ............................ 1
461
+ retro_cyclic_train_iters ........................ None
462
+ retro_encoder_attention_dropout ................. 0.1
463
+ retro_encoder_hidden_dropout .................... 0.1
464
+ retro_encoder_layers ............................ 2
465
+ retro_num_neighbors ............................. 2
466
+ retro_num_retrieved_chunks ...................... 2
467
+ retro_project_dir ............................... None
468
+ retro_verify_neighbor_count ..................... True
469
+ reuse_grad_buf_for_mxfp8_param_ag ............... False
470
+ rope_scaling_factor ............................. 8.0
471
+ rope_type ....................................... None
472
+ rotary_base ..................................... 50000
473
+ rotary_interleaved .............................. False
474
+ rotary_percent .................................. 1.0
475
+ rotary_scaling_factor ........................... 32.0
476
+ rotary_seq_len_interpolation_factor ............. None
477
+ run_workload_inspector_server ................... False
478
+ sample_rate ..................................... 1.0
479
+ save ............................................ None
480
+ save_interval ................................... None
481
+ save_retain_interval ............................ None
482
+ scatter_gather_tensors_in_pipeline .............. True
483
+ seed ............................................ 1234
484
+ seq_length ...................................... 4096
485
+ sequence_parallel ............................... False
486
+ sft ............................................. False
487
+ sft_tokenizer_prompt_format ..................... nemotron-h-aligned
488
+ sgd_momentum .................................... 0.9
489
+ short_seq_prob .................................. 0.1
490
+ skip_train ...................................... False
491
+ skipped_train_samples ........................... 0
492
+ spec ............................................ None
493
+ split ........................................... 100,0,0
494
+ squared_relu .................................... False
495
+ start_weight_decay .............................. 0.1
496
+ straggler_ctrlr_port ............................ 65535
497
+ straggler_minmax_count .......................... 1
498
+ suggested_communication_unit_size ............... None
499
+ swiglu .......................................... True
500
+ swin_backbone_type .............................. tiny
501
+ symmetric_ar_type ............................... None
502
+ te_rng_tracker .................................. True
503
+ tensor_model_parallel_size ...................... 1
504
+ tensorboard_dir ................................. None
505
+ tensorboard_log_interval ........................ 1
506
+ tensorboard_queue_size .......................... 1000
507
+ test_data_path .................................. None
508
+ test_mode ....................................... False
509
+ tiktoken_num_special_tokens ..................... 1000
510
+ tiktoken_pattern ................................ None
511
+ tiktoken_special_tokens ......................... None
512
+ timing_log_level ................................ 0
513
+ timing_log_option ............................... minmax
514
+ titles_data_path ................................ None
515
+ tokenizer_model ................................. None
516
+ tokenizer_type .................................. GPT2BPETokenizer
517
+ torch_fsdp2_reshard_after_forward ............... True
518
+ tp_comm_bootstrap_backend ....................... nccl
519
+ tp_comm_bulk_dgrad .............................. True
520
+ tp_comm_bulk_wgrad .............................. True
521
+ tp_comm_overlap ................................. False
522
+ tp_comm_overlap_ag .............................. True
523
+ tp_comm_overlap_cfg ............................. None
524
+ tp_comm_overlap_rs .............................. True
525
+ tp_comm_overlap_rs_dgrad ........................ False
526
+ tp_comm_split_ag ................................ True
527
+ tp_comm_split_rs ................................ True
528
+ train_data_path ................................. None
529
+ train_iters ..................................... 12
530
+ train_samples ................................... None
531
+ train_sync_interval ............................. None
532
+ transformer_impl ................................ transformer_engine
533
+ transformer_pipeline_model_parallel_size ........ 1
534
+ untie_embeddings_and_output_weights ............. True
535
+ use_checkpoint_args ............................. False
536
+ use_checkpoint_opt_param_scheduler .............. False
537
+ use_cpu_initialization .......................... None
538
+ use_custom_fsdp ................................. False
539
+ use_dist_ckpt ................................... True
540
+ use_dist_ckpt_deprecated ........................ False
541
+ use_distributed_optimizer ....................... True
542
+ use_flash_attn .................................. False
543
+ use_legacy_models ............................... False
544
+ use_mp_args_from_checkpoint_args ................ False
545
+ use_one_sent_docs ............................... False
546
+ use_persistent_ckpt_worker ...................... False
547
+ use_precision_aware_optimizer ................... False
548
+ use_pytorch_profiler ............................ False
549
+ use_ring_exchange_p2p ........................... False
550
+ use_rope_scaling ................................ False
551
+ use_rotary_position_embeddings .................. False
552
+ use_sharp ....................................... False
553
+ use_tokenizer_model_from_checkpoint_args ........ True
554
+ use_torch_fsdp2 ................................. False
555
+ use_torch_optimizer_for_cpu_offload ............. False
556
+ use_tp_pp_dp_mapping ............................ False
557
+ v_head_dim ...................................... 128
558
+ valid_data_path ................................. None
559
+ variable_seq_lengths ............................ False
560
+ virtual_pipeline_model_parallel_size ............ None
561
+ vision_backbone_type ............................ vit
562
+ vision_pretraining .............................. False
563
+ vision_pretraining_type ......................... classify
564
+ vocab_extra_ids ................................. 0
565
+ vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json
566
+ vocab_size ...................................... None
567
+ wandb_exp_name ..................................
568
+ wandb_project ...................................
569
+ wandb_save_dir ..................................
570
+ weight_decay .................................... 0.1
571
+ weight_decay_incr_style ......................... constant
572
+ wgrad_deferral_limit ............................ 0
573
+ world_size ...................................... 8
574
+ yaml_cfg ........................................ None
575
+ -------------------- end of arguments ---------------------
576
+ INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1
577
+ > building GPT2BPETokenizer tokenizer ...
578
+ > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304)
579
+ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS
580
+ > initializing torch distributed ...
581
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
582
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
583
+ [Gloo] Rank [Gloo] Rank 2 is connected to 7 peer ranks. 1Expected number of connected peer ranks is : is connected to 77
584
+ peer ranks. Expected number of connected peer ranks is : 7
585
+ [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
586
+ [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
587
+ [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
588
+ [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
589
+ [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
590
+ [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
591
+ [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
592
+ [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
593
+ [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
594
+ [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 25467 is connected to is connected to is connected to is connected to is connected to 77777 peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77777
595
+
596
+
597
+
598
+
599
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
600
+ > initialized tensor model parallel with size 1
601
+ > initialized pipeline model parallel with size 1
602
+ > setting random seeds to 1234 ...
603
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
604
+ [Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00
605
+
606
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
607
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
608
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
609
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
610
+ [rank6]:[W921 07:49:55.489030863 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
611
+ [rank3]:[W921 07:49:55.489174158 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
612
+ [rank2]:[W921 07:49:55.489180926 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
613
+ [rank1]:[W921 07:49:55.489186964 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
614
+ [rank5]:[W921 07:49:55.489214517 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
615
+ [rank7]:[W921 07:49:55.489255123 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
616
+ [rank4]:[W921 07:49:55.489344079 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
617
+ > compiling dataset index builder ...
618
+ make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets'
619
+ make: Nothing to be done for 'default'.
620
+ make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets'
621
+ >>> done with dataset index builder. Compilation time: 0.036 seconds
622
+ WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations.
623
+ > compiling and loading fused kernels ...
624
+ [rank0]:[W921 07:49:55.651074971 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
625
+ >>> done with compiling and loading fused kernels. Compilation time: 4.296 seconds
626
+ time to initialize megatron (seconds): 7.944
627
+ [after megatron is initialized] datetime: 2026-09-21 07:50:01
628
+ building GPT model ...
629
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
630
+ warnings.warn(
631
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
632
+ warnings.warn(
633
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
634
+ warnings.warn(
635
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
636
+ warnings.warn(
637
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
638
+ warnings.warn(
639
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
640
+ warnings.warn(
641
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
642
+ warnings.warn(
643
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
644
+ warnings.warn(
645
+ [INFO] Use MNNVL fabric: False
646
+ [WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8.
647
+ NVSHMEM v3.7.2
648
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
649
+
650
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
651
+
652
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
653
+
654
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
655
+
656
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
657
+
658
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
659
+
660
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
661
+
662
+ /dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
663
+
664
+ UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive).
665
+ > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 5594455040
666
+ INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False)
667
+ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 9
668
+ Params for bucket 1 (360579072 elements, 360579072 padded size):
669
+ module.output_layer.weight
670
+ Params for bucket 2 (44047360 elements, 44047360 padded size):
671
+ module.decoder.final_layernorm.weight
672
+ module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight
673
+ module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight
674
+ Params for bucket 3 (45163520 elements, 45163520 padded size):
675
+ module.decoder.layers.1.mlp.router.weight
676
+ module.decoder.layers.1.pre_mlp_layernorm.weight
677
+ module.decoder.layers.1.self_attention.linear_kv_up_proj.weight
678
+ module.decoder.layers.1.self_attention.linear_q_up_proj.layer_norm_weight
679
+ module.decoder.layers.1.self_attention.linear_q_down_proj.weight
680
+ module.decoder.layers.1.self_attention.linear_kv_down_proj.weight
681
+ module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight
682
+ module.decoder.layers.1.self_attention.linear_q_up_proj.weight
683
+ Params for bucket 4 (58720256 elements, 58720256 padded size):
684
+ module.decoder.layers.1.self_attention.linear_proj.weight
685
+ Params for bucket 5 (132127744 elements, 132127744 padded size):
686
+ module.decoder.layers.1.input_layernorm.weight
687
+ module.decoder.layers.0.mlp.linear_fc2.weight
688
+ Params for bucket 6 (264241152 elements, 264241152 padded size):
689
+ module.decoder.layers.0.mlp.linear_fc1.weight
690
+ Params for bucket 7 (42411008 elements, 42411008 padded size):
691
+ module.decoder.layers.0.self_attention.linear_kv_up_proj.weight
692
+ module.decoder.layers.0.self_attention.linear_kv_down_proj.weight
693
+ module.decoder.layers.0.self_attention.linear_q_down_proj.weight
694
+ module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight
695
+ module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight
696
+ module.decoder.layers.0.self_attention.linear_q_up_proj.weight
697
+ module.decoder.layers.0.self_attention.linear_q_up_proj.layer_norm_weight
698
+ Params for bucket 8 (58720256 elements, 58720256 padded size):
699
+ module.decoder.layers.0.self_attention.linear_proj.weight
700
+ Params for bucket 9 (360586240 elements, 360586240 padded size):
701
+ module.decoder.layers.0.input_layernorm.weight
702
+ module.embedding.word_embeddings.weight
703
+ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40
704
+ Params for bucket 1 (44040192 elements, 44040192 padded size):
705
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight47
706
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight45
707
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight46
708
+ Params for bucket 2 (44040192 elements, 44040192 padded size):
709
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight44
710
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight42
711
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight43
712
+ Params for bucket 3 (44040192 elements, 44040192 padded size):
713
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight41
714
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight39
715
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight40
716
+ Params for bucket 4 (44040192 elements, 44040192 padded size):
717
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight38
718
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight37
719
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight36
720
+ Params for bucket 5 (44040192 elements, 44040192 padded size):
721
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight35
722
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight34
723
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight33
724
+ Params for bucket 6 (44040192 elements, 44040192 padded size):
725
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight32
726
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight31
727
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight30
728
+ Params for bucket 7 (44040192 elements, 44040192 padded size):
729
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight29
730
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight28
731
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight27
732
+ Params for bucket 8 (44040192 elements, 44040192 padded size):
733
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight26
734
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight25
735
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight24
736
+ Params for bucket 9 (44040192 elements, 44040192 padded size):
737
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight23
738
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight21
739
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight22
740
+ Params for bucket 10 (44040192 elements, 44040192 padded size):
741
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight20
742
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight18
743
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight19
744
+ Params for bucket 11 (44040192 elements, 44040192 padded size):
745
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight17
746
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight15
747
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight16
748
+ Params for bucket 12 (44040192 elements, 44040192 padded size):
749
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight14
750
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight12
751
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight13
752
+ Params for bucket 13 (44040192 elements, 44040192 padded size):
753
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight11
754
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight9
755
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight10
756
+ Params for bucket 14 (44040192 elements, 44040192 padded size):
757
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight8
758
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight6
759
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight7
760
+ Params for bucket 15 (44040192 elements, 44040192 padded size):
761
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight5
762
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight3
763
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight4
764
+ Params for bucket 16 (44040192 elements, 44040192 padded size):
765
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight2
766
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight0
767
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight1
768
+ Params for bucket 17 (58720256 elements, 58720256 padded size):
769
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight47
770
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight46
771
+ Params for bucket 18 (58720256 elements, 58720256 padded size):
772
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight45
773
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight44
774
+ Params for bucket 19 (58720256 elements, 58720256 padded size):
775
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight43
776
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight42
777
+ Params for bucket 20 (58720256 elements, 58720256 padded size):
778
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight41
779
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight40
780
+ Params for bucket 21 (58720256 elements, 58720256 padded size):
781
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight39
782
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight38
783
+ Params for bucket 22 (58720256 elements, 58720256 padded size):
784
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight37
785
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight36
786
+ Params for bucket 23 (58720256 elements, 58720256 padded size):
787
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight35
788
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight34
789
+ Params for bucket 24 (58720256 elements, 58720256 padded size):
790
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight33
791
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight32
792
+ Params for bucket 25 (58720256 elements, 58720256 padded size):
793
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight31
794
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight30
795
+ Params for bucket 26 (58720256 elements, 58720256 padded size):
796
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight29
797
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight28
798
+ Params for bucket 27 (58720256 elements, 58720256 padded size):
799
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight27
800
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight26
801
+ Params for bucket 28 (58720256 elements, 58720256 padded size):
802
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight25
803
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight24
804
+ Params for bucket 29 (58720256 elements, 58720256 padded size):
805
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight23
806
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight22
807
+ Params for bucket 30 (58720256 elements, 58720256 padded size):
808
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight21
809
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight20
810
+ Params for bucket 31 (58720256 elements, 58720256 padded size):
811
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight19
812
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight18
813
+ Params for bucket 32 (58720256 elements, 58720256 padded size):
814
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight17
815
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight16
816
+ Params for bucket 33 (58720256 elements, 58720256 padded size):
817
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight15
818
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight14
819
+ Params for bucket 34 (58720256 elements, 58720256 padded size):
820
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight13
821
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight12
822
+ Params for bucket 35 (58720256 elements, 58720256 padded size):
823
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight11
824
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight10
825
+ Params for bucket 36 (58720256 elements, 58720256 padded size):
826
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight9
827
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight8
828
+ Params for bucket 37 (58720256 elements, 58720256 padded size):
829
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight7
830
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight6
831
+ Params for bucket 38 (58720256 elements, 58720256 padded size):
832
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight5
833
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight4
834
+ Params for bucket 39 (58720256 elements, 58720256 padded size):
835
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight3
836
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight2
837
+ Params for bucket 40 (58720256 elements, 58720256 padded size):
838
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight1
839
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight0
840
+ INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=<megatron.core.timers.Timers object at 0x76f5d7cfea20>, config_logger_dir='')
841
+ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine
842
+ [after model, optimizer, and learning rate scheduler are built] datetime: 2026-09-21 07:50:05
843
+ > building train, validation, and test datasets ...
844
+ > datasets target sizes (minimum size):
845
+ train: 96
846
+ validation: 0
847
+ test: 0
848
+ INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None]
849
+ > building train, validation, and test datasets for GPT ...
850
+ INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=<megatron.training.tokenizer.tokenizer._GPT2BPETokenizer object at 0x76f5d7df6ae0>, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None)
851
+ INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx
852
+ INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths
853
+ INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers
854
+ INFO:megatron.core.datasets.indexed_dataset: Extract the document indices
855
+ INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029
856
+ INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350
857
+ INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices
858
+ INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy
859
+ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy
860
+ INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy
861
+ INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073
862
+ > finished creating GPT datasets ...
863
+ [after dataloaders are built] datetime: 2026-09-21 07:50:05
864
+ done with setup ...
865
+ (min, max) time across ranks (ms):
866
+ model-and-optimizer-setup ......................: (3319.76, 3323.88)
867
+ train/valid/test-data-iterators-setup ..........: (71.88, 92.46)
868
+ training ...
869
+ Overwriting rerun_state_machine.current_iteration from -1 to 0...
870
+ [before the start of training step] datetime: 2026-09-21 07:50:05
871
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
872
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
873
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
874
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
875
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
876
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
877
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
878
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
879
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
880
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
881
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
882
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
883
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
884
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
885
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
886
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
887
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
888
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
889
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
890
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
891
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
892
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
893
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
894
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
895
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
896
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
897
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
898
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
899
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
900
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
901
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
902
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
903
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
904
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
905
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
906
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
907
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
908
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
909
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
910
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
911
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
912
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
913
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
914
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
915
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
916
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
917
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
918
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
919
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
920
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
921
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
922
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
923
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
924
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
925
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
926
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
927
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
928
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
929
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
930
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
931
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
932
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
933
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
934
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
935
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
936
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
937
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
938
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
939
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
940
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
941
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
942
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
943
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
944
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
945
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
946
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
947
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
948
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
949
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
950
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
951
+ [2026-09-21 07:50:34] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 29238.8 | throughput per GPU (TFLOP/s/GPU): 1.2 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.227924E+01 | loss scale: 1.0 | grad norm: 12.617 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36224 | min safe capacity: 1.1055 |Number of parameters in transformer block in billions: 17.55
952
+
953
+ Number of parameters in embedding layers in billions: 0.72
954
+ Total number of parameters in billions: 18.28
955
+ Number of parameters in most loaded shard in billions: 18.2753
956
+ Theoretical memory footprints: weight and optimizer=130715.23 MB
957
+ [Rank 0] (after 1 iterations) memory (MB) | allocated: 49136.10400390625 | max allocated: 49136.1044921875 | reserved: 54394.0 | max reserved: 54394.0
958
+ /workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
959
+ warnings.warn(
960
+ /workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
961
+ warnings.warn(
962
+ /workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
963
+ warnings.warn(
964
+ /workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
965
+ warnings.warn(
966
+ /workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
967
+ warnings.warn(
968
+ /workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
969
+ warnings.warn(
970
+ /workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
971
+ warnings.warn(
972
+ /workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
973
+ warnings.warn(
974
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
975
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
976
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
977
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
978
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
979
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
980
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
981
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
982
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
983
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
984
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
985
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
986
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
987
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
988
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
989
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
990
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
991
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
992
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
993
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
994
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
995
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
996
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
997
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
998
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
999
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1000
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1001
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1002
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1003
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1004
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1005
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1006
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1007
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1008
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1009
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1010
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1011
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1012
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1013
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1014
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1015
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1016
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1017
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1018
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1019
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1020
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1021
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1022
+ [2026-09-21 07:50:34] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 164.0 | throughput per GPU (TFLOP/s/GPU): 215.7 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.227088E+01 | loss scale: 1.0 | grad norm: 12.534 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36224 | min safe capacity: 1.1055 |
1023
+ [2026-09-21 07:50:35] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 111.7 | throughput per GPU (TFLOP/s/GPU): 316.7 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.088328E+01 | loss scale: 1.0 | grad norm: 11.023 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36224 | min safe capacity: 1.1055 |
1024
+ Capture CUDA graph for training!!!
1025
+ CUDA graph capture done!!!
1026
+ [2026-09-21 07:50:36] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 1248.8 | throughput per GPU (TFLOP/s/GPU): 28.3 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.448181E+01 | loss scale: 1.0 | grad norm: 72.911 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36608 | min safe capacity: 1.1172 |
1027
+ [2026-09-21 07:50:36] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 110.2 | throughput per GPU (TFLOP/s/GPU): 320.9 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.808459E+01 | loss scale: 1.0 | grad norm: 108.268 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
1028
+ [2026-09-21 07:50:36] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 112.2 | throughput per GPU (TFLOP/s/GPU): 315.3 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.491727E+01 | loss scale: 1.0 | grad norm: 83.473 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
1029
+ [2026-09-21 07:50:36] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 114.3 | throughput per GPU (TFLOP/s/GPU): 309.5 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.143662E+01 | loss scale: 1.0 | grad norm: 26.627 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
1030
+ [2026-09-21 07:50:36] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 108.9 | throughput per GPU (TFLOP/s/GPU): 324.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.166943E+01 | loss scale: 1.0 | grad norm: 12.800 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
1031
+ [2026-09-21 07:50:36] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 109.0 | throughput per GPU (TFLOP/s/GPU): 324.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.131635E+01 | loss scale: 1.0 | grad norm: 10.441 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
1032
+ [2026-09-21 07:50:36] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 109.1 | throughput per GPU (TFLOP/s/GPU): 324.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.096943E+01 | loss scale: 1.0 | grad norm: 9.147 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
1033
+ [2026-09-21 07:50:37] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 108.6 | throughput per GPU (TFLOP/s/GPU): 325.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.073911E+01 | loss scale: 1.0 | grad norm: 9.755 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
1034
+ [2026-09-21 07:50:37] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 108.5 | throughput per GPU (TFLOP/s/GPU): 326.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.048286E+01 | loss scale: 1.0 | grad norm: 8.843 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
1035
+ [after training is done] datetime: 2026-09-21 07:50:37
1036
+ W0921 07:50:37.341000 527457 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers
1037
+ W0921 07:50:37.342000 527457 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 527563 closing signal SIGTERM
1038
+ W0921 07:50:37.342000 527457 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 527564 closing signal SIGTERM
1039
+ W0921 07:50:37.343000 527457 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 527565 closing signal SIGTERM
1040
+ W0921 07:50:37.344000 527457 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 527566 closing signal SIGTERM
1041
+ W0921 07:50:37.345000 527457 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 527567 closing signal SIGTERM
1042
+ W0921 07:50:37.346000 527457 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 527568 closing signal SIGTERM
1043
+ W0921 07:50:37.348000 527457 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 527569 closing signal SIGTERM
1044
+ W0921 07:50:37.349000 527457 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 527570 closing signal SIGTERM
1045
+ Traceback (most recent call last):
1046
+ File "/usr/local/bin/torchrun", line 8, in <module>
1047
+ sys.exit(main())
1048
+ ^^^^^^
1049
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
1050
+ return f(*args, **kwargs)
1051
+ ^^^^^^^^^^^^^^^^^^
1052
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main
1053
+ run(args)
1054
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run
1055
+ elastic_launch(
1056
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__
1057
+ return launch_agent(self._config, self._entrypoint, list(args))
1058
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1059
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent
1060
+ result = agent.run()
1061
+ ^^^^^^^^^^^
1062
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
1063
+ result = f(*args, **kwargs)
1064
+ ^^^^^^^^^^^^^^^^^^
1065
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run
1066
+ result = self._invoke_run(role)
1067
+ ^^^^^^^^^^^^^^^^^^^^^^
1068
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run
1069
+ time.sleep(monitor_interval)
1070
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler
1071
+ raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
1072
+ torch.distributed.elastic.multiprocessing.api.SignalException: Process 527457 got signal: 15
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/results.tsv ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
2
+ KIMI_K2_L2 ultraep matched 48 4096 8 1.17578125 none 1 OK 110.06 108.20 90375 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep1
3
+ KIMI_K2_L2 ultraep matched 48 4096 8 1.17578125 none 2 OK 110.08 108.20 90375 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep2
4
+ KIMI_K2_L2 ultraep matched 48 4096 8 1.17578125 none 3 OK 110.08 108.20 90375 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep3