diff --git a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/manifest.txt b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/manifest.txt new file mode 100644 index 0000000000000000000000000000000000000000..7aad9af25a348e363947c328bf5ef3b63cc02761 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/manifest.txt @@ -0,0 +1,19 @@ +timestamp: 2026-09-21T07:37:08+00:00 +host: sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0 +gpu: NVIDIA B200 x8 +driver: 580.95.05 +study_scripts_branch: granularity-ultraep +study_scripts_commit: 05dd2cb +campaign: ultraep +backend: ultraep +phase: tierbprobe +gbs: 8 +seq: 4096 +configs: G1 G2 G3 G4 +capacity: 1.25 +repeats: 3 +nsys: 1 +megatron_repo: /workspace/Megatron-UltraEP +megatron_branch_required: ultraep-token-drop-analysis +megatron_branch_actual: ultraep-token-drop-analysis +megatron_commit: eaf829e7 diff --git a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/pip_freeze.txt b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/pip_freeze.txt new file mode 100644 index 0000000000000000000000000000000000000000..4a5dfe1c1d96cdd9f602f2679d5f5a722d5c2afb --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/pip_freeze.txt @@ -0,0 +1,317 @@ +absl-py==2.3.0 +aiohappyeyeballs @ file:///rapids/aiohappyeyeballs-2.6.1-py3-none-any.whl#sha256=f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 +aiohttp @ file:///rapids/aiohttp-3.12.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=94f98e0e5a49f89b252e115844f756c04fc8050f38252a32a3dd994ce8121f10 +aiosignal @ file:///rapids/aiosignal-1.3.2-py2.py3-none-any.whl#sha256=45cde58e409a301715980c2b01d0c28bdde3770d8290b5eb2173759d9acb31a5 +annotated-types==0.7.0 +anyio==4.9.0 +apex @ file:///opt/pytorch/apex/dist/apex-0.1-cp312-cp312-linux_x86_64.whl#sha256=a1508e4a386362e9c8f46c92ccada9110e421212b964c694235d0fd5a224b170 +argon2-cffi==25.1.0 +argon2-cffi-bindings==21.2.0 +arrow==1.3.0 +asciitree @ file:///rapids/asciitree-0.3.3-py3-none-any.whl#sha256=3e507ccfaeb417823158091fdb9d26b0205f6a59ba680b479a71db65c73d01d4 +asttokens==3.0.0 +astunparse==1.6.3 +async-lru==2.0.5 +attrs==25.3.0 +audioread==3.0.1 +babel==2.17.0 +beautifulsoup4==4.13.4 +black==25.1.0 +bleach==6.2.0 +blis==0.7.11 +cachetools @ file:///rapids/cachetools-6.0.0-py3-none-any.whl#sha256=82e73ba88f7b30228b5507dce1a1f878498fc669d972aef2dde4f3a3c24f103e +catalogue==2.0.10 +certifi==2025.4.26 +cffi==1.17.1 +charset-normalizer==3.4.2 +click==8.5.0 +cloudpathlib==0.21.1 +cloudpickle @ file:///rapids/cloudpickle-3.1.1-py3-none-any.whl#sha256=c8c5a44295039331ee9dad40ba100a9c7297b6f988e50e87ccdf3765a668350e +cmake==3.31.6 +comm==0.2.2 +confection==0.1.5 +contourpy==1.3.2 +cuda-bindings @ file:///rapids/cuda_bindings-12.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=ff0e28d1e34758654b9c961e1f55e4786e49aee6a4dbceaf3cc24c46c672df7e +cuda-python @ file:///rapids/cuda_python-12.9.0-py3-none-any.whl#sha256=926acba49b2c0a0374c61b7c98f337c085199cf51cdfe4d6423c4129c20547a7 +cudf @ file:///rapids/cudf-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b779a8e8c0e8577b9b560895958496da211bfac49cdc4f11a49e8d18cb82e348 +cudf-polars @ file:///rapids/cudf_polars-25.4.0-py3-none-any.whl#sha256=227b6c5147e93ea86282c8f7219118f916769fa2ca0cdd931e97a148aa66b869 +cugraph @ file:///rapids/cugraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b837c2dc2f7b235c702e4f8df64479b8fcbb342f848516d5faf0cdc4b5f31f29 +cugraph-service-client @ file:///rapids/cugraph_service_client-25.4.0-py3-none-any.whl#sha256=7e568c3e41154b9752f5173555ffcc282ac52d72343feeb79653b2e2b1232470 +cugraph-service-server @ file:///rapids/cugraph_service_server-25.4.0-py3-none-any.whl#sha256=a2830dcffafcccb16e12ebf85b98f8f247869d38c01ab4dd8664a1e7439d31be +cuml @ file:///rapids/cuml-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=1e90a464a18b094e24797f1676688c5d7696c4cf97c934504214a53d5aa09aa5 +cupy-cuda12x @ file:///rapids/cupy_cuda12x-13.3.0-cp312-cp312-linux_x86_64.whl#sha256=6a2dfa0aa1e8189a10d1534276c08f1d2f73b747fd28bfbe7b9f4ef66d9af73f +cuvs @ file:///rapids/cuvs-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b12f6b9f80e053a3b38542c2947d2516e468c02cad54a13d8b1a4a61d5aa7914 +cycler==0.12.1 +cymem==2.0.11 +Cython==3.1.1 +dask @ file:///rapids/dask-2025.2.0-py3-none-any.whl#sha256=f0fdeef6ceb0a06569d456c9e704f220f7f54e80f3a6ea42ab98cea6bc642b6e +dask-cuda @ file:///rapids/dask_cuda-25.4.0-py3-none-any.whl#sha256=442d87da0a95d00ca2a262b64c76e785a32ad905c82000fa2227932f81390e19 +dask-cudf @ file:///rapids/dask_cudf-25.4.0-py3-none-any.whl#sha256=5e7cea0b26daea17da6e0bb457435e24460f76ba740358a3a18e239d6015c95c +datasets==5.0.1 +debugpy==1.8.14 +decorator==5.2.1 +-e git+https://github.com/deepseek-ai/DeepEP.git@34152ae28f80bcc3ee38d7a12cb2ad87cfd4ea72#egg=deep_ep +defusedxml==0.7.1 +dill==0.4.1 +distributed @ file:///rapids/distributed-2025.2.0-py3-none-any.whl#sha256=368462084be47092d0a835f6c64c9b852da64dc88302b76d5f93bd5131228c67 +distributed-ucxx @ file:///rapids/distributed_ucxx-0.43.0-py3-none-any.whl#sha256=94f60d0ccb0c94ba8dbfa1eaf455874d18aaab17e2ae218c38590e9fdc2d721d +dm-tree==0.1.9 +einops==0.8.1 +execnet==2.1.1 +executing==2.2.0 +expecttest==0.3.0 +fasteners @ file:///rapids/fasteners-0.19-py3-none-any.whl#sha256=758819cb5d94cdedf4e836988b74de396ceacb8e2794d21f82d131fd9ee77237 +fastjsonschema==2.21.1 +fastrlock @ file:///rapids/fastrlock-0.8.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl#sha256=5f13ec08f1adb1aa916c384b05ecb7dbebb8df9ea81abd045f60941c6283a670 +filelock==3.18.0 +flash_attn @ file:///opt/pytorch/flash_attn-2.7.4.post1-cp312-cp312-linux_x86_64.whl#sha256=a10455d6c56b387785b7413fd0bbcb392b42b3d4f4b88fda5d86cc8a9ce9182a +fonttools==4.58.1 +fqdn==1.5.1 +frozenlist @ file:///rapids/frozenlist-1.6.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=4e1be9111cb6756868ac242b3c2bd1f09d9aea09846e4f5c23715e7afb647103 +fsspec==2025.5.1 +gast==0.6.0 +grpcio==1.62.1 +h11==0.16.0 +hf-xet==1.6.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.32.0 +hypothesis==6.130.8 +idna==3.10 +importlib_metadata==8.7.0 +iniconfig==2.1.0 +intel-openmp==2021.4.0 +ipykernel==6.29.5 +ipython==9.3.0 +ipython_pygments_lexers==1.1.1 +isoduration==20.11.0 +isort==6.0.1 +jedi==0.19.2 +Jinja2==3.1.6 +joblib==1.5.1 +json5==0.12.0 +jsonpointer==3.0.0 +jsonschema==4.24.0 +jsonschema-specifications==2025.4.1 +jupyter-events==0.12.0 +jupyter-lsp==2.2.5 +jupyter_client==8.6.3 +jupyter_core==5.8.1 +jupyter_server==2.16.0 +jupyter_server_terminals==0.5.3 +jupyterlab==4.4.3 +jupyterlab_code_formatter==3.0.2 +jupyterlab_pygments==0.3.0 +jupyterlab_server==2.27.3 +jupyterlab_tensorboard_pro @ file:///builder/jupyterlab_tensorboard_pro-4.0.0-py2.py3-none-any.whl#sha256=6bfcb5ced3358eb72150f4b2ab91312faef2f0ca14d845e8379d8fa52df6902e +jupytext==1.17.2 +kiwisolver==1.4.8 +kvikio @ file:///rapids/kvikio-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=8b40fd8ea9e4a2974c04b50b7fc26b06094ac807f9a233281d0bafbaa71b1d85 +langcodes==3.5.0 +language_data==1.3.0 +lazy_loader==0.4 +libcudf @ file:///rapids/libcudf-25.4.0-py3-none-linux_x86_64.whl#sha256=238320b4958c1737bba2a609673e2539a78216b811b5498a68e00395d9b021a4 +libcugraph @ file:///rapids/libcugraph-25.4.0-py3-none-linux_x86_64.whl#sha256=29085e901b2f7569ff2653bb1355fe6fad20f2fcaa585f5649da9867d8df766d +libcuml @ file:///rapids/libcuml-25.4.0-py3-none-linux_x86_64.whl#sha256=8e2e444c72b0e995e15a48da37ff12fb410a47a2baa69ba86ccc9b1a3db91e8c +libcuvs @ file:///rapids/libcuvs-25.4.0-py3-none-linux_x86_64.whl#sha256=e2f8db61c92b769c590dd752f43439ddd62b2f6592a3c337362b6c1ef220c3b9 +libkvikio @ file:///rapids/libkvikio-25.4.0-py3-none-linux_x86_64.whl#sha256=da8a5713b05fc478bf40529a3e0fb61a2d605bb5a70ca4d461f96a6005dbe8b7 +libraft @ file:///rapids/libraft-25.4.0-py3-none-linux_x86_64.whl#sha256=b921dec005aa4a321dd156776a18376feb47c96a5b67eec125c8a8e19edd1e24 +librmm @ file:///rapids/librmm-25.4.0-py3-none-linux_x86_64.whl#sha256=ff951811bda729d4fa1109ab906225ae1f489a6ec2eeab5d1b674cc34bd4422f +librmm-cu12 @ file:///rapids/librmm_cu12-25.4.0-py3-none-any.whl#sha256=29296ef2b0e90679a0db280e86015be12d6ab9f4e62a181aa085f62dbadd06b3 +librosa==0.11.0 +libucxx @ file:///rapids/libucxx-0.43.0-py3-none-linux_x86_64.whl#sha256=f15849d7ac2fb68f0fd8998838a13321ddcfb0c594f7e6cbc401a3bcc4750566 +lightning-thunder @ file:///opt/pytorch/lightning-thunder +lightning-utilities==0.14.3 +lintrunner==0.12.7 +llvmlite==0.42.0 +locket @ file:///rapids/locket-1.0.0-py2.py3-none-any.whl#sha256=b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3 +looseversion==1.3.0 +marisa-trie==1.2.1 +Markdown==3.8 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +matplotlib==3.10.3 +matplotlib-inline==0.1.7 +mdit-py-plugins==0.4.2 +mdurl==0.1.2 +-e git+ssh://git@github.com/sriram-17-17/my-Megatron-MoE-Echo.git@b099fd8494b09e8c08ad34e26abf91fb6c83636d#egg=megatron_core +mistune==3.1.3 +mkl==2021.1.1 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +ml_dtypes==0.6.0 +mock==5.2.0 +-e git+ssh://git@github.com/sriram-17-17/MoonEP.git@d8bc6667fad7177e6e09b637d773fa67f08bae93#egg=moonep +mpmath==1.3.0 +msgpack==1.1.0 +multidict @ file:///rapids/multidict-6.4.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=a920f9cf2abdf6e493c519492d892c362007f113c94da4c239ae88429835bad1 +multiprocess==0.70.19 +murmurhash==1.0.13 +mypy_extensions==1.1.0 +nbclient==0.10.2 +nbconvert==7.16.6 +nbformat==5.10.4 +nest-asyncio==1.6.0 +networkx==3.5 +ninja==1.11.1.4 +notebook==7.4.3 +notebook_shim==0.2.4 +numba==0.59.1 +numba-cuda @ file:///rapids/numba_cuda-0.4.0-py3-none-any.whl#sha256=5b95a5f42068771e37142944d44fe844fa87f1b2f89051461a4cb0af91bcc28a +numcodecs @ file:///rapids/numcodecs-0.13.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=da2230484e6102e5fa3cc1a5dd37ca1f92dfbd183d91662074d6f7574e3e8f53 +numpy==1.26.4 +nvdlfw_inspect @ git+https://github.com/NVIDIA/nvidia-dlfw-inspect.git@6b60eb0e675606fb2fbbfcb12667ecbec75eaf87 +nvfuser==0.2.27a0+9bf5aca +nvidia-cuda-cccl-cu12==12.9.27 +nvidia-cudnn-frontend @ file:///opt/pytorch/lightning-thunder/tmp/cudnn_frontend +nvidia-cutlass-dsl==4.4.2 +nvidia-cutlass-dsl-libs-base==4.4.2 +nvidia-dali-cuda120==1.50.0 +nvidia-ml-py @ file:///rapids/nvidia_ml_py-12.575.51-py3-none-any.whl#sha256=eb8641800d98ce40a22f479873f34b482e214a7e80349c63be51c3919845446e +nvidia-modelopt==0.29.0 +nvidia-modelopt-core==0.29.0 +nvidia-nvcomp-cu12==4.2.0.14 +nvidia-nvimgcodec-cu12==0.5.0.13 +nvidia-nvjpeg-cu12==12.4.0.16 +nvidia-nvjpeg2k-cu12==0.8.1.40 +nvidia-nvshmem-cu12==3.7.2 +nvidia-nvtiff-cu12==0.5.0.67 +nvidia-resiliency-ext==0.4.0 +nvtx @ file:///rapids/nvtx-0.2.11-cp312-cp312-linux_x86_64.whl#sha256=73521e37cba42b4db0b3cd68fc5eb814e55c500a61eeed6edcdf6746315ac53e +nx-cugraph @ file:///rapids/nx_cugraph-25.4.0-py3-none-any.whl#sha256=e625444486cc899a77bd48e7da76a2edd95c0006dd95fbf323745feae7f33111 +onnx @ file:///opt/pytorch/pytorch/third_party/onnx +onnx-ir==1.0.0 +onnxscript==0.7.2 +opt_einsum==3.4.0 +optree==0.16.0 +overrides==7.7.0 +packaging==23.2 +pandas @ file:///rapids/pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319 +pandocfilters==1.5.1 +parso==0.8.4 +partd @ file:///rapids/partd-1.4.2-py3-none-any.whl#sha256=978e4ac767ec4ba5b86c6eaa52e5a2a3bc748a2ca839e8cc798f1cc6ce6efb0f +pathspec==0.12.1 +pexpect==4.9.0 +pillow==11.2.1 +platformdirs==4.3.8 +pluggy==1.6.0 +ply @ file:///rapids/ply-3.11-py2.py3-none-any.whl#sha256=096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce +polars @ file:///rapids/polars-1.25.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=f7fcbb4f476784384ccda48757fca4e8c2e2c5a0a3aef3717aaf56aee4e30e09 +polygraphy==0.49.20 +pooch==1.8.2 +preshed==3.0.10 +prometheus_client==0.22.1 +prompt_toolkit==3.0.51 +propcache @ file:///rapids/propcache-0.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=563f9d8c03ad645597b8d010ef4e9eab359faeb11a0a2ac9f7b4bc8c28ebef25 +protobuf==4.24.4 +psutil==7.0.0 +ptyprocess==0.7.0 +PuLP==3.2.1 +pure_eval==0.2.3 +pyarrow==25.0.1 +pybind11==2.13.6 +pybind11_global==2.13.6 +pycocotools @ git+https://github.com/nvidia/cocoapi.git@08350a157b41f0755a536441b5abec89ab934c5b#subdirectory=PythonAPI +pycparser==2.22 +pydantic==2.11.5 +pydantic_core==2.33.2 +Pygments==2.19.1 +pylibcudf @ file:///rapids/pylibcudf-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=401b8bfd022d06f84e120779ca7c3d01a28b68762aaa8f7873b10d6b1975cf6a +pylibcugraph @ file:///rapids/pylibcugraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=061c7389a88c815b54fd5d3a3584fb87bf693e5602f5be335260d0ba575ba557 +pylibcugraphops @ file:///rapids/pylibcugraphops-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=a25ef4b22166c80718d5c7e5bddb755e81a2fe340fa5becd0ac36ec3edfb208a +pylibraft @ file:///rapids/pylibraft-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=f2ecd23fe51f9c2caa87bbbe85cb32fcaff4ecfb52ceb9a9991bc50f82782c2e +pylibwholegraph @ file:///rapids/pylibwholegraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b82913d1e38bda76d8c272fbd8cc9072d987209d3d47cc93a464a37c2925aca2 +pynvjitlink @ file:///rapids/pynvjitlink-0.3.0-cp312-cp312-linux_x86_64.whl#sha256=066caf1314730643d3b48c9847bd65789defdd6a7c78993176eb3d8bbe3f4919 +pynvml @ file:///rapids/pynvml-12.0.0-py3-none-any.whl#sha256=fdff84b62a27dbe98e08e1a647eb77342bef1aebe0878bcd15e99a83fcbecb9e +pyparsing==3.2.3 +pytest==8.1.1 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==15.1 +pytest-shard==0.1.2 +pytest-xdist==3.7.0 +python-dateutil==2.9.0.post0 +python-hostlist==2.2.1 +python-json-logger==3.3.0 +pytorch-triton @ file:///tmp/dist/pytorch_triton-3.3.0%2Bgit96316ce52.nvinternal-cp312-cp312-linux_x86_64.whl#sha256=6d6921772b9dfd0243c055e0d054de42a113e92cabcfdd1d3e80a5ad9c08cf7d +pytz @ file:///rapids/pytz-2023.4-py2.py3-none-any.whl#sha256=f90ef520d95e7c46951105338d918664ebfd6f1d995bd7d153127ce90efafa6a +PyYAML==6.0.2 +pyzmq==26.4.0 +raft-dask @ file:///rapids/raft_dask-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=36cf37ba6ce35454c4729c2cb597a360f22780eefb11d514a16a07137da5eb6c +rapids-dask-dependency @ file:///rapids/rapids_dask_dependency-25.4.0a0-py3-none-any.whl#sha256=484faca626e7760112a7d8ad3426f5bdd720b609006d8dca87c2607076a348bd +rapids-logger @ file:///rapids/rapids_logger-0.1.18-py3-none-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=3617b2d86291ac6cb957542895f20fc9737ace29773ca90337e6fb04c73bd120 +referencing==0.36.2 +regex==2024.11.6 +requests==2.32.3 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rich==14.0.0 +rmm @ file:///rapids/rmm-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=1824bcab3836296666980a997ddcb9c4bca24dfb873012cfa0a7ecea583ca712 +rpds-py==0.25.1 +safetensors==0.5.3 +scikit-learn==1.6.1 +scipy==1.15.3 +Send2Trash==1.8.3 +setuptools==78.1.1 +shellingham==1.5.4 +six==1.16.0 +smart-open==7.1.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.13.1 +soupsieve==2.7 +soxr==0.5.0.post1 +spacy==3.7.5 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +srsly==2.5.1 +stack-data==0.6.3 +sympy==1.14.0 +tabulate==0.9.0 +tbb==2021.13.1 +tblib @ file:///rapids/tblib-3.1.0-py3-none-any.whl#sha256=670bb4582578134b3d81a84afa1b016128b429f3d48e6cbbaecc9d15675e984e +tensorboard==2.16.2 +tensorboard-data-server==0.7.2 +tensorrt @ file:///workspace/TensorRT-10.11.0.33/python/tensorrt-10.11.0.33-cp312-none-linux_x86_64.whl#sha256=c2f18f72758a3d54a71901495c8fda9e7094a8bcdd2a5b6b58967fbf05733b3a +terminado==0.18.1 +thinc==8.2.5 +threadpoolctl==3.6.0 +thriftpy2 @ file:///rapids/thriftpy2-0.5.2-cp312-cp312-linux_x86_64.whl#sha256=c0c7d939bdc5ccf7739154959938445d5fdf72f1d6e12bd563947a36f67e2147 +tinycss2==1.4.0 +toolz @ file:///rapids/toolz-1.0.0-py3-none-any.whl#sha256=292c8f1c4e7516bf9086f8850935c799a874039c8bcf959d47b600e4c44a6236 +torch @ file:///opt/transfer/torch-2.8.0a0%2B5228986c39.nv25.6-29946474-cp312-cp312-linux_x86_64.whl#sha256=7a3c9ff5288d8102a31296b3d62491f3f47c8820a5c9aaefc7dc0fe20e1e8ad5 +torch_tensorrt @ file:///opt/pytorch/torch_tensorrt/dist/torch_tensorrt-2.8.0a0-cp312-cp312-linux_x86_64.whl#sha256=26f4d7a04e7e6e017a5b917b8b98b83720bb2bb0195f48d25f52d567d286bfe1 +torchao @ file:///opt/pytorch/ao +torchprofile==0.0.4 +torchvision @ file:///opt/transfer/torchvision-0.22.0a0%2B95f10a4e-29946474-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=746a3c674cc61680a86f4a14b374b21af1b5c21f629c42765b40c9494f6b234d +tornado==6.5.1 +tqdm==4.67.1 +traitlets==5.14.3 +transformer_engine @ file:///pvcdata/aiinfra/b-aghosh/wheels/transformer_engine-2.10.0.dev0%2B10f6128d-cp312-cp312-linux_x86_64.whl#sha256=35cf5f4b83b2baad864e12e512bb422e0929409ac05610812fa521d4942d0189 +treelite @ file:///rapids/treelite-4.4.1-py3-none-linux_x86_64.whl#sha256=174741724205540a1a5b37da81483aeff55d89548f42b3461cf07acac7ae96eb +typer==0.16.0 +types-dataclasses==0.6.6 +types-python-dateutil==2.9.0.20250516 +typing-inspection==0.4.1 +typing_extensions==4.14.0 +tzdata @ file:///rapids/tzdata-2025.2-py2.py3-none-any.whl#sha256=1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 +ucx-py @ file:///rapids/ucx_py-0.43.0-cp312-cp312-linux_x86_64.whl#sha256=4a452fd8d8c983283ada3d49086e3657dc2ee94fbad08efdd708d15ab6e0b73a +ucxx @ file:///rapids/ucxx-0.43.0-cp312-cp312-linux_x86_64.whl#sha256=f1b4e55cb45c66fe88a2d8f9c2b87751165f3cfec21fbdc9c47c4ddff75e8a5f +-e git+https://github.com/Dots-Infra/UltraEP.git@94cab099b44fffa99a82fea99e7c12d89cf65e4f#egg=ultra_ep +uri-template==1.3.0 +urllib3==2.0.7 +wasabi==1.1.3 +wcwidth==0.2.13 +weasel==0.4.1 +webcolors==24.11.1 +webencodings==0.5.1 +websocket-client==1.8.0 +Werkzeug==3.1.3 +wheel==0.45.1 +wrapt==1.17.2 +xdoctest==1.0.2 +xgboost @ file:///rapids/xgboost-2.1.4-py3-none-manylinux_2_39_x86_64.whl#sha256=25eeca27f4048fc491940b3a0b4b5782ee48c33cb3aec059cef71f8da542930b +xxhash==4.0.1 +yarl @ file:///rapids/yarl-1.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=bf099e2432131093cc611623e0b0bcc399b8cddd9a91eded8bfb50402ec35018 +zarr @ file:///rapids/zarr-2.18.7-py3-none-any.whl#sha256=ac3dc4033e9ae4e9d7b5e27c97ea3eaf1003cc0a07f010bd83d5134bf8c4b223 +zict @ file:///rapids/zict-3.0.0-py2.py3-none-any.whl#sha256=5796e36bd0e0cc8cf0fbc1ace6a68912611c1dbd74750a3f3026b9b9d6a327ae +zipp==3.22.0 diff --git a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/provenance.txt b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/provenance.txt new file mode 100644 index 0000000000000000000000000000000000000000..bc51ae034063928316c858bd1ccfbdf8f2f6651e --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/provenance.txt @@ -0,0 +1,90 @@ +============================================================== +PROVENANCE: ultraep__tierbprobe__gbs8 +============================================================== +captured_at : 2026-09-21T07:36:53+00:00 +hostname : sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0 +kernel : 6.8.0-1043-nvidia +cwd : /workspace/blackwell_run_scripts +command : run_ladder.sh --phase tierbprobe --gbs 8 --out /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe --nsys + +---- GPU / driver -------------------------------------------- + 0, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 1, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 2, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 3, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 4, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 5, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 6, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 7, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + persistence/clocks: + 0, 120 MHz, 1965 MHz, 1000.00 W, 29 + 1, 1965 MHz, 1965 MHz, 1000.00 W, 35 + 2, 120 MHz, 1965 MHz, 1000.00 W, 35 + 3, 120 MHz, 1965 MHz, 1000.00 W, 30 + 4, 1965 MHz, 1965 MHz, 1000.00 W, 30 + 5, 120 MHz, 1965 MHz, 1000.00 W, 34 + 6, 120 MHz, 1965 MHz, 1000.00 W, 34 + 7, 120 MHz, 1965 MHz, 1000.00 W, 30 + + NVLink topology: + GPU0 GPU1 GPU2 GPU3 GPU4 GPU5 GPU6 GPU7 NIC0 NIC1 NIC2 NIC3 NIC4 NIC5 NIC6 NIC7 NIC8 CPU Affinity NUMA Affinity GPU NUMA ID + GPU0 X NV18 NV18 NV18 NV18 NV18 NV18 NV18 PIX NODE SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A + GPU1 NV18 X NV18 NV18 NV18 NV18 NV18 NV18 NODE PIX SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A + GPU2 NV18 NV18 X NV18 NV18 NV18 NV18 NV18 SYS SYS PIX NODE SYS SYS SYS SYS SYS 28-55 1 N/A + GPU3 NV18 NV18 NV18 X NV18 NV18 NV18 NV18 SYS SYS NODE PIX SYS SYS SYS SYS SYS 28-55 1 N/A + GPU4 NV18 NV18 NV18 NV18 X NV18 NV18 NV18 SYS SYS SYS SYS PIX NODE SYS SYS SYS 56-83 2 N/A + GPU5 NV18 NV18 NV18 NV18 NV18 X NV18 NV18 SYS SYS SYS SYS NODE PIX SYS SYS SYS 56-83 2 N/A + GPU6 NV18 NV18 NV18 NV18 NV18 NV18 X NV18 SYS SYS SYS SYS SYS SYS PIX NODE SYS 84-111 3 N/A + GPU7 NV18 NV18 NV18 NV18 NV18 NV18 NV18 X SYS SYS SYS SYS SYS SYS NODE PIX SYS 84-111 3 N/A + NIC0 PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS SYS NODE + NIC1 NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS SYS NODE + NIC2 SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS + NIC3 SYS SYS NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS + NIC4 SYS SYS SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS + unavailable + +---- toolchain ----------------------------------------------- + nvcc : release 12.9, V12.9.86 + gcc : gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 + python : 3.12.3 + torch : 2.8.0a0+5228986c39.nv25.06 cuda=12.9 nccl=2.27.3 + torch arch : ['sm_75', 'sm_80', 'sm_86', 'sm_90', 'sm_100', 'sm_120', 'compute_120'] + transformer_engine: 2.10.0.dev0+10f6128d [/usr/local/lib/python3.12/dist-packages/transformer_engine/__init__.py] + deep_ep : 1.2.1+34152ae [/workspace/DeepEP/deep_ep/__init__.py] + ultra_ep : 1.0.0 [/workspace/UltraEP/ultra_ep/__init__.py] + moonep : 0.0.1 [/workspace/MoonEP/moonep/__init__.py] + megatron : 0.14.0rc5 [None] + flash_attn : 2.7.4.post1 [/usr/local/lib/python3.12/dist-packages/flash_attn/__init__.py] + numpy : 1.26.4 [/usr/local/lib/python3.12/dist-packages/numpy/__init__.py] + huggingface_hub: 1.32.0 [/usr/local/lib/python3.12/dist-packages/huggingface_hub/__init__.py] + +---- repositories -------------------------------------------- + Megatron-MoE-Echo token-drop-analysis b099fd8494b0 + Megatron-UltraEP ultraep-token-drop-analysis eaf829e7ad21 + UltraEP main 94cab099b44f + DeepEP HEAD 34152ae28f80 (+3 untracked: build output) + MoonEP megatron-ddp-symmetric-params d8bc6667fad7 + blackwell_run_scripts granularity-ultraep 05dd2cbfb1eb + +---- environment --------------------------------------------- + CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0 + CUDA_DRIVER_VERSION=575.57.08 + CUDA_HOME=/usr/local/cuda + CUDA_MODULE_LOADING=LAZY + CUDA_VERSION=12.9.1.010 + HF_REMOTE_ROOT=granularity_next_b200/tierb_models/attempts/KIMI_K2_L2 + NCCL_VERSION=2.27.3 + NVSHMEM_VERSION=3.2.5 + PYTORCH_BUILD_NUMBER=0 + PYTORCH_BUILD_VERSION=2.8.0a0+5228986 + PYTORCH_HOME=/opt/pytorch/pytorch + PYTORCH_VERSION=2.8.0a0+5228986 + TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1 + TORCH_CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0+PTX + TORCH_NCCL_USE_COMM_NONBLOCKING=0 + +---- shared library resolution ------------------------------- + libnvshmem_host.so: not in ldconfig + libnccl.so: /lib/x86_64-linux-gnu/libnccl.so.2 + lrwxrwxrwx 1 root root 20 Sep 21 04:57 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so -> libnvshmem_host.so.3 + -rw-r--r-- 1 root root 179045744 Sep 21 04:54 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so.3 diff --git a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p0625/rep1/mem_samples.csv b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p0625/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..285c75f326f7df7dbd9ae1d426f44ad70d387d00 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p0625/rep1/mem_samples.csv @@ -0,0 +1,184 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 2 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 27 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 819 +2, 819 +3, 819 +4, 819 +5, 821 +6, 821 +7, 819 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 2359 +1, 2359 +2, 2359 +3, 2359 +4, 2359 +5, 2359 +6, 2359 +7, 2359 +0, 4289 +1, 4289 +2, 4289 +3, 4289 +4, 4289 +5, 4289 +6, 4289 +7, 4289 +0, 50789 +1, 50795 +2, 50819 +3, 50783 +4, 50825 +5, 50821 +6, 50835 +7, 50809 +0, 60285 +1, 60287 +2, 60285 +3, 60285 +4, 60285 +5, 60285 +6, 60285 +7, 60285 +0, 61249 +1, 61251 +2, 61249 +3, 61249 +4, 61249 +5, 61249 +6, 61249 +7, 61249 +0, 61249 +1, 61251 +2, 61249 +3, 61249 +4, 61249 +5, 61249 +6, 61249 +7, 61249 +0, 62983 +1, 63027 +2, 63375 +3, 63187 +4, 63043 +5, 62809 +6, 63549 +7, 63579 +0, 64241 +1, 64243 +2, 64255 +3, 64241 +4, 64241 +5, 64241 +6, 64241 +7, 64241 +0, 65741 +1, 65743 +2, 65755 +3, 65741 +4, 65741 +5, 65741 +6, 65741 +7, 65741 +0, 65741 +1, 65743 +2, 65755 +3, 65741 +4, 65741 +5, 65741 +6, 65741 +7, 65741 +0, 65747 +1, 65747 +2, 65761 +3, 65747 +4, 65747 +5, 65747 +6, 65747 +7, 65747 +0, 68035 +1, 68029 +2, 68061 +3, 68023 +4, 68029 +5, 68043 +6, 68071 +7, 68045 +0, 68843 +1, 68849 +2, 68861 +3, 68845 +4, 68857 +5, 68853 +6, 68857 +7, 68865 +0, 75547 +1, 75543 +2, 75671 +3, 75549 +4, 75515 +5, 75605 +6, 75595 +7, 75603 +0, 89497 +1, 89513 +2, 89497 +3, 89497 +4, 89497 +5, 89483 +6, 89497 +7, 89497 +0, 15428 +1, 13828 +2, 13828 +3, 13828 +4, 13828 +5, 13828 +6, 14148 +7, 88275 diff --git a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p0625/rep1/run.env b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p0625/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..2f9aabac3cbb02bbcf793414ae8c99c1d11ee958 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p0625/rep1/run.env @@ -0,0 +1,102 @@ +backend=ultraep +config=KIMI_K2_L2 +family=tierB +source_model=Kimi-K2 +standard_num_layers=61 +base_rung= +repo=/workspace/Megatron-UltraEP +branch=ultraep-token-drop-analysis +commit=eaf829e7 +num_layers=2 +hidden=7168 +ffn_hidden=18432 +moe_ffn_hidden=2048 +num_experts=384 +topk=8 +experts_per_rank=48 +expert_params=44040192 +heads=64 +kv_groups=64 +kv_channels=64 +qk_layernorm=1 +shared_ffn=2048 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=1 +q_lora_rank=1536 +kv_lora_rank=512 +qk_head_dim=128 +qk_pos_emb_head_dim=64 +v_head_dim=128 +dense_first_layers=1 +moe_layers=1 +rotary_base=50000 +rotary_percent=1.0 +rotary_scaling=32 +mscale=1.0 +mscale_all_dim=1.0 +qkv_bias=0 +router_balancing=seq_aux_loss +aux_loss_coeff=0 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +router_groups=1 +router_group_topk=1 +router_score_function=sigmoid +router_expert_bias=1 +router_bias_update_rate=1e-3 +router_topk_scaling_factor=2.827 +shared_expert_gate=0 +apply_layernorm_1p=0 +attention_output_gate=0 +linear_attention_type= +linear_attention_freq= +linear_conv_kernel_dim= +linear_key_head_dim= +linear_value_head_dim= +linear_num_key_heads= +linear_num_value_heads= +mtp_num_layers=0 +mtp_loss_scaling_factor=0.1 +no_weight_decay_cond_type= +max_position_embeddings=4096 +init_method_std=0.02 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=682 +slot_policy=matched +slots_per_rank=48 +capacity_mode=static +capacity=1.0625 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=85 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p0625/rep1/train.log b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p0625/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..fd0033ffe4f9736caefd253919512d62e88853fe --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p0625/rep1/train.log @@ -0,0 +1,1652 @@ +[07:38:14] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[07:38:14] WARN: ultraep-token-drop-analysis lacks --moe-router-fusion; KIMI_K2_L2 runs unfused (timing only, routing identical) +============================================================== + KIMI_K2_L2 / ultraep (slots/rank=48 policy=matched) + E=384 K=8 H'=2048 m=48 + L=2 seq=4096 gbs=8 (1 microbatches) + cap_mode=static cap=1.0625 cg=0 recompute=none + memory_dump=none snapshots=/workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p0625/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ eaf829e7 +============================================================== +W0921 07:38:15.705000 479764 torch/distributed/run.py:766] +W0921 07:38:15.705000 479764 torch/distributed/run.py:766] ***************************************** +W0921 07:38:15.705000 479764 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0921 07:38:15.705000 479764 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 2 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 18432 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... False + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 7168 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.02 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 512 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 4096 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 2048 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 1] + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 48 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.0625 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... True + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... 1 + moe_router_load_balancing_type .................. seq_aux_loss + moe_router_num_groups ........................... 1 + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... sigmoid + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. 2.827 + moe_shared_expert_intermediate_size ............. 2048 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 1.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... True + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 64 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 384 + num_layers ...................................... 2 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 1 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... 1536 + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 50000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 32.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 6 is connected to 75 peer ranks. is connected to [Gloo] Rank Expected number of connected peer ranks is : 777 peer ranks. is connected to +Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank4]:[W921 07:38:32.561974173 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W921 07:38:32.564142787 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W921 07:38:32.564142499 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W921 07:38:32.564190252 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W921 07:38:32.564217319 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W921 07:38:32.564257594 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W921 07:38:32.564281968 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.037 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W921 07:38:32.749939594 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.721 seconds +time to initialize megatron (seconds): 7.621 +[after megatron is initialized] datetime: 2026-09-21 07:38:38 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 5594455040 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 9 +Params for bucket 1 (360579072 elements, 360579072 padded size): + module.output_layer.weight +Params for bucket 2 (44047360 elements, 44047360 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight +Params for bucket 3 (45163520 elements, 45163520 padded size): + module.decoder.layers.1.self_attention.linear_kv_up_proj.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_q_up_proj.weight + module.decoder.layers.1.self_attention.linear_q_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_q_down_proj.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight + module.decoder.layers.1.pre_mlp_layernorm.weight +Params for bucket 4 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight +Params for bucket 5 (132127744 elements, 132127744 padded size): + module.decoder.layers.1.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc2.weight +Params for bucket 6 (264241152 elements, 264241152 padded size): + module.decoder.layers.0.mlp.linear_fc1.weight +Params for bucket 7 (42411008 elements, 42411008 padded size): + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.self_attention.linear_q_up_proj.weight + module.decoder.layers.0.self_attention.linear_q_up_proj.layer_norm_weight + module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.decoder.layers.0.self_attention.linear_q_down_proj.weight +Params for bucket 8 (58720256 elements, 58720256 padded size): + module.decoder.layers.0.self_attention.linear_proj.weight +Params for bucket 9 (360586240 elements, 360586240 padded size): + module.decoder.layers.0.input_layernorm.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight47 + module.decoder.layers.1.mlp.experts.linear_fc2.weight45 + module.decoder.layers.1.mlp.experts.linear_fc2.weight46 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight44 + module.decoder.layers.1.mlp.experts.linear_fc2.weight42 + module.decoder.layers.1.mlp.experts.linear_fc2.weight43 +Params for bucket 3 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight41 + module.decoder.layers.1.mlp.experts.linear_fc2.weight39 + module.decoder.layers.1.mlp.experts.linear_fc2.weight40 +Params for bucket 4 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight38 + module.decoder.layers.1.mlp.experts.linear_fc2.weight36 + module.decoder.layers.1.mlp.experts.linear_fc2.weight37 +Params for bucket 5 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight35 + module.decoder.layers.1.mlp.experts.linear_fc2.weight33 + module.decoder.layers.1.mlp.experts.linear_fc2.weight34 +Params for bucket 6 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight32 + module.decoder.layers.1.mlp.experts.linear_fc2.weight30 + module.decoder.layers.1.mlp.experts.linear_fc2.weight31 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight29 + module.decoder.layers.1.mlp.experts.linear_fc2.weight27 + module.decoder.layers.1.mlp.experts.linear_fc2.weight28 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight26 + module.decoder.layers.1.mlp.experts.linear_fc2.weight24 + module.decoder.layers.1.mlp.experts.linear_fc2.weight25 +Params for bucket 9 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight23 + module.decoder.layers.1.mlp.experts.linear_fc2.weight21 + module.decoder.layers.1.mlp.experts.linear_fc2.weight22 +Params for bucket 10 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight20 + module.decoder.layers.1.mlp.experts.linear_fc2.weight19 + module.decoder.layers.1.mlp.experts.linear_fc2.weight18 +Params for bucket 11 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight17 + module.decoder.layers.1.mlp.experts.linear_fc2.weight16 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 +Params for bucket 12 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 +Params for bucket 15 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 +Params for bucket 16 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 +Params for bucket 17 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight47 + module.decoder.layers.1.mlp.experts.linear_fc1.weight46 +Params for bucket 18 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight45 + module.decoder.layers.1.mlp.experts.linear_fc1.weight44 +Params for bucket 19 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight43 + module.decoder.layers.1.mlp.experts.linear_fc1.weight42 +Params for bucket 20 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight41 + module.decoder.layers.1.mlp.experts.linear_fc1.weight40 +Params for bucket 21 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight39 + module.decoder.layers.1.mlp.experts.linear_fc1.weight38 +Params for bucket 22 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight37 + module.decoder.layers.1.mlp.experts.linear_fc1.weight36 +Params for bucket 23 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight35 + module.decoder.layers.1.mlp.experts.linear_fc1.weight34 +Params for bucket 24 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight33 + module.decoder.layers.1.mlp.experts.linear_fc1.weight32 +Params for bucket 25 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight31 + module.decoder.layers.1.mlp.experts.linear_fc1.weight30 +Params for bucket 26 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight29 + module.decoder.layers.1.mlp.experts.linear_fc1.weight28 +Params for bucket 27 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight27 + module.decoder.layers.1.mlp.experts.linear_fc1.weight26 +Params for bucket 28 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight25 + module.decoder.layers.1.mlp.experts.linear_fc1.weight24 +Params for bucket 29 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight23 + module.decoder.layers.1.mlp.experts.linear_fc1.weight22 +Params for bucket 30 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight21 + module.decoder.layers.1.mlp.experts.linear_fc1.weight20 +Params for bucket 31 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight19 + module.decoder.layers.1.mlp.experts.linear_fc1.weight18 +Params for bucket 32 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight17 + module.decoder.layers.1.mlp.experts.linear_fc1.weight16 +Params for bucket 33 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 +Params for bucket 34 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 +Params for bucket 35 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 +Params for bucket 36 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 +Params for bucket 37 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 +Params for bucket 38 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 +Params for bucket 39 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 +Params for bucket 40 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-09-21 07:38:41 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-09-21 07:38:42 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3484.90, 3492.28) + train/valid/test-data-iterators-setup ..........: (76.80, 91.11) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-09-21 07:38:42 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=34816 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=34816 expected_recv=33098 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[28774, 33214, 27953, 31005, 35336, 30433, 39027, 36402] avg=32768.0000 max=39027 max_rank=6 min=27953 min_rank=2 imbalance_ratio=1.191010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=34816 expected_recv=31766 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] avg=32768.0000 max=33098 max_rank=0 min=31766 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=5929 max_load_reduction_pct=0.151920 moved_to_replicas=11595 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=34816 expected_recv=31766 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[28774, 33214, 27953, 31005, 35336, 30433, 39027, 36402] avg=32768.0000 max=39027 max_rank=6 min=27953 min_rank=2 imbalance_ratio=1.191010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34560 slack_slots=256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] avg=32768.0000 max=33098 max_rank=0 min=31766 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=5929 max_load_reduction_pct=0.151920 moved_to_replicas=11595 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=1 total_expected_recv=33098 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[28774, 33214, 27953, 31005, 35336, 30433, 39027, 36402] avg=32768.0000 max=39027 max_rank=6 min=27953 min_rank=2 imbalance_ratio=1.191010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[28774, 33214, 27953, 31005, 35336, 30433, 39027, 36402] avg=32768.0000 max=39027 max_rank=6 min=27953 min_rank=2 imbalance_ratio=1.191010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=1 total_expected_recv=33095 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33095 required_capacity=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[28774, 33214, 27953, 31005, 35336, 30433, 39027, 36402] avg=32768.0000 max=39027 max_rank=6 min=27953 min_rank=2 imbalance_ratio=1.191010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] avg=32768.0000 max=33098 max_rank=0 min=31766 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=5929 max_load_reduction_pct=0.151920 moved_to_replicas=11595 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] avg=32768.0000 max=33098 max_rank=0 min=31766 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=5929 max_load_reduction_pct=0.151920 moved_to_replicas=11595 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[28774, 33214, 27953, 31005, 35336, 30433, 39027, 36402] avg=32768.0000 max=39027 max_rank=6 min=27953 min_rank=2 imbalance_ratio=1.191010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] avg=32768.0000 max=33098 max_rank=0 min=31766 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=5929 max_load_reduction_pct=0.151920 moved_to_replicas=11595 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] avg=32768.0000 max=33098 max_rank=0 min=31766 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=5929 max_load_reduction_pct=0.151920 moved_to_replicas=11595 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=1 total_expected_recv=31766 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=256 peak_demand=31766 required_capacity=0.969421 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=1 total_expected_recv=33095 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33095 required_capacity=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[28774, 33214, 27953, 31005, 35336, 30433, 39027, 36402] avg=32768.0000 max=39027 max_rank=6 min=27953 min_rank=2 imbalance_ratio=1.191010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=34816 expected_recv=31802 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] avg=32768.0000 max=33098 max_rank=0 min=31766 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=5929 max_load_reduction_pct=0.151920 moved_to_replicas=11595 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=34816 expected_recv=31802 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[28774, 33214, 27953, 31005, 35336, 30433, 39027, 36402] avg=32768.0000 max=39027 max_rank=6 min=27953 min_rank=2 imbalance_ratio=1.191010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] avg=32768.0000 max=33098 max_rank=0 min=31766 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=5929 max_load_reduction_pct=0.151920 moved_to_replicas=11595 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=1 total_expected_recv=31802 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=31802 required_capacity=0.970520 + [2026-09-21 07:39:10] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 28677.6 | throughput per GPU (TFLOP/s/GPU): 1.2 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.227764E+01 | loss scale: 1.0 | grad norm: 12.625 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34816 | min safe capacity: 1.0625 |Number of parameters in transformer block in billions: 17.55 + +Number of parameters in embedding layers in billions: 0.72 +Total number of parameters in billions: 18.28 +Number of parameters in most loaded shard in billions: 18.2753 +Theoretical memory footprints: weight and optimizer=130715.23 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 49119.92529296875 | max allocated: 49119.92578125 | reserved: 54172.0 | max reserved: 54172.0 +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=34816 expected_recv=31670 expected_overflow=0 exact_recv_per_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=34816 expected_recv=32236 expected_overflow=0 exact_recv_per_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=34816 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=34816 expected_recv=31670 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=34816 expected_recv=32236 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=34816 expected_recv=32757 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[29410, 33844, 28405, 30208, 34500, 31179, 38439, 36159] avg=32768.0000 max=38439 max_rank=6 min=28405 min_rank=2 imbalance_ratio=1.173065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[29410, 33844, 28405, 30208, 34500, 31179, 38439, 36159] avg=32768.0000 max=38439 max_rank=6 min=28405 min_rank=2 imbalance_ratio=1.173065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[29410, 33844, 28405, 30208, 34500, 31179, 38439, 36159] avg=32768.0000 max=38439 max_rank=6 min=28405 min_rank=2 imbalance_ratio=1.173065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[29410, 33844, 28405, 30208, 34500, 31179, 38439, 36159] avg=32768.0000 max=38439 max_rank=6 min=28405 min_rank=2 imbalance_ratio=1.173065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[29410, 33844, 28405, 30208, 34500, 31179, 38439, 36159] avg=32768.0000 max=38439 max_rank=6 min=28405 min_rank=2 imbalance_ratio=1.173065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[29410, 33844, 28405, 30208, 34500, 31179, 38439, 36159] avg=32768.0000 max=38439 max_rank=6 min=28405 min_rank=2 imbalance_ratio=1.173065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31670 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=5342 max_load_reduction_pct=0.138973 moved_to_replicas=10558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[29410, 33844, 28405, 30208, 34500, 31179, 38439, 36159] avg=32768.0000 max=38439 max_rank=6 min=28405 min_rank=2 imbalance_ratio=1.173065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31670 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=5342 max_load_reduction_pct=0.138973 moved_to_replicas=10558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31670 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=5342 max_load_reduction_pct=0.138973 moved_to_replicas=10558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31670 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=5342 max_load_reduction_pct=0.138973 moved_to_replicas=10558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31670 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=5342 max_load_reduction_pct=0.138973 moved_to_replicas=10558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31670 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=5342 max_load_reduction_pct=0.138973 moved_to_replicas=10558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[29410, 33844, 28405, 30208, 34500, 31179, 38439, 36159] avg=32768.0000 max=38439 max_rank=6 min=28405 min_rank=2 imbalance_ratio=1.173065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31670 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=5342 max_load_reduction_pct=0.138973 moved_to_replicas=10558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=1 total_expected_recv=31670 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=31670 required_capacity=0.966492 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=1 total_expected_recv=32757 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=32757 required_capacity=0.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=1 total_expected_recv=32236 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=32236 required_capacity=0.983765 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31670 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=5342 max_load_reduction_pct=0.138973 moved_to_replicas=10558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=1 total_expected_recv=33095 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33095 required_capacity=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=1 total_expected_recv=33097 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=1 total_expected_recv=33097 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33097 required_capacity=1.010040 + [2026-09-21 07:39:11] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 234.9 | throughput per GPU (TFLOP/s/GPU): 150.6 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.227066E+01 | loss scale: 1.0 | grad norm: 12.545 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34816 | min safe capacity: 1.0625 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32237, 33095, 32810, 33095, 33097, 31618, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32237, 33095, 32810, 33095, 33097, 31618, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=34816 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32237, 33095, 32810, 33095, 33097, 31618, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=34816 expected_recv=32237 expected_overflow=0 exact_recv_per_rank=[32237, 33095, 32810, 33095, 33097, 31618, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=34816 expected_recv=31618 expected_overflow=0 exact_recv_per_rank=[32237, 33095, 32810, 33095, 33097, 31618, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32237, 33095, 32810, 33095, 33097, 31618, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32237, 33095, 32810, 33095, 33097, 31618, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=34816 expected_recv=32810 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=34816 expected_recv=32237 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32237, 33095, 32810, 33095, 33097, 31618, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=34816 expected_recv=31618 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[28926, 33784, 29401, 30705, 35522, 30894, 37093, 35819] avg=32768.0000 max=37093 max_rank=6 min=28926 min_rank=0 imbalance_ratio=1.131989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[28926, 33784, 29401, 30705, 35522, 30894, 37093, 35819] avg=32768.0000 max=37093 max_rank=6 min=28926 min_rank=0 imbalance_ratio=1.131989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[28926, 33784, 29401, 30705, 35522, 30894, 37093, 35819] avg=32768.0000 max=37093 max_rank=6 min=28926 min_rank=0 imbalance_ratio=1.131989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[28926, 33784, 29401, 30705, 35522, 30894, 37093, 35819] avg=32768.0000 max=37093 max_rank=6 min=28926 min_rank=0 imbalance_ratio=1.131989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[28926, 33784, 29401, 30705, 35522, 30894, 37093, 35819] avg=32768.0000 max=37093 max_rank=6 min=28926 min_rank=0 imbalance_ratio=1.131989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[28926, 33784, 29401, 30705, 35522, 30894, 37093, 35819] avg=32768.0000 max=37093 max_rank=6 min=28926 min_rank=0 imbalance_ratio=1.131989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[28926, 33784, 29401, 30705, 35522, 30894, 37093, 35819] avg=32768.0000 max=37093 max_rank=6 min=28926 min_rank=0 imbalance_ratio=1.131989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[32237, 33095, 32810, 33095, 33097, 31618, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31618 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=3996 max_load_reduction_pct=0.107729 moved_to_replicas=9834 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[32237, 33095, 32810, 33095, 33097, 31618, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31618 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=3996 max_load_reduction_pct=0.107729 moved_to_replicas=9834 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[32237, 33095, 32810, 33095, 33097, 31618, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31618 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=3996 max_load_reduction_pct=0.107729 moved_to_replicas=9834 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[32237, 33095, 32810, 33095, 33097, 31618, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31618 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=3996 max_load_reduction_pct=0.107729 moved_to_replicas=9834 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[28926, 33784, 29401, 30705, 35522, 30894, 37093, 35819] avg=32768.0000 max=37093 max_rank=6 min=28926 min_rank=0 imbalance_ratio=1.131989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[32237, 33095, 32810, 33095, 33097, 31618, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31618 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=3996 max_load_reduction_pct=0.107729 moved_to_replicas=9834 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[32237, 33095, 32810, 33095, 33097, 31618, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31618 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=3996 max_load_reduction_pct=0.107729 moved_to_replicas=9834 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[32237, 33095, 32810, 33095, 33097, 31618, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31618 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=3996 max_load_reduction_pct=0.107729 moved_to_replicas=9834 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[32237, 33095, 32810, 33095, 33097, 31618, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31618 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=3996 max_load_reduction_pct=0.107729 moved_to_replicas=9834 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=1 total_expected_recv=32810 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=32810 required_capacity=1.001282 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=1 total_expected_recv=33095 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33095 required_capacity=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=1 total_expected_recv=32237 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=32237 required_capacity=0.983795 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=1 total_expected_recv=31618 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=31618 required_capacity=0.964905 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=1 total_expected_recv=33095 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33095 required_capacity=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=1 total_expected_recv=33097 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33097 required_capacity=1.010040 + [2026-09-21 07:39:11] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 122.4 | throughput per GPU (TFLOP/s/GPU): 289.0 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.088622E+01 | loss scale: 1.0 | grad norm: 11.029 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34816 | min safe capacity: 1.0625 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=34816 expected_recv=32610 expected_overflow=0 exact_recv_per_rank=[33096, 32610, 33096, 33007, 33095, 32931, 31213, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32610, 33096, 33007, 33095, 32931, 31213, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32610, 33096, 33007, 33095, 32931, 31213, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=34816 expected_recv=31213 expected_overflow=0 exact_recv_per_rank=[33096, 32610, 33096, 33007, 33095, 32931, 31213, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=34816 expected_recv=32931 expected_overflow=0 exact_recv_per_rank=[33096, 32610, 33096, 33007, 33095, 32931, 31213, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32610, 33096, 33007, 33095, 32931, 31213, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32610, 33096, 33007, 33095, 32931, 31213, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=34816 expected_recv=33007 expected_overflow=0 exact_recv_per_rank=[33096, 32610, 33096, 33007, 33095, 32931, 31213, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=34816 expected_recv=32610 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=34816 expected_recv=32931 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=34816 expected_recv=31213 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34432 slack_slots=384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=34816 expected_recv=33007 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[26936, 32610, 35904, 33007, 33885, 30123, 30007, 39672] avg=32768.0000 max=39672 max_rank=7 min=26936 min_rank=0 imbalance_ratio=1.210693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[26936, 32610, 35904, 33007, 33885, 30123, 30007, 39672] avg=32768.0000 max=39672 max_rank=7 min=26936 min_rank=0 imbalance_ratio=1.210693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[26936, 32610, 35904, 33007, 33885, 30123, 30007, 39672] avg=32768.0000 max=39672 max_rank=7 min=26936 min_rank=0 imbalance_ratio=1.210693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[26936, 32610, 35904, 33007, 33885, 30123, 30007, 39672] avg=32768.0000 max=39672 max_rank=7 min=26936 min_rank=0 imbalance_ratio=1.210693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[26936, 32610, 35904, 33007, 33885, 30123, 30007, 39672] avg=32768.0000 max=39672 max_rank=7 min=26936 min_rank=0 imbalance_ratio=1.210693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[26936, 32610, 35904, 33007, 33885, 30123, 30007, 39672] avg=32768.0000 max=39672 max_rank=7 min=26936 min_rank=0 imbalance_ratio=1.210693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[26936, 32610, 35904, 33007, 33885, 30123, 30007, 39672] avg=32768.0000 max=39672 max_rank=7 min=26936 min_rank=0 imbalance_ratio=1.210693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[26936, 32610, 35904, 33007, 33885, 30123, 30007, 39672] avg=32768.0000 max=39672 max_rank=7 min=26936 min_rank=0 imbalance_ratio=1.210693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[33096, 32610, 33096, 33007, 33095, 32931, 31213, 33096] avg=32768.0000 max=33096 max_rank=0 min=31213 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=6576 max_load_reduction_pct=0.165759 moved_to_replicas=10173 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[33096, 32610, 33096, 33007, 33095, 32931, 31213, 33096] avg=32768.0000 max=33096 max_rank=0 min=31213 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=6576 max_load_reduction_pct=0.165759 moved_to_replicas=10173 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[33096, 32610, 33096, 33007, 33095, 32931, 31213, 33096] avg=32768.0000 max=33096 max_rank=0 min=31213 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=6576 max_load_reduction_pct=0.165759 moved_to_replicas=10173 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[33096, 32610, 33096, 33007, 33095, 32931, 31213, 33096] avg=32768.0000 max=33096 max_rank=0 min=31213 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=6576 max_load_reduction_pct=0.165759 moved_to_replicas=10173 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[33096, 32610, 33096, 33007, 33095, 32931, 31213, 33096] avg=32768.0000 max=33096 max_rank=0 min=31213 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=6576 max_load_reduction_pct=0.165759 moved_to_replicas=10173 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[33096, 32610, 33096, 33007, 33095, 32931, 31213, 33096] avg=32768.0000 max=33096 max_rank=0 min=31213 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=6576 max_load_reduction_pct=0.165759 moved_to_replicas=10173 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[33096, 32610, 33096, 33007, 33095, 32931, 31213, 33096] avg=32768.0000 max=33096 max_rank=0 min=31213 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=6576 max_load_reduction_pct=0.165759 moved_to_replicas=10173 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[33096, 32610, 33096, 33007, 33095, 32931, 31213, 33096] avg=32768.0000 max=33096 max_rank=0 min=31213 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=6576 max_load_reduction_pct=0.165759 moved_to_replicas=10173 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=1 total_expected_recv=32610 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=32610 required_capacity=0.995178 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=1 total_expected_recv=31213 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=384 peak_demand=31213 required_capacity=0.952545 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=1 total_expected_recv=32931 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=32931 required_capacity=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=1 total_expected_recv=33095 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33095 required_capacity=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=1 total_expected_recv=33007 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33007 required_capacity=1.007294 + [2026-09-21 07:39:11] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 119.9 | throughput per GPU (TFLOP/s/GPU): 295.0 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.448883E+01 | loss scale: 1.0 | grad norm: 73.051 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34816 | min safe capacity: 1.0625 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31774, 33096, 31793, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=34816 expected_recv=31793 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31774, 33096, 31793, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31774, 33096, 31793, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31774, 33096, 31793, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31774, 33096, 31793, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31774, 33096, 31793, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31774, 33096, 31793, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=34816 expected_recv=31774 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31774, 33096, 31793, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=34816 expected_recv=31793 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=34816 expected_recv=31774 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[43926, 33542, 42099, 31328, 30455, 28693, 23187, 28914] avg=32768.0000 max=43926 max_rank=0 min=23187 min_rank=6 imbalance_ratio=1.340515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[43926, 33542, 42099, 31328, 30455, 28693, 23187, 28914] avg=32768.0000 max=43926 max_rank=0 min=23187 min_rank=6 imbalance_ratio=1.340515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[43926, 33542, 42099, 31328, 30455, 28693, 23187, 28914] avg=32768.0000 max=43926 max_rank=0 min=23187 min_rank=6 imbalance_ratio=1.340515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[43926, 33542, 42099, 31328, 30455, 28693, 23187, 28914] avg=32768.0000 max=43926 max_rank=0 min=23187 min_rank=6 imbalance_ratio=1.340515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[43926, 33542, 42099, 31328, 30455, 28693, 23187, 28914] avg=32768.0000 max=43926 max_rank=0 min=23187 min_rank=6 imbalance_ratio=1.340515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[43926, 33542, 42099, 31328, 30455, 28693, 23187, 28914] avg=32768.0000 max=43926 max_rank=0 min=23187 min_rank=6 imbalance_ratio=1.340515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[43926, 33542, 42099, 31328, 30455, 28693, 23187, 28914] avg=32768.0000 max=43926 max_rank=0 min=23187 min_rank=6 imbalance_ratio=1.340515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31774, 33096, 31793, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31774 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=10829 max_load_reduction_pct=0.246528 moved_to_replicas=20279 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[43926, 33542, 42099, 31328, 30455, 28693, 23187, 28914] avg=32768.0000 max=43926 max_rank=0 min=23187 min_rank=6 imbalance_ratio=1.340515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31774, 33096, 31793, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31774 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=10829 max_load_reduction_pct=0.246528 moved_to_replicas=20279 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31774, 33096, 31793, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31774 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=10829 max_load_reduction_pct=0.246528 moved_to_replicas=20279 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31774, 33096, 31793, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31774 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=10829 max_load_reduction_pct=0.246528 moved_to_replicas=20279 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31774, 33096, 31793, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31774 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=10829 max_load_reduction_pct=0.246528 moved_to_replicas=20279 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31774, 33096, 31793, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31774 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=10829 max_load_reduction_pct=0.246528 moved_to_replicas=20279 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31774, 33096, 31793, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31774 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=10829 max_load_reduction_pct=0.246528 moved_to_replicas=20279 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 31774, 33096, 31793, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31774 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=10829 max_load_reduction_pct=0.246528 moved_to_replicas=20279 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=1 total_expected_recv=31793 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=31793 required_capacity=0.970245 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=1 total_expected_recv=33097 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=1 total_expected_recv=31774 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=31774 required_capacity=0.969666 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 + [2026-09-21 07:39:11] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 123.9 | throughput per GPU (TFLOP/s/GPU): 285.6 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.809777E+01 | loss scale: 1.0 | grad norm: 108.439 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34816 | min safe capacity: 1.0625 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32980, 33096, 31937, 33096, 31747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=34816 expected_recv=31937 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32980, 33096, 31937, 33096, 31747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=34816 expected_recv=31747 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32980, 33096, 31937, 33096, 31747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32980, 33096, 31937, 33096, 31747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32980, 33096, 31937, 33096, 31747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32980, 33096, 31937, 33096, 31747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32980, 33096, 31937, 33096, 31747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=34816 expected_recv=32980 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32980, 33096, 31937, 33096, 31747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=34816 expected_recv=31937 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=34816 expected_recv=31747 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=34816 expected_recv=32980 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[46095, 40773, 38651, 30218, 26713, 31183, 22284, 26227] avg=32768.0000 max=46095 max_rank=0 min=22284 min_rank=6 imbalance_ratio=1.406708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[46095, 40773, 38651, 30218, 26713, 31183, 22284, 26227] avg=32768.0000 max=46095 max_rank=0 min=22284 min_rank=6 imbalance_ratio=1.406708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[46095, 40773, 38651, 30218, 26713, 31183, 22284, 26227] avg=32768.0000 max=46095 max_rank=0 min=22284 min_rank=6 imbalance_ratio=1.406708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[46095, 40773, 38651, 30218, 26713, 31183, 22284, 26227] avg=32768.0000 max=46095 max_rank=0 min=22284 min_rank=6 imbalance_ratio=1.406708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[46095, 40773, 38651, 30218, 26713, 31183, 22284, 26227] avg=32768.0000 max=46095 max_rank=0 min=22284 min_rank=6 imbalance_ratio=1.406708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[46095, 40773, 38651, 30218, 26713, 31183, 22284, 26227] avg=32768.0000 max=46095 max_rank=0 min=22284 min_rank=6 imbalance_ratio=1.406708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[46095, 40773, 38651, 30218, 26713, 31183, 22284, 26227] avg=32768.0000 max=46095 max_rank=0 min=22284 min_rank=6 imbalance_ratio=1.406708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 32980, 33096, 31937, 33096, 31747] avg=32768.0000 max=33096 max_rank=0 min=31747 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=12999 max_load_reduction_pct=0.282005 moved_to_replicas=26231 active_clone_links=10 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[46095, 40773, 38651, 30218, 26713, 31183, 22284, 26227] avg=32768.0000 max=46095 max_rank=0 min=22284 min_rank=6 imbalance_ratio=1.406708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 32980, 33096, 31937, 33096, 31747] avg=32768.0000 max=33096 max_rank=0 min=31747 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=12999 max_load_reduction_pct=0.282005 moved_to_replicas=26231 active_clone_links=10 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 32980, 33096, 31937, 33096, 31747] avg=32768.0000 max=33096 max_rank=0 min=31747 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=12999 max_load_reduction_pct=0.282005 moved_to_replicas=26231 active_clone_links=10 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 32980, 33096, 31937, 33096, 31747] avg=32768.0000 max=33096 max_rank=0 min=31747 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=12999 max_load_reduction_pct=0.282005 moved_to_replicas=26231 active_clone_links=10 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 32980, 33096, 31937, 33096, 31747] avg=32768.0000 max=33096 max_rank=0 min=31747 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=12999 max_load_reduction_pct=0.282005 moved_to_replicas=26231 active_clone_links=10 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 32980, 33096, 31937, 33096, 31747] avg=32768.0000 max=33096 max_rank=0 min=31747 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=12999 max_load_reduction_pct=0.282005 moved_to_replicas=26231 active_clone_links=10 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 32980, 33096, 31937, 33096, 31747] avg=32768.0000 max=33096 max_rank=0 min=31747 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=12999 max_load_reduction_pct=0.282005 moved_to_replicas=26231 active_clone_links=10 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 32980, 33096, 31937, 33096, 31747] avg=32768.0000 max=33096 max_rank=0 min=31747 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=12999 max_load_reduction_pct=0.282005 moved_to_replicas=26231 active_clone_links=10 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=1 total_expected_recv=31747 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=31747 required_capacity=0.968842 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=1 total_expected_recv=31937 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=31937 required_capacity=0.974640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=1 total_expected_recv=32980 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=32980 required_capacity=1.006470 + [2026-09-21 07:39:11] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 126.8 | throughput per GPU (TFLOP/s/GPU): 279.0 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.493599E+01 | loss scale: 1.0 | grad norm: 83.780 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34816 | min safe capacity: 1.0625 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=34816 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 33096, 33096, 32114, 32694, 32230, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=34816 expected_recv=32230 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 33096, 33096, 32114, 32694, 32230, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 33096, 33096, 32114, 32694, 32230, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 33096, 33096, 32114, 32694, 32230, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=34816 expected_recv=32720 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 33096, 33096, 32114, 32694, 32230, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=34816 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 33096, 33096, 32114, 32694, 32230, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=34816 expected_recv=32114 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 33096, 33096, 32114, 32694, 32230, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=34816 expected_recv=33098 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=34816 expected_recv=32230 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=34816 expected_recv=32720 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 33096, 33096, 32114, 32694, 32230, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=34816 expected_recv=32694 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=34816 expected_recv=32114 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[37276, 44107, 33528, 29674, 28256, 32694, 28965, 27644] avg=32768.0000 max=44107 max_rank=1 min=27644 min_rank=7 imbalance_ratio=1.346039 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[37276, 44107, 33528, 29674, 28256, 32694, 28965, 27644] avg=32768.0000 max=44107 max_rank=1 min=27644 min_rank=7 imbalance_ratio=1.346039 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[37276, 44107, 33528, 29674, 28256, 32694, 28965, 27644] avg=32768.0000 max=44107 max_rank=1 min=27644 min_rank=7 imbalance_ratio=1.346039 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[37276, 44107, 33528, 29674, 28256, 32694, 28965, 27644] avg=32768.0000 max=44107 max_rank=1 min=27644 min_rank=7 imbalance_ratio=1.346039 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[37276, 44107, 33528, 29674, 28256, 32694, 28965, 27644] avg=32768.0000 max=44107 max_rank=1 min=27644 min_rank=7 imbalance_ratio=1.346039 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[37276, 44107, 33528, 29674, 28256, 32694, 28965, 27644] avg=32768.0000 max=44107 max_rank=1 min=27644 min_rank=7 imbalance_ratio=1.346039 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[37276, 44107, 33528, 29674, 28256, 32694, 28965, 27644] avg=32768.0000 max=44107 max_rank=1 min=27644 min_rank=7 imbalance_ratio=1.346039 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[33098, 33096, 33096, 33096, 32114, 32694, 32230, 32720] avg=32768.0000 max=33098 max_rank=0 min=32114 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=11009 max_load_reduction_pct=0.249598 moved_to_replicas=15623 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[33098, 33096, 33096, 33096, 32114, 32694, 32230, 32720] avg=32768.0000 max=33098 max_rank=0 min=32114 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=11009 max_load_reduction_pct=0.249598 moved_to_replicas=15623 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[37276, 44107, 33528, 29674, 28256, 32694, 28965, 27644] avg=32768.0000 max=44107 max_rank=1 min=27644 min_rank=7 imbalance_ratio=1.346039 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[33098, 33096, 33096, 33096, 32114, 32694, 32230, 32720] avg=32768.0000 max=33098 max_rank=0 min=32114 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=11009 max_load_reduction_pct=0.249598 moved_to_replicas=15623 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[33098, 33096, 33096, 33096, 32114, 32694, 32230, 32720] avg=32768.0000 max=33098 max_rank=0 min=32114 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=11009 max_load_reduction_pct=0.249598 moved_to_replicas=15623 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[33098, 33096, 33096, 33096, 32114, 32694, 32230, 32720] avg=32768.0000 max=33098 max_rank=0 min=32114 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=11009 max_load_reduction_pct=0.249598 moved_to_replicas=15623 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[33098, 33096, 33096, 33096, 32114, 32694, 32230, 32720] avg=32768.0000 max=33098 max_rank=0 min=32114 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=11009 max_load_reduction_pct=0.249598 moved_to_replicas=15623 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[33098, 33096, 33096, 33096, 32114, 32694, 32230, 32720] avg=32768.0000 max=33098 max_rank=0 min=32114 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=11009 max_load_reduction_pct=0.249598 moved_to_replicas=15623 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[33098, 33096, 33096, 33096, 32114, 32694, 32230, 32720] avg=32768.0000 max=33098 max_rank=0 min=32114 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=11009 max_load_reduction_pct=0.249598 moved_to_replicas=15623 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=1 total_expected_recv=33098 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=1 total_expected_recv=32230 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=32230 required_capacity=0.983582 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=1 total_expected_recv=32720 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=32720 required_capacity=0.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=1 total_expected_recv=32694 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=32694 required_capacity=0.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=1 total_expected_recv=32114 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=32114 required_capacity=0.980042 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 + [2026-09-21 07:39:11] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 128.7 | throughput per GPU (TFLOP/s/GPU): 274.9 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.142802E+01 | loss scale: 1.0 | grad norm: 26.678 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34816 | min safe capacity: 1.0625 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32209, 32023, 33096, 33096, 32431, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=34816 expected_recv=32431 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32209, 32023, 33096, 33096, 32431, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32209, 32023, 33096, 33096, 32431, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32209, 32023, 33096, 33096, 32431, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32209, 32023, 33096, 33096, 32431, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=34816 expected_recv=32209 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32209, 32023, 33096, 33096, 32431, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32209, 32023, 33096, 33096, 32431, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=34816 expected_recv=32431 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=34816 expected_recv=32023 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32209, 32023, 33096, 33096, 32431, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=34816 expected_recv=32209 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=34816 expected_recv=32023 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[39700, 40090, 28980, 31487, 29223, 33607, 27938, 31119] avg=32768.0000 max=40090 max_rank=1 min=27938 min_rank=6 imbalance_ratio=1.223450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[39700, 40090, 28980, 31487, 29223, 33607, 27938, 31119] avg=32768.0000 max=40090 max_rank=1 min=27938 min_rank=6 imbalance_ratio=1.223450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[39700, 40090, 28980, 31487, 29223, 33607, 27938, 31119] avg=32768.0000 max=40090 max_rank=1 min=27938 min_rank=6 imbalance_ratio=1.223450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[39700, 40090, 28980, 31487, 29223, 33607, 27938, 31119] avg=32768.0000 max=40090 max_rank=1 min=27938 min_rank=6 imbalance_ratio=1.223450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[39700, 40090, 28980, 31487, 29223, 33607, 27938, 31119] avg=32768.0000 max=40090 max_rank=1 min=27938 min_rank=6 imbalance_ratio=1.223450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[39700, 40090, 28980, 31487, 29223, 33607, 27938, 31119] avg=32768.0000 max=40090 max_rank=1 min=27938 min_rank=6 imbalance_ratio=1.223450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[39700, 40090, 28980, 31487, 29223, 33607, 27938, 31119] avg=32768.0000 max=40090 max_rank=1 min=27938 min_rank=6 imbalance_ratio=1.223450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[39700, 40090, 28980, 31487, 29223, 33607, 27938, 31119] avg=32768.0000 max=40090 max_rank=1 min=27938 min_rank=6 imbalance_ratio=1.223450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[33097, 33096, 32209, 32023, 33096, 33096, 32431, 33096] avg=32768.0000 max=33097 max_rank=0 min=32023 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=6993 max_load_reduction_pct=0.174433 moved_to_replicas=14109 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[33097, 33096, 32209, 32023, 33096, 33096, 32431, 33096] avg=32768.0000 max=33097 max_rank=0 min=32023 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=6993 max_load_reduction_pct=0.174433 moved_to_replicas=14109 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[33097, 33096, 32209, 32023, 33096, 33096, 32431, 33096] avg=32768.0000 max=33097 max_rank=0 min=32023 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=6993 max_load_reduction_pct=0.174433 moved_to_replicas=14109 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[33097, 33096, 32209, 32023, 33096, 33096, 32431, 33096] avg=32768.0000 max=33097 max_rank=0 min=32023 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=6993 max_load_reduction_pct=0.174433 moved_to_replicas=14109 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[33097, 33096, 32209, 32023, 33096, 33096, 32431, 33096] avg=32768.0000 max=33097 max_rank=0 min=32023 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=6993 max_load_reduction_pct=0.174433 moved_to_replicas=14109 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[33097, 33096, 32209, 32023, 33096, 33096, 32431, 33096] avg=32768.0000 max=33097 max_rank=0 min=32023 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=6993 max_load_reduction_pct=0.174433 moved_to_replicas=14109 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[33097, 33096, 32209, 32023, 33096, 33096, 32431, 33096] avg=32768.0000 max=33097 max_rank=0 min=32023 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=6993 max_load_reduction_pct=0.174433 moved_to_replicas=14109 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[33097, 33096, 32209, 32023, 33096, 33096, 32431, 33096] avg=32768.0000 max=33097 max_rank=0 min=32023 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=6993 max_load_reduction_pct=0.174433 moved_to_replicas=14109 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=1 total_expected_recv=33097 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=1 total_expected_recv=32431 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=32431 required_capacity=0.989716 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=1 total_expected_recv=32209 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=32209 required_capacity=0.982941 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=1 total_expected_recv=32023 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=32023 required_capacity=0.977264 + [2026-09-21 07:39:11] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 126.4 | throughput per GPU (TFLOP/s/GPU): 279.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.166129E+01 | loss scale: 1.0 | grad norm: 12.390 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34816 | min safe capacity: 1.0625 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32186, 32878, 33096, 33096, 31928, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32186, 32878, 33096, 33096, 31928, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=34816 expected_recv=31928 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32186, 32878, 33096, 33096, 31928, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32186, 32878, 33096, 33096, 31928, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32186, 32878, 33096, 33096, 31928, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=34816 expected_recv=32878 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32186, 32878, 33096, 33096, 31928, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=34816 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32186, 32878, 33096, 33096, 31928, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=34816 expected_recv=31928 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=34816 expected_recv=32186 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32186, 32878, 33096, 33096, 31928, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=34816 expected_recv=32878 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=34816 expected_recv=32768 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=34816 expected_recv=32186 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[39332, 38248, 28314, 31201, 30387, 34244, 28798, 31620] avg=32768.0000 max=39332 max_rank=0 min=28314 min_rank=2 imbalance_ratio=1.200317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[39332, 38248, 28314, 31201, 30387, 34244, 28798, 31620] avg=32768.0000 max=39332 max_rank=0 min=28314 min_rank=2 imbalance_ratio=1.200317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[39332, 38248, 28314, 31201, 30387, 34244, 28798, 31620] avg=32768.0000 max=39332 max_rank=0 min=28314 min_rank=2 imbalance_ratio=1.200317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[39332, 38248, 28314, 31201, 30387, 34244, 28798, 31620] avg=32768.0000 max=39332 max_rank=0 min=28314 min_rank=2 imbalance_ratio=1.200317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[39332, 38248, 28314, 31201, 30387, 34244, 28798, 31620] avg=32768.0000 max=39332 max_rank=0 min=28314 min_rank=2 imbalance_ratio=1.200317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[39332, 38248, 28314, 31201, 30387, 34244, 28798, 31620] avg=32768.0000 max=39332 max_rank=0 min=28314 min_rank=2 imbalance_ratio=1.200317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[39332, 38248, 28314, 31201, 30387, 34244, 28798, 31620] avg=32768.0000 max=39332 max_rank=0 min=28314 min_rank=2 imbalance_ratio=1.200317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[33096, 33096, 32186, 32878, 33096, 33096, 31928, 32768] avg=32768.0000 max=33096 max_rank=0 min=31928 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=6236 max_load_reduction_pct=0.158548 moved_to_replicas=12536 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[39332, 38248, 28314, 31201, 30387, 34244, 28798, 31620] avg=32768.0000 max=39332 max_rank=0 min=28314 min_rank=2 imbalance_ratio=1.200317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[33096, 33096, 32186, 32878, 33096, 33096, 31928, 32768] avg=32768.0000 max=33096 max_rank=0 min=31928 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=6236 max_load_reduction_pct=0.158548 moved_to_replicas=12536 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[33096, 33096, 32186, 32878, 33096, 33096, 31928, 32768] avg=32768.0000 max=33096 max_rank=0 min=31928 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=6236 max_load_reduction_pct=0.158548 moved_to_replicas=12536 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[33096, 33096, 32186, 32878, 33096, 33096, 31928, 32768] avg=32768.0000 max=33096 max_rank=0 min=31928 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=6236 max_load_reduction_pct=0.158548 moved_to_replicas=12536 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[33096, 33096, 32186, 32878, 33096, 33096, 31928, 32768] avg=32768.0000 max=33096 max_rank=0 min=31928 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=6236 max_load_reduction_pct=0.158548 moved_to_replicas=12536 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[33096, 33096, 32186, 32878, 33096, 33096, 31928, 32768] avg=32768.0000 max=33096 max_rank=0 min=31928 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=6236 max_load_reduction_pct=0.158548 moved_to_replicas=12536 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[33096, 33096, 32186, 32878, 33096, 33096, 31928, 32768] avg=32768.0000 max=33096 max_rank=0 min=31928 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=6236 max_load_reduction_pct=0.158548 moved_to_replicas=12536 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[33096, 33096, 32186, 32878, 33096, 33096, 31928, 32768] avg=32768.0000 max=33096 max_rank=0 min=31928 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=6236 max_load_reduction_pct=0.158548 moved_to_replicas=12536 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=1 total_expected_recv=31928 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=31928 required_capacity=0.974365 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=1 total_expected_recv=32878 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=32878 required_capacity=1.003357 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=1 total_expected_recv=32768 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=32768 required_capacity=1.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=1 total_expected_recv=32186 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=32186 required_capacity=0.982239 + [2026-09-21 07:39:11] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 125.3 | throughput per GPU (TFLOP/s/GPU): 282.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.129806E+01 | loss scale: 1.0 | grad norm: 10.486 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34816 | min safe capacity: 1.0625 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32434, 32790, 33095, 33096, 31858, 32680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=34816 expected_recv=31858 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32434, 32790, 33095, 33096, 31858, 32680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=34816 expected_recv=32434 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32434, 32790, 33095, 33096, 31858, 32680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32434, 32790, 33095, 33096, 31858, 32680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=34816 expected_recv=32680 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32434, 32790, 33095, 33096, 31858, 32680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32434, 32790, 33095, 33096, 31858, 32680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=34816 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32434, 32790, 33095, 33096, 31858, 32680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=34816 expected_recv=31858 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32434, 32790, 33095, 33096, 31858, 32680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=34816 expected_recv=32434 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=34816 expected_recv=32680 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34560 slack_slots=256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=34816 expected_recv=32790 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[38379, 37490, 29222, 32790, 30340, 33926, 28147, 31850] avg=32768.0000 max=38379 max_rank=0 min=28147 min_rank=6 imbalance_ratio=1.171234 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[38379, 37490, 29222, 32790, 30340, 33926, 28147, 31850] avg=32768.0000 max=38379 max_rank=0 min=28147 min_rank=6 imbalance_ratio=1.171234 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[38379, 37490, 29222, 32790, 30340, 33926, 28147, 31850] avg=32768.0000 max=38379 max_rank=0 min=28147 min_rank=6 imbalance_ratio=1.171234 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[38379, 37490, 29222, 32790, 30340, 33926, 28147, 31850] avg=32768.0000 max=38379 max_rank=0 min=28147 min_rank=6 imbalance_ratio=1.171234 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[38379, 37490, 29222, 32790, 30340, 33926, 28147, 31850] avg=32768.0000 max=38379 max_rank=0 min=28147 min_rank=6 imbalance_ratio=1.171234 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[38379, 37490, 29222, 32790, 30340, 33926, 28147, 31850] avg=32768.0000 max=38379 max_rank=0 min=28147 min_rank=6 imbalance_ratio=1.171234 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[33095, 33096, 32434, 32790, 33095, 33096, 31858, 32680] avg=32768.0000 max=33096 max_rank=1 min=31858 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=5283 max_load_reduction_pct=0.137653 moved_to_replicas=10507 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[33095, 33096, 32434, 32790, 33095, 33096, 31858, 32680] avg=32768.0000 max=33096 max_rank=1 min=31858 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=5283 max_load_reduction_pct=0.137653 moved_to_replicas=10507 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[38379, 37490, 29222, 32790, 30340, 33926, 28147, 31850] avg=32768.0000 max=38379 max_rank=0 min=28147 min_rank=6 imbalance_ratio=1.171234 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[38379, 37490, 29222, 32790, 30340, 33926, 28147, 31850] avg=32768.0000 max=38379 max_rank=0 min=28147 min_rank=6 imbalance_ratio=1.171234 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[33095, 33096, 32434, 32790, 33095, 33096, 31858, 32680] avg=32768.0000 max=33096 max_rank=1 min=31858 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=5283 max_load_reduction_pct=0.137653 moved_to_replicas=10507 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[33095, 33096, 32434, 32790, 33095, 33096, 31858, 32680] avg=32768.0000 max=33096 max_rank=1 min=31858 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=5283 max_load_reduction_pct=0.137653 moved_to_replicas=10507 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[33095, 33096, 32434, 32790, 33095, 33096, 31858, 32680] avg=32768.0000 max=33096 max_rank=1 min=31858 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=5283 max_load_reduction_pct=0.137653 moved_to_replicas=10507 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[33095, 33096, 32434, 32790, 33095, 33096, 31858, 32680] avg=32768.0000 max=33096 max_rank=1 min=31858 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=5283 max_load_reduction_pct=0.137653 moved_to_replicas=10507 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[33095, 33096, 32434, 32790, 33095, 33096, 31858, 32680] avg=32768.0000 max=33096 max_rank=1 min=31858 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=5283 max_load_reduction_pct=0.137653 moved_to_replicas=10507 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=1 total_expected_recv=33095 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33095 required_capacity=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=1 total_expected_recv=31858 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=256 peak_demand=31858 required_capacity=0.972229 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[33095, 33096, 32434, 32790, 33095, 33096, 31858, 32680] avg=32768.0000 max=33096 max_rank=1 min=31858 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=5283 max_load_reduction_pct=0.137653 moved_to_replicas=10507 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=1 total_expected_recv=32434 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=32434 required_capacity=0.989807 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=1 total_expected_recv=32680 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=32680 required_capacity=0.997314 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=1 total_expected_recv=33095 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33095 required_capacity=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=1 total_expected_recv=32790 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=32790 required_capacity=1.000671 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 + [2026-09-21 07:39:12] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 122.7 | throughput per GPU (TFLOP/s/GPU): 288.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.095494E+01 | loss scale: 1.0 | grad norm: 9.303 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34816 | min safe capacity: 1.0625 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32435, 33096, 32042, 33096, 32540, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=34816 expected_recv=32435 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32435, 33096, 32042, 33096, 32540, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=34816 expected_recv=32540 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32435, 33096, 32042, 33096, 32540, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=34816 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32435, 33096, 32042, 33096, 32540, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32435, 33096, 32042, 33096, 32540, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32435, 33096, 32042, 33096, 32540, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=34816 expected_recv=32042 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32435, 33096, 32042, 33096, 32540, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32435, 33096, 32042, 33096, 32540, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=34816 expected_recv=32435 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=34816 expected_recv=32540 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=34816 expected_recv=32742 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=34816 expected_recv=32042 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[38647, 36890, 28896, 33258, 28925, 34380, 29690, 31458] avg=32768.0000 max=38647 max_rank=0 min=28896 min_rank=2 imbalance_ratio=1.179413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[38647, 36890, 28896, 33258, 28925, 34380, 29690, 31458] avg=32768.0000 max=38647 max_rank=0 min=28896 min_rank=2 imbalance_ratio=1.179413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[38647, 36890, 28896, 33258, 28925, 34380, 29690, 31458] avg=32768.0000 max=38647 max_rank=0 min=28896 min_rank=2 imbalance_ratio=1.179413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[38647, 36890, 28896, 33258, 28925, 34380, 29690, 31458] avg=32768.0000 max=38647 max_rank=0 min=28896 min_rank=2 imbalance_ratio=1.179413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[38647, 36890, 28896, 33258, 28925, 34380, 29690, 31458] avg=32768.0000 max=38647 max_rank=0 min=28896 min_rank=2 imbalance_ratio=1.179413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[38647, 36890, 28896, 33258, 28925, 34380, 29690, 31458] avg=32768.0000 max=38647 max_rank=0 min=28896 min_rank=2 imbalance_ratio=1.179413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[38647, 36890, 28896, 33258, 28925, 34380, 29690, 31458] avg=32768.0000 max=38647 max_rank=0 min=28896 min_rank=2 imbalance_ratio=1.179413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[38647, 36890, 28896, 33258, 28925, 34380, 29690, 31458] avg=32768.0000 max=38647 max_rank=0 min=28896 min_rank=2 imbalance_ratio=1.179413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[33097, 33096, 32435, 33096, 32042, 33096, 32540, 32742] avg=32768.0000 max=33097 max_rank=0 min=32042 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=5550 max_load_reduction_pct=0.143608 moved_to_replicas=10791 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[33097, 33096, 32435, 33096, 32042, 33096, 32540, 32742] avg=32768.0000 max=33097 max_rank=0 min=32042 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=5550 max_load_reduction_pct=0.143608 moved_to_replicas=10791 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[33097, 33096, 32435, 33096, 32042, 33096, 32540, 32742] avg=32768.0000 max=33097 max_rank=0 min=32042 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=5550 max_load_reduction_pct=0.143608 moved_to_replicas=10791 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[33097, 33096, 32435, 33096, 32042, 33096, 32540, 32742] avg=32768.0000 max=33097 max_rank=0 min=32042 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=5550 max_load_reduction_pct=0.143608 moved_to_replicas=10791 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[33097, 33096, 32435, 33096, 32042, 33096, 32540, 32742] avg=32768.0000 max=33097 max_rank=0 min=32042 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=5550 max_load_reduction_pct=0.143608 moved_to_replicas=10791 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[33097, 33096, 32435, 33096, 32042, 33096, 32540, 32742] avg=32768.0000 max=33097 max_rank=0 min=32042 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=5550 max_load_reduction_pct=0.143608 moved_to_replicas=10791 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[33097, 33096, 32435, 33096, 32042, 33096, 32540, 32742] avg=32768.0000 max=33097 max_rank=0 min=32042 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=5550 max_load_reduction_pct=0.143608 moved_to_replicas=10791 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[33097, 33096, 32435, 33096, 32042, 33096, 32540, 32742] avg=32768.0000 max=33097 max_rank=0 min=32042 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=5550 max_load_reduction_pct=0.143608 moved_to_replicas=10791 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=1 total_expected_recv=33097 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=1 total_expected_recv=32435 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=32435 required_capacity=0.989838 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=1 total_expected_recv=32742 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=32742 required_capacity=0.999207 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=1 total_expected_recv=32540 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=32540 required_capacity=0.993042 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=1 total_expected_recv=32042 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=32042 required_capacity=0.977844 + [2026-09-21 07:39:12] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 122.0 | throughput per GPU (TFLOP/s/GPU): 289.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.068848E+01 | loss scale: 1.0 | grad norm: 8.651 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34816 | min safe capacity: 1.0625 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32226, 33096, 31889, 33096, 32549, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=34816 expected_recv=32549 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32226, 33096, 31889, 33096, 32549, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=34816 expected_recv=32226 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32226, 33096, 31889, 33096, 32549, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32226, 33096, 31889, 33096, 32549, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32226, 33096, 31889, 33096, 32549, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32226, 33096, 31889, 33096, 32549, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=34816 expected_recv=32549 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32226, 33096, 31889, 33096, 32549, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=34816 expected_recv=32226 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[37315, 35878, 29524, 33708, 29284, 36047, 29279, 31109] avg=32768.0000 max=37315 max_rank=0 min=29279 min_rank=6 imbalance_ratio=1.138763 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[37315, 35878, 29524, 33708, 29284, 36047, 29279, 31109] avg=32768.0000 max=37315 max_rank=0 min=29279 min_rank=6 imbalance_ratio=1.138763 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[37315, 35878, 29524, 33708, 29284, 36047, 29279, 31109] avg=32768.0000 max=37315 max_rank=0 min=29279 min_rank=6 imbalance_ratio=1.138763 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[37315, 35878, 29524, 33708, 29284, 36047, 29279, 31109] avg=32768.0000 max=37315 max_rank=0 min=29279 min_rank=6 imbalance_ratio=1.138763 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[37315, 35878, 29524, 33708, 29284, 36047, 29279, 31109] avg=32768.0000 max=37315 max_rank=0 min=29279 min_rank=6 imbalance_ratio=1.138763 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[37315, 35878, 29524, 33708, 29284, 36047, 29279, 31109] avg=32768.0000 max=37315 max_rank=0 min=29279 min_rank=6 imbalance_ratio=1.138763 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 32226, 33096, 31889, 33096, 32549, 33096] avg=32768.0000 max=33096 max_rank=0 min=31889 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=4219 max_load_reduction_pct=0.113064 moved_to_replicas=10564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[37315, 35878, 29524, 33708, 29284, 36047, 29279, 31109] avg=32768.0000 max=37315 max_rank=0 min=29279 min_rank=6 imbalance_ratio=1.138763 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 32226, 33096, 31889, 33096, 32549, 33096] avg=32768.0000 max=33096 max_rank=0 min=31889 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=4219 max_load_reduction_pct=0.113064 moved_to_replicas=10564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 32226, 33096, 31889, 33096, 32549, 33096] avg=32768.0000 max=33096 max_rank=0 min=31889 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=4219 max_load_reduction_pct=0.113064 moved_to_replicas=10564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 32226, 33096, 31889, 33096, 32549, 33096] avg=32768.0000 max=33096 max_rank=0 min=31889 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=4219 max_load_reduction_pct=0.113064 moved_to_replicas=10564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 32226, 33096, 31889, 33096, 32549, 33096] avg=32768.0000 max=33096 max_rank=0 min=31889 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=4219 max_load_reduction_pct=0.113064 moved_to_replicas=10564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 32226, 33096, 31889, 33096, 32549, 33096] avg=32768.0000 max=33096 max_rank=0 min=31889 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=4219 max_load_reduction_pct=0.113064 moved_to_replicas=10564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 32226, 33096, 31889, 33096, 32549, 33096] avg=32768.0000 max=33096 max_rank=0 min=31889 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=4219 max_load_reduction_pct=0.113064 moved_to_replicas=10564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=1 total_expected_recv=32549 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=32549 required_capacity=0.993317 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=1 total_expected_recv=32226 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=32226 required_capacity=0.983459 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=34816 expected_recv=31889 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32226, 33096, 31889, 33096, 32549, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=34816 expected_recv=31889 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=34816 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[37315, 35878, 29524, 33708, 29284, 36047, 29279, 31109] avg=32768.0000 max=37315 max_rank=0 min=29279 min_rank=6 imbalance_ratio=1.138763 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 32226, 33096, 31889, 33096, 32549, 33096] avg=32768.0000 max=33096 max_rank=0 min=31889 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=4219 max_load_reduction_pct=0.113064 moved_to_replicas=10564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=1 total_expected_recv=31889 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=0 peak_demand=31889 required_capacity=0.973175 + [2026-09-21 07:39:12] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 123.6 | throughput per GPU (TFLOP/s/GPU): 286.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.046317E+01 | loss scale: 1.0 | grad norm: 9.772 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34816 | min safe capacity: 1.0625 | +[after training is done] datetime: 2026-09-21 07:39:12 +[rank0]:[W921 07:39:13.539891932 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0921 07:39:14.366000 479764 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0921 07:39:14.367000 479764 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 479871 closing signal SIGTERM +W0921 07:39:14.367000 479764 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 479872 closing signal SIGTERM +W0921 07:39:14.369000 479764 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 479873 closing signal SIGTERM +W0921 07:39:14.369000 479764 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 479874 closing signal SIGTERM +W0921 07:39:14.370000 479764 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 479875 closing signal SIGTERM +W0921 07:39:14.371000 479764 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 479876 closing signal SIGTERM +W0921 07:39:14.372000 479764 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 479877 closing signal SIGTERM +W0921 07:39:14.373000 479764 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 479878 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 479764 got signal: 15 diff --git a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25/rep1/mem_samples.csv b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..fe32710633803933b40750582e636ec41cc169de --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25/rep1/mem_samples.csv @@ -0,0 +1,192 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 819 +2, 821 +3, 819 +4, 821 +5, 819 +6, 821 +7, 819 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 2359 +1, 2359 +2, 2359 +3, 2359 +4, 2359 +5, 2359 +6, 2359 +7, 2359 +0, 4289 +1, 4289 +2, 4289 +3, 4289 +4, 4289 +5, 4289 +6, 4289 +7, 4289 +0, 50801 +1, 50799 +2, 50805 +3, 50793 +4, 50873 +5, 50849 +6, 50833 +7, 50821 +0, 60251 +1, 60229 +2, 60227 +3, 60251 +4, 60283 +5, 60233 +6, 60251 +7, 60283 +0, 61249 +1, 61251 +2, 61249 +3, 61249 +4, 61249 +5, 61249 +6, 61249 +7, 61249 +0, 61249 +1, 61251 +2, 61249 +3, 61249 +4, 61249 +5, 61249 +6, 61249 +7, 61249 +0, 62641 +1, 62771 +2, 62539 +3, 62351 +4, 62669 +5, 62583 +6, 62901 +7, 62569 +0, 64591 +1, 64593 +2, 64591 +3, 64591 +4, 64591 +5, 64591 +6, 64591 +7, 64591 +0, 66091 +1, 66093 +2, 66091 +3, 66091 +4, 66091 +5, 66091 +6, 66091 +7, 66091 +0, 66091 +1, 66093 +2, 66091 +3, 66091 +4, 66091 +5, 66091 +6, 66091 +7, 66091 +0, 66097 +1, 66097 +2, 66097 +3, 66097 +4, 66097 +5, 66097 +6, 66097 +7, 66097 +0, 68339 +1, 68391 +2, 68415 +3, 68365 +4, 68389 +5, 68369 +6, 68379 +7, 68479 +0, 69327 +1, 69319 +2, 69333 +3, 69339 +4, 69299 +5, 69331 +6, 69339 +7, 69301 +0, 75875 +1, 75897 +2, 75931 +3, 75891 +4, 75883 +5, 75935 +6, 75953 +7, 75893 +0, 89825 +1, 89813 +2, 89811 +3, 89825 +4, 89825 +5, 89811 +6, 89825 +7, 89825 +0, 15748 +1, 88591 +2, 88589 +3, 88603 +4, 88603 +5, 88589 +6, 88603 +7, 88603 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25/rep1/run.env b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..9f99bf61d036077a2fecad5d76c4930668c1e6fb --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25/rep1/run.env @@ -0,0 +1,102 @@ +backend=ultraep +config=KIMI_K2_L2 +family=tierB +source_model=Kimi-K2 +standard_num_layers=61 +base_rung= +repo=/workspace/Megatron-UltraEP +branch=ultraep-token-drop-analysis +commit=eaf829e7 +num_layers=2 +hidden=7168 +ffn_hidden=18432 +moe_ffn_hidden=2048 +num_experts=384 +topk=8 +experts_per_rank=48 +expert_params=44040192 +heads=64 +kv_groups=64 +kv_channels=64 +qk_layernorm=1 +shared_ffn=2048 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=1 +q_lora_rank=1536 +kv_lora_rank=512 +qk_head_dim=128 +qk_pos_emb_head_dim=64 +v_head_dim=128 +dense_first_layers=1 +moe_layers=1 +rotary_base=50000 +rotary_percent=1.0 +rotary_scaling=32 +mscale=1.0 +mscale_all_dim=1.0 +qkv_bias=0 +router_balancing=seq_aux_loss +aux_loss_coeff=0 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +router_groups=1 +router_group_topk=1 +router_score_function=sigmoid +router_expert_bias=1 +router_bias_update_rate=1e-3 +router_topk_scaling_factor=2.827 +shared_expert_gate=0 +apply_layernorm_1p=0 +attention_output_gate=0 +linear_attention_type= +linear_attention_freq= +linear_conv_kernel_dim= +linear_key_head_dim= +linear_value_head_dim= +linear_num_key_heads= +linear_num_value_heads= +mtp_num_layers=0 +mtp_loss_scaling_factor=0.1 +no_weight_decay_cond_type= +max_position_embeddings=4096 +init_method_std=0.02 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=682 +slot_policy=matched +slots_per_rank=48 +capacity_mode=static +capacity=1.25 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=85 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25/rep1/train.log b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..589f3e3b87ba6de82adb2913366d4b63022f2e0a --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25/rep1/train.log @@ -0,0 +1,1652 @@ +[07:37:08] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[07:37:08] WARN: ultraep-token-drop-analysis lacks --moe-router-fusion; KIMI_K2_L2 runs unfused (timing only, routing identical) +============================================================== + KIMI_K2_L2 / ultraep (slots/rank=48 policy=matched) + E=384 K=8 H'=2048 m=48 + L=2 seq=4096 gbs=8 (1 microbatches) + cap_mode=static cap=1.25 cg=0 recompute=none + memory_dump=none snapshots=/workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ eaf829e7 +============================================================== +W0921 07:37:10.264000 476709 torch/distributed/run.py:766] +W0921 07:37:10.264000 476709 torch/distributed/run.py:766] ***************************************** +W0921 07:37:10.264000 476709 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0921 07:37:10.264000 476709 torch/distributed/run.py:766] ***************************************** +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 2 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 18432 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... False + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 7168 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.02 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 512 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 4096 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 2048 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 1] + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 48 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... True + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... 1 + moe_router_load_balancing_type .................. seq_aux_loss + moe_router_num_groups ........................... 1 + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... sigmoid + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. 2.827 + moe_shared_expert_intermediate_size ............. 2048 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 1.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... True + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 64 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 384 + num_layers ...................................... 2 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 1 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... 1536 + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 50000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 32.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank [Gloo] Rank 0 is connected to 47 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 5 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 02 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank [Gloo] Rank +76 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank6]:[W921 07:37:27.972696174 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W921 07:37:27.972695846 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W921 07:37:27.976048242 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W921 07:37:27.978837798 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W921 07:37:27.980722565 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W921 07:37:27.980750122 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W921 07:37:27.980793375 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.035 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W921 07:37:27.128308590 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.923 seconds +time to initialize megatron (seconds): 7.404 +[after megatron is initialized] datetime: 2026-09-21 07:37:33 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 5594455040 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 9 +Params for bucket 1 (360579072 elements, 360579072 padded size): + module.output_layer.weight +Params for bucket 2 (44047360 elements, 44047360 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight +Params for bucket 3 (45163520 elements, 45163520 padded size): + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.weight + module.decoder.layers.1.self_attention.linear_q_down_proj.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_q_up_proj.weight + module.decoder.layers.1.self_attention.linear_q_up_proj.layer_norm_weight +Params for bucket 4 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight +Params for bucket 5 (132127744 elements, 132127744 padded size): + module.decoder.layers.1.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc2.weight +Params for bucket 6 (264241152 elements, 264241152 padded size): + module.decoder.layers.0.mlp.linear_fc1.weight +Params for bucket 7 (42411008 elements, 42411008 padded size): + module.decoder.layers.0.self_attention.linear_q_up_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.0.self_attention.linear_q_up_proj.layer_norm_weight + module.decoder.layers.0.self_attention.linear_q_down_proj.weight + module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight +Params for bucket 8 (58720256 elements, 58720256 padded size): + module.decoder.layers.0.self_attention.linear_proj.weight +Params for bucket 9 (360586240 elements, 360586240 padded size): + module.decoder.layers.0.input_layernorm.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight47 + module.decoder.layers.1.mlp.experts.linear_fc2.weight45 + module.decoder.layers.1.mlp.experts.linear_fc2.weight46 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight44 + module.decoder.layers.1.mlp.experts.linear_fc2.weight43 + module.decoder.layers.1.mlp.experts.linear_fc2.weight42 +Params for bucket 3 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight41 + module.decoder.layers.1.mlp.experts.linear_fc2.weight40 + module.decoder.layers.1.mlp.experts.linear_fc2.weight39 +Params for bucket 4 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight38 + module.decoder.layers.1.mlp.experts.linear_fc2.weight37 + module.decoder.layers.1.mlp.experts.linear_fc2.weight36 +Params for bucket 5 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight35 + module.decoder.layers.1.mlp.experts.linear_fc2.weight34 + module.decoder.layers.1.mlp.experts.linear_fc2.weight33 +Params for bucket 6 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight32 + module.decoder.layers.1.mlp.experts.linear_fc2.weight31 + module.decoder.layers.1.mlp.experts.linear_fc2.weight30 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight29 + module.decoder.layers.1.mlp.experts.linear_fc2.weight28 + module.decoder.layers.1.mlp.experts.linear_fc2.weight27 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight26 + module.decoder.layers.1.mlp.experts.linear_fc2.weight24 + module.decoder.layers.1.mlp.experts.linear_fc2.weight25 +Params for bucket 9 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight23 + module.decoder.layers.1.mlp.experts.linear_fc2.weight21 + module.decoder.layers.1.mlp.experts.linear_fc2.weight22 +Params for bucket 10 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight20 + module.decoder.layers.1.mlp.experts.linear_fc2.weight18 + module.decoder.layers.1.mlp.experts.linear_fc2.weight19 +Params for bucket 11 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight17 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight16 +Params for bucket 12 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 +Params for bucket 15 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 +Params for bucket 16 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 +Params for bucket 17 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight47 + module.decoder.layers.1.mlp.experts.linear_fc1.weight46 +Params for bucket 18 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight45 + module.decoder.layers.1.mlp.experts.linear_fc1.weight44 +Params for bucket 19 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight43 + module.decoder.layers.1.mlp.experts.linear_fc1.weight42 +Params for bucket 20 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight41 + module.decoder.layers.1.mlp.experts.linear_fc1.weight40 +Params for bucket 21 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight39 + module.decoder.layers.1.mlp.experts.linear_fc1.weight38 +Params for bucket 22 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight37 + module.decoder.layers.1.mlp.experts.linear_fc1.weight36 +Params for bucket 23 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight35 + module.decoder.layers.1.mlp.experts.linear_fc1.weight34 +Params for bucket 24 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight33 + module.decoder.layers.1.mlp.experts.linear_fc1.weight32 +Params for bucket 25 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight31 + module.decoder.layers.1.mlp.experts.linear_fc1.weight30 +Params for bucket 26 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight29 + module.decoder.layers.1.mlp.experts.linear_fc1.weight28 +Params for bucket 27 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight27 + module.decoder.layers.1.mlp.experts.linear_fc1.weight26 +Params for bucket 28 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight25 + module.decoder.layers.1.mlp.experts.linear_fc1.weight24 +Params for bucket 29 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight23 + module.decoder.layers.1.mlp.experts.linear_fc1.weight22 +Params for bucket 30 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight21 + module.decoder.layers.1.mlp.experts.linear_fc1.weight20 +Params for bucket 31 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight19 + module.decoder.layers.1.mlp.experts.linear_fc1.weight18 +Params for bucket 32 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight17 + module.decoder.layers.1.mlp.experts.linear_fc1.weight16 +Params for bucket 33 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 +Params for bucket 34 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 +Params for bucket 35 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 +Params for bucket 36 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 +Params for bucket 37 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 +Params for bucket 38 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 +Params for bucket 39 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 +Params for bucket 40 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-09-21 07:37:36 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-09-21 07:37:36 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3430.29, 3433.79) + train/valid/test-data-iterators-setup ..........: (74.26, 94.45) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-09-21 07:37:36 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33098 actual_valid_recv=35712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35712 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=40960 expected_recv=31766 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[28774, 33214, 27953, 31005, 35336, 30433, 39027, 36402] avg=32768.0000 max=39027 max_rank=6 min=27953 min_rank=2 imbalance_ratio=1.191010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=40960 expected_recv=31766 actual_valid_recv=34560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=35840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] avg=32768.0000 max=33098 max_rank=0 min=31766 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=5929 max_load_reduction_pct=0.151920 moved_to_replicas=11595 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34560 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=35968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35840 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=36608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=35968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35968 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35968 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36608 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=1 total_expected_recv=33098 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5248 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[28774, 33214, 27953, 31005, 35336, 30433, 39027, 36402] avg=32768.0000 max=39027 max_rank=6 min=27953 min_rank=2 imbalance_ratio=1.191010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[28774, 33214, 27953, 31005, 35336, 30433, 39027, 36402] avg=32768.0000 max=39027 max_rank=6 min=27953 min_rank=2 imbalance_ratio=1.191010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[28774, 33214, 27953, 31005, 35336, 30433, 39027, 36402] avg=32768.0000 max=39027 max_rank=6 min=27953 min_rank=2 imbalance_ratio=1.191010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[28774, 33214, 27953, 31005, 35336, 30433, 39027, 36402] avg=32768.0000 max=39027 max_rank=6 min=27953 min_rank=2 imbalance_ratio=1.191010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[28774, 33214, 27953, 31005, 35336, 30433, 39027, 36402] avg=32768.0000 max=39027 max_rank=6 min=27953 min_rank=2 imbalance_ratio=1.191010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[28774, 33214, 27953, 31005, 35336, 30433, 39027, 36402] avg=32768.0000 max=39027 max_rank=6 min=27953 min_rank=2 imbalance_ratio=1.191010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] avg=32768.0000 max=33098 max_rank=0 min=31766 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=5929 max_load_reduction_pct=0.151920 moved_to_replicas=11595 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] avg=32768.0000 max=33098 max_rank=0 min=31766 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=5929 max_load_reduction_pct=0.151920 moved_to_replicas=11595 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] avg=32768.0000 max=33098 max_rank=0 min=31766 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=5929 max_load_reduction_pct=0.151920 moved_to_replicas=11595 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] avg=32768.0000 max=33098 max_rank=0 min=31766 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=5929 max_load_reduction_pct=0.151920 moved_to_replicas=11595 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] avg=32768.0000 max=33098 max_rank=0 min=31766 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=5929 max_load_reduction_pct=0.151920 moved_to_replicas=11595 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] avg=32768.0000 max=33098 max_rank=0 min=31766 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=5929 max_load_reduction_pct=0.151920 moved_to_replicas=11595 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=1 total_expected_recv=31766 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=6400 peak_demand=31766 required_capacity=0.969421 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5120 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4992 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=1 total_expected_recv=33095 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4992 peak_demand=33095 required_capacity=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=1 total_expected_recv=33095 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4352 peak_demand=33095 required_capacity=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=40960 expected_recv=31802 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=40960 expected_recv=31802 actual_valid_recv=35200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35200 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[28774, 33214, 27953, 31005, 35336, 30433, 39027, 36402] avg=32768.0000 max=39027 max_rank=6 min=27953 min_rank=2 imbalance_ratio=1.191010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[33098, 33096, 31802, 31766, 33095, 33096, 33095, 33096] avg=32768.0000 max=33098 max_rank=0 min=31766 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=5929 max_load_reduction_pct=0.151920 moved_to_replicas=11595 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=1 total_expected_recv=31802 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5760 peak_demand=31802 required_capacity=0.970520 + [2026-09-21 07:38:05] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 28648.3 | throughput per GPU (TFLOP/s/GPU): 1.2 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.227924E+01 | loss scale: 1.0 | grad norm: 12.617 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36224 | min safe capacity: 1.1055 | +Number of parameters in transformer block in billions: 17.55 +Number of parameters in embedding layers in billions: 0.72 +Total number of parameters in billions: 18.28 +Number of parameters in most loaded shard in billions: 18.2753 +Theoretical memory footprints: weight and optimizer=130715.23 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 49120.48779296875 | max allocated: 49120.48828125 | reserved: 54522.0 | max reserved: 54522.0 +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=40960 expected_recv=31670 expected_overflow=0 exact_recv_per_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=40960 expected_recv=31670 actual_valid_recv=34944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=35840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32236 expected_overflow=0 exact_recv_per_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34944 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35840 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=36096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32236 actual_valid_recv=35072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36096 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35072 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=36096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[29410, 33844, 28405, 30208, 34500, 31179, 38439, 36159] avg=32768.0000 max=38439 max_rank=6 min=28405 min_rank=2 imbalance_ratio=1.173065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[29410, 33844, 28405, 30208, 34500, 31179, 38439, 36159] avg=32768.0000 max=38439 max_rank=6 min=28405 min_rank=2 imbalance_ratio=1.173065 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36096 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[29410, 33844, 28405, 30208, 34500, 31179, 38439, 36159] avg=32768.0000 max=38439 max_rank=6 min=28405 min_rank=2 imbalance_ratio=1.173065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[29410, 33844, 28405, 30208, 34500, 31179, 38439, 36159] avg=32768.0000 max=38439 max_rank=6 min=28405 min_rank=2 imbalance_ratio=1.173065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[29410, 33844, 28405, 30208, 34500, 31179, 38439, 36159] avg=32768.0000 max=38439 max_rank=6 min=28405 min_rank=2 imbalance_ratio=1.173065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[29410, 33844, 28405, 30208, 34500, 31179, 38439, 36159] avg=32768.0000 max=38439 max_rank=6 min=28405 min_rank=2 imbalance_ratio=1.173065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31670 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=5342 max_load_reduction_pct=0.138973 moved_to_replicas=10558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31670 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=5342 max_load_reduction_pct=0.138973 moved_to_replicas=10558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[29410, 33844, 28405, 30208, 34500, 31179, 38439, 36159] avg=32768.0000 max=38439 max_rank=6 min=28405 min_rank=2 imbalance_ratio=1.173065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31670 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=5342 max_load_reduction_pct=0.138973 moved_to_replicas=10558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31670 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=5342 max_load_reduction_pct=0.138973 moved_to_replicas=10558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31670 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=5342 max_load_reduction_pct=0.138973 moved_to_replicas=10558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[29410, 33844, 28405, 30208, 34500, 31179, 38439, 36159] avg=32768.0000 max=38439 max_rank=6 min=28405 min_rank=2 imbalance_ratio=1.173065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31670 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=5342 max_load_reduction_pct=0.138973 moved_to_replicas=10558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=1 total_expected_recv=32757 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5120 peak_demand=32757 required_capacity=0.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=1 total_expected_recv=31670 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=6016 peak_demand=31670 required_capacity=0.966492 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31670 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=5342 max_load_reduction_pct=0.138973 moved_to_replicas=10558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[31670, 33096, 32236, 33097, 33097, 32757, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31670 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=5342 max_load_reduction_pct=0.138973 moved_to_replicas=10558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=1 total_expected_recv=33097 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=1 total_expected_recv=33097 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4864 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=1 total_expected_recv=32236 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5888 peak_demand=32236 required_capacity=0.983765 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=1 total_expected_recv=33095 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4864 peak_demand=33095 required_capacity=1.009979 + [2026-09-21 07:38:05] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 189.5 | throughput per GPU (TFLOP/s/GPU): 186.7 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.227088E+01 | loss scale: 1.0 | grad norm: 12.534 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36224 | min safe capacity: 1.1055 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32357 expected_overflow=0 exact_recv_per_rank=[32357, 33096, 32696, 33096, 33096, 31611, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=40960 expected_recv=31611 expected_overflow=0 exact_recv_per_rank=[32357, 33096, 32696, 33096, 33096, 31611, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32357, 33096, 32696, 33096, 33096, 31611, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32357, 33096, 32696, 33096, 33096, 31611, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32357, 33096, 32696, 33096, 33096, 31611, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32357, 33096, 32696, 33096, 33096, 31611, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32357, 33096, 32696, 33096, 33096, 31611, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32357 actual_valid_recv=35456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=40960 expected_recv=31611 actual_valid_recv=35200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32696 expected_overflow=0 exact_recv_per_rank=[32357, 33096, 32696, 33096, 33096, 31611, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=35840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35456 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35200 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=36096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=36096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35840 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36096 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36096 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32696 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[29011, 33745, 29470, 30817, 35499, 30838, 37051, 35713] avg=32768.0000 max=37051 max_rank=6 min=29011 min_rank=0 imbalance_ratio=1.130707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[29011, 33745, 29470, 30817, 35499, 30838, 37051, 35713] avg=32768.0000 max=37051 max_rank=6 min=29011 min_rank=0 imbalance_ratio=1.130707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[29011, 33745, 29470, 30817, 35499, 30838, 37051, 35713] avg=32768.0000 max=37051 max_rank=6 min=29011 min_rank=0 imbalance_ratio=1.130707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[29011, 33745, 29470, 30817, 35499, 30838, 37051, 35713] avg=32768.0000 max=37051 max_rank=6 min=29011 min_rank=0 imbalance_ratio=1.130707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[29011, 33745, 29470, 30817, 35499, 30838, 37051, 35713] avg=32768.0000 max=37051 max_rank=6 min=29011 min_rank=0 imbalance_ratio=1.130707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[29011, 33745, 29470, 30817, 35499, 30838, 37051, 35713] avg=32768.0000 max=37051 max_rank=6 min=29011 min_rank=0 imbalance_ratio=1.130707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[29011, 33745, 29470, 30817, 35499, 30838, 37051, 35713] avg=32768.0000 max=37051 max_rank=6 min=29011 min_rank=0 imbalance_ratio=1.130707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[32357, 33096, 32696, 33096, 33096, 31611, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31611 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=3954 max_load_reduction_pct=0.106718 moved_to_replicas=9624 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[32357, 33096, 32696, 33096, 33096, 31611, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31611 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=3954 max_load_reduction_pct=0.106718 moved_to_replicas=9624 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[29011, 33745, 29470, 30817, 35499, 30838, 37051, 35713] avg=32768.0000 max=37051 max_rank=6 min=29011 min_rank=0 imbalance_ratio=1.130707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[32357, 33096, 32696, 33096, 33096, 31611, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31611 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=3954 max_load_reduction_pct=0.106718 moved_to_replicas=9624 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[32357, 33096, 32696, 33096, 33096, 31611, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31611 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=3954 max_load_reduction_pct=0.106718 moved_to_replicas=9624 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[32357, 33096, 32696, 33096, 33096, 31611, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31611 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=3954 max_load_reduction_pct=0.106718 moved_to_replicas=9624 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[32357, 33096, 32696, 33096, 33096, 31611, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31611 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=3954 max_load_reduction_pct=0.106718 moved_to_replicas=9624 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[32357, 33096, 32696, 33096, 33096, 31611, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31611 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=3954 max_load_reduction_pct=0.106718 moved_to_replicas=9624 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[32357, 33096, 32696, 33096, 33096, 31611, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31611 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=3954 max_load_reduction_pct=0.106718 moved_to_replicas=9624 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=1 total_expected_recv=32357 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5504 peak_demand=32357 required_capacity=0.987457 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=1 total_expected_recv=31611 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5760 peak_demand=31611 required_capacity=0.964691 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=1 total_expected_recv=33095 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=33095 required_capacity=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5120 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4864 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=1 total_expected_recv=33097 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4864 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=1 total_expected_recv=32696 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=32696 required_capacity=0.997803 + [2026-09-21 07:38:05] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 122.5 | throughput per GPU (TFLOP/s/GPU): 288.7 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.088328E+01 | loss scale: 1.0 | grad norm: 11.023 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36224 | min safe capacity: 1.1055 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=40960 expected_recv=31238 expected_overflow=0 exact_recv_per_rank=[33096, 32580, 33096, 32944, 33095, 31238, 32999, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32944 expected_overflow=0 exact_recv_per_rank=[33096, 32580, 33096, 32944, 33095, 31238, 32999, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32999 expected_overflow=0 exact_recv_per_rank=[33096, 32580, 33096, 32944, 33095, 31238, 32999, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32580, 33096, 32944, 33095, 31238, 32999, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32580 expected_overflow=0 exact_recv_per_rank=[33096, 32580, 33096, 32944, 33095, 31238, 32999, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32580, 33096, 32944, 33095, 31238, 32999, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32580, 33096, 32944, 33095, 31238, 32999, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=40960 expected_recv=31238 actual_valid_recv=34432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32944 actual_valid_recv=35840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32580, 33096, 32944, 33095, 31238, 32999, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32999 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=36608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32580 actual_valid_recv=35584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34432 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=35968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35840 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=35968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36608 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35584 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35968 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35968 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[27013, 32580, 35975, 32944, 33937, 29906, 30120, 39669] avg=32768.0000 max=39669 max_rank=7 min=27013 min_rank=0 imbalance_ratio=1.210602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[27013, 32580, 35975, 32944, 33937, 29906, 30120, 39669] avg=32768.0000 max=39669 max_rank=7 min=27013 min_rank=0 imbalance_ratio=1.210602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[27013, 32580, 35975, 32944, 33937, 29906, 30120, 39669] avg=32768.0000 max=39669 max_rank=7 min=27013 min_rank=0 imbalance_ratio=1.210602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[27013, 32580, 35975, 32944, 33937, 29906, 30120, 39669] avg=32768.0000 max=39669 max_rank=7 min=27013 min_rank=0 imbalance_ratio=1.210602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[27013, 32580, 35975, 32944, 33937, 29906, 30120, 39669] avg=32768.0000 max=39669 max_rank=7 min=27013 min_rank=0 imbalance_ratio=1.210602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[27013, 32580, 35975, 32944, 33937, 29906, 30120, 39669] avg=32768.0000 max=39669 max_rank=7 min=27013 min_rank=0 imbalance_ratio=1.210602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[27013, 32580, 35975, 32944, 33937, 29906, 30120, 39669] avg=32768.0000 max=39669 max_rank=7 min=27013 min_rank=0 imbalance_ratio=1.210602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[33096, 32580, 33096, 32944, 33095, 31238, 32999, 33096] avg=32768.0000 max=33096 max_rank=0 min=31238 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=6573 max_load_reduction_pct=0.165696 moved_to_replicas=10293 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[27013, 32580, 35975, 32944, 33937, 29906, 30120, 39669] avg=32768.0000 max=39669 max_rank=7 min=27013 min_rank=0 imbalance_ratio=1.210602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[33096, 32580, 33096, 32944, 33095, 31238, 32999, 33096] avg=32768.0000 max=33096 max_rank=0 min=31238 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=6573 max_load_reduction_pct=0.165696 moved_to_replicas=10293 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[33096, 32580, 33096, 32944, 33095, 31238, 32999, 33096] avg=32768.0000 max=33096 max_rank=0 min=31238 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=6573 max_load_reduction_pct=0.165696 moved_to_replicas=10293 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[33096, 32580, 33096, 32944, 33095, 31238, 32999, 33096] avg=32768.0000 max=33096 max_rank=0 min=31238 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=6573 max_load_reduction_pct=0.165696 moved_to_replicas=10293 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[33096, 32580, 33096, 32944, 33095, 31238, 32999, 33096] avg=32768.0000 max=33096 max_rank=0 min=31238 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=6573 max_load_reduction_pct=0.165696 moved_to_replicas=10293 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[33096, 32580, 33096, 32944, 33095, 31238, 32999, 33096] avg=32768.0000 max=33096 max_rank=0 min=31238 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=6573 max_load_reduction_pct=0.165696 moved_to_replicas=10293 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[33096, 32580, 33096, 32944, 33095, 31238, 32999, 33096] avg=32768.0000 max=33096 max_rank=0 min=31238 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=6573 max_load_reduction_pct=0.165696 moved_to_replicas=10293 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[33096, 32580, 33096, 32944, 33095, 31238, 32999, 33096] avg=32768.0000 max=33096 max_rank=0 min=31238 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=6573 max_load_reduction_pct=0.165696 moved_to_replicas=10293 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=1 total_expected_recv=31238 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=6528 peak_demand=31238 required_capacity=0.953308 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4352 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=1 total_expected_recv=32944 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5120 peak_demand=32944 required_capacity=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=1 total_expected_recv=32999 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=32999 required_capacity=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4992 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=1 total_expected_recv=32580 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5376 peak_demand=32580 required_capacity=0.994263 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=1 total_expected_recv=33095 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4992 peak_demand=33095 required_capacity=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=33096 required_capacity=1.010010 + [2026-09-21 07:38:05] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 119.8 | throughput per GPU (TFLOP/s/GPU): 295.4 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.448181E+01 | loss scale: 1.0 | grad norm: 72.911 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36608 | min safe capacity: 1.1172 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 32010, 33096, 31557, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32010 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 32010, 33096, 31557, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 32010, 33096, 31557, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 32010, 33096, 31557, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=40960 expected_recv=31557 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 32010, 33096, 31557, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 32010, 33096, 31557, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 32010, 33096, 31557, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 32010, 33096, 31557, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32010 actual_valid_recv=35712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=36096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=36736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=40960 expected_recv=31557 actual_valid_recv=34944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35712 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=36736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36096 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36736 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34944 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36736 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[43539, 33599, 41781, 31508, 30819, 28282, 23523, 29093] avg=32768.0000 max=43539 max_rank=0 min=23523 min_rank=6 imbalance_ratio=1.328705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[43539, 33599, 41781, 31508, 30819, 28282, 23523, 29093] avg=32768.0000 max=43539 max_rank=0 min=23523 min_rank=6 imbalance_ratio=1.328705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[43539, 33599, 41781, 31508, 30819, 28282, 23523, 29093] avg=32768.0000 max=43539 max_rank=0 min=23523 min_rank=6 imbalance_ratio=1.328705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[43539, 33599, 41781, 31508, 30819, 28282, 23523, 29093] avg=32768.0000 max=43539 max_rank=0 min=23523 min_rank=6 imbalance_ratio=1.328705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33097, 33095, 32010, 33096, 31557, 33096, 33097] avg=32768.0000 max=33097 max_rank=1 min=31557 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10442 max_load_reduction_pct=0.239831 moved_to_replicas=19631 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[43539, 33599, 41781, 31508, 30819, 28282, 23523, 29093] avg=32768.0000 max=43539 max_rank=0 min=23523 min_rank=6 imbalance_ratio=1.328705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[43539, 33599, 41781, 31508, 30819, 28282, 23523, 29093] avg=32768.0000 max=43539 max_rank=0 min=23523 min_rank=6 imbalance_ratio=1.328705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[43539, 33599, 41781, 31508, 30819, 28282, 23523, 29093] avg=32768.0000 max=43539 max_rank=0 min=23523 min_rank=6 imbalance_ratio=1.328705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33097, 33095, 32010, 33096, 31557, 33096, 33097] avg=32768.0000 max=33097 max_rank=1 min=31557 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10442 max_load_reduction_pct=0.239831 moved_to_replicas=19631 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[43539, 33599, 41781, 31508, 30819, 28282, 23523, 29093] avg=32768.0000 max=43539 max_rank=0 min=23523 min_rank=6 imbalance_ratio=1.328705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33097, 33095, 32010, 33096, 31557, 33096, 33097] avg=32768.0000 max=33097 max_rank=1 min=31557 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10442 max_load_reduction_pct=0.239831 moved_to_replicas=19631 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33097, 33095, 32010, 33096, 31557, 33096, 33097] avg=32768.0000 max=33097 max_rank=1 min=31557 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10442 max_load_reduction_pct=0.239831 moved_to_replicas=19631 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33097, 33095, 32010, 33096, 31557, 33096, 33097] avg=32768.0000 max=33097 max_rank=1 min=31557 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10442 max_load_reduction_pct=0.239831 moved_to_replicas=19631 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33097, 33095, 32010, 33096, 31557, 33096, 33097] avg=32768.0000 max=33097 max_rank=1 min=31557 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10442 max_load_reduction_pct=0.239831 moved_to_replicas=19631 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33097, 33095, 32010, 33096, 31557, 33096, 33097] avg=32768.0000 max=33097 max_rank=1 min=31557 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10442 max_load_reduction_pct=0.239831 moved_to_replicas=19631 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33097, 33095, 32010, 33096, 31557, 33096, 33097] avg=32768.0000 max=33097 max_rank=1 min=31557 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=10442 max_load_reduction_pct=0.239831 moved_to_replicas=19631 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=1 total_expected_recv=32010 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5248 peak_demand=32010 required_capacity=0.976868 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=1 total_expected_recv=33097 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4864 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=1 total_expected_recv=31557 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=6016 peak_demand=31557 required_capacity=0.963043 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=1 total_expected_recv=33097 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4224 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4224 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=1 total_expected_recv=33095 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=33095 required_capacity=1.009979 + [2026-09-21 07:38:05] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 124.0 | throughput per GPU (TFLOP/s/GPU): 285.3 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.808459E+01 | loss scale: 1.0 | grad norm: 108.268 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32057 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33097, 32057, 32988, 31619] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33097, 32057, 32988, 31619] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33097, 32057, 32988, 31619] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=40960 expected_recv=31619 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33097, 32057, 32988, 31619] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33097, 32057, 32988, 31619] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32988 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33097, 32057, 32988, 31619] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33097, 32057, 32988, 31619] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32057 actual_valid_recv=35200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33097, 32057, 32988, 31619] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=40960 expected_recv=31619 actual_valid_recv=35200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35200 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32988 actual_valid_recv=36736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35200 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36736 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[45405, 40911, 38777, 30397, 26999, 30951, 22281, 26423] avg=32768.0000 max=45405 max_rank=0 min=22281 min_rank=6 imbalance_ratio=1.385651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[45405, 40911, 38777, 30397, 26999, 30951, 22281, 26423] avg=32768.0000 max=45405 max_rank=0 min=22281 min_rank=6 imbalance_ratio=1.385651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[45405, 40911, 38777, 30397, 26999, 30951, 22281, 26423] avg=32768.0000 max=45405 max_rank=0 min=22281 min_rank=6 imbalance_ratio=1.385651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[45405, 40911, 38777, 30397, 26999, 30951, 22281, 26423] avg=32768.0000 max=45405 max_rank=0 min=22281 min_rank=6 imbalance_ratio=1.385651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[45405, 40911, 38777, 30397, 26999, 30951, 22281, 26423] avg=32768.0000 max=45405 max_rank=0 min=22281 min_rank=6 imbalance_ratio=1.385651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[45405, 40911, 38777, 30397, 26999, 30951, 22281, 26423] avg=32768.0000 max=45405 max_rank=0 min=22281 min_rank=6 imbalance_ratio=1.385651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33097, 32057, 32988, 31619] avg=32768.0000 max=33097 max_rank=4 min=31619 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12308 max_load_reduction_pct=0.271071 moved_to_replicas=25805 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[45405, 40911, 38777, 30397, 26999, 30951, 22281, 26423] avg=32768.0000 max=45405 max_rank=0 min=22281 min_rank=6 imbalance_ratio=1.385651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[45405, 40911, 38777, 30397, 26999, 30951, 22281, 26423] avg=32768.0000 max=45405 max_rank=0 min=22281 min_rank=6 imbalance_ratio=1.385651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33097, 32057, 32988, 31619] avg=32768.0000 max=33097 max_rank=4 min=31619 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12308 max_load_reduction_pct=0.271071 moved_to_replicas=25805 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33097, 32057, 32988, 31619] avg=32768.0000 max=33097 max_rank=4 min=31619 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12308 max_load_reduction_pct=0.271071 moved_to_replicas=25805 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33097, 32057, 32988, 31619] avg=32768.0000 max=33097 max_rank=4 min=31619 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12308 max_load_reduction_pct=0.271071 moved_to_replicas=25805 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33097, 32057, 32988, 31619] avg=32768.0000 max=33097 max_rank=4 min=31619 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12308 max_load_reduction_pct=0.271071 moved_to_replicas=25805 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33097, 32057, 32988, 31619] avg=32768.0000 max=33097 max_rank=4 min=31619 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12308 max_load_reduction_pct=0.271071 moved_to_replicas=25805 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=1 total_expected_recv=32057 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5760 peak_demand=32057 required_capacity=0.978302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33097, 32057, 32988, 31619] avg=32768.0000 max=33097 max_rank=4 min=31619 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12308 max_load_reduction_pct=0.271071 moved_to_replicas=25805 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33097, 32057, 32988, 31619] avg=32768.0000 max=33097 max_rank=4 min=31619 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=12308 max_load_reduction_pct=0.271071 moved_to_replicas=25805 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=1 total_expected_recv=33095 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=33095 required_capacity=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=1 total_expected_recv=31619 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5760 peak_demand=31619 required_capacity=0.964935 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=1 total_expected_recv=32988 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4224 peak_demand=32988 required_capacity=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=1 total_expected_recv=33097 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=33097 required_capacity=1.010040 + [2026-09-21 07:38:06] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 125.7 | throughput per GPU (TFLOP/s/GPU): 281.4 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.491727E+01 | loss scale: 1.0 | grad norm: 83.473 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 32038, 33096, 32168, 32457] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 32038, 33096, 32168, 32457] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 32038, 33096, 32168, 32457] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32168 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 32038, 33096, 32168, 32457] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 32038, 33096, 32168, 32457] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32038 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 32038, 33096, 32168, 32457] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 32038, 33096, 32168, 32457] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=35968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=36096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32457 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 32038, 33096, 32168, 32457] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32168 actual_valid_recv=35456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=35968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35968 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32038 actual_valid_recv=35072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36096 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35456 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35968 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32457 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35072 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[36887, 43890, 33486, 29862, 28534, 33100, 28809, 27576] avg=32768.0000 max=43890 max_rank=1 min=27576 min_rank=7 imbalance_ratio=1.339417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[36887, 43890, 33486, 29862, 28534, 33100, 28809, 27576] avg=32768.0000 max=43890 max_rank=1 min=27576 min_rank=7 imbalance_ratio=1.339417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[36887, 43890, 33486, 29862, 28534, 33100, 28809, 27576] avg=32768.0000 max=43890 max_rank=1 min=27576 min_rank=7 imbalance_ratio=1.339417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[36887, 43890, 33486, 29862, 28534, 33100, 28809, 27576] avg=32768.0000 max=43890 max_rank=1 min=27576 min_rank=7 imbalance_ratio=1.339417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[36887, 43890, 33486, 29862, 28534, 33100, 28809, 27576] avg=32768.0000 max=43890 max_rank=1 min=27576 min_rank=7 imbalance_ratio=1.339417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[36887, 43890, 33486, 29862, 28534, 33100, 28809, 27576] avg=32768.0000 max=43890 max_rank=1 min=27576 min_rank=7 imbalance_ratio=1.339417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[36887, 43890, 33486, 29862, 28534, 33100, 28809, 27576] avg=32768.0000 max=43890 max_rank=1 min=27576 min_rank=7 imbalance_ratio=1.339417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[36887, 43890, 33486, 29862, 28534, 33100, 28809, 27576] avg=32768.0000 max=43890 max_rank=1 min=27576 min_rank=7 imbalance_ratio=1.339417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 32038, 33096, 32168, 32457] avg=32768.0000 max=33097 max_rank=2 min=32038 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=10793 max_load_reduction_pct=0.245910 moved_to_replicas=14979 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 32038, 33096, 32168, 32457] avg=32768.0000 max=33097 max_rank=2 min=32038 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=10793 max_load_reduction_pct=0.245910 moved_to_replicas=14979 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 32038, 33096, 32168, 32457] avg=32768.0000 max=33097 max_rank=2 min=32038 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=10793 max_load_reduction_pct=0.245910 moved_to_replicas=14979 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 32038, 33096, 32168, 32457] avg=32768.0000 max=33097 max_rank=2 min=32038 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=10793 max_load_reduction_pct=0.245910 moved_to_replicas=14979 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 32038, 33096, 32168, 32457] avg=32768.0000 max=33097 max_rank=2 min=32038 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=10793 max_load_reduction_pct=0.245910 moved_to_replicas=14979 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 32038, 33096, 32168, 32457] avg=32768.0000 max=33097 max_rank=2 min=32038 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=10793 max_load_reduction_pct=0.245910 moved_to_replicas=14979 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 32038, 33096, 32168, 32457] avg=32768.0000 max=33097 max_rank=2 min=32038 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=10793 max_load_reduction_pct=0.245910 moved_to_replicas=14979 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 32038, 33096, 32168, 32457] avg=32768.0000 max=33097 max_rank=2 min=32038 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=10793 max_load_reduction_pct=0.245910 moved_to_replicas=14979 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4992 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4864 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=1 total_expected_recv=32168 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5504 peak_demand=32168 required_capacity=0.981689 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=1 total_expected_recv=32038 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5888 peak_demand=32038 required_capacity=0.977722 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=1 total_expected_recv=33097 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4992 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=1 total_expected_recv=32457 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=32457 required_capacity=0.990509 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=33096 required_capacity=1.010010 + [2026-09-21 07:38:06] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 129.2 | throughput per GPU (TFLOP/s/GPU): 273.9 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.143662E+01 | loss scale: 1.0 | grad norm: 26.627 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 32483, 32987, 33096, 32282, 32009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 32483, 32987, 33096, 32282, 32009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 32483, 32987, 33096, 32282, 32009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 32483, 32987, 33096, 32282, 32009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32282 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 32483, 32987, 33096, 32282, 32009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32483 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 32483, 32987, 33096, 32282, 32009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32009 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 32483, 32987, 33096, 32282, 32009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=35968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32987 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 32483, 32987, 33096, 32282, 32009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32282 actual_valid_recv=35072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32483 actual_valid_recv=35328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35968 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32009 actual_valid_recv=35328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35072 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35328 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32987 actual_valid_recv=35840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35328 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35840 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[39628, 39486, 28919, 31588, 29172, 33991, 28016, 31344] avg=32768.0000 max=39628 max_rank=0 min=28016 min_rank=6 imbalance_ratio=1.209351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[39628, 39486, 28919, 31588, 29172, 33991, 28016, 31344] avg=32768.0000 max=39628 max_rank=0 min=28016 min_rank=6 imbalance_ratio=1.209351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[39628, 39486, 28919, 31588, 29172, 33991, 28016, 31344] avg=32768.0000 max=39628 max_rank=0 min=28016 min_rank=6 imbalance_ratio=1.209351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[39628, 39486, 28919, 31588, 29172, 33991, 28016, 31344] avg=32768.0000 max=39628 max_rank=0 min=28016 min_rank=6 imbalance_ratio=1.209351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[39628, 39486, 28919, 31588, 29172, 33991, 28016, 31344] avg=32768.0000 max=39628 max_rank=0 min=28016 min_rank=6 imbalance_ratio=1.209351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[39628, 39486, 28919, 31588, 29172, 33991, 28016, 31344] avg=32768.0000 max=39628 max_rank=0 min=28016 min_rank=6 imbalance_ratio=1.209351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[39628, 39486, 28919, 31588, 29172, 33991, 28016, 31344] avg=32768.0000 max=39628 max_rank=0 min=28016 min_rank=6 imbalance_ratio=1.209351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[33096, 33096, 33095, 32483, 32987, 33096, 32282, 32009] avg=32768.0000 max=33096 max_rank=0 min=32009 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=6532 max_load_reduction_pct=0.164833 moved_to_replicas=13817 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[39628, 39486, 28919, 31588, 29172, 33991, 28016, 31344] avg=32768.0000 max=39628 max_rank=0 min=28016 min_rank=6 imbalance_ratio=1.209351 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[33096, 33096, 33095, 32483, 32987, 33096, 32282, 32009] avg=32768.0000 max=33096 max_rank=0 min=32009 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=6532 max_load_reduction_pct=0.164833 moved_to_replicas=13817 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[33096, 33096, 33095, 32483, 32987, 33096, 32282, 32009] avg=32768.0000 max=33096 max_rank=0 min=32009 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=6532 max_load_reduction_pct=0.164833 moved_to_replicas=13817 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[33096, 33096, 33095, 32483, 32987, 33096, 32282, 32009] avg=32768.0000 max=33096 max_rank=0 min=32009 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=6532 max_load_reduction_pct=0.164833 moved_to_replicas=13817 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[33096, 33096, 33095, 32483, 32987, 33096, 32282, 32009] avg=32768.0000 max=33096 max_rank=0 min=32009 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=6532 max_load_reduction_pct=0.164833 moved_to_replicas=13817 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[33096, 33096, 33095, 32483, 32987, 33096, 32282, 32009] avg=32768.0000 max=33096 max_rank=0 min=32009 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=6532 max_load_reduction_pct=0.164833 moved_to_replicas=13817 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[33096, 33096, 33095, 32483, 32987, 33096, 32282, 32009] avg=32768.0000 max=33096 max_rank=0 min=32009 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=6532 max_load_reduction_pct=0.164833 moved_to_replicas=13817 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[33096, 33096, 33095, 32483, 32987, 33096, 32282, 32009] avg=32768.0000 max=33096 max_rank=0 min=32009 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=6532 max_load_reduction_pct=0.164833 moved_to_replicas=13817 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=1 total_expected_recv=33095 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=33095 required_capacity=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4992 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=1 total_expected_recv=32282 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5888 peak_demand=32282 required_capacity=0.985168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=1 total_expected_recv=32483 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5632 peak_demand=32483 required_capacity=0.991302 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=1 total_expected_recv=32009 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5632 peak_demand=32009 required_capacity=0.976837 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=1 total_expected_recv=32987 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5120 peak_demand=32987 required_capacity=1.006683 + [2026-09-21 07:38:06] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 122.9 | throughput per GPU (TFLOP/s/GPU): 287.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.166943E+01 | loss scale: 1.0 | grad norm: 12.800 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31943, 32655, 33096, 33096, 32787, 32375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31943, 32655, 33096, 33096, 32787, 32375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31943, 32655, 33096, 33096, 32787, 32375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31943, 32655, 33096, 33096, 32787, 32375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=36736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=40960 expected_recv=31943 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31943, 32655, 33096, 33096, 32787, 32375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31943, 32655, 33096, 33096, 32787, 32375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32375 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31943, 32655, 33096, 33096, 32787, 32375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=35712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31943, 32655, 33096, 33096, 32787, 32375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=35968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36736 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=40960 expected_recv=31943 actual_valid_recv=34816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35712 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35968 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32375 actual_valid_recv=35456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=34816 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=36096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35456 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36096 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[38829, 38031, 29123, 31129, 30693, 33847, 28868, 31624] avg=32768.0000 max=38829 max_rank=0 min=28868 min_rank=6 imbalance_ratio=1.184967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[38829, 38031, 29123, 31129, 30693, 33847, 28868, 31624] avg=32768.0000 max=38829 max_rank=0 min=28868 min_rank=6 imbalance_ratio=1.184967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[38829, 38031, 29123, 31129, 30693, 33847, 28868, 31624] avg=32768.0000 max=38829 max_rank=0 min=28868 min_rank=6 imbalance_ratio=1.184967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[38829, 38031, 29123, 31129, 30693, 33847, 28868, 31624] avg=32768.0000 max=38829 max_rank=0 min=28868 min_rank=6 imbalance_ratio=1.184967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[38829, 38031, 29123, 31129, 30693, 33847, 28868, 31624] avg=32768.0000 max=38829 max_rank=0 min=28868 min_rank=6 imbalance_ratio=1.184967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[38829, 38031, 29123, 31129, 30693, 33847, 28868, 31624] avg=32768.0000 max=38829 max_rank=0 min=28868 min_rank=6 imbalance_ratio=1.184967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[33095, 33097, 31943, 32655, 33096, 33096, 32787, 32375] avg=32768.0000 max=33097 max_rank=1 min=31943 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=5732 max_load_reduction_pct=0.147622 moved_to_replicas=11419 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[38829, 38031, 29123, 31129, 30693, 33847, 28868, 31624] avg=32768.0000 max=38829 max_rank=0 min=28868 min_rank=6 imbalance_ratio=1.184967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[33095, 33097, 31943, 32655, 33096, 33096, 32787, 32375] avg=32768.0000 max=33097 max_rank=1 min=31943 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=5732 max_load_reduction_pct=0.147622 moved_to_replicas=11419 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[38829, 38031, 29123, 31129, 30693, 33847, 28868, 31624] avg=32768.0000 max=38829 max_rank=0 min=28868 min_rank=6 imbalance_ratio=1.184967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[33095, 33097, 31943, 32655, 33096, 33096, 32787, 32375] avg=32768.0000 max=33097 max_rank=1 min=31943 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=5732 max_load_reduction_pct=0.147622 moved_to_replicas=11419 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[33095, 33097, 31943, 32655, 33096, 33096, 32787, 32375] avg=32768.0000 max=33097 max_rank=1 min=31943 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=5732 max_load_reduction_pct=0.147622 moved_to_replicas=11419 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[33095, 33097, 31943, 32655, 33096, 33096, 32787, 32375] avg=32768.0000 max=33097 max_rank=1 min=31943 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=5732 max_load_reduction_pct=0.147622 moved_to_replicas=11419 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[33095, 33097, 31943, 32655, 33096, 33096, 32787, 32375] avg=32768.0000 max=33097 max_rank=1 min=31943 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=5732 max_load_reduction_pct=0.147622 moved_to_replicas=11419 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[33095, 33097, 31943, 32655, 33096, 33096, 32787, 32375] avg=32768.0000 max=33097 max_rank=1 min=31943 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=5732 max_load_reduction_pct=0.147622 moved_to_replicas=11419 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=1 total_expected_recv=33095 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4224 peak_demand=33095 required_capacity=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[33095, 33097, 31943, 32655, 33096, 33096, 32787, 32375] avg=32768.0000 max=33097 max_rank=1 min=31943 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=5732 max_load_reduction_pct=0.147622 moved_to_replicas=11419 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=1 total_expected_recv=32655 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5248 peak_demand=32655 required_capacity=0.996552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4992 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=1 total_expected_recv=32787 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=32787 required_capacity=1.000580 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=1 total_expected_recv=31943 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=6144 peak_demand=31943 required_capacity=0.974823 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=1 total_expected_recv=33097 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=1 total_expected_recv=32375 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5504 peak_demand=32375 required_capacity=0.988007 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4864 peak_demand=33096 required_capacity=1.010010 + [2026-09-21 07:38:06] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 123.4 | throughput per GPU (TFLOP/s/GPU): 286.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.131635E+01 | loss scale: 1.0 | grad norm: 10.441 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32760, 32476, 33096, 33096, 32056, 32467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32760, 32476, 33096, 33096, 32056, 32467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32476 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32760, 32476, 33096, 33096, 32056, 32467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32467 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32760, 32476, 33096, 33096, 32056, 32467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32056 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32760, 32476, 33096, 33096, 32056, 32467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32760, 32476, 33096, 33096, 32056, 32467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=36736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=36096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32760, 32476, 33096, 33096, 32056, 32467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 32760, 32476, 33096, 33096, 32056, 32467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32476 actual_valid_recv=35968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36096 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32467 actual_valid_recv=35456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36736 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32056 actual_valid_recv=35072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=35584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35968 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35456 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35072 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35584 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[37910, 37644, 29661, 32476, 30546, 33579, 28344, 31984] avg=32768.0000 max=37910 max_rank=0 min=28344 min_rank=6 imbalance_ratio=1.156921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[37910, 37644, 29661, 32476, 30546, 33579, 28344, 31984] avg=32768.0000 max=37910 max_rank=0 min=28344 min_rank=6 imbalance_ratio=1.156921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[37910, 37644, 29661, 32476, 30546, 33579, 28344, 31984] avg=32768.0000 max=37910 max_rank=0 min=28344 min_rank=6 imbalance_ratio=1.156921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[37910, 37644, 29661, 32476, 30546, 33579, 28344, 31984] avg=32768.0000 max=37910 max_rank=0 min=28344 min_rank=6 imbalance_ratio=1.156921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33097, 32760, 32476, 33096, 33096, 32056, 32467] avg=32768.0000 max=33097 max_rank=1 min=32056 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=4813 max_load_reduction_pct=0.126959 moved_to_replicas=9845 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[37910, 37644, 29661, 32476, 30546, 33579, 28344, 31984] avg=32768.0000 max=37910 max_rank=0 min=28344 min_rank=6 imbalance_ratio=1.156921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[37910, 37644, 29661, 32476, 30546, 33579, 28344, 31984] avg=32768.0000 max=37910 max_rank=0 min=28344 min_rank=6 imbalance_ratio=1.156921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33097, 32760, 32476, 33096, 33096, 32056, 32467] avg=32768.0000 max=33097 max_rank=1 min=32056 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=4813 max_load_reduction_pct=0.126959 moved_to_replicas=9845 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[37910, 37644, 29661, 32476, 30546, 33579, 28344, 31984] avg=32768.0000 max=37910 max_rank=0 min=28344 min_rank=6 imbalance_ratio=1.156921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[37910, 37644, 29661, 32476, 30546, 33579, 28344, 31984] avg=32768.0000 max=37910 max_rank=0 min=28344 min_rank=6 imbalance_ratio=1.156921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33097, 32760, 32476, 33096, 33096, 32056, 32467] avg=32768.0000 max=33097 max_rank=1 min=32056 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=4813 max_load_reduction_pct=0.126959 moved_to_replicas=9845 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33097, 32760, 32476, 33096, 33096, 32056, 32467] avg=32768.0000 max=33097 max_rank=1 min=32056 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=4813 max_load_reduction_pct=0.126959 moved_to_replicas=9845 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33097, 32760, 32476, 33096, 33096, 32056, 32467] avg=32768.0000 max=33097 max_rank=1 min=32056 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=4813 max_load_reduction_pct=0.126959 moved_to_replicas=9845 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33097, 32760, 32476, 33096, 33096, 32056, 32467] avg=32768.0000 max=33097 max_rank=1 min=32056 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=4813 max_load_reduction_pct=0.126959 moved_to_replicas=9845 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4864 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33097, 32760, 32476, 33096, 33096, 32056, 32467] avg=32768.0000 max=33097 max_rank=1 min=32056 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=4813 max_load_reduction_pct=0.126959 moved_to_replicas=9845 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33097, 32760, 32476, 33096, 33096, 32056, 32467] avg=32768.0000 max=33097 max_rank=1 min=32056 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=4813 max_load_reduction_pct=0.126959 moved_to_replicas=9845 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4224 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=1 total_expected_recv=32467 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5504 peak_demand=32467 required_capacity=0.990814 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=1 total_expected_recv=32476 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4992 peak_demand=32476 required_capacity=0.991089 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=1 total_expected_recv=32056 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5888 peak_demand=32056 required_capacity=0.978271 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=1 total_expected_recv=32760 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5376 peak_demand=32760 required_capacity=0.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=1 total_expected_recv=33097 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=33097 required_capacity=1.010040 + [2026-09-21 07:38:06] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 123.3 | throughput per GPU (TFLOP/s/GPU): 286.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.096943E+01 | loss scale: 1.0 | grad norm: 9.147 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32242, 33096, 32782, 33096, 32048, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32048 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32242, 33096, 32782, 33096, 32048, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32688 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32242, 33096, 32782, 33096, 32048, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32242, 33096, 32782, 33096, 32048, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32242, 33096, 32782, 33096, 32048, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32242 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32242, 33096, 32782, 33096, 32048, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32242, 33096, 32782, 33096, 32048, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32048 actual_valid_recv=35456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32242, 33096, 32782, 33096, 32048, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32688 actual_valid_recv=35200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=35712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=35840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32242 actual_valid_recv=35584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35456 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35200 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35712 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35840 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=35712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35584 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35712 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[38265, 36744, 29450, 33443, 29211, 34278, 29247, 31506] avg=32768.0000 max=38265 max_rank=0 min=29211 min_rank=4 imbalance_ratio=1.167755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[38265, 36744, 29450, 33443, 29211, 34278, 29247, 31506] avg=32768.0000 max=38265 max_rank=0 min=29211 min_rank=4 imbalance_ratio=1.167755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[38265, 36744, 29450, 33443, 29211, 34278, 29247, 31506] avg=32768.0000 max=38265 max_rank=0 min=29211 min_rank=4 imbalance_ratio=1.167755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[38265, 36744, 29450, 33443, 29211, 34278, 29247, 31506] avg=32768.0000 max=38265 max_rank=0 min=29211 min_rank=4 imbalance_ratio=1.167755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[38265, 36744, 29450, 33443, 29211, 34278, 29247, 31506] avg=32768.0000 max=38265 max_rank=0 min=29211 min_rank=4 imbalance_ratio=1.167755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[38265, 36744, 29450, 33443, 29211, 34278, 29247, 31506] avg=32768.0000 max=38265 max_rank=0 min=29211 min_rank=4 imbalance_ratio=1.167755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[38265, 36744, 29450, 33443, 29211, 34278, 29247, 31506] avg=32768.0000 max=38265 max_rank=0 min=29211 min_rank=4 imbalance_ratio=1.167755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[33095, 33097, 32242, 33096, 32782, 33096, 32048, 32688] avg=32768.0000 max=33097 max_rank=1 min=32048 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=5168 max_load_reduction_pct=0.135058 moved_to_replicas=10346 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[38265, 36744, 29450, 33443, 29211, 34278, 29247, 31506] avg=32768.0000 max=38265 max_rank=0 min=29211 min_rank=4 imbalance_ratio=1.167755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[33095, 33097, 32242, 33096, 32782, 33096, 32048, 32688] avg=32768.0000 max=33097 max_rank=1 min=32048 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=5168 max_load_reduction_pct=0.135058 moved_to_replicas=10346 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[33095, 33097, 32242, 33096, 32782, 33096, 32048, 32688] avg=32768.0000 max=33097 max_rank=1 min=32048 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=5168 max_load_reduction_pct=0.135058 moved_to_replicas=10346 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[33095, 33097, 32242, 33096, 32782, 33096, 32048, 32688] avg=32768.0000 max=33097 max_rank=1 min=32048 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=5168 max_load_reduction_pct=0.135058 moved_to_replicas=10346 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[33095, 33097, 32242, 33096, 32782, 33096, 32048, 32688] avg=32768.0000 max=33097 max_rank=1 min=32048 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=5168 max_load_reduction_pct=0.135058 moved_to_replicas=10346 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[33095, 33097, 32242, 33096, 32782, 33096, 32048, 32688] avg=32768.0000 max=33097 max_rank=1 min=32048 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=5168 max_load_reduction_pct=0.135058 moved_to_replicas=10346 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[33095, 33097, 32242, 33096, 32782, 33096, 32048, 32688] avg=32768.0000 max=33097 max_rank=1 min=32048 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=5168 max_load_reduction_pct=0.135058 moved_to_replicas=10346 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[33095, 33097, 32242, 33096, 32782, 33096, 32048, 32688] avg=32768.0000 max=33097 max_rank=1 min=32048 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=5168 max_load_reduction_pct=0.135058 moved_to_replicas=10346 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=1 total_expected_recv=33095 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=33095 required_capacity=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5120 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=1 total_expected_recv=32688 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5760 peak_demand=32688 required_capacity=0.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=1 total_expected_recv=32048 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5504 peak_demand=32048 required_capacity=0.978027 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=1 total_expected_recv=32782 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5248 peak_demand=32782 required_capacity=1.000427 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=1 total_expected_recv=33097 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=1 total_expected_recv=32242 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5376 peak_demand=32242 required_capacity=0.983948 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5248 peak_demand=33096 required_capacity=1.010010 + [2026-09-21 07:38:06] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 122.4 | throughput per GPU (TFLOP/s/GPU): 289.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.073911E+01 | loss scale: 1.0 | grad norm: 9.755 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31986, 33096, 33096, 33096, 32629, 32049] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31986, 33096, 33096, 33096, 32629, 32049] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31986, 33096, 33096, 33096, 32629, 32049] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32629 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31986, 33096, 33096, 33096, 32629, 32049] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32049 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31986, 33096, 33096, 33096, 32629, 32049] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31986, 33096, 33096, 33096, 32629, 32049] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=40960 expected_recv=31986 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31986, 33096, 33096, 33096, 32629, 32049] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=35968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=36608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31986, 33096, 33096, 33096, 32629, 32049] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32629 actual_valid_recv=35840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32049 actual_valid_recv=35072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35968 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36608 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=35840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=40960 expected_recv=31986 actual_valid_recv=35328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35840 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35072 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35840 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35328 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[36918, 35655, 29855, 33672, 29804, 35831, 29330, 31079] avg=32768.0000 max=36918 max_rank=0 min=29330 min_rank=6 imbalance_ratio=1.126648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[36918, 35655, 29855, 33672, 29804, 35831, 29330, 31079] avg=32768.0000 max=36918 max_rank=0 min=29330 min_rank=6 imbalance_ratio=1.126648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[36918, 35655, 29855, 33672, 29804, 35831, 29330, 31079] avg=32768.0000 max=36918 max_rank=0 min=29330 min_rank=6 imbalance_ratio=1.126648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[36918, 35655, 29855, 33672, 29804, 35831, 29330, 31079] avg=32768.0000 max=36918 max_rank=0 min=29330 min_rank=6 imbalance_ratio=1.126648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[36918, 35655, 29855, 33672, 29804, 35831, 29330, 31079] avg=32768.0000 max=36918 max_rank=0 min=29330 min_rank=6 imbalance_ratio=1.126648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[36918, 35655, 29855, 33672, 29804, 35831, 29330, 31079] avg=32768.0000 max=36918 max_rank=0 min=29330 min_rank=6 imbalance_ratio=1.126648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[36918, 35655, 29855, 33672, 29804, 35831, 29330, 31079] avg=32768.0000 max=36918 max_rank=0 min=29330 min_rank=6 imbalance_ratio=1.126648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[36918, 35655, 29855, 33672, 29804, 35831, 29330, 31079] avg=32768.0000 max=36918 max_rank=0 min=29330 min_rank=6 imbalance_ratio=1.126648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 31986, 33096, 33096, 33096, 32629, 32049] avg=32768.0000 max=33096 max_rank=0 min=31986 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=3822 max_load_reduction_pct=0.103527 moved_to_replicas=9692 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 31986, 33096, 33096, 33096, 32629, 32049] avg=32768.0000 max=33096 max_rank=0 min=31986 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=3822 max_load_reduction_pct=0.103527 moved_to_replicas=9692 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 31986, 33096, 33096, 33096, 32629, 32049] avg=32768.0000 max=33096 max_rank=0 min=31986 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=3822 max_load_reduction_pct=0.103527 moved_to_replicas=9692 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 31986, 33096, 33096, 33096, 32629, 32049] avg=32768.0000 max=33096 max_rank=0 min=31986 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=3822 max_load_reduction_pct=0.103527 moved_to_replicas=9692 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 31986, 33096, 33096, 33096, 32629, 32049] avg=32768.0000 max=33096 max_rank=0 min=31986 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=3822 max_load_reduction_pct=0.103527 moved_to_replicas=9692 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 31986, 33096, 33096, 33096, 32629, 32049] avg=32768.0000 max=33096 max_rank=0 min=31986 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=3822 max_load_reduction_pct=0.103527 moved_to_replicas=9692 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 31986, 33096, 33096, 33096, 32629, 32049] avg=32768.0000 max=33096 max_rank=0 min=31986 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=3822 max_load_reduction_pct=0.103527 moved_to_replicas=9692 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 31986, 33096, 33096, 33096, 32629, 32049] avg=32768.0000 max=33096 max_rank=0 min=31986 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=3822 max_load_reduction_pct=0.103527 moved_to_replicas=9692 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4352 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4992 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=1 total_expected_recv=32629 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5120 peak_demand=32629 required_capacity=0.995758 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=1 total_expected_recv=32049 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5888 peak_demand=32049 required_capacity=0.978058 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5120 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=1 total_expected_recv=31986 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5632 peak_demand=31986 required_capacity=0.976135 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=33096 required_capacity=1.010010 + [2026-09-21 07:38:06] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 122.8 | throughput per GPU (TFLOP/s/GPU): 288.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.048286E+01 | loss scale: 1.0 | grad norm: 8.843 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 | +[after training is done] datetime: 2026-09-21 07:38:06 +[rank0]:[W921 07:38:08.172325392 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0921 07:38:08.917000 476709 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0921 07:38:08.917000 476709 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 476816 closing signal SIGTERM +W0921 07:38:08.918000 476709 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 476817 closing signal SIGTERM +W0921 07:38:08.919000 476709 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 476818 closing signal SIGTERM +W0921 07:38:08.921000 476709 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 476819 closing signal SIGTERM +W0921 07:38:08.921000 476709 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 476820 closing signal SIGTERM +W0921 07:38:08.922000 476709 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 476821 closing signal SIGTERM +W0921 07:38:08.923000 476709 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 476822 closing signal SIGTERM +W0921 07:38:08.923000 476709 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 476823 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 476709 got signal: 15 diff --git a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/mem_samples.csv b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..85b3b1197b2785ffce021ab674f0cae2c5d5ab49 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/mem_samples.csv @@ -0,0 +1,192 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 2 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 629 +2, 27 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1927 +1, 1927 +2, 1927 +3, 1927 +4, 1927 +5, 1927 +6, 1927 +7, 1927 +0, 3733 +1, 3755 +2, 3739 +3, 3749 +4, 3733 +5, 3739 +6, 3757 +7, 3747 +0, 50559 +1, 50559 +2, 50559 +3, 50559 +4, 50559 +5, 50559 +6, 50559 +7, 50559 +0, 55537 +1, 55537 +2, 55537 +3, 55537 +4, 55537 +5, 55537 +6, 55537 +7, 55537 +0, 60339 +1, 60361 +2, 60379 +3, 60389 +4, 60359 +5, 60379 +6, 60399 +7, 60389 +0, 61249 +1, 61251 +2, 61249 +3, 61249 +4, 61249 +5, 61249 +6, 61249 +7, 61249 +0, 61249 +1, 61251 +2, 61249 +3, 61249 +4, 61249 +5, 61249 +6, 61249 +7, 61249 +0, 64577 +1, 64565 +2, 64591 +3, 63961 +4, 63873 +5, 64563 +6, 63889 +7, 64577 +0, 64577 +1, 64565 +2, 64591 +3, 64577 +4, 64577 +5, 64563 +6, 64577 +7, 64577 +0, 66077 +1, 66065 +2, 66091 +3, 66077 +4, 66077 +5, 66063 +6, 66077 +7, 66077 +0, 66077 +1, 66065 +2, 66091 +3, 66077 +4, 66077 +5, 66063 +6, 66077 +7, 66077 +0, 66083 +1, 66069 +2, 66097 +3, 66083 +4, 66083 +5, 66069 +6, 66083 +7, 66083 +0, 68369 +1, 68413 +2, 68403 +3, 68403 +4, 68359 +5, 68373 +6, 68445 +7, 68395 +0, 69169 +1, 69155 +2, 69183 +3, 69169 +4, 69169 +5, 69155 +6, 69169 +7, 69169 +0, 75897 +1, 75941 +2, 75913 +3, 75923 +4, 75855 +5, 75901 +6, 75959 +7, 75929 +0, 89825 +1, 89813 +2, 89811 +3, 89825 +4, 89825 +5, 89811 +6, 89825 +7, 89825 +0, 15458 +1, 13828 +2, 13828 +3, 13828 +4, 13828 +5, 13828 +6, 14154 +7, 88809 diff --git a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/run.env b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..d95a5b10993fb9df086d3cbedfef1b369737c4c9 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/run.env @@ -0,0 +1,102 @@ +backend=ultraep +config=KIMI_K2_L2 +family=tierB +source_model=Kimi-K2 +standard_num_layers=61 +base_rung= +repo=/workspace/Megatron-UltraEP +branch=ultraep-token-drop-analysis +commit=eaf829e7 +num_layers=2 +hidden=7168 +ffn_hidden=18432 +moe_ffn_hidden=2048 +num_experts=384 +topk=8 +experts_per_rank=48 +expert_params=44040192 +heads=64 +kv_groups=64 +kv_channels=64 +qk_layernorm=1 +shared_ffn=2048 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=1 +q_lora_rank=1536 +kv_lora_rank=512 +qk_head_dim=128 +qk_pos_emb_head_dim=64 +v_head_dim=128 +dense_first_layers=1 +moe_layers=1 +rotary_base=50000 +rotary_percent=1.0 +rotary_scaling=32 +mscale=1.0 +mscale_all_dim=1.0 +qkv_bias=0 +router_balancing=seq_aux_loss +aux_loss_coeff=0 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +router_groups=1 +router_group_topk=1 +router_score_function=sigmoid +router_expert_bias=1 +router_bias_update_rate=1e-3 +router_topk_scaling_factor=2.827 +shared_expert_gate=0 +apply_layernorm_1p=0 +attention_output_gate=0 +linear_attention_type= +linear_attention_freq= +linear_conv_kernel_dim= +linear_key_head_dim= +linear_value_head_dim= +linear_num_key_heads= +linear_num_value_heads= +mtp_num_layers=0 +mtp_loss_scaling_factor=0.1 +no_weight_decay_cond_type= +max_position_embeddings=4096 +init_method_std=0.02 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=682 +slot_policy=matched +slots_per_rank=48 +capacity_mode=static +capacity=1.25 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=1 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=85 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..d6fe6b898851437e906d772b292b344a0354aefd --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log @@ -0,0 +1,1652 @@ +[07:39:18] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[07:39:18] WARN: ultraep-token-drop-analysis lacks --moe-router-fusion; KIMI_K2_L2 runs unfused (timing only, routing identical) +============================================================== + KIMI_K2_L2 / ultraep (slots/rank=48 policy=matched) + E=384 K=8 H'=2048 m=48 + L=2 seq=4096 gbs=8 (1 microbatches) + cap_mode=static cap=1.25 cg=0 recompute=none + memory_dump=none snapshots=/workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ eaf829e7 +============================================================== +W0921 07:39:20.043000 482786 torch/distributed/run.py:766] +W0921 07:39:20.043000 482786 torch/distributed/run.py:766] ***************************************** +W0921 07:39:20.043000 482786 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0921 07:39:20.043000 482786 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 2 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 18432 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... False + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 7168 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.02 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 512 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 4096 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 2048 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 1] + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 48 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... True + moe_router_force_load_balancing ................. True + moe_router_group_topk ........................... 1 + moe_router_load_balancing_type .................. seq_aux_loss + moe_router_num_groups ........................... 1 + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... sigmoid + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. 2.827 + moe_shared_expert_intermediate_size ............. 2048 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 1.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... True + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 64 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 384 + num_layers ...................................... 2 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 1 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... 1536 + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 50000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 32.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 64 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 1274563 is connected to is connected to is connected to is connected to is connected to is connected to is connected to 7777777 peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7777 +777 + + + + + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank1]:[W921 07:39:37.984691918 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W921 07:39:37.984753087 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W921 07:39:37.984778593 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W921 07:39:37.984826508 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W921 07:39:37.984867292 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W921 07:39:37.986372878 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W921 07:39:37.987009388 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.036 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W921 07:39:37.146428976 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.957 seconds +time to initialize megatron (seconds): 7.333 +[after megatron is initialized] datetime: 2026-09-21 07:39:43 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 5594455040 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 9 +Params for bucket 1 (360579072 elements, 360579072 padded size): + module.output_layer.weight +Params for bucket 2 (44047360 elements, 44047360 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight +Params for bucket 3 (45163520 elements, 45163520 padded size): + module.decoder.layers.1.self_attention.linear_q_up_proj.weight + module.decoder.layers.1.self_attention.linear_q_up_proj.layer_norm_weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight + module.decoder.layers.1.self_attention.linear_q_down_proj.weight +Params for bucket 4 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight +Params for bucket 5 (132127744 elements, 132127744 padded size): + module.decoder.layers.1.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc2.weight +Params for bucket 6 (264241152 elements, 264241152 padded size): + module.decoder.layers.0.mlp.linear_fc1.weight +Params for bucket 7 (42411008 elements, 42411008 padded size): + module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.self_attention.linear_q_up_proj.weight + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.0.self_attention.linear_q_up_proj.layer_norm_weight + module.decoder.layers.0.self_attention.linear_q_down_proj.weight +Params for bucket 8 (58720256 elements, 58720256 padded size): + module.decoder.layers.0.self_attention.linear_proj.weight +Params for bucket 9 (360586240 elements, 360586240 padded size): + module.decoder.layers.0.input_layernorm.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight47 + module.decoder.layers.1.mlp.experts.linear_fc2.weight45 + module.decoder.layers.1.mlp.experts.linear_fc2.weight46 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight44 + module.decoder.layers.1.mlp.experts.linear_fc2.weight42 + module.decoder.layers.1.mlp.experts.linear_fc2.weight43 +Params for bucket 3 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight41 + module.decoder.layers.1.mlp.experts.linear_fc2.weight39 + module.decoder.layers.1.mlp.experts.linear_fc2.weight40 +Params for bucket 4 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight38 + module.decoder.layers.1.mlp.experts.linear_fc2.weight36 + module.decoder.layers.1.mlp.experts.linear_fc2.weight37 +Params for bucket 5 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight35 + module.decoder.layers.1.mlp.experts.linear_fc2.weight33 + module.decoder.layers.1.mlp.experts.linear_fc2.weight34 +Params for bucket 6 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight32 + module.decoder.layers.1.mlp.experts.linear_fc2.weight30 + module.decoder.layers.1.mlp.experts.linear_fc2.weight31 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight29 + module.decoder.layers.1.mlp.experts.linear_fc2.weight27 + module.decoder.layers.1.mlp.experts.linear_fc2.weight28 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight26 + module.decoder.layers.1.mlp.experts.linear_fc2.weight25 + module.decoder.layers.1.mlp.experts.linear_fc2.weight24 +Params for bucket 9 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight23 + module.decoder.layers.1.mlp.experts.linear_fc2.weight22 + module.decoder.layers.1.mlp.experts.linear_fc2.weight21 +Params for bucket 10 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight20 + module.decoder.layers.1.mlp.experts.linear_fc2.weight19 + module.decoder.layers.1.mlp.experts.linear_fc2.weight18 +Params for bucket 11 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight17 + module.decoder.layers.1.mlp.experts.linear_fc2.weight16 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 +Params for bucket 12 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 +Params for bucket 15 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 +Params for bucket 16 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 +Params for bucket 17 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight47 + module.decoder.layers.1.mlp.experts.linear_fc1.weight46 +Params for bucket 18 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight45 + module.decoder.layers.1.mlp.experts.linear_fc1.weight44 +Params for bucket 19 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight43 + module.decoder.layers.1.mlp.experts.linear_fc1.weight42 +Params for bucket 20 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight41 + module.decoder.layers.1.mlp.experts.linear_fc1.weight40 +Params for bucket 21 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight39 + module.decoder.layers.1.mlp.experts.linear_fc1.weight38 +Params for bucket 22 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight37 + module.decoder.layers.1.mlp.experts.linear_fc1.weight36 +Params for bucket 23 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight35 + module.decoder.layers.1.mlp.experts.linear_fc1.weight34 +Params for bucket 24 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight33 + module.decoder.layers.1.mlp.experts.linear_fc1.weight32 +Params for bucket 25 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight31 + module.decoder.layers.1.mlp.experts.linear_fc1.weight30 +Params for bucket 26 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight29 + module.decoder.layers.1.mlp.experts.linear_fc1.weight28 +Params for bucket 27 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight27 + module.decoder.layers.1.mlp.experts.linear_fc1.weight26 +Params for bucket 28 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight25 + module.decoder.layers.1.mlp.experts.linear_fc1.weight24 +Params for bucket 29 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight23 + module.decoder.layers.1.mlp.experts.linear_fc1.weight22 +Params for bucket 30 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight21 + module.decoder.layers.1.mlp.experts.linear_fc1.weight20 +Params for bucket 31 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight19 + module.decoder.layers.1.mlp.experts.linear_fc1.weight18 +Params for bucket 32 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight17 + module.decoder.layers.1.mlp.experts.linear_fc1.weight16 +Params for bucket 33 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 +Params for bucket 34 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 +Params for bucket 35 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 +Params for bucket 36 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 +Params for bucket 37 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 +Params for bucket 38 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 +Params for bucket 39 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 +Params for bucket 40 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-09-21 07:39:46 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-09-21 07:39:46 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3392.65, 3396.83) + train/valid/test-data-iterators-setup ..........: (78.09, 93.46) +[before the start of training step] datetime: 2026-09-21 07:39:46 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32672 expected_overflow=0 exact_recv_per_rank=[32672, 32772, 32789, 32840, 32953, 32686, 32844, 32588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32686 expected_overflow=0 exact_recv_per_rank=[32672, 32772, 32789, 32840, 32953, 32686, 32844, 32588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32672, 32772, 32789, 32840, 32953, 32686, 32844, 32588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32672 actual_valid_recv=36736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32953 expected_overflow=0 exact_recv_per_rank=[32672, 32772, 32789, 32840, 32953, 32686, 32844, 32588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32844 expected_overflow=0 exact_recv_per_rank=[32672, 32772, 32789, 32840, 32953, 32686, 32844, 32588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32672, 32772, 32789, 32840, 32953, 32686, 32844, 32588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32588 expected_overflow=0 exact_recv_per_rank=[32672, 32772, 32789, 32840, 32953, 32686, 32844, 32588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36736 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=36736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32686 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32844 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32953 actual_valid_recv=36864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36736 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32588 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36864 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[32672, 32772, 32789, 32840, 32953, 32686, 32844, 32588] avg=32768.0000 max=32953 max_rank=4 min=32588 min_rank=7 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[32672, 32772, 32789, 32840, 32953, 32686, 32844, 32588] avg=32768.0000 max=32953 max_rank=4 min=32588 min_rank=7 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[32672, 32772, 32789, 32840, 32953, 32686, 32844, 32588] avg=32768.0000 max=32953 max_rank=4 min=32588 min_rank=7 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[32672, 32772, 32789, 32840, 32953, 32686, 32844, 32588] avg=32768.0000 max=32953 max_rank=4 min=32588 min_rank=7 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[32672, 32772, 32789, 32840, 32953, 32686, 32844, 32588] avg=32768.0000 max=32953 max_rank=4 min=32588 min_rank=7 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[32672, 32772, 32789, 32840, 32953, 32686, 32844, 32588] avg=32768.0000 max=32953 max_rank=4 min=32588 min_rank=7 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[32672, 32772, 32789, 32840, 32953, 32686, 32844, 32588] avg=32768.0000 max=32953 max_rank=4 min=32588 min_rank=7 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[32672, 32772, 32789, 32840, 32953, 32686, 32844, 32588] avg=32768.0000 max=32953 max_rank=4 min=32588 min_rank=7 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[32672, 32772, 32789, 32840, 32953, 32686, 32844, 32588] avg=32768.0000 max=32953 max_rank=4 min=32588 min_rank=7 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[32672, 32772, 32789, 32840, 32953, 32686, 32844, 32588] avg=32768.0000 max=32953 max_rank=4 min=32588 min_rank=7 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[32672, 32772, 32789, 32840, 32953, 32686, 32844, 32588] avg=32768.0000 max=32953 max_rank=4 min=32588 min_rank=7 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[32672, 32772, 32789, 32840, 32953, 32686, 32844, 32588] avg=32768.0000 max=32953 max_rank=4 min=32588 min_rank=7 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[32672, 32772, 32789, 32840, 32953, 32686, 32844, 32588] avg=32768.0000 max=32953 max_rank=4 min=32588 min_rank=7 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[32672, 32772, 32789, 32840, 32953, 32686, 32844, 32588] avg=32768.0000 max=32953 max_rank=4 min=32588 min_rank=7 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=1 total_expected_recv=32672 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4224 peak_demand=32672 required_capacity=0.997070 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=1 total_expected_recv=32789 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4224 peak_demand=32789 required_capacity=1.000641 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=1 total_expected_recv=32844 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=32844 required_capacity=1.002319 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=1 total_expected_recv=32686 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=32686 required_capacity=0.997498 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=1 total_expected_recv=32772 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=32772 required_capacity=1.000122 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=1 total_expected_recv=32953 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4096 peak_demand=32953 required_capacity=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=1 total_expected_recv=32588 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=32588 required_capacity=0.994507 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32840 expected_overflow=0 exact_recv_per_rank=[32672, 32772, 32789, 32840, 32953, 32686, 32844, 32588] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32840 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[32672, 32772, 32789, 32840, 32953, 32686, 32844, 32588] avg=32768.0000 max=32953 max_rank=4 min=32588 min_rank=7 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[32672, 32772, 32789, 32840, 32953, 32686, 32844, 32588] avg=32768.0000 max=32953 max_rank=4 min=32588 min_rank=7 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=1 total_expected_recv=32840 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=32840 required_capacity=1.002197 +Number of parameters in transformer block in billions: 17.55 [2026-09-21 07:40:15] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 28616.2 | throughput per GPU (TFLOP/s/GPU): 1.2 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.228655E+01 | loss scale: 1.0 | grad norm: 12.484 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36224 | min safe capacity: 1.1055 | + +Number of parameters in embedding layers in billions: 0.72 +Total number of parameters in billions: 18.28 +Number of parameters in most loaded shard in billions: 18.2753 +Theoretical memory footprints: weight and optimizer=130715.23 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 49120.73779296875 | max allocated: 49120.73828125 | reserved: 54508.0 | max reserved: 54508.0 +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33091 expected_overflow=0 exact_recv_per_rank=[33091, 32749, 32859, 32319, 32955, 32994, 32693, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32994 expected_overflow=0 exact_recv_per_rank=[33091, 32749, 32859, 32319, 32955, 32994, 32693, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32955 expected_overflow=0 exact_recv_per_rank=[33091, 32749, 32859, 32319, 32955, 32994, 32693, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[33091, 32749, 32859, 32319, 32955, 32994, 32693, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33091 actual_valid_recv=36736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32994 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32693 expected_overflow=0 exact_recv_per_rank=[33091, 32749, 32859, 32319, 32955, 32994, 32693, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32859 expected_overflow=0 exact_recv_per_rank=[33091, 32749, 32859, 32319, 32955, 32994, 32693, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32955 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36736 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32484 expected_overflow=0 exact_recv_per_rank=[33091, 32749, 32859, 32319, 32955, 32994, 32693, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32319 expected_overflow=0 exact_recv_per_rank=[33091, 32749, 32859, 32319, 32955, 32994, 32693, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32693 actual_valid_recv=36608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32859 actual_valid_recv=36608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32484 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36608 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32319 actual_valid_recv=36096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36608 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36096 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[33091, 32749, 32859, 32319, 32955, 32994, 32693, 32484] avg=32768.0000 max=33091 max_rank=0 min=32319 min_rank=3 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[33091, 32749, 32859, 32319, 32955, 32994, 32693, 32484] avg=32768.0000 max=33091 max_rank=0 min=32319 min_rank=3 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[33091, 32749, 32859, 32319, 32955, 32994, 32693, 32484] avg=32768.0000 max=33091 max_rank=0 min=32319 min_rank=3 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[33091, 32749, 32859, 32319, 32955, 32994, 32693, 32484] avg=32768.0000 max=33091 max_rank=0 min=32319 min_rank=3 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[33091, 32749, 32859, 32319, 32955, 32994, 32693, 32484] avg=32768.0000 max=33091 max_rank=0 min=32319 min_rank=3 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[33091, 32749, 32859, 32319, 32955, 32994, 32693, 32484] avg=32768.0000 max=33091 max_rank=0 min=32319 min_rank=3 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[33091, 32749, 32859, 32319, 32955, 32994, 32693, 32484] avg=32768.0000 max=33091 max_rank=0 min=32319 min_rank=3 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[33091, 32749, 32859, 32319, 32955, 32994, 32693, 32484] avg=32768.0000 max=33091 max_rank=0 min=32319 min_rank=3 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[33091, 32749, 32859, 32319, 32955, 32994, 32693, 32484] avg=32768.0000 max=33091 max_rank=0 min=32319 min_rank=3 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[33091, 32749, 32859, 32319, 32955, 32994, 32693, 32484] avg=32768.0000 max=33091 max_rank=0 min=32319 min_rank=3 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[33091, 32749, 32859, 32319, 32955, 32994, 32693, 32484] avg=32768.0000 max=33091 max_rank=0 min=32319 min_rank=3 imbalance_ratio=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[33091, 32749, 32859, 32319, 32955, 32994, 32693, 32484] avg=32768.0000 max=33091 max_rank=0 min=32319 min_rank=3 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[33091, 32749, 32859, 32319, 32955, 32994, 32693, 32484] avg=32768.0000 max=33091 max_rank=0 min=32319 min_rank=3 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[33091, 32749, 32859, 32319, 32955, 32994, 32693, 32484] avg=32768.0000 max=33091 max_rank=0 min=32319 min_rank=3 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=1 total_expected_recv=33091 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4224 peak_demand=33091 required_capacity=1.009857 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=1 total_expected_recv=32994 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=32994 required_capacity=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[33091, 32749, 32859, 32319, 32955, 32994, 32693, 32484] avg=32768.0000 max=33091 max_rank=0 min=32319 min_rank=3 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[33091, 32749, 32859, 32319, 32955, 32994, 32693, 32484] avg=32768.0000 max=33091 max_rank=0 min=32319 min_rank=3 imbalance_ratio=1.009857 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=1 total_expected_recv=32955 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=32955 required_capacity=1.005707 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=1 total_expected_recv=32749 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=32749 required_capacity=0.999420 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=1 total_expected_recv=32693 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4352 peak_demand=32693 required_capacity=0.997711 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=1 total_expected_recv=32859 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4352 peak_demand=32859 required_capacity=1.002777 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=1 total_expected_recv=32484 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=32484 required_capacity=0.991333 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=1 total_expected_recv=32319 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4864 peak_demand=32319 required_capacity=0.986298 + [2026-09-21 07:40:15] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 186.8 | throughput per GPU (TFLOP/s/GPU): 189.4 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.227169E+01 | loss scale: 1.0 | grad norm: 12.418 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36480 | min safe capacity: 1.1133 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[32671, 32916, 32804, 33095, 32611, 32712, 32690, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32671 expected_overflow=0 exact_recv_per_rank=[32671, 32916, 32804, 33095, 32611, 32712, 32690, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32690 expected_overflow=0 exact_recv_per_rank=[32671, 32916, 32804, 33095, 32611, 32712, 32690, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32916 expected_overflow=0 exact_recv_per_rank=[32671, 32916, 32804, 33095, 32611, 32712, 32690, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32611 expected_overflow=0 exact_recv_per_rank=[32671, 32916, 32804, 33095, 32611, 32712, 32690, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32645 expected_overflow=0 exact_recv_per_rank=[32671, 32916, 32804, 33095, 32611, 32712, 32690, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[32671, 32916, 32804, 33095, 32611, 32712, 32690, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=36864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32671 actual_valid_recv=36736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32671, 32916, 32804, 33095, 32611, 32712, 32690, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32690 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32916 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32611 actual_valid_recv=36096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36864 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36736 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32645 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36096 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[32342, 32916, 32804, 33424, 32611, 32712, 32690, 32645] avg=32768.0000 max=33424 max_rank=3 min=32342 min_rank=0 imbalance_ratio=1.020020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[32342, 32916, 32804, 33424, 32611, 32712, 32690, 32645] avg=32768.0000 max=33424 max_rank=3 min=32342 min_rank=0 imbalance_ratio=1.020020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[32342, 32916, 32804, 33424, 32611, 32712, 32690, 32645] avg=32768.0000 max=33424 max_rank=3 min=32342 min_rank=0 imbalance_ratio=1.020020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[32342, 32916, 32804, 33424, 32611, 32712, 32690, 32645] avg=32768.0000 max=33424 max_rank=3 min=32342 min_rank=0 imbalance_ratio=1.020020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[32342, 32916, 32804, 33424, 32611, 32712, 32690, 32645] avg=32768.0000 max=33424 max_rank=3 min=32342 min_rank=0 imbalance_ratio=1.020020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[32342, 32916, 32804, 33424, 32611, 32712, 32690, 32645] avg=32768.0000 max=33424 max_rank=3 min=32342 min_rank=0 imbalance_ratio=1.020020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[32342, 32916, 32804, 33424, 32611, 32712, 32690, 32645] avg=32768.0000 max=33424 max_rank=3 min=32342 min_rank=0 imbalance_ratio=1.020020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[32342, 32916, 32804, 33424, 32611, 32712, 32690, 32645] avg=32768.0000 max=33424 max_rank=3 min=32342 min_rank=0 imbalance_ratio=1.020020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[32671, 32916, 32804, 33095, 32611, 32712, 32690, 32645] avg=32768.0000 max=33095 max_rank=3 min=32611 min_rank=4 imbalance_ratio=1.009979 max_load_reduction=329 max_load_reduction_pct=0.009843 moved_to_replicas=328 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[32671, 32916, 32804, 33095, 32611, 32712, 32690, 32645] avg=32768.0000 max=33095 max_rank=3 min=32611 min_rank=4 imbalance_ratio=1.009979 max_load_reduction=329 max_load_reduction_pct=0.009843 moved_to_replicas=328 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[32671, 32916, 32804, 33095, 32611, 32712, 32690, 32645] avg=32768.0000 max=33095 max_rank=3 min=32611 min_rank=4 imbalance_ratio=1.009979 max_load_reduction=329 max_load_reduction_pct=0.009843 moved_to_replicas=328 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[32671, 32916, 32804, 33095, 32611, 32712, 32690, 32645] avg=32768.0000 max=33095 max_rank=3 min=32611 min_rank=4 imbalance_ratio=1.009979 max_load_reduction=329 max_load_reduction_pct=0.009843 moved_to_replicas=328 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[32671, 32916, 32804, 33095, 32611, 32712, 32690, 32645] avg=32768.0000 max=33095 max_rank=3 min=32611 min_rank=4 imbalance_ratio=1.009979 max_load_reduction=329 max_load_reduction_pct=0.009843 moved_to_replicas=328 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[32671, 32916, 32804, 33095, 32611, 32712, 32690, 32645] avg=32768.0000 max=33095 max_rank=3 min=32611 min_rank=4 imbalance_ratio=1.009979 max_load_reduction=329 max_load_reduction_pct=0.009843 moved_to_replicas=328 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[32671, 32916, 32804, 33095, 32611, 32712, 32690, 32645] avg=32768.0000 max=33095 max_rank=3 min=32611 min_rank=4 imbalance_ratio=1.009979 max_load_reduction=329 max_load_reduction_pct=0.009843 moved_to_replicas=328 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[32671, 32916, 32804, 33095, 32611, 32712, 32690, 32645] avg=32768.0000 max=33095 max_rank=3 min=32611 min_rank=4 imbalance_ratio=1.009979 max_load_reduction=329 max_load_reduction_pct=0.009843 moved_to_replicas=328 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=1 total_expected_recv=32712 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4096 peak_demand=32712 required_capacity=0.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=1 total_expected_recv=32671 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4224 peak_demand=32671 required_capacity=0.997040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=1 total_expected_recv=32916 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=32916 required_capacity=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=1 total_expected_recv=32690 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=32690 required_capacity=0.997620 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=1 total_expected_recv=32611 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4864 peak_demand=32611 required_capacity=0.995209 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=1 total_expected_recv=32645 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=32645 required_capacity=0.996246 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=1 total_expected_recv=33095 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=33095 required_capacity=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=1 total_expected_recv=32804 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=32804 required_capacity=1.001099 + [2026-09-21 07:40:15] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 119.1 | throughput per GPU (TFLOP/s/GPU): 297.1 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.092447E+01 | loss scale: 1.0 | grad norm: 10.964 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36480 | min safe capacity: 1.1133 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32928, 32578, 32571, 32591, 32583, 32898, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32928 expected_overflow=0 exact_recv_per_rank=[33097, 32928, 32578, 32571, 32591, 32583, 32898, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32583 expected_overflow=0 exact_recv_per_rank=[33097, 32928, 32578, 32571, 32591, 32583, 32898, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32898 expected_overflow=0 exact_recv_per_rank=[33097, 32928, 32578, 32571, 32591, 32583, 32898, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32578 expected_overflow=0 exact_recv_per_rank=[33097, 32928, 32578, 32571, 32591, 32583, 32898, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33097 actual_valid_recv=36608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32571 expected_overflow=0 exact_recv_per_rank=[33097, 32928, 32578, 32571, 32591, 32583, 32898, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32898 expected_overflow=0 exact_recv_per_rank=[33097, 32928, 32578, 32571, 32591, 32583, 32898, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32928 actual_valid_recv=36608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32583 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32578 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32898 actual_valid_recv=36608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36608 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32591 expected_overflow=0 exact_recv_per_rank=[33097, 32928, 32578, 32571, 32591, 32583, 32898, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36608 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32571 actual_valid_recv=36096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32898 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36608 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36096 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32591 actual_valid_recv=35840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35840 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[33231, 32928, 32578, 32571, 32591, 32449, 32898, 32898] avg=32768.0000 max=33231 max_rank=0 min=32449 min_rank=5 imbalance_ratio=1.014130 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[33231, 32928, 32578, 32571, 32591, 32449, 32898, 32898] avg=32768.0000 max=33231 max_rank=0 min=32449 min_rank=5 imbalance_ratio=1.014130 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[33231, 32928, 32578, 32571, 32591, 32449, 32898, 32898] avg=32768.0000 max=33231 max_rank=0 min=32449 min_rank=5 imbalance_ratio=1.014130 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[33231, 32928, 32578, 32571, 32591, 32449, 32898, 32898] avg=32768.0000 max=33231 max_rank=0 min=32449 min_rank=5 imbalance_ratio=1.014130 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[33231, 32928, 32578, 32571, 32591, 32449, 32898, 32898] avg=32768.0000 max=33231 max_rank=0 min=32449 min_rank=5 imbalance_ratio=1.014130 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[33231, 32928, 32578, 32571, 32591, 32449, 32898, 32898] avg=32768.0000 max=33231 max_rank=0 min=32449 min_rank=5 imbalance_ratio=1.014130 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[33231, 32928, 32578, 32571, 32591, 32449, 32898, 32898] avg=32768.0000 max=33231 max_rank=0 min=32449 min_rank=5 imbalance_ratio=1.014130 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[33097, 32928, 32578, 32571, 32591, 32583, 32898, 32898] avg=32768.0000 max=33097 max_rank=0 min=32571 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=134 max_load_reduction_pct=0.004032 moved_to_replicas=135 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[33097, 32928, 32578, 32571, 32591, 32583, 32898, 32898] avg=32768.0000 max=33097 max_rank=0 min=32571 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=134 max_load_reduction_pct=0.004032 moved_to_replicas=135 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[33231, 32928, 32578, 32571, 32591, 32449, 32898, 32898] avg=32768.0000 max=33231 max_rank=0 min=32449 min_rank=5 imbalance_ratio=1.014130 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[33097, 32928, 32578, 32571, 32591, 32583, 32898, 32898] avg=32768.0000 max=33097 max_rank=0 min=32571 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=134 max_load_reduction_pct=0.004032 moved_to_replicas=135 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[33097, 32928, 32578, 32571, 32591, 32583, 32898, 32898] avg=32768.0000 max=33097 max_rank=0 min=32571 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=134 max_load_reduction_pct=0.004032 moved_to_replicas=135 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[33097, 32928, 32578, 32571, 32591, 32583, 32898, 32898] avg=32768.0000 max=33097 max_rank=0 min=32571 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=134 max_load_reduction_pct=0.004032 moved_to_replicas=135 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[33097, 32928, 32578, 32571, 32591, 32583, 32898, 32898] avg=32768.0000 max=33097 max_rank=0 min=32571 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=134 max_load_reduction_pct=0.004032 moved_to_replicas=135 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[33097, 32928, 32578, 32571, 32591, 32583, 32898, 32898] avg=32768.0000 max=33097 max_rank=0 min=32571 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=134 max_load_reduction_pct=0.004032 moved_to_replicas=135 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=1 total_expected_recv=33097 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4352 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[33097, 32928, 32578, 32571, 32591, 32583, 32898, 32898] avg=32768.0000 max=33097 max_rank=0 min=32571 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=134 max_load_reduction_pct=0.004032 moved_to_replicas=135 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=1 total_expected_recv=32583 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=32583 required_capacity=0.994354 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=1 total_expected_recv=32928 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4352 peak_demand=32928 required_capacity=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=1 total_expected_recv=32578 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=32578 required_capacity=0.994202 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=1 total_expected_recv=32898 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4352 peak_demand=32898 required_capacity=1.003967 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=1 total_expected_recv=32571 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4864 peak_demand=32571 required_capacity=0.993988 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=1 total_expected_recv=32898 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=32898 required_capacity=1.003967 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=1 total_expected_recv=32591 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5120 peak_demand=32591 required_capacity=0.994598 + [2026-09-21 07:40:15] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 118.6 | throughput per GPU (TFLOP/s/GPU): 298.3 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.476595E+01 | loss scale: 1.0 | grad norm: 74.385 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36480 | min safe capacity: 1.1133 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33067 expected_overflow=0 exact_recv_per_rank=[32967, 32395, 32641, 33067, 32617, 33082, 32356, 33019] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33019 expected_overflow=0 exact_recv_per_rank=[32967, 32395, 32641, 33067, 32617, 33082, 32356, 33019] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32641 expected_overflow=0 exact_recv_per_rank=[32967, 32395, 32641, 33067, 32617, 33082, 32356, 33019] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32356 expected_overflow=0 exact_recv_per_rank=[32967, 32395, 32641, 33067, 32617, 33082, 32356, 33019] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33067 actual_valid_recv=36864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33019 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32641 actual_valid_recv=35712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36864 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35712 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32356 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[32967, 32395, 32641, 33067, 32617, 33082, 32356, 33019] avg=32768.0000 max=33082 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.009583 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[32967, 32395, 32641, 33067, 32617, 33082, 32356, 33019] avg=32768.0000 max=33082 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.009583 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[32967, 32395, 32641, 33067, 32617, 33082, 32356, 33019] avg=32768.0000 max=33082 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.009583 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[32967, 32395, 32641, 33067, 32617, 33082, 32356, 33019] avg=32768.0000 max=33082 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.009583 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[32967, 32395, 32641, 33067, 32617, 33082, 32356, 33019] avg=32768.0000 max=33082 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.009583 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[32967, 32395, 32641, 33067, 32617, 33082, 32356, 33019] avg=32768.0000 max=33082 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.009583 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[32967, 32395, 32641, 33067, 32617, 33082, 32356, 33019] avg=32768.0000 max=33082 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.009583 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[32967, 32395, 32641, 33067, 32617, 33082, 32356, 33019] avg=32768.0000 max=33082 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.009583 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33082 expected_overflow=0 exact_recv_per_rank=[32967, 32395, 32641, 33067, 32617, 33082, 32356, 33019] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=1 total_expected_recv=33019 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=33019 required_capacity=1.007660 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=1 total_expected_recv=33067 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4096 peak_demand=33067 required_capacity=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=1 total_expected_recv=32356 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=32356 required_capacity=0.987427 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=1 total_expected_recv=32641 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5248 peak_demand=32641 required_capacity=0.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32967 expected_overflow=0 exact_recv_per_rank=[32967, 32395, 32641, 33067, 32617, 33082, 32356, 33019] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32395 expected_overflow=0 exact_recv_per_rank=[32967, 32395, 32641, 33067, 32617, 33082, 32356, 33019] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32967 actual_valid_recv=36736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=40960 expected_recv=33082 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32395 actual_valid_recv=35840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36736 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35840 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[32967, 32395, 32641, 33067, 32617, 33082, 32356, 33019] avg=32768.0000 max=33082 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.009583 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[32967, 32395, 32641, 33067, 32617, 33082, 32356, 33019] avg=32768.0000 max=33082 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.009583 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32617 expected_overflow=0 exact_recv_per_rank=[32967, 32395, 32641, 33067, 32617, 33082, 32356, 33019] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[32967, 32395, 32641, 33067, 32617, 33082, 32356, 33019] avg=32768.0000 max=33082 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.009583 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[32967, 32395, 32641, 33067, 32617, 33082, 32356, 33019] avg=32768.0000 max=33082 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.009583 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[32967, 32395, 32641, 33067, 32617, 33082, 32356, 33019] avg=32768.0000 max=33082 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.009583 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32617 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[32967, 32395, 32641, 33067, 32617, 33082, 32356, 33019] avg=32768.0000 max=33082 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.009583 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=1 total_expected_recv=32967 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4224 peak_demand=32967 required_capacity=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=1 total_expected_recv=32395 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5120 peak_demand=32395 required_capacity=0.988617 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=1 total_expected_recv=33082 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=33082 required_capacity=1.009583 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[32967, 32395, 32641, 33067, 32617, 33082, 32356, 33019] avg=32768.0000 max=33082 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.009583 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[32967, 32395, 32641, 33067, 32617, 33082, 32356, 33019] avg=32768.0000 max=33082 max_rank=5 min=32356 min_rank=6 imbalance_ratio=1.009583 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=1 total_expected_recv=32617 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=32617 required_capacity=0.995392 + [2026-09-21 07:40:15] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 118.1 | throughput per GPU (TFLOP/s/GPU): 299.6 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.821806E+01 | loss scale: 1.0 | grad norm: 104.113 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36480 | min safe capacity: 1.1133 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32954 expected_overflow=0 exact_recv_per_rank=[32954, 32535, 32996, 32376, 32951, 32503, 33089, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32503 expected_overflow=0 exact_recv_per_rank=[32954, 32535, 32996, 32376, 32951, 32503, 33089, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32535 expected_overflow=0 exact_recv_per_rank=[32954, 32535, 32996, 32376, 32951, 32503, 33089, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33089 expected_overflow=0 exact_recv_per_rank=[32954, 32535, 32996, 32376, 32951, 32503, 33089, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[32954, 32535, 32996, 32376, 32951, 32503, 33089, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32951 expected_overflow=0 exact_recv_per_rank=[32954, 32535, 32996, 32376, 32951, 32503, 33089, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32376 expected_overflow=0 exact_recv_per_rank=[32954, 32535, 32996, 32376, 32951, 32503, 33089, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32996 expected_overflow=0 exact_recv_per_rank=[32954, 32535, 32996, 32376, 32951, 32503, 33089, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32954 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32503 actual_valid_recv=36608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33089 actual_valid_recv=36608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32535 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32951 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32376 actual_valid_recv=35840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36608 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36608 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32996 actual_valid_recv=36736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35840 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36736 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[32954, 32535, 32996, 32376, 32951, 32503, 33089, 32740] avg=32768.0000 max=33089 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.009796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[32954, 32535, 32996, 32376, 32951, 32503, 33089, 32740] avg=32768.0000 max=33089 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.009796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[32954, 32535, 32996, 32376, 32951, 32503, 33089, 32740] avg=32768.0000 max=33089 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.009796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[32954, 32535, 32996, 32376, 32951, 32503, 33089, 32740] avg=32768.0000 max=33089 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.009796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[32954, 32535, 32996, 32376, 32951, 32503, 33089, 32740] avg=32768.0000 max=33089 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.009796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[32954, 32535, 32996, 32376, 32951, 32503, 33089, 32740] avg=32768.0000 max=33089 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.009796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[32954, 32535, 32996, 32376, 32951, 32503, 33089, 32740] avg=32768.0000 max=33089 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.009796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[32954, 32535, 32996, 32376, 32951, 32503, 33089, 32740] avg=32768.0000 max=33089 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.009796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[32954, 32535, 32996, 32376, 32951, 32503, 33089, 32740] avg=32768.0000 max=33089 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.009796 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[32954, 32535, 32996, 32376, 32951, 32503, 33089, 32740] avg=32768.0000 max=33089 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.009796 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[32954, 32535, 32996, 32376, 32951, 32503, 33089, 32740] avg=32768.0000 max=33089 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.009796 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[32954, 32535, 32996, 32376, 32951, 32503, 33089, 32740] avg=32768.0000 max=33089 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.009796 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[32954, 32535, 32996, 32376, 32951, 32503, 33089, 32740] avg=32768.0000 max=33089 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.009796 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[32954, 32535, 32996, 32376, 32951, 32503, 33089, 32740] avg=32768.0000 max=33089 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.009796 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[32954, 32535, 32996, 32376, 32951, 32503, 33089, 32740] avg=32768.0000 max=33089 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.009796 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[32954, 32535, 32996, 32376, 32951, 32503, 33089, 32740] avg=32768.0000 max=33089 max_rank=6 min=32376 min_rank=3 imbalance_ratio=1.009796 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=1 total_expected_recv=33089 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4352 peak_demand=33089 required_capacity=1.009796 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=1 total_expected_recv=32503 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4352 peak_demand=32503 required_capacity=0.991913 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=1 total_expected_recv=32954 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=32954 required_capacity=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=1 total_expected_recv=32535 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=32535 required_capacity=0.992889 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=1 total_expected_recv=32951 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=32951 required_capacity=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=1 total_expected_recv=32740 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=32740 required_capacity=0.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=1 total_expected_recv=32996 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4224 peak_demand=32996 required_capacity=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=1 total_expected_recv=32376 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5120 peak_demand=32376 required_capacity=0.988037 + [2026-09-21 07:40:15] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 116.2 | throughput per GPU (TFLOP/s/GPU): 304.4 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.480657E+01 | loss scale: 1.0 | grad norm: 83.391 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36480 | min safe capacity: 1.1133 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32714 expected_overflow=0 exact_recv_per_rank=[32828, 32856, 32831, 32537, 32732, 32714, 32703, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32537 expected_overflow=0 exact_recv_per_rank=[32828, 32856, 32831, 32537, 32732, 32714, 32703, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32828 expected_overflow=0 exact_recv_per_rank=[32828, 32856, 32831, 32537, 32732, 32714, 32703, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32856 expected_overflow=0 exact_recv_per_rank=[32828, 32856, 32831, 32537, 32732, 32714, 32703, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[32828, 32856, 32831, 32537, 32732, 32714, 32703, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32943 expected_overflow=0 exact_recv_per_rank=[32828, 32856, 32831, 32537, 32732, 32714, 32703, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32714 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32703 expected_overflow=0 exact_recv_per_rank=[32828, 32856, 32831, 32537, 32732, 32714, 32703, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32537 actual_valid_recv=36096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32828 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32831 expected_overflow=0 exact_recv_per_rank=[32828, 32856, 32831, 32537, 32732, 32714, 32703, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32856 actual_valid_recv=36096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32732 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36096 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32943 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32703 actual_valid_recv=36736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36096 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32831 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36736 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[32828, 32856, 32831, 32537, 32732, 32714, 32703, 32943] avg=32768.0000 max=32943 max_rank=7 min=32537 min_rank=3 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[32828, 32856, 32831, 32537, 32732, 32714, 32703, 32943] avg=32768.0000 max=32943 max_rank=7 min=32537 min_rank=3 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[32828, 32856, 32831, 32537, 32732, 32714, 32703, 32943] avg=32768.0000 max=32943 max_rank=7 min=32537 min_rank=3 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[32828, 32856, 32831, 32537, 32732, 32714, 32703, 32943] avg=32768.0000 max=32943 max_rank=7 min=32537 min_rank=3 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[32828, 32856, 32831, 32537, 32732, 32714, 32703, 32943] avg=32768.0000 max=32943 max_rank=7 min=32537 min_rank=3 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[32828, 32856, 32831, 32537, 32732, 32714, 32703, 32943] avg=32768.0000 max=32943 max_rank=7 min=32537 min_rank=3 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[32828, 32856, 32831, 32537, 32732, 32714, 32703, 32943] avg=32768.0000 max=32943 max_rank=7 min=32537 min_rank=3 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[32828, 32856, 32831, 32537, 32732, 32714, 32703, 32943] avg=32768.0000 max=32943 max_rank=7 min=32537 min_rank=3 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[32828, 32856, 32831, 32537, 32732, 32714, 32703, 32943] avg=32768.0000 max=32943 max_rank=7 min=32537 min_rank=3 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[32828, 32856, 32831, 32537, 32732, 32714, 32703, 32943] avg=32768.0000 max=32943 max_rank=7 min=32537 min_rank=3 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[32828, 32856, 32831, 32537, 32732, 32714, 32703, 32943] avg=32768.0000 max=32943 max_rank=7 min=32537 min_rank=3 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[32828, 32856, 32831, 32537, 32732, 32714, 32703, 32943] avg=32768.0000 max=32943 max_rank=7 min=32537 min_rank=3 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[32828, 32856, 32831, 32537, 32732, 32714, 32703, 32943] avg=32768.0000 max=32943 max_rank=7 min=32537 min_rank=3 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[32828, 32856, 32831, 32537, 32732, 32714, 32703, 32943] avg=32768.0000 max=32943 max_rank=7 min=32537 min_rank=3 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[32828, 32856, 32831, 32537, 32732, 32714, 32703, 32943] avg=32768.0000 max=32943 max_rank=7 min=32537 min_rank=3 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=1 total_expected_recv=32714 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=32714 required_capacity=0.998352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[32828, 32856, 32831, 32537, 32732, 32714, 32703, 32943] avg=32768.0000 max=32943 max_rank=7 min=32537 min_rank=3 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=1 total_expected_recv=32828 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=32828 required_capacity=1.001831 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=1 total_expected_recv=32537 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4864 peak_demand=32537 required_capacity=0.992950 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=1 total_expected_recv=32856 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4864 peak_demand=32856 required_capacity=1.002686 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=1 total_expected_recv=32732 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=32732 required_capacity=0.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=1 total_expected_recv=32943 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=32943 required_capacity=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=1 total_expected_recv=32703 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4224 peak_demand=32703 required_capacity=0.998016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=1 total_expected_recv=32831 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=32831 required_capacity=1.001923 + [2026-09-21 07:40:16] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 116.2 | throughput per GPU (TFLOP/s/GPU): 304.3 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.131050E+01 | loss scale: 1.0 | grad norm: 25.264 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36480 | min safe capacity: 1.1133 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32423 expected_overflow=0 exact_recv_per_rank=[32423, 32708, 32844, 32793, 32976, 32792, 32740, 32868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[32423, 32708, 32844, 32793, 32976, 32792, 32740, 32868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32740 expected_overflow=0 exact_recv_per_rank=[32423, 32708, 32844, 32793, 32976, 32792, 32740, 32868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32708 expected_overflow=0 exact_recv_per_rank=[32423, 32708, 32844, 32793, 32976, 32792, 32740, 32868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32976 expected_overflow=0 exact_recv_per_rank=[32423, 32708, 32844, 32793, 32976, 32792, 32740, 32868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32423 actual_valid_recv=36096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32423, 32708, 32844, 32793, 32976, 32792, 32740, 32868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32792 actual_valid_recv=36096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32868 expected_overflow=0 exact_recv_per_rank=[32423, 32708, 32844, 32793, 32976, 32792, 32740, 32868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32844 expected_overflow=0 exact_recv_per_rank=[32423, 32708, 32844, 32793, 32976, 32792, 32740, 32868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32740 actual_valid_recv=35968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32708 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36096 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32976 actual_valid_recv=36096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36096 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35968 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32868 actual_valid_recv=35968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36096 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32844 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35968 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[32423, 32708, 32844, 32793, 32976, 32792, 32740, 32868] avg=32768.0000 max=32976 max_rank=4 min=32423 min_rank=0 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[32423, 32708, 32844, 32793, 32976, 32792, 32740, 32868] avg=32768.0000 max=32976 max_rank=4 min=32423 min_rank=0 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[32423, 32708, 32844, 32793, 32976, 32792, 32740, 32868] avg=32768.0000 max=32976 max_rank=4 min=32423 min_rank=0 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[32423, 32708, 32844, 32793, 32976, 32792, 32740, 32868] avg=32768.0000 max=32976 max_rank=4 min=32423 min_rank=0 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[32423, 32708, 32844, 32793, 32976, 32792, 32740, 32868] avg=32768.0000 max=32976 max_rank=4 min=32423 min_rank=0 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[32423, 32708, 32844, 32793, 32976, 32792, 32740, 32868] avg=32768.0000 max=32976 max_rank=4 min=32423 min_rank=0 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[32423, 32708, 32844, 32793, 32976, 32792, 32740, 32868] avg=32768.0000 max=32976 max_rank=4 min=32423 min_rank=0 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[32423, 32708, 32844, 32793, 32976, 32792, 32740, 32868] avg=32768.0000 max=32976 max_rank=4 min=32423 min_rank=0 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[32423, 32708, 32844, 32793, 32976, 32792, 32740, 32868] avg=32768.0000 max=32976 max_rank=4 min=32423 min_rank=0 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[32423, 32708, 32844, 32793, 32976, 32792, 32740, 32868] avg=32768.0000 max=32976 max_rank=4 min=32423 min_rank=0 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[32423, 32708, 32844, 32793, 32976, 32792, 32740, 32868] avg=32768.0000 max=32976 max_rank=4 min=32423 min_rank=0 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[32423, 32708, 32844, 32793, 32976, 32792, 32740, 32868] avg=32768.0000 max=32976 max_rank=4 min=32423 min_rank=0 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[32423, 32708, 32844, 32793, 32976, 32792, 32740, 32868] avg=32768.0000 max=32976 max_rank=4 min=32423 min_rank=0 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[32423, 32708, 32844, 32793, 32976, 32792, 32740, 32868] avg=32768.0000 max=32976 max_rank=4 min=32423 min_rank=0 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[32423, 32708, 32844, 32793, 32976, 32792, 32740, 32868] avg=32768.0000 max=32976 max_rank=4 min=32423 min_rank=0 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[32423, 32708, 32844, 32793, 32976, 32792, 32740, 32868] avg=32768.0000 max=32976 max_rank=4 min=32423 min_rank=0 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=1 total_expected_recv=32792 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4864 peak_demand=32792 required_capacity=1.000732 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=1 total_expected_recv=32423 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4864 peak_demand=32423 required_capacity=0.989471 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=1 total_expected_recv=32740 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4992 peak_demand=32740 required_capacity=0.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=1 total_expected_recv=32708 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=32708 required_capacity=0.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=1 total_expected_recv=32976 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4864 peak_demand=32976 required_capacity=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=1 total_expected_recv=32793 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=32793 required_capacity=1.000763 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=1 total_expected_recv=32868 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4992 peak_demand=32868 required_capacity=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=1 total_expected_recv=32844 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=32844 required_capacity=1.002319 + [2026-09-21 07:40:16] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 115.9 | throughput per GPU (TFLOP/s/GPU): 305.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.159143E+01 | loss scale: 1.0 | grad norm: 12.156 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36480 | min safe capacity: 1.1133 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32642 expected_overflow=0 exact_recv_per_rank=[32642, 32471, 32644, 33052, 33095, 32589, 32985, 32666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32471 expected_overflow=0 exact_recv_per_rank=[32642, 32471, 32644, 33052, 33095, 32589, 32985, 32666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32589 expected_overflow=0 exact_recv_per_rank=[32642, 32471, 32644, 33052, 33095, 32589, 32985, 32666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32642, 32471, 32644, 33052, 33095, 32589, 32985, 32666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33052 expected_overflow=0 exact_recv_per_rank=[32642, 32471, 32644, 33052, 33095, 32589, 32985, 32666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32985 expected_overflow=0 exact_recv_per_rank=[32642, 32471, 32644, 33052, 33095, 32589, 32985, 32666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32666 expected_overflow=0 exact_recv_per_rank=[32642, 32471, 32644, 33052, 33095, 32589, 32985, 32666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32642 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32471 actual_valid_recv=36096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32644 expected_overflow=0 exact_recv_per_rank=[32642, 32471, 32644, 33052, 33095, 32589, 32985, 32666] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32589 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33095 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33052 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36096 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32985 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32666 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32644 actual_valid_recv=35968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35968 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[32642, 32471, 32644, 33052, 33351, 32589, 32985, 32410] avg=32768.0000 max=33351 max_rank=4 min=32410 min_rank=7 imbalance_ratio=1.017792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[32642, 32471, 32644, 33052, 33351, 32589, 32985, 32410] avg=32768.0000 max=33351 max_rank=4 min=32410 min_rank=7 imbalance_ratio=1.017792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[32642, 32471, 32644, 33052, 33351, 32589, 32985, 32410] avg=32768.0000 max=33351 max_rank=4 min=32410 min_rank=7 imbalance_ratio=1.017792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[32642, 32471, 32644, 33052, 33351, 32589, 32985, 32410] avg=32768.0000 max=33351 max_rank=4 min=32410 min_rank=7 imbalance_ratio=1.017792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[32642, 32471, 32644, 33052, 33351, 32589, 32985, 32410] avg=32768.0000 max=33351 max_rank=4 min=32410 min_rank=7 imbalance_ratio=1.017792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[32642, 32471, 32644, 33052, 33351, 32589, 32985, 32410] avg=32768.0000 max=33351 max_rank=4 min=32410 min_rank=7 imbalance_ratio=1.017792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[32642, 32471, 32644, 33052, 33351, 32589, 32985, 32410] avg=32768.0000 max=33351 max_rank=4 min=32410 min_rank=7 imbalance_ratio=1.017792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[32642, 32471, 32644, 33052, 33351, 32589, 32985, 32410] avg=32768.0000 max=33351 max_rank=4 min=32410 min_rank=7 imbalance_ratio=1.017792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[32642, 32471, 32644, 33052, 33095, 32589, 32985, 32666] avg=32768.0000 max=33095 max_rank=4 min=32471 min_rank=1 imbalance_ratio=1.009979 max_load_reduction=256 max_load_reduction_pct=0.007676 moved_to_replicas=255 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[32642, 32471, 32644, 33052, 33095, 32589, 32985, 32666] avg=32768.0000 max=33095 max_rank=4 min=32471 min_rank=1 imbalance_ratio=1.009979 max_load_reduction=256 max_load_reduction_pct=0.007676 moved_to_replicas=255 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[32642, 32471, 32644, 33052, 33095, 32589, 32985, 32666] avg=32768.0000 max=33095 max_rank=4 min=32471 min_rank=1 imbalance_ratio=1.009979 max_load_reduction=256 max_load_reduction_pct=0.007676 moved_to_replicas=255 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[32642, 32471, 32644, 33052, 33095, 32589, 32985, 32666] avg=32768.0000 max=33095 max_rank=4 min=32471 min_rank=1 imbalance_ratio=1.009979 max_load_reduction=256 max_load_reduction_pct=0.007676 moved_to_replicas=255 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[32642, 32471, 32644, 33052, 33095, 32589, 32985, 32666] avg=32768.0000 max=33095 max_rank=4 min=32471 min_rank=1 imbalance_ratio=1.009979 max_load_reduction=256 max_load_reduction_pct=0.007676 moved_to_replicas=255 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[32642, 32471, 32644, 33052, 33095, 32589, 32985, 32666] avg=32768.0000 max=33095 max_rank=4 min=32471 min_rank=1 imbalance_ratio=1.009979 max_load_reduction=256 max_load_reduction_pct=0.007676 moved_to_replicas=255 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[32642, 32471, 32644, 33052, 33095, 32589, 32985, 32666] avg=32768.0000 max=33095 max_rank=4 min=32471 min_rank=1 imbalance_ratio=1.009979 max_load_reduction=256 max_load_reduction_pct=0.007676 moved_to_replicas=255 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[32642, 32471, 32644, 33052, 33095, 32589, 32985, 32666] avg=32768.0000 max=33095 max_rank=4 min=32471 min_rank=1 imbalance_ratio=1.009979 max_load_reduction=256 max_load_reduction_pct=0.007676 moved_to_replicas=255 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=1 total_expected_recv=32642 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=32642 required_capacity=0.996155 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=1 total_expected_recv=32471 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4864 peak_demand=32471 required_capacity=0.990936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=1 total_expected_recv=33095 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=33095 required_capacity=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=1 total_expected_recv=32589 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=32589 required_capacity=0.994537 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=1 total_expected_recv=33052 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=33052 required_capacity=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=1 total_expected_recv=32985 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=32985 required_capacity=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=1 total_expected_recv=32644 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4992 peak_demand=32644 required_capacity=0.996216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=1 total_expected_recv=32666 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=32666 required_capacity=0.996887 + [2026-09-21 07:40:16] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 119.0 | throughput per GPU (TFLOP/s/GPU): 297.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.108660E+01 | loss scale: 1.0 | grad norm: 10.329 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36480 | min safe capacity: 1.1133 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32794, 32678, 32667, 32394, 32693, 32793, 33029, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32667 expected_overflow=0 exact_recv_per_rank=[32794, 32678, 32667, 32394, 32693, 32793, 33029, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32794, 32678, 32667, 32394, 32693, 32793, 33029, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32794, 32678, 32667, 32394, 32693, 32793, 33029, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32794, 32678, 32667, 32394, 32693, 32793, 33029, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32693 expected_overflow=0 exact_recv_per_rank=[32794, 32678, 32667, 32394, 32693, 32793, 33029, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32394 expected_overflow=0 exact_recv_per_rank=[32794, 32678, 32667, 32394, 32693, 32793, 33029, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=35968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32667 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33029 expected_overflow=0 exact_recv_per_rank=[32794, 32678, 32667, 32394, 32693, 32793, 33029, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=36864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35968 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32693 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32394 actual_valid_recv=35840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36864 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=40960 expected_recv=33029 actual_valid_recv=36608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=35840 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36608 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[32794, 32678, 32667, 32290, 32693, 32793, 33029, 33200] avg=32768.0000 max=33200 max_rank=7 min=32290 min_rank=3 imbalance_ratio=1.013184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[32794, 32678, 32667, 32290, 32693, 32793, 33029, 33200] avg=32768.0000 max=33200 max_rank=7 min=32290 min_rank=3 imbalance_ratio=1.013184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[32794, 32678, 32667, 32290, 32693, 32793, 33029, 33200] avg=32768.0000 max=33200 max_rank=7 min=32290 min_rank=3 imbalance_ratio=1.013184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[32794, 32678, 32667, 32290, 32693, 32793, 33029, 33200] avg=32768.0000 max=33200 max_rank=7 min=32290 min_rank=3 imbalance_ratio=1.013184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[32794, 32678, 32667, 32290, 32693, 32793, 33029, 33200] avg=32768.0000 max=33200 max_rank=7 min=32290 min_rank=3 imbalance_ratio=1.013184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[32794, 32678, 32667, 32290, 32693, 32793, 33029, 33200] avg=32768.0000 max=33200 max_rank=7 min=32290 min_rank=3 imbalance_ratio=1.013184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[32794, 32678, 32667, 32290, 32693, 32793, 33029, 33200] avg=32768.0000 max=33200 max_rank=7 min=32290 min_rank=3 imbalance_ratio=1.013184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[32794, 32678, 32667, 32394, 32693, 32793, 33029, 33096] avg=32768.0000 max=33096 max_rank=7 min=32394 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=104 max_load_reduction_pct=0.003133 moved_to_replicas=104 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[32794, 32678, 32667, 32394, 32693, 32793, 33029, 33096] avg=32768.0000 max=33096 max_rank=7 min=32394 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=104 max_load_reduction_pct=0.003133 moved_to_replicas=104 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[32794, 32678, 32667, 32394, 32693, 32793, 33029, 33096] avg=32768.0000 max=33096 max_rank=7 min=32394 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=104 max_load_reduction_pct=0.003133 moved_to_replicas=104 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[32794, 32678, 32667, 32290, 32693, 32793, 33029, 33200] avg=32768.0000 max=33200 max_rank=7 min=32290 min_rank=3 imbalance_ratio=1.013184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[32794, 32678, 32667, 32394, 32693, 32793, 33029, 33096] avg=32768.0000 max=33096 max_rank=7 min=32394 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=104 max_load_reduction_pct=0.003133 moved_to_replicas=104 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[32794, 32678, 32667, 32394, 32693, 32793, 33029, 33096] avg=32768.0000 max=33096 max_rank=7 min=32394 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=104 max_load_reduction_pct=0.003133 moved_to_replicas=104 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[32794, 32678, 32667, 32394, 32693, 32793, 33029, 33096] avg=32768.0000 max=33096 max_rank=7 min=32394 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=104 max_load_reduction_pct=0.003133 moved_to_replicas=104 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[32794, 32678, 32667, 32394, 32693, 32793, 33029, 33096] avg=32768.0000 max=33096 max_rank=7 min=32394 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=104 max_load_reduction_pct=0.003133 moved_to_replicas=104 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=1 total_expected_recv=32678 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4992 peak_demand=32678 required_capacity=0.997253 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=1 total_expected_recv=32794 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=32794 required_capacity=1.000793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[32794, 32678, 32667, 32394, 32693, 32793, 33029, 33096] avg=32768.0000 max=33096 max_rank=7 min=32394 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=104 max_load_reduction_pct=0.003133 moved_to_replicas=104 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=1 total_expected_recv=32667 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=32667 required_capacity=0.996918 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=1 total_expected_recv=32793 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=32793 required_capacity=1.000763 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=1 total_expected_recv=33096 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4096 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=1 total_expected_recv=32693 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=32693 required_capacity=0.997711 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=1 total_expected_recv=32394 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=5120 peak_demand=32394 required_capacity=0.988586 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=1 total_expected_recv=33029 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4352 peak_demand=33029 required_capacity=1.007965 + [2026-09-21 07:40:16] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 119.0 | throughput per GPU (TFLOP/s/GPU): 297.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.082569E+01 | loss scale: 1.0 | grad norm: 8.435 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36864 | min safe capacity: 1.1250 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32910 expected_overflow=0 exact_recv_per_rank=[32910, 32357, 32741, 32869, 32603, 32943, 32870, 32851] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32357 expected_overflow=0 exact_recv_per_rank=[32910, 32357, 32741, 32869, 32603, 32943, 32870, 32851] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32603 expected_overflow=0 exact_recv_per_rank=[32910, 32357, 32741, 32869, 32603, 32943, 32870, 32851] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32943 expected_overflow=0 exact_recv_per_rank=[32910, 32357, 32741, 32869, 32603, 32943, 32870, 32851] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32851 expected_overflow=0 exact_recv_per_rank=[32910, 32357, 32741, 32869, 32603, 32943, 32870, 32851] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32870 expected_overflow=0 exact_recv_per_rank=[32910, 32357, 32741, 32869, 32603, 32943, 32870, 32851] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32910, 32357, 32741, 32869, 32603, 32943, 32870, 32851] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32910 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32869 expected_overflow=0 exact_recv_per_rank=[32910, 32357, 32741, 32869, 32603, 32943, 32870, 32851] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32357 actual_valid_recv=36096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32603 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32943 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36096 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32851 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32870 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32869 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[32910, 32357, 32741, 32869, 32603, 32943, 32870, 32851] avg=32768.0000 max=32943 max_rank=5 min=32357 min_rank=1 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[32910, 32357, 32741, 32869, 32603, 32943, 32870, 32851] avg=32768.0000 max=32943 max_rank=5 min=32357 min_rank=1 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[32910, 32357, 32741, 32869, 32603, 32943, 32870, 32851] avg=32768.0000 max=32943 max_rank=5 min=32357 min_rank=1 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[32910, 32357, 32741, 32869, 32603, 32943, 32870, 32851] avg=32768.0000 max=32943 max_rank=5 min=32357 min_rank=1 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[32910, 32357, 32741, 32869, 32603, 32943, 32870, 32851] avg=32768.0000 max=32943 max_rank=5 min=32357 min_rank=1 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[32910, 32357, 32741, 32869, 32603, 32943, 32870, 32851] avg=32768.0000 max=32943 max_rank=5 min=32357 min_rank=1 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[32910, 32357, 32741, 32869, 32603, 32943, 32870, 32851] avg=32768.0000 max=32943 max_rank=5 min=32357 min_rank=1 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[32910, 32357, 32741, 32869, 32603, 32943, 32870, 32851] avg=32768.0000 max=32943 max_rank=5 min=32357 min_rank=1 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[32910, 32357, 32741, 32869, 32603, 32943, 32870, 32851] avg=32768.0000 max=32943 max_rank=5 min=32357 min_rank=1 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[32910, 32357, 32741, 32869, 32603, 32943, 32870, 32851] avg=32768.0000 max=32943 max_rank=5 min=32357 min_rank=1 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[32910, 32357, 32741, 32869, 32603, 32943, 32870, 32851] avg=32768.0000 max=32943 max_rank=5 min=32357 min_rank=1 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[32910, 32357, 32741, 32869, 32603, 32943, 32870, 32851] avg=32768.0000 max=32943 max_rank=5 min=32357 min_rank=1 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[32910, 32357, 32741, 32869, 32603, 32943, 32870, 32851] avg=32768.0000 max=32943 max_rank=5 min=32357 min_rank=1 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[32910, 32357, 32741, 32869, 32603, 32943, 32870, 32851] avg=32768.0000 max=32943 max_rank=5 min=32357 min_rank=1 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[32910, 32357, 32741, 32869, 32603, 32943, 32870, 32851] avg=32768.0000 max=32943 max_rank=5 min=32357 min_rank=1 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[32910, 32357, 32741, 32869, 32603, 32943, 32870, 32851] avg=32768.0000 max=32943 max_rank=5 min=32357 min_rank=1 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=1 total_expected_recv=32910 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=32910 required_capacity=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=1 total_expected_recv=32357 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4864 peak_demand=32357 required_capacity=0.987457 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=1 total_expected_recv=32943 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=32943 required_capacity=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=1 total_expected_recv=32603 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=32603 required_capacity=0.994965 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=1 total_expected_recv=32851 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=32851 required_capacity=1.002533 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=1 total_expected_recv=32870 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=32870 required_capacity=1.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=1 total_expected_recv=32869 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=32869 required_capacity=1.003082 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=1 total_expected_recv=32741 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=32741 required_capacity=0.999176 + [2026-09-21 07:40:16] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 116.2 | throughput per GPU (TFLOP/s/GPU): 304.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.067696E+01 | loss scale: 1.0 | grad norm: 8.493 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36864 | min safe capacity: 1.1250 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32761, 33053, 33057, 32318, 32476, 32519, 33004, 32956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32956 expected_overflow=0 exact_recv_per_rank=[32761, 33053, 33057, 32318, 32476, 32519, 33004, 32956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32519 expected_overflow=0 exact_recv_per_rank=[32761, 33053, 33057, 32318, 32476, 32519, 33004, 32956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32476 expected_overflow=0 exact_recv_per_rank=[32761, 33053, 33057, 32318, 32476, 32519, 33004, 32956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33053 expected_overflow=0 exact_recv_per_rank=[32761, 33053, 33057, 32318, 32476, 32519, 33004, 32956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=36480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33004 expected_overflow=0 exact_recv_per_rank=[32761, 33053, 33057, 32318, 32476, 32519, 33004, 32956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33057 expected_overflow=0 exact_recv_per_rank=[32761, 33053, 33057, 32318, 32476, 32519, 33004, 32956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32956 actual_valid_recv=36736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32318 expected_overflow=0 exact_recv_per_rank=[32761, 33053, 33057, 32318, 32476, 32519, 33004, 32956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32519 actual_valid_recv=36096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36480 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32476 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33053 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33004 actual_valid_recv=36736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36736 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=40960 expected_recv=33057 actual_valid_recv=36352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36096 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32318 actual_valid_recv=36224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36736 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36352 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=36224 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[32761, 33053, 33057, 32318, 32476, 32519, 33004, 32956] avg=32768.0000 max=33057 max_rank=2 min=32318 min_rank=3 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[32761, 33053, 33057, 32318, 32476, 32519, 33004, 32956] avg=32768.0000 max=33057 max_rank=2 min=32318 min_rank=3 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[32761, 33053, 33057, 32318, 32476, 32519, 33004, 32956] avg=32768.0000 max=33057 max_rank=2 min=32318 min_rank=3 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[32761, 33053, 33057, 32318, 32476, 32519, 33004, 32956] avg=32768.0000 max=33057 max_rank=2 min=32318 min_rank=3 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[32761, 33053, 33057, 32318, 32476, 32519, 33004, 32956] avg=32768.0000 max=33057 max_rank=2 min=32318 min_rank=3 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[32761, 33053, 33057, 32318, 32476, 32519, 33004, 32956] avg=32768.0000 max=33057 max_rank=2 min=32318 min_rank=3 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[32761, 33053, 33057, 32318, 32476, 32519, 33004, 32956] avg=32768.0000 max=33057 max_rank=2 min=32318 min_rank=3 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[32761, 33053, 33057, 32318, 32476, 32519, 33004, 32956] avg=32768.0000 max=33057 max_rank=2 min=32318 min_rank=3 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[32761, 33053, 33057, 32318, 32476, 32519, 33004, 32956] avg=32768.0000 max=33057 max_rank=2 min=32318 min_rank=3 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[32761, 33053, 33057, 32318, 32476, 32519, 33004, 32956] avg=32768.0000 max=33057 max_rank=2 min=32318 min_rank=3 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[32761, 33053, 33057, 32318, 32476, 32519, 33004, 32956] avg=32768.0000 max=33057 max_rank=2 min=32318 min_rank=3 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[32761, 33053, 33057, 32318, 32476, 32519, 33004, 32956] avg=32768.0000 max=33057 max_rank=2 min=32318 min_rank=3 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[32761, 33053, 33057, 32318, 32476, 32519, 33004, 32956] avg=32768.0000 max=33057 max_rank=2 min=32318 min_rank=3 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[32761, 33053, 33057, 32318, 32476, 32519, 33004, 32956] avg=32768.0000 max=33057 max_rank=2 min=32318 min_rank=3 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[32761, 33053, 33057, 32318, 32476, 32519, 33004, 32956] avg=32768.0000 max=33057 max_rank=2 min=32318 min_rank=3 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[32761, 33053, 33057, 32318, 32476, 32519, 33004, 32956] avg=32768.0000 max=33057 max_rank=2 min=32318 min_rank=3 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=1 total_expected_recv=32761 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4480 peak_demand=32761 required_capacity=0.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=1 total_expected_recv=32956 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4224 peak_demand=32956 required_capacity=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=1 total_expected_recv=32519 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4864 peak_demand=32519 required_capacity=0.992401 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=1 total_expected_recv=32476 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=32476 required_capacity=0.991089 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=1 total_expected_recv=33053 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=33053 required_capacity=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=1 total_expected_recv=33004 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4224 peak_demand=33004 required_capacity=1.007202 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=1 total_expected_recv=33057 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4608 peak_demand=33057 required_capacity=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=1 total_expected_recv=32318 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=4736 peak_demand=32318 required_capacity=0.986267 + [2026-09-21 07:40:16] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 116.0 | throughput per GPU (TFLOP/s/GPU): 304.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.043214E+01 | loss scale: 1.0 | grad norm: 8.163 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36864 | min safe capacity: 1.1250 | +[after training is done] datetime: 2026-09-21 07:40:16 +[rank0]:[W921 07:40:17.931013272 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0921 07:40:18.709000 482786 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0921 07:40:18.709000 482786 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 482893 closing signal SIGTERM +W0921 07:40:18.710000 482786 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 482894 closing signal SIGTERM +W0921 07:40:18.711000 482786 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 482895 closing signal SIGTERM +W0921 07:40:18.712000 482786 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 482896 closing signal SIGTERM +W0921 07:40:18.713000 482786 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 482897 closing signal SIGTERM +W0921 07:40:18.714000 482786 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 482898 closing signal SIGTERM +W0921 07:40:18.715000 482786 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 482899 closing signal SIGTERM +W0921 07:40:18.715000 482786 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 482900 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 482786 got signal: 15 diff --git a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv new file mode 100644 index 0000000000000000000000000000000000000000..7d85bb32ddd033632e20e9e4a914b8cc9cb30fa9 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv @@ -0,0 +1,4 @@ +model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir +KIMI_K2_L2 ultraep measure 1.25 OK 89825 1 0.0 1.010071 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25/rep1 +KIMI_K2_L2 ultraep validate 1.0625 OK 89513 1 0.0 1.010071 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p0625/rep1 +KIMI_K2_L2 ultraep forced 1.25 OK 89825 1 0.0 1.01004 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1 diff --git a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv new file mode 100644 index 0000000000000000000000000000000000000000..64e5630fe122433aa852092d5cb488ca3747d1f1 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv @@ -0,0 +1,2 @@ +model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir +KIMI_K2_L2 ultraep SELECTED 1.0625 1.010071 0.0 89513 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p0625/rep1 diff --git a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.json b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..69347b37d281e74e119a03af62a021f3c4a1ee7a --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.json @@ -0,0 +1,233 @@ +[ + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "0", + "backend": "ultraep", + "branch": "ultraep-token-drop-analysis", + "capacity": "1.0625", + "capacity_mode": "static", + "commit": "eaf829e7", + "config": "KIMI_K2_L2", + "cuda_graph": "0", + "dense_first_layers": "1", + "drop_expected_assignments": 3145728.0, + "drop_rate_pct_max": 0.0, + "drop_rate_pct_mean": 0.0, + "drop_rate_pct_min": 0.0, + "drop_rate_pct_n": 96, + "drop_rate_pct_overall": 0.0, + "dropped_tokens_sum_all_records": 0.0, + "duplicate_vector_mismatches": 0, + "experts_per_rank": "48", + "force_balance": "0", + "global_batch_size": "8", + "global_snapshots": 12, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "1", + "num_experts": "384", + "num_layers": "2", + "observed_dropped_tokens": 0.0, + "peak_demand_max": 33098.0, + "peak_demand_mean": 32768.0, + "peak_demand_min": 31213.0, + "peak_demand_n": 96, + "peak_rank_overflow_max": 0.0, + "peak_rank_overflow_mean": 0.0, + "peak_rank_overflow_min": 0.0, + "peak_rank_overflow_n": 96, + "post_physical_rank_cv_max": 0.018551700753199882, + "post_physical_rank_cv_mean": 0.014736188098268745, + "post_physical_rank_cv_min": 0.01142687710780522, + "post_physical_rank_cv_n": 12, + "post_physical_rank_imax_max": 1.01007080078125, + "post_physical_rank_imax_mean": 1.0100326538085938, + "post_physical_rank_imax_min": 1.010009765625, + "post_physical_rank_imax_n": 12, + "pre_home_rank_cv_max": 0.2350733530310411, + "pre_home_rank_cv_mean": 0.13086798523811746, + "pre_home_rank_cv_min": 0.0905917891947787, + "pre_home_rank_cv_n": 12, + "pre_home_rank_imax_max": 1.406707763671875, + "pre_home_rank_imax_mean": 1.2260996500651042, + "pre_home_rank_imax_min": 1.131988525390625, + "pre_home_rank_imax_n": 12, + "records": { + "drop_dispatch": 96, + "drop_experts": 96, + "drop_setup": 96, + "ultraep_drop": 96, + "ultraep_post": 96, + "ultraep_pre": 96 + }, + "required_capacity_max": 1.010071, + "required_capacity_mean": 1.0000000833333333, + "required_capacity_min": 0.952545, + "required_capacity_n": 96, + "router_balancing": "seq_aux_loss", + "run_dir": "KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p0625/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.282004555808656, + "valid_max_rank_reduction_mean": 0.17248486518149106, + "valid_max_rank_reduction_min": 0.10772922114684712, + "valid_max_rank_reduction_n": 12, + "zero_drop": 1 + }, + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "0", + "backend": "ultraep", + "branch": "ultraep-token-drop-analysis", + "capacity": "1.25", + "capacity_mode": "static", + "commit": "eaf829e7", + "config": "KIMI_K2_L2", + "cuda_graph": "0", + "dense_first_layers": "1", + "drop_expected_assignments": 3145728.0, + "drop_rate_pct_max": 0.0, + "drop_rate_pct_mean": 0.0, + "drop_rate_pct_min": 0.0, + "drop_rate_pct_n": 96, + "drop_rate_pct_overall": 0.0, + "dropped_tokens_sum_all_records": 0.0, + "duplicate_vector_mismatches": 0, + "experts_per_rank": "48", + "force_balance": "0", + "global_batch_size": "8", + "global_snapshots": 12, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "1", + "num_experts": "384", + "num_layers": "2", + "observed_dropped_tokens": 0.0, + "peak_demand_max": 33098.0, + "peak_demand_mean": 32768.0, + "peak_demand_min": 31238.0, + "peak_demand_n": 96, + "peak_rank_overflow_max": 0.0, + "peak_rank_overflow_mean": 0.0, + "peak_rank_overflow_min": 0.0, + "peak_rank_overflow_n": 96, + "post_physical_rank_cv_max": 0.018343139886457832, + "post_physical_rank_cv_mean": 0.014707084226669935, + "post_physical_rank_cv_min": 0.011387320530400494, + "post_physical_rank_cv_n": 12, + "post_physical_rank_imax_max": 1.01007080078125, + "post_physical_rank_imax_mean": 1.0100351969401042, + "post_physical_rank_imax_min": 1.010009765625, + "post_physical_rank_imax_n": 12, + "pre_home_rank_cv_max": 0.23003629950316515, + "pre_home_rank_cv_mean": 0.12688407124108866, + "pre_home_rank_cv_min": 0.08877670743785306, + "pre_home_rank_cv_n": 12, + "pre_home_rank_imax_max": 1.385650634765625, + "pre_home_rank_imax_mean": 1.2170664469401042, + "pre_home_rank_imax_min": 1.12664794921875, + "pre_home_rank_imax_n": 12, + "records": { + "drop_dispatch": 96, + "drop_experts": 96, + "drop_setup": 96, + "ultraep_drop": 96, + "ultraep_post": 96, + "ultraep_pre": 96 + }, + "required_capacity_max": 1.010071, + "required_capacity_mean": 1.0000000208333335, + "required_capacity_min": 0.953308, + "required_capacity_n": 96, + "router_balancing": "seq_aux_loss", + "run_dir": "KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.271071467900011, + "valid_max_rank_reduction_mean": 0.16650987487084534, + "valid_max_rank_reduction_min": 0.10352673492605233, + "valid_max_rank_reduction_n": 12, + "zero_drop": 1 + }, + { + "arm": "forced", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "0", + "backend": "ultraep", + "branch": "ultraep-token-drop-analysis", + "capacity": "1.25", + "capacity_mode": "static", + "commit": "eaf829e7", + "config": "KIMI_K2_L2", + "cuda_graph": "0", + "dense_first_layers": "1", + "drop_expected_assignments": 3145728.0, + "drop_rate_pct_max": 0.0, + "drop_rate_pct_mean": 0.0, + "drop_rate_pct_min": 0.0, + "drop_rate_pct_n": 96, + "drop_rate_pct_overall": 0.0, + "dropped_tokens_sum_all_records": 0.0, + "duplicate_vector_mismatches": 0, + "experts_per_rank": "48", + "force_balance": "1", + "global_batch_size": "8", + "global_snapshots": 12, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "1", + "num_experts": "384", + "num_layers": "2", + "observed_dropped_tokens": 0.0, + "peak_demand_max": 33097.0, + "peak_demand_mean": 32768.0, + "peak_demand_min": 32318.0, + "peak_demand_n": 96, + "peak_rank_overflow_max": 0.0, + "peak_rank_overflow_mean": 0.0, + "peak_rank_overflow_min": 0.0, + "peak_rank_overflow_n": 96, + "post_physical_rank_cv_max": 0.008620891733745346, + "post_physical_rank_cv_mean": 0.00609096140081787, + "post_physical_rank_cv_min": 0.0033077098989994554, + "post_physical_rank_cv_n": 12, + "post_physical_rank_imax_max": 1.010040283203125, + "post_physical_rank_imax_mean": 1.0083948771158855, + "post_physical_rank_imax_min": 1.005340576171875, + "post_physical_rank_imax_n": 12, + "pre_home_rank_cv_max": 0.009327227347999425, + "pre_home_rank_cv_mean": 0.006896456793407019, + "pre_home_rank_cv_min": 0.0033077098989994554, + "pre_home_rank_cv_n": 12, + "pre_home_rank_imax_max": 1.02001953125, + "pre_home_rank_imax_mean": 1.0104878743489583, + "pre_home_rank_imax_min": 1.005340576171875, + "pre_home_rank_imax_n": 12, + "records": { + "drop_dispatch": 96, + "drop_experts": 96, + "drop_setup": 96, + "ultraep_drop": 96, + "ultraep_post": 96, + "ultraep_pre": 96 + }, + "required_capacity_max": 1.01004, + "required_capacity_mean": 0.9999999895833334, + "required_capacity_min": 0.986267, + "required_capacity_n": 96, + "router_balancing": "seq_aux_loss", + "run_dir": "KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.009843226424126376, + "valid_max_rank_reduction_mean": 0.002057005642129182, + "valid_max_rank_reduction_min": 0.0, + "valid_max_rank_reduction_n": 12, + "zero_drop": 1 + } +] \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv new file mode 100644 index 0000000000000000000000000000000000000000..7e84b225a5197000a55815a6b12062faf6736869 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv @@ -0,0 +1,4 @@ +run_dir arm config backend num_experts topk experts_per_rank num_layers moe_layers dense_first_layers seq_length global_batch_size capacity_mode capacity force_balance cuda_graph branch commit router_balancing aux_loss_coeff shared_expert_overlap assignment_sum_mismatches drop_expected_assignments drop_rate_pct_max drop_rate_pct_mean drop_rate_pct_min drop_rate_pct_n drop_rate_pct_overall dropped_tokens_sum_all_records duplicate_vector_mismatches global_snapshots incomplete_snapshots malformed_records observed_dropped_tokens peak_demand_max peak_demand_mean peak_demand_min peak_demand_n peak_rank_overflow_max peak_rank_overflow_mean peak_rank_overflow_min peak_rank_overflow_n post_physical_rank_cv_max post_physical_rank_cv_mean post_physical_rank_cv_min post_physical_rank_cv_n post_physical_rank_imax_max post_physical_rank_imax_mean post_physical_rank_imax_min post_physical_rank_imax_n pre_home_rank_cv_max pre_home_rank_cv_mean pre_home_rank_cv_min pre_home_rank_cv_n pre_home_rank_imax_max pre_home_rank_imax_mean pre_home_rank_imax_min pre_home_rank_imax_n required_capacity_max required_capacity_mean required_capacity_min required_capacity_n valid_max_rank_reduction_max valid_max_rank_reduction_mean valid_max_rank_reduction_min valid_max_rank_reduction_n zero_drop +KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p0625/rep1 natural KIMI_K2_L2 ultraep 384 8 48 2 1 1 4096 8 static 1.0625 0 0 ultraep-token-drop-analysis eaf829e7 seq_aux_loss 0 0 0 3145728.0 0.0 0.0 0.0 96 0.0 0.0 0 12 0 0 0.0 33098.0 32768.0 31213.0 96 0.0 0.0 0.0 96 0.018551700753199882 0.014736188098268745 0.01142687710780522 12 1.01007080078125 1.0100326538085938 1.010009765625 12 0.2350733530310411 0.13086798523811746 0.0905917891947787 12 1.406707763671875 1.2260996500651042 1.131988525390625 12 1.010071 1.0000000833333333 0.952545 96 0.282004555808656 0.17248486518149106 0.10772922114684712 12 1 +KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25/rep1 natural KIMI_K2_L2 ultraep 384 8 48 2 1 1 4096 8 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 seq_aux_loss 0 0 0 3145728.0 0.0 0.0 0.0 96 0.0 0.0 0 12 0 0 0.0 33098.0 32768.0 31238.0 96 0.0 0.0 0.0 96 0.018343139886457832 0.014707084226669935 0.011387320530400494 12 1.01007080078125 1.0100351969401042 1.010009765625 12 0.23003629950316515 0.12688407124108866 0.08877670743785306 12 1.385650634765625 1.2170664469401042 1.12664794921875 12 1.010071 1.0000000208333335 0.953308 96 0.271071467900011 0.16650987487084534 0.10352673492605233 12 1 +KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1 forced KIMI_K2_L2 ultraep 384 8 48 2 1 1 4096 8 static 1.25 1 0 ultraep-token-drop-analysis eaf829e7 seq_aux_loss 0 0 0 3145728.0 0.0 0.0 0.0 96 0.0 0.0 0 12 0 0 0.0 33097.0 32768.0 32318.0 96 0.0 0.0 0.0 96 0.008620891733745346 0.00609096140081787 0.0033077098989994554 12 1.010040283203125 1.0083948771158855 1.005340576171875 12 0.009327227347999425 0.006896456793407019 0.0033077098989994554 12 1.02001953125 1.0104878743489583 1.005340576171875 12 1.01004 0.9999999895833334 0.986267 96 0.009843226424126376 0.002057005642129182 0.0 12 1 diff --git a/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/results.tsv b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/results.tsv new file mode 100644 index 0000000000000000000000000000000000000000..ba7f04201ffaafc8285850a4a1b8a37c980bfb87 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/results.tsv @@ -0,0 +1,4 @@ +config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir +KIMI_K2_L2 ultraep matched 48 4096 8 1.25 none 1 OK 124.24 122.40 89825 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25/rep1 +KIMI_K2_L2 ultraep matched 48 4096 8 1.0625 none 1 OK 125.07 122.00 89513 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p0625/rep1 +KIMI_K2_L2 ultraep matched 48 4096 8 1.25 none+forcebal 1 OK 116.93 115.90 89825 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierbprobe/tierbprobe_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1